| { |
| "release": "v2.0.0", |
| "official_replication_evaluation_date": "2026-07-27", |
| "multiturn_diagnostic_date": "2026-08-04", |
| "official_replication": { |
| "protocol": "vistec-AI/thaillm-medical-post-training@73772633663dfe02eff558a85eacbac9f617d329", |
| "med_iq": { |
| "examples": 200, |
| "rollouts_per_example": 3, |
| "judge_model": "deepseek/deepseek-v4-flash", |
| "v2": { |
| "format_reward": 1.0, |
| "citations_reward": 0.6782222222222223, |
| "response_reward": 0.7583333333333333, |
| "combined_reward": 1.5365555555555557, |
| "judge_errors": 0 |
| }, |
| "v1": { |
| "format_reward": 0.9566666666666667, |
| "citations_reward": 0.6228968253968254, |
| "response_reward": 0.6316666666666667, |
| "combined_reward": 1.3502301587301588, |
| "judge_errors": 0 |
| } |
| }, |
| "tooluse": { |
| "examples": 5122, |
| "rollouts_per_example": 3, |
| "v2": { |
| "pass_at_1_accuracy": 0.9991539763113367, |
| "pass_at_1_trigger_f1": 1.0, |
| "pass_at_1_macro_f1": 0.9939327426620813, |
| "pass_at_3_accuracy": 0.9994142912924638, |
| "rollout_errors": 0 |
| }, |
| "v1": { |
| "pass_at_1_accuracy": 0.9035532994923858, |
| "pass_at_1_trigger_f1": 0.8753987433542773, |
| "pass_at_1_macro_f1": 0.7829973019437224, |
| "pass_at_3_accuracy": 0.9478719250292854, |
| "rollout_errors": 3 |
| } |
| } |
| }, |
| "multiturn_diagnostic": { |
| "run_id": "20260804_223124", |
| "scenarios": 10, |
| "turns_per_scenario": 8, |
| "repeats": 2, |
| "responses_per_profile": 160, |
| "v1_published_style": { |
| "flag_rate": 0.46875, |
| "mean_repeated_12gram_fraction": 0.2005, |
| "mean_max_previous_similarity": 0.4083, |
| "length_caps": 61 |
| }, |
| "v1_stabilized": { |
| "flag_rate": 0.33125, |
| "mean_repeated_12gram_fraction": 0.1153, |
| "mean_max_previous_similarity": 0.3934, |
| "length_caps": 36 |
| }, |
| "v2_published_style": { |
| "flag_rate": 0.03125, |
| "mean_repeated_12gram_fraction": 0.0063, |
| "mean_max_previous_similarity": 0.302, |
| "length_caps": 0 |
| }, |
| "v2_stabilized": { |
| "flag_rate": 0.0, |
| "mean_repeated_12gram_fraction": 0.0078, |
| "mean_max_previous_similarity": 0.2199, |
| "length_caps": 0 |
| } |
| }, |
| "caveats": [ |
| "The official benchmark used the published temperature-0.4 protocol, not the stabilized decoding defaults.", |
| "The comparison is protocol-aligned; the v1 control did not exactly reproduce every value on the previous public model card.", |
| "Historical evaluation manifests recorded the selected model path but not a weight-file hash; release staging created the first content-addressed fingerprint of that output.", |
| "Multi-turn flags are automated degeneration heuristics, not medical-correctness scores." |
| ] |
| } |
|
|
