{ "paper": "Burstiness Was Measured Wrong, and Prompting Cannot Aim It", "source": "https://openreview.net/forum?id=1E20ig92Zi", "delta_definition": "human minus model", "models": [ { "key": "deepseek-v4-pro", "name": "DeepSeek V4 Pro", "scored": { "label": "deepseek-v4-norm", "dims": { "B_n": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.035, "median_delta_95ci": [ 0.011, 0.053 ], "wilcoxon_z": 4.14, "wilcoxon_p": 3e-05, "matched_rank_biserial": 0.338, "paired_dz": 0.336 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.029, "median_delta_95ci": [ 0.009, 0.049 ], "wilcoxon_z": 2.91, "wilcoxon_p": 0.00366, "matched_rank_biserial": 0.237, "paired_dz": 0.216 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.0, "median_delta_95ci": [ -0.012, 0.026 ], "wilcoxon_z": 0.56, "wilcoxon_p": 0.57212, "matched_rank_biserial": 0.046, "paired_dz": 0.047 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.013, "median_delta_95ci": [ -0.01, 0.04 ], "wilcoxon_z": 1.8, "wilcoxon_p": 0.07171, "matched_rank_biserial": 0.147, "paired_dz": 0.149 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.018, "median_delta_95ci": [ -0.032, 0.003 ], "wilcoxon_z": -1.18, "wilcoxon_p": 0.23804, "matched_rank_biserial": -0.096, "paired_dz": -0.071 } }, "M": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.025, "median_delta_95ci": [ -0.058, 0.035 ], "wilcoxon_z": -0.91, "wilcoxon_p": 0.36077, "matched_rank_biserial": -0.075, "paired_dz": -0.073 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.012, "median_delta_95ci": [ -0.044, 0.056 ], "wilcoxon_z": 0.16, "wilcoxon_p": 0.87204, "matched_rank_biserial": 0.013, "paired_dz": 0.004 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.006, "median_delta_95ci": [ -0.033, 0.059 ], "wilcoxon_z": 0.89, "wilcoxon_p": 0.37373, "matched_rank_biserial": 0.073, "paired_dz": 0.065 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.003, "median_delta_95ci": [ -0.047, 0.046 ], "wilcoxon_z": 0.38, "wilcoxon_p": 0.70524, "matched_rank_biserial": 0.031, "paired_dz": 0.033 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.027, "median_delta_95ci": [ -0.0, 0.068 ], "wilcoxon_z": 1.77, "wilcoxon_p": 0.07726, "matched_rank_biserial": 0.144, "paired_dz": 0.12 } }, "gini": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.018, "median_delta_95ci": [ 0.009, 0.029 ], "wilcoxon_z": 4.27, "wilcoxon_p": 2e-05, "matched_rank_biserial": 0.348, "paired_dz": 0.33 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.014, "median_delta_95ci": [ 0.001, 0.022 ], "wilcoxon_z": 2.51, "wilcoxon_p": 0.01212, "matched_rank_biserial": 0.205, "paired_dz": 0.178 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.005, "median_delta_95ci": [ -0.01, 0.008 ], "wilcoxon_z": -0.2, "wilcoxon_p": 0.84522, "matched_rank_biserial": -0.016, "paired_dz": -0.006 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.003, "median_delta_95ci": [ -0.01, 0.015 ], "wilcoxon_z": 0.96, "wilcoxon_p": 0.33815, "matched_rank_biserial": 0.078, "paired_dz": 0.084 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.01, "median_delta_95ci": [ -0.018, -0.0 ], "wilcoxon_z": -1.37, "wilcoxon_p": 0.17061, "matched_rank_biserial": -0.112, "paired_dz": -0.084 } }, "seg_entropy": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.05, "median_delta_95ci": [ -0.091, 0.016 ], "wilcoxon_z": -1.48, "wilcoxon_p": 0.14016, "matched_rank_biserial": -0.12, "paired_dz": -0.108 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": -0.068, "median_delta_95ci": [ -0.117, -0.021 ], "wilcoxon_z": -3.19, "wilcoxon_p": 0.00143, "matched_rank_biserial": -0.26, "paired_dz": -0.237 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.145, "median_delta_95ci": [ -0.181, -0.116 ], "wilcoxon_z": -5.76, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.469, "paired_dz": -0.449 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.117, "median_delta_95ci": [ -0.177, -0.066 ], "wilcoxon_z": -4.41, "wilcoxon_p": 1e-05, "matched_rank_biserial": -0.36, "paired_dz": -0.322 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.139, "median_delta_95ci": [ -0.192, -0.1 ], "wilcoxon_z": -6.05, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.493, "paired_dz": -0.463 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.277, "it1": 0.336, "it2": 0.334 }, "fewshot": { "it0": 0.212, "it1": 0.235, "it2": 0.198 }, "instruction": { "it0": 0.022, "it1": -0.045, "it2": 0.139 }, "plan": { "it0": 0.155, "it1": 0.142, "it2": 0.118 }, "self_refine": { "it0": -0.1, "it1": -0.064, "it2": -0.09 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 8, "n_holm": 6, "n_bh": 8, "lost_under_bh": [], "lost_under_holm": [ "fewshot/B_n (p=0.004)", "fewshot/gini (p=0.012)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 3e-05, "rb": 0.338, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.00366, "rb": 0.237, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.57212, "rb": 0.046, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.07171, "rb": 0.147, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.23804, "rb": -0.096, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.36077, "rb": -0.075, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "fewshot", "p": 0.87204, "rb": 0.013, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.37373, "rb": 0.073, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.70524, "rb": 0.031, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "self_refine", "p": 0.07726, "rb": 0.144, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "control", "p": 2e-05, "rb": 0.348, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.01212, "rb": 0.205, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "gini", "mode": "instruction", "p": 0.84522, "rb": -0.016, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 0.33815, "rb": 0.078, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.17061, "rb": -0.112, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.14016, "rb": -0.12, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.00143, "rb": -0.26, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.0, "rb": -0.469, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 1e-05, "rb": -0.36, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 0.0, "rb": -0.493, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false } ] } }, { "key": "gemini-3.1-pro", "name": "Gemini 3.1 Pro", "scored": { "label": "gemini-3.1-pro", "dims": { "B_n": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.128, "median_delta_95ci": [ 0.102, 0.14 ], "wilcoxon_z": 10.44, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.852, "paired_dz": 0.998 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.169, "median_delta_95ci": [ 0.148, 0.183 ], "wilcoxon_z": 11.57, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.944, "paired_dz": 1.271 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.071, "median_delta_95ci": [ 0.05, 0.09 ], "wilcoxon_z": 6.83, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.557, "paired_dz": 0.546 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.07, "median_delta_95ci": [ 0.049, 0.088 ], "wilcoxon_z": 7.13, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.581, "paired_dz": 0.561 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.039, "median_delta_95ci": [ 0.016, 0.053 ], "wilcoxon_z": 3.33, "wilcoxon_p": 0.00086, "matched_rank_biserial": 0.272, "paired_dz": 0.26 } }, "M": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.043, "median_delta_95ci": [ -0.012, 0.097 ], "wilcoxon_z": 2.22, "wilcoxon_p": 0.02621, "matched_rank_biserial": 0.181, "paired_dz": 0.168 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.051, "median_delta_95ci": [ 0.009, 0.088 ], "wilcoxon_z": 2.94, "wilcoxon_p": 0.00324, "matched_rank_biserial": 0.24, "paired_dz": 0.228 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.109, "median_delta_95ci": [ 0.037, 0.146 ], "wilcoxon_z": 4.65, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.379, "paired_dz": 0.336 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.114, "median_delta_95ci": [ 0.067, 0.162 ], "wilcoxon_z": 5.72, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.466, "paired_dz": 0.429 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.047, "median_delta_95ci": [ -0.013, 0.09 ], "wilcoxon_z": 1.46, "wilcoxon_p": 0.14314, "matched_rank_biserial": 0.119, "paired_dz": 0.083 } }, "gini": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.056, "median_delta_95ci": [ 0.047, 0.066 ], "wilcoxon_z": 9.95, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.812, "paired_dz": 0.908 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.08, "median_delta_95ci": [ 0.07, 0.091 ], "wilcoxon_z": 11.35, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.926, "paired_dz": 1.195 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.026, "median_delta_95ci": [ 0.019, 0.037 ], "wilcoxon_z": 5.8, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.473, "paired_dz": 0.45 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.028, "median_delta_95ci": [ 0.017, 0.043 ], "wilcoxon_z": 5.88, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.479, "paired_dz": 0.448 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.013, "median_delta_95ci": [ 0.003, 0.024 ], "wilcoxon_z": 2.59, "wilcoxon_p": 0.00952, "matched_rank_biserial": 0.211, "paired_dz": 0.201 } }, "seg_entropy": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.062, "median_delta_95ci": [ -0.105, -0.003 ], "wilcoxon_z": -2.31, "wilcoxon_p": 0.02083, "matched_rank_biserial": -0.188, "paired_dz": -0.148 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.117, "median_delta_95ci": [ 0.059, 0.153 ], "wilcoxon_z": 4.74, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.387, "paired_dz": 0.332 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.053, "median_delta_95ci": [ -0.088, -0.012 ], "wilcoxon_z": -1.95, "wilcoxon_p": 0.05062, "matched_rank_biserial": -0.159, "paired_dz": -0.104 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.026, "median_delta_95ci": [ -0.061, 0.04 ], "wilcoxon_z": -0.79, "wilcoxon_p": 0.42985, "matched_rank_biserial": -0.064, "paired_dz": -0.036 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.051, "median_delta_95ci": [ -0.1, 0.023 ], "wilcoxon_z": -1.43, "wilcoxon_p": 0.15166, "matched_rank_biserial": -0.117, "paired_dz": -0.085 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.793, "it1": 0.835, "it2": 0.782 }, "fewshot": { "it0": 0.914, "it1": 0.895, "it2": 0.925 }, "instruction": { "it0": 0.494, "it1": 0.542, "it2": 0.507 }, "plan": { "it0": 0.56, "it1": 0.541, "it2": 0.454 }, "self_refine": { "it0": 0.227, "it1": 0.24, "it2": 0.28 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 16, "n_holm": 13, "n_bh": 16, "lost_under_bh": [], "lost_under_holm": [ "control/M (p=0.026)", "self_refine/gini (p=0.010)", "control/seg_entropy (p=0.021)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.0, "rb": 0.852, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.0, "rb": 0.944, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.0, "rb": 0.557, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.0, "rb": 0.581, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.00086, "rb": 0.272, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.02621, "rb": 0.181, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "M", "mode": "fewshot", "p": 0.00324, "rb": 0.24, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.0, "rb": 0.379, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.0, "rb": 0.466, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "self_refine", "p": 0.14314, "rb": 0.119, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "control", "p": 0.0, "rb": 0.812, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.0, "rb": 0.926, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 0.0, "rb": 0.473, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 0.0, "rb": 0.479, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.00952, "rb": 0.211, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.02083, "rb": -0.188, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.0, "rb": 0.387, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.05062, "rb": -0.159, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 0.42985, "rb": -0.064, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 0.15166, "rb": -0.117, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false } ] } }, { "key": "gemma-3-27b", "name": "Gemma 3 27B", "scored": { "label": "gemma-3-27b", "dims": { "B_n": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.069, "median_delta_95ci": [ 0.046, 0.099 ], "wilcoxon_z": 7.12, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.581, "paired_dz": 0.575 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.063, "median_delta_95ci": [ 0.047, 0.1 ], "wilcoxon_z": 7.95, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.649, "paired_dz": 0.666 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.049, "median_delta_95ci": [ 0.031, 0.067 ], "wilcoxon_z": 5.42, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.442, "paired_dz": 0.419 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.061, "median_delta_95ci": [ 0.046, 0.081 ], "wilcoxon_z": 7.0, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.571, "paired_dz": 0.553 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.029, "median_delta_95ci": [ 0.009, 0.048 ], "wilcoxon_z": 3.63, "wilcoxon_p": 0.00028, "matched_rank_biserial": 0.296, "paired_dz": 0.315 } }, "M": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.023, "median_delta_95ci": [ -0.046, 0.076 ], "wilcoxon_z": 1.4, "wilcoxon_p": 0.16165, "matched_rank_biserial": 0.114, "paired_dz": 0.113 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.005, "median_delta_95ci": [ -0.048, 0.06 ], "wilcoxon_z": 0.21, "wilcoxon_p": 0.83472, "matched_rank_biserial": 0.017, "paired_dz": 0.019 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.04, "median_delta_95ci": [ -0.008, 0.08 ], "wilcoxon_z": 1.25, "wilcoxon_p": 0.21239, "matched_rank_biserial": 0.102, "paired_dz": 0.07 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.003, "median_delta_95ci": [ -0.039, 0.054 ], "wilcoxon_z": 0.39, "wilcoxon_p": 0.6962, "matched_rank_biserial": 0.032, "paired_dz": 0.031 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.048, "median_delta_95ci": [ -0.011, 0.087 ], "wilcoxon_z": 2.62, "wilcoxon_p": 0.00874, "matched_rank_biserial": 0.214, "paired_dz": 0.194 } }, "gini": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.037, "median_delta_95ci": [ 0.023, 0.05 ], "wilcoxon_z": 7.16, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.583, "paired_dz": 0.57 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.033, "median_delta_95ci": [ 0.023, 0.048 ], "wilcoxon_z": 7.74, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.632, "paired_dz": 0.646 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.025, "median_delta_95ci": [ 0.012, 0.033 ], "wilcoxon_z": 5.09, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.415, "paired_dz": 0.388 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.033, "median_delta_95ci": [ 0.02, 0.044 ], "wilcoxon_z": 6.74, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.55, "paired_dz": 0.526 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.015, "median_delta_95ci": [ 0.006, 0.023 ], "wilcoxon_z": 3.51, "wilcoxon_p": 0.00045, "matched_rank_biserial": 0.286, "paired_dz": 0.306 } }, "seg_entropy": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.063, "median_delta_95ci": [ 0.011, 0.11 ], "wilcoxon_z": 3.08, "wilcoxon_p": 0.00209, "matched_rank_biserial": 0.251, "paired_dz": 0.238 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.145, "median_delta_95ci": [ 0.094, 0.176 ], "wilcoxon_z": 5.91, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.482, "paired_dz": 0.476 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.086, "median_delta_95ci": [ 0.036, 0.134 ], "wilcoxon_z": 4.46, "wilcoxon_p": 1e-05, "matched_rank_biserial": 0.364, "paired_dz": 0.336 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.129, "median_delta_95ci": [ 0.089, 0.192 ], "wilcoxon_z": 5.57, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.454, "paired_dz": 0.438 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.094, "median_delta_95ci": [ 0.024, 0.145 ], "wilcoxon_z": 3.97, "wilcoxon_p": 7e-05, "matched_rank_biserial": 0.324, "paired_dz": 0.307 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.589, "it1": 0.532, "it2": 0.476 }, "fewshot": { "it0": 0.562, "it1": 0.564, "it2": 0.641 }, "instruction": { "it0": 0.379, "it1": 0.41, "it2": 0.404 }, "plan": { "it0": 0.544, "it1": 0.486, "it2": 0.494 }, "self_refine": { "it0": 0.28, "it1": 0.267, "it2": 0.29 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 16, "n_holm": 16, "n_bh": 16, "lost_under_bh": [], "lost_under_holm": [], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.0, "rb": 0.581, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.0, "rb": 0.649, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.0, "rb": 0.442, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.0, "rb": 0.571, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.00028, "rb": 0.296, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.16165, "rb": 0.114, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "fewshot", "p": 0.83472, "rb": 0.017, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.21239, "rb": 0.102, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.6962, "rb": 0.032, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "self_refine", "p": 0.00874, "rb": 0.214, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "control", "p": 0.0, "rb": 0.583, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.0, "rb": 0.632, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 0.0, "rb": 0.415, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 0.0, "rb": 0.55, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.00045, "rb": 0.286, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.00209, "rb": 0.251, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.0, "rb": 0.482, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 1e-05, "rb": 0.364, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 0.0, "rb": 0.454, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 7e-05, "rb": 0.324, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false } ] } }, { "key": "glm-5.2", "name": "GLM-5.2", "scored": { "label": "glm-5.2", "dims": { "B_n": { "control": { "n_pairs": 198, "median_delta_h_minus_ai": 0.126, "median_delta_95ci": [ 0.102, 0.15 ], "wilcoxon_z": 10.38, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.851, "paired_dz": 0.991 }, "fewshot": { "n_pairs": 197, "median_delta_h_minus_ai": 0.148, "median_delta_95ci": [ 0.123, 0.169 ], "wilcoxon_z": 11.41, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.937, "paired_dz": 1.209 }, "instruction": { "n_pairs": 198, "median_delta_h_minus_ai": 0.061, "median_delta_95ci": [ 0.047, 0.082 ], "wilcoxon_z": 6.99, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.573, "paired_dz": 0.561 }, "plan": { "n_pairs": 197, "median_delta_h_minus_ai": 0.08, "median_delta_95ci": [ 0.062, 0.099 ], "wilcoxon_z": 7.9, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.649, "paired_dz": 0.656 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.011, "median_delta_95ci": [ -0.01, 0.038 ], "wilcoxon_z": 1.63, "wilcoxon_p": 0.10359, "matched_rank_biserial": 0.133, "paired_dz": 0.153 } }, "M": { "control": { "n_pairs": 198, "median_delta_h_minus_ai": 0.053, "median_delta_95ci": [ -0.001, 0.091 ], "wilcoxon_z": 2.34, "wilcoxon_p": 0.01945, "matched_rank_biserial": 0.192, "paired_dz": 0.17 }, "fewshot": { "n_pairs": 197, "median_delta_h_minus_ai": 0.022, "median_delta_95ci": [ -0.026, 0.05 ], "wilcoxon_z": 0.9, "wilcoxon_p": 0.36587, "matched_rank_biserial": 0.074, "paired_dz": 0.064 }, "instruction": { "n_pairs": 198, "median_delta_h_minus_ai": 0.076, "median_delta_95ci": [ 0.007, 0.134 ], "wilcoxon_z": 2.96, "wilcoxon_p": 0.00303, "matched_rank_biserial": 0.243, "paired_dz": 0.225 }, "plan": { "n_pairs": 197, "median_delta_h_minus_ai": 0.034, "median_delta_95ci": [ 0.01, 0.065 ], "wilcoxon_z": 2.49, "wilcoxon_p": 0.0128, "matched_rank_biserial": 0.205, "paired_dz": 0.197 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.019, "median_delta_95ci": [ -0.057, 0.027 ], "wilcoxon_z": -0.29, "wilcoxon_p": 0.77151, "matched_rank_biserial": -0.024, "paired_dz": -0.014 } }, "gini": { "control": { "n_pairs": 198, "median_delta_h_minus_ai": 0.058, "median_delta_95ci": [ 0.048, 0.072 ], "wilcoxon_z": 10.06, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.824, "paired_dz": 0.927 }, "fewshot": { "n_pairs": 197, "median_delta_h_minus_ai": 0.068, "median_delta_95ci": [ 0.058, 0.081 ], "wilcoxon_z": 11.13, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.915, "paired_dz": 1.129 }, "instruction": { "n_pairs": 198, "median_delta_h_minus_ai": 0.029, "median_delta_95ci": [ 0.014, 0.037 ], "wilcoxon_z": 6.19, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.507, "paired_dz": 0.49 }, "plan": { "n_pairs": 197, "median_delta_h_minus_ai": 0.034, "median_delta_95ci": [ 0.024, 0.047 ], "wilcoxon_z": 7.13, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.586, "paired_dz": 0.568 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.002, "median_delta_95ci": [ -0.01, 0.017 ], "wilcoxon_z": 1.04, "wilcoxon_p": 0.29627, "matched_rank_biserial": 0.085, "paired_dz": 0.109 } }, "seg_entropy": { "control": { "n_pairs": 198, "median_delta_h_minus_ai": 0.068, "median_delta_95ci": [ 0.018, 0.12 ], "wilcoxon_z": 3.03, "wilcoxon_p": 0.00246, "matched_rank_biserial": 0.248, "paired_dz": 0.251 }, "fewshot": { "n_pairs": 197, "median_delta_h_minus_ai": 0.221, "median_delta_95ci": [ 0.16, 0.28 ], "wilcoxon_z": 9.03, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.742, "paired_dz": 0.8 }, "instruction": { "n_pairs": 198, "median_delta_h_minus_ai": 0.064, "median_delta_95ci": [ 0.011, 0.138 ], "wilcoxon_z": 3.19, "wilcoxon_p": 0.00143, "matched_rank_biserial": 0.261, "paired_dz": 0.245 }, "plan": { "n_pairs": 197, "median_delta_h_minus_ai": 0.091, "median_delta_95ci": [ 0.067, 0.138 ], "wilcoxon_z": 3.92, "wilcoxon_p": 9e-05, "matched_rank_biserial": 0.322, "paired_dz": 0.312 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.05, "median_delta_95ci": [ 0.029, 0.133 ], "wilcoxon_z": 4.09, "wilcoxon_p": 4e-05, "matched_rank_biserial": 0.333, "paired_dz": 0.331 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.785, "it1": 0.811, "it2": 0.823 }, "fewshot": { "it0": 0.863, "it1": 0.892, "it2": 0.912 }, "instruction": { "it0": 0.551, "it1": 0.516, "it2": 0.485 }, "plan": { "it0": 0.613, "it1": 0.568, "it2": 0.568 }, "self_refine": { "it0": 0.07, "it1": 0.181, "it2": 0.114 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 16, "n_holm": 14, "n_bh": 16, "lost_under_bh": [], "lost_under_holm": [ "control/M (p=0.019)", "plan/M (p=0.013)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.0, "rb": 0.851, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.0, "rb": 0.937, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.0, "rb": 0.573, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.0, "rb": 0.649, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.10359, "rb": 0.133, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.01945, "rb": 0.192, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "M", "mode": "fewshot", "p": 0.36587, "rb": 0.074, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.00303, "rb": 0.243, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.0128, "rb": 0.205, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "M", "mode": "self_refine", "p": 0.77151, "rb": -0.024, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "control", "p": 0.0, "rb": 0.824, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.0, "rb": 0.915, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 0.0, "rb": 0.507, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 0.0, "rb": 0.586, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.29627, "rb": 0.085, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.00246, "rb": 0.248, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.0, "rb": 0.742, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.00143, "rb": 0.261, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 9e-05, "rb": 0.322, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 4e-05, "rb": 0.333, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false } ] } }, { "key": "grok-4.3", "name": "Grok 4.3", "scored": { "label": "grok-4.3-norm", "dims": { "B_n": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.027, "median_delta_95ci": [ 0.005, 0.044 ], "wilcoxon_z": 3.31, "wilcoxon_p": 0.00094, "matched_rank_biserial": 0.27, "paired_dz": 0.25 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.108, "median_delta_95ci": [ 0.085, 0.133 ], "wilcoxon_z": 9.34, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.762, "paired_dz": 0.803 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.05, "median_delta_95ci": [ 0.03, 0.068 ], "wilcoxon_z": 5.18, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.423, "paired_dz": 0.399 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.061, "median_delta_95ci": [ 0.038, 0.085 ], "wilcoxon_z": 6.8, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.555, "paired_dz": 0.53 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.019, "median_delta_95ci": [ -0.003, 0.042 ], "wilcoxon_z": 2.28, "wilcoxon_p": 0.02287, "matched_rank_biserial": 0.186, "paired_dz": 0.162 } }, "M": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.004, "median_delta_95ci": [ -0.059, 0.047 ], "wilcoxon_z": 0.1, "wilcoxon_p": 0.9203, "matched_rank_biserial": 0.008, "paired_dz": 0.014 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.001, "median_delta_95ci": [ -0.057, 0.025 ], "wilcoxon_z": -0.62, "wilcoxon_p": 0.53616, "matched_rank_biserial": -0.05, "paired_dz": -0.047 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.028, "median_delta_95ci": [ -0.017, 0.069 ], "wilcoxon_z": 1.19, "wilcoxon_p": 0.2337, "matched_rank_biserial": 0.097, "paired_dz": 0.069 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.023, "median_delta_95ci": [ -0.037, 0.054 ], "wilcoxon_z": 0.39, "wilcoxon_p": 0.6971, "matched_rank_biserial": 0.032, "paired_dz": 0.004 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.028, "median_delta_95ci": [ -0.029, 0.086 ], "wilcoxon_z": 1.69, "wilcoxon_p": 0.09057, "matched_rank_biserial": 0.138, "paired_dz": 0.114 } }, "gini": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.01, "median_delta_95ci": [ 0.002, 0.022 ], "wilcoxon_z": 2.82, "wilcoxon_p": 0.00481, "matched_rank_biserial": 0.23, "paired_dz": 0.212 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.052, "median_delta_95ci": [ 0.039, 0.067 ], "wilcoxon_z": 9.11, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.743, "paired_dz": 0.769 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.026, "median_delta_95ci": [ 0.015, 0.036 ], "wilcoxon_z": 4.83, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.394, "paired_dz": 0.351 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.026, "median_delta_95ci": [ 0.017, 0.038 ], "wilcoxon_z": 6.08, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.496, "paired_dz": 0.467 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.009, "median_delta_95ci": [ -0.008, 0.018 ], "wilcoxon_z": 1.69, "wilcoxon_p": 0.09081, "matched_rank_biserial": 0.138, "paired_dz": 0.126 } }, "seg_entropy": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.046, "median_delta_95ci": [ -0.115, 0.004 ], "wilcoxon_z": -1.91, "wilcoxon_p": 0.0565, "matched_rank_biserial": -0.156, "paired_dz": -0.133 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.18, "median_delta_95ci": [ 0.104, 0.24 ], "wilcoxon_z": 6.87, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.561, "paired_dz": 0.551 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.062, "median_delta_95ci": [ 0.032, 0.102 ], "wilcoxon_z": 2.32, "wilcoxon_p": 0.0203, "matched_rank_biserial": 0.189, "paired_dz": 0.168 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.136, "median_delta_95ci": [ 0.055, 0.185 ], "wilcoxon_z": 4.35, "wilcoxon_p": 1e-05, "matched_rank_biserial": 0.355, "paired_dz": 0.341 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.055, "median_delta_95ci": [ 0.004, 0.123 ], "wilcoxon_z": 2.62, "wilcoxon_p": 0.00886, "matched_rank_biserial": 0.213, "paired_dz": 0.206 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.141, "it1": 0.324, "it2": 0.266 }, "fewshot": { "it0": 0.631, "it1": 0.692, "it2": 0.738 }, "instruction": { "it0": 0.402, "it1": 0.369, "it2": 0.364 }, "plan": { "it0": 0.528, "it1": 0.507, "it2": 0.461 }, "self_refine": { "it0": 0.131, "it1": 0.166, "it2": 0.183 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 13, "n_holm": 9, "n_bh": 13, "lost_under_bh": [], "lost_under_holm": [ "self_refine/B_n (p=0.023)", "control/gini (p=0.005)", "instruction/seg_entropy (p=0.020)", "self_refine/seg_entropy (p=0.009)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.00094, "rb": 0.27, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.0, "rb": 0.762, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.0, "rb": 0.423, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.0, "rb": 0.555, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.02287, "rb": 0.186, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "M", "mode": "control", "p": 0.9203, "rb": 0.008, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "fewshot", "p": 0.53616, "rb": -0.05, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.2337, "rb": 0.097, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.6971, "rb": 0.032, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "self_refine", "p": 0.09057, "rb": 0.138, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "control", "p": 0.00481, "rb": 0.23, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.0, "rb": 0.743, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 0.0, "rb": 0.394, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 0.0, "rb": 0.496, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.09081, "rb": 0.138, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.0565, "rb": -0.156, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.0, "rb": 0.561, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.0203, "rb": 0.189, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "seg_entropy", "mode": "plan", "p": 1e-05, "rb": 0.355, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 0.00886, "rb": 0.213, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": false } ] } }, { "key": "grok-4.5", "name": "Grok 4.5", "scored": { "label": "grok-4.5", "dims": { "B_n": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.009, "median_delta_95ci": [ -0.036, 0.003 ], "wilcoxon_z": -1.72, "wilcoxon_p": 0.08624, "matched_rank_biserial": -0.14, "paired_dz": -0.097 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.077, "median_delta_95ci": [ 0.058, 0.112 ], "wilcoxon_z": 8.51, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.694, "paired_dz": 0.723 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.025, "median_delta_95ci": [ -0.045, -0.006 ], "wilcoxon_z": -2.44, "wilcoxon_p": 0.01482, "matched_rank_biserial": -0.199, "paired_dz": -0.136 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.009, "median_delta_95ci": [ -0.023, 0.019 ], "wilcoxon_z": -0.29, "wilcoxon_p": 0.76964, "matched_rank_biserial": -0.024, "paired_dz": 0.012 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.059, "median_delta_95ci": [ -0.085, -0.024 ], "wilcoxon_z": -5.04, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.411, "paired_dz": -0.372 } }, "M": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.023, "median_delta_95ci": [ -0.067, 0.044 ], "wilcoxon_z": -0.85, "wilcoxon_p": 0.39304, "matched_rank_biserial": -0.07, "paired_dz": -0.077 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": -0.002, "median_delta_95ci": [ -0.045, 0.038 ], "wilcoxon_z": -0.56, "wilcoxon_p": 0.57544, "matched_rank_biserial": -0.046, "paired_dz": -0.052 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.048, "median_delta_95ci": [ 0.008, 0.086 ], "wilcoxon_z": 2.65, "wilcoxon_p": 0.00813, "matched_rank_biserial": 0.216, "paired_dz": 0.19 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.021, "median_delta_95ci": [ -0.04, 0.061 ], "wilcoxon_z": 0.7, "wilcoxon_p": 0.48445, "matched_rank_biserial": 0.057, "paired_dz": 0.048 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.071, "median_delta_95ci": [ 0.024, 0.095 ], "wilcoxon_z": 3.75, "wilcoxon_p": 0.00017, "matched_rank_biserial": 0.306, "paired_dz": 0.275 } }, "gini": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.011, "median_delta_95ci": [ -0.024, 0.001 ], "wilcoxon_z": -2.09, "wilcoxon_p": 0.03704, "matched_rank_biserial": -0.17, "paired_dz": -0.126 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.039, "median_delta_95ci": [ 0.03, 0.056 ], "wilcoxon_z": 8.31, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.678, "paired_dz": 0.699 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.014, "median_delta_95ci": [ -0.024, -0.001 ], "wilcoxon_z": -2.51, "wilcoxon_p": 0.01216, "matched_rank_biserial": -0.204, "paired_dz": -0.147 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.005, "median_delta_95ci": [ -0.014, 0.007 ], "wilcoxon_z": -0.61, "wilcoxon_p": 0.53858, "matched_rank_biserial": -0.05, "paired_dz": -0.014 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.033, "median_delta_95ci": [ -0.047, -0.019 ], "wilcoxon_z": -5.44, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.444, "paired_dz": -0.403 } }, "seg_entropy": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.249, "median_delta_95ci": [ -0.307, -0.165 ], "wilcoxon_z": -9.13, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.745, "paired_dz": -0.81 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.089, "median_delta_95ci": [ 0.047, 0.123 ], "wilcoxon_z": 3.46, "wilcoxon_p": 0.00054, "matched_rank_biserial": 0.282, "paired_dz": 0.254 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.099, "median_delta_95ci": [ -0.144, -0.063 ], "wilcoxon_z": -4.88, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.398, "paired_dz": -0.384 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.062, "median_delta_95ci": [ -0.13, -0.013 ], "wilcoxon_z": -2.67, "wilcoxon_p": 0.00754, "matched_rank_biserial": -0.218, "paired_dz": -0.211 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.133, "median_delta_95ci": [ -0.178, -0.075 ], "wilcoxon_z": -5.37, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.438, "paired_dz": -0.424 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": -0.144, "it1": -0.144, "it2": -0.058 }, "fewshot": { "it0": 0.674, "it1": 0.632, "it2": 0.549 }, "instruction": { "it0": -0.108, "it1": -0.12, "it2": -0.243 }, "plan": { "it0": 0.048, "it1": -0.091, "it2": 0.024 }, "self_refine": { "it0": -0.314, "it1": -0.422, "it2": -0.346 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 14, "n_holm": 9, "n_bh": 13, "lost_under_bh": [ "control/gini (p=0.037)" ], "lost_under_holm": [ "instruction/B_n (p=0.015)", "instruction/M (p=0.008)", "control/gini (p=0.037)", "instruction/gini (p=0.012)", "plan/seg_entropy (p=0.008)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.08624, "rb": -0.14, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.0, "rb": 0.694, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.01482, "rb": -0.199, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "B_n", "mode": "plan", "p": 0.76964, "rb": -0.024, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.0, "rb": -0.411, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.39304, "rb": -0.07, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "fewshot", "p": 0.57544, "rb": -0.046, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.00813, "rb": 0.216, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.48445, "rb": 0.057, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "self_refine", "p": 0.00017, "rb": 0.306, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "control", "p": 0.03704, "rb": -0.17, "raw_sig_05": true, "holm_survives": false, "bh_survives": false, "fragile": true }, { "metric": "gini", "mode": "fewshot", "p": 0.0, "rb": 0.678, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 0.01216, "rb": -0.204, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "gini", "mode": "plan", "p": 0.53858, "rb": -0.05, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.0, "rb": -0.444, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.0, "rb": -0.745, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.00054, "rb": 0.282, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.0, "rb": -0.398, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 0.00754, "rb": -0.218, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 0.0, "rb": -0.438, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false } ] } }, { "key": "kimi-k2.6", "name": "Kimi K2.6", "scored": { "label": "kimi-k2.6", "dims": { "B_n": { "control": { "n_pairs": 199, "median_delta_h_minus_ai": 0.095, "median_delta_95ci": [ 0.07, 0.12 ], "wilcoxon_z": 8.41, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.687, "paired_dz": 0.703 }, "fewshot": { "n_pairs": 199, "median_delta_h_minus_ai": 0.123, "median_delta_95ci": [ 0.112, 0.152 ], "wilcoxon_z": 10.17, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.831, "paired_dz": 0.965 }, "instruction": { "n_pairs": 199, "median_delta_h_minus_ai": -0.006, "median_delta_95ci": [ -0.034, 0.02 ], "wilcoxon_z": -0.05, "wilcoxon_p": 0.95882, "matched_rank_biserial": -0.004, "paired_dz": 0.015 }, "plan": { "n_pairs": 199, "median_delta_h_minus_ai": 0.026, "median_delta_95ci": [ 0.007, 0.054 ], "wilcoxon_z": 2.95, "wilcoxon_p": 0.00319, "matched_rank_biserial": 0.241, "paired_dz": 0.229 }, "self_refine": { "n_pairs": 198, "median_delta_h_minus_ai": -0.07, "median_delta_95ci": [ -0.091, -0.035 ], "wilcoxon_z": -6.94, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.569, "paired_dz": -0.536 } }, "M": { "control": { "n_pairs": 199, "median_delta_h_minus_ai": -0.025, "median_delta_95ci": [ -0.069, 0.011 ], "wilcoxon_z": -1.08, "wilcoxon_p": 0.28042, "matched_rank_biserial": -0.088, "paired_dz": -0.078 }, "fewshot": { "n_pairs": 199, "median_delta_h_minus_ai": -0.073, "median_delta_95ci": [ -0.124, -0.048 ], "wilcoxon_z": -3.15, "wilcoxon_p": 0.00166, "matched_rank_biserial": -0.257, "paired_dz": -0.249 }, "instruction": { "n_pairs": 199, "median_delta_h_minus_ai": -0.014, "median_delta_95ci": [ -0.039, 0.04 ], "wilcoxon_z": 0.94, "wilcoxon_p": 0.34635, "matched_rank_biserial": 0.077, "paired_dz": 0.097 }, "plan": { "n_pairs": 199, "median_delta_h_minus_ai": 0.055, "median_delta_95ci": [ -0.013, 0.109 ], "wilcoxon_z": 2.09, "wilcoxon_p": 0.03652, "matched_rank_biserial": 0.171, "paired_dz": 0.138 }, "self_refine": { "n_pairs": 198, "median_delta_h_minus_ai": -0.052, "median_delta_95ci": [ -0.105, -0.007 ], "wilcoxon_z": -2.83, "wilcoxon_p": 0.00471, "matched_rank_biserial": -0.232, "paired_dz": -0.219 } }, "gini": { "control": { "n_pairs": 199, "median_delta_h_minus_ai": 0.043, "median_delta_95ci": [ 0.034, 0.056 ], "wilcoxon_z": 7.7, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.629, "paired_dz": 0.623 }, "fewshot": { "n_pairs": 199, "median_delta_h_minus_ai": 0.063, "median_delta_95ci": [ 0.051, 0.072 ], "wilcoxon_z": 9.66, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.789, "paired_dz": 0.884 }, "instruction": { "n_pairs": 199, "median_delta_h_minus_ai": -0.014, "median_delta_95ci": [ -0.025, 0.0 ], "wilcoxon_z": -1.73, "wilcoxon_p": 0.08346, "matched_rank_biserial": -0.142, "paired_dz": -0.112 }, "plan": { "n_pairs": 199, "median_delta_h_minus_ai": 0.006, "median_delta_95ci": [ -0.006, 0.02 ], "wilcoxon_z": 1.14, "wilcoxon_p": 0.25547, "matched_rank_biserial": 0.093, "paired_dz": 0.102 }, "self_refine": { "n_pairs": 198, "median_delta_h_minus_ai": -0.04, "median_delta_95ci": [ -0.052, -0.026 ], "wilcoxon_z": -7.11, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.583, "paired_dz": -0.564 } }, "seg_entropy": { "control": { "n_pairs": 199, "median_delta_h_minus_ai": -0.048, "median_delta_95ci": [ -0.094, 0.003 ], "wilcoxon_z": -1.98, "wilcoxon_p": 0.04765, "matched_rank_biserial": -0.162, "paired_dz": -0.128 }, "fewshot": { "n_pairs": 199, "median_delta_h_minus_ai": 0.152, "median_delta_95ci": [ 0.1, 0.196 ], "wilcoxon_z": 6.76, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.553, "paired_dz": 0.538 }, "instruction": { "n_pairs": 199, "median_delta_h_minus_ai": -0.024, "median_delta_95ci": [ -0.064, 0.03 ], "wilcoxon_z": -1.29, "wilcoxon_p": 0.19634, "matched_rank_biserial": -0.106, "paired_dz": -0.11 }, "plan": { "n_pairs": 199, "median_delta_h_minus_ai": 0.032, "median_delta_95ci": [ -0.047, 0.077 ], "wilcoxon_z": 0.6, "wilcoxon_p": 0.55102, "matched_rank_biserial": 0.049, "paired_dz": 0.041 }, "self_refine": { "n_pairs": 198, "median_delta_h_minus_ai": 0.061, "median_delta_95ci": [ -0.011, 0.1 ], "wilcoxon_z": 1.92, "wilcoxon_p": 0.05542, "matched_rank_biserial": 0.157, "paired_dz": 0.113 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.626, "it1": 0.674, "it2": 0.581 }, "fewshot": { "it0": 0.737, "it1": 0.812, "it2": 0.752 }, "instruction": { "it0": 0.048, "it1": -0.008, "it2": -0.01 }, "plan": { "it0": 0.291, "it1": 0.208, "it2": 0.126 }, "self_refine": { "it0": -0.489, "it1": -0.461, "it2": -0.563 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 12, "n_holm": 9, "n_bh": 10, "lost_under_bh": [ "plan/M (p=0.037)", "control/seg_entropy (p=0.048)" ], "lost_under_holm": [ "plan/M (p=0.037)", "self_refine/M (p=0.005)", "control/seg_entropy (p=0.048)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.0, "rb": 0.687, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.0, "rb": 0.831, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.95882, "rb": -0.004, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.00319, "rb": 0.241, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.0, "rb": -0.569, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.28042, "rb": -0.088, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "fewshot", "p": 0.00166, "rb": -0.257, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.34635, "rb": 0.077, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.03652, "rb": 0.171, "raw_sig_05": true, "holm_survives": false, "bh_survives": false, "fragile": true }, { "metric": "M", "mode": "self_refine", "p": 0.00471, "rb": -0.232, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "control", "p": 0.0, "rb": 0.629, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.0, "rb": 0.789, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 0.08346, "rb": -0.142, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 0.25547, "rb": 0.093, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.0, "rb": -0.583, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.04765, "rb": -0.162, "raw_sig_05": true, "holm_survives": false, "bh_survives": false, "fragile": true }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.0, "rb": 0.553, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.19634, "rb": -0.106, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 0.55102, "rb": 0.049, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 0.05542, "rb": 0.157, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false } ] } }, { "key": "opus-4.8", "name": "Claude Opus 4.8", "scored": { "label": "opus-4.8", "dims": { "B_n": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.119, "median_delta_95ci": [ 0.091, 0.141 ], "wilcoxon_z": 9.8, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.799, "paired_dz": 0.885 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.039, "median_delta_95ci": [ 0.011, 0.058 ], "wilcoxon_z": 4.11, "wilcoxon_p": 4e-05, "matched_rank_biserial": 0.335, "paired_dz": 0.33 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.037, "median_delta_95ci": [ -0.06, -0.013 ], "wilcoxon_z": -3.12, "wilcoxon_p": 0.00184, "matched_rank_biserial": -0.254, "paired_dz": -0.212 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.0, "median_delta_95ci": [ -0.022, 0.022 ], "wilcoxon_z": 0.18, "wilcoxon_p": 0.86053, "matched_rank_biserial": 0.014, "paired_dz": 0.032 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.065, "median_delta_95ci": [ -0.088, -0.045 ], "wilcoxon_z": -6.18, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.504, "paired_dz": -0.475 } }, "M": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.069, "median_delta_95ci": [ -0.124, -0.032 ], "wilcoxon_z": -2.98, "wilcoxon_p": 0.0029, "matched_rank_biserial": -0.243, "paired_dz": -0.196 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": -0.045, "median_delta_95ci": [ -0.108, -0.001 ], "wilcoxon_z": -3.09, "wilcoxon_p": 0.00201, "matched_rank_biserial": -0.252, "paired_dz": -0.261 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.117, "median_delta_95ci": [ -0.16, -0.066 ], "wilcoxon_z": -4.66, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.38, "paired_dz": -0.346 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.059, "median_delta_95ci": [ -0.11, -0.009 ], "wilcoxon_z": -3.04, "wilcoxon_p": 0.00235, "matched_rank_biserial": -0.248, "paired_dz": -0.227 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.039, "median_delta_95ci": [ 0.001, 0.08 ], "wilcoxon_z": 2.16, "wilcoxon_p": 0.0308, "matched_rank_biserial": 0.176, "paired_dz": 0.159 } }, "gini": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.054, "median_delta_95ci": [ 0.045, 0.07 ], "wilcoxon_z": 9.43, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.769, "paired_dz": 0.827 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.012, "median_delta_95ci": [ 0.002, 0.03 ], "wilcoxon_z": 3.2, "wilcoxon_p": 0.00135, "matched_rank_biserial": 0.261, "paired_dz": 0.251 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.031, "median_delta_95ci": [ -0.043, -0.018 ], "wilcoxon_z": -4.42, "wilcoxon_p": 1e-05, "matched_rank_biserial": -0.36, "paired_dz": -0.315 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.004, "median_delta_95ci": [ -0.024, 0.004 ], "wilcoxon_z": -1.24, "wilcoxon_p": 0.21509, "matched_rank_biserial": -0.101, "paired_dz": -0.075 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.043, "median_delta_95ci": [ -0.054, -0.033 ], "wilcoxon_z": -7.23, "wilcoxon_p": 0.0, "matched_rank_biserial": -0.59, "paired_dz": -0.58 } }, "seg_entropy": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.116, "median_delta_95ci": [ 0.053, 0.173 ], "wilcoxon_z": 4.76, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.388, "paired_dz": 0.359 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.253, "median_delta_95ci": [ 0.202, 0.298 ], "wilcoxon_z": 9.16, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.747, "paired_dz": 0.801 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.032, "median_delta_95ci": [ -0.043, 0.072 ], "wilcoxon_z": 1.07, "wilcoxon_p": 0.28293, "matched_rank_biserial": 0.088, "paired_dz": 0.069 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.108, "median_delta_95ci": [ 0.062, 0.16 ], "wilcoxon_z": 4.23, "wilcoxon_p": 2e-05, "matched_rank_biserial": 0.345, "paired_dz": 0.313 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": -0.005, "median_delta_95ci": [ -0.049, 0.048 ], "wilcoxon_z": 0.14, "wilcoxon_p": 0.88648, "matched_rank_biserial": 0.012, "paired_dz": 0.016 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.743, "it1": 0.767, "it2": 0.76 }, "fewshot": { "it0": 0.289, "it1": 0.326, "it2": 0.305 }, "instruction": { "it0": -0.199, "it1": -0.239, "it2": -0.235 }, "plan": { "it0": -0.009, "it1": 0.05, "it2": 0.007 }, "self_refine": { "it0": -0.456, "it1": -0.493, "it2": -0.431 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 16, "n_holm": 15, "n_bh": 16, "lost_under_bh": [], "lost_under_holm": [ "self_refine/M (p=0.031)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.0, "rb": 0.799, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 4e-05, "rb": 0.335, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 0.00184, "rb": -0.254, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.86053, "rb": 0.014, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.0, "rb": -0.504, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.0029, "rb": -0.243, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "fewshot", "p": 0.00201, "rb": -0.252, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.0, "rb": -0.38, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "plan", "p": 0.00235, "rb": -0.248, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "self_refine", "p": 0.0308, "rb": 0.176, "raw_sig_05": true, "holm_survives": false, "bh_survives": true, "fragile": true }, { "metric": "gini", "mode": "control", "p": 0.0, "rb": 0.769, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.00135, "rb": 0.261, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 1e-05, "rb": -0.36, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 0.21509, "rb": -0.101, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.0, "rb": -0.59, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.0, "rb": 0.388, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.0, "rb": 0.747, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.28293, "rb": 0.088, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 2e-05, "rb": 0.345, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 0.88648, "rb": 0.012, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false } ] } }, { "key": "qwen3.6-27b", "name": "Qwen3.6-27B", "scored": { "label": "qwen3.6-27b-norm", "dims": { "B_n": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.097, "median_delta_95ci": [ 0.071, 0.116 ], "wilcoxon_z": 8.25, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.673, "paired_dz": 0.702 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.119, "median_delta_95ci": [ 0.094, 0.138 ], "wilcoxon_z": 9.75, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.795, "paired_dz": 0.89 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.028, "median_delta_95ci": [ 0.013, 0.046 ], "wilcoxon_z": 4.21, "wilcoxon_p": 3e-05, "matched_rank_biserial": 0.343, "paired_dz": 0.34 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.045, "median_delta_95ci": [ 0.023, 0.065 ], "wilcoxon_z": 4.96, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.405, "paired_dz": 0.403 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.039, "median_delta_95ci": [ 0.018, 0.063 ], "wilcoxon_z": 3.12, "wilcoxon_p": 0.00179, "matched_rank_biserial": 0.255, "paired_dz": 0.257 } }, "M": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.007, "median_delta_95ci": [ -0.048, 0.038 ], "wilcoxon_z": -0.64, "wilcoxon_p": 0.5202, "matched_rank_biserial": -0.052, "paired_dz": -0.048 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": -0.071, "median_delta_95ci": [ -0.117, -0.016 ], "wilcoxon_z": -3.13, "wilcoxon_p": 0.00175, "matched_rank_biserial": -0.255, "paired_dz": -0.227 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.052, "median_delta_95ci": [ -0.095, -0.003 ], "wilcoxon_z": -2.0, "wilcoxon_p": 0.04512, "matched_rank_biserial": -0.163, "paired_dz": -0.126 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": -0.028, "median_delta_95ci": [ -0.061, 0.007 ], "wilcoxon_z": -0.79, "wilcoxon_p": 0.427, "matched_rank_biserial": -0.065, "paired_dz": -0.052 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.028, "median_delta_95ci": [ -0.013, 0.064 ], "wilcoxon_z": 1.09, "wilcoxon_p": 0.27535, "matched_rank_biserial": 0.089, "paired_dz": 0.089 } }, "gini": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": 0.046, "median_delta_95ci": [ 0.034, 0.056 ], "wilcoxon_z": 8.24, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.672, "paired_dz": 0.695 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.06, "median_delta_95ci": [ 0.045, 0.07 ], "wilcoxon_z": 9.62, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.785, "paired_dz": 0.861 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": 0.01, "median_delta_95ci": [ 0.002, 0.024 ], "wilcoxon_z": 3.5, "wilcoxon_p": 0.00047, "matched_rank_biserial": 0.285, "paired_dz": 0.292 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.017, "median_delta_95ci": [ 0.008, 0.033 ], "wilcoxon_z": 4.31, "wilcoxon_p": 2e-05, "matched_rank_biserial": 0.351, "paired_dz": 0.353 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.021, "median_delta_95ci": [ 0.007, 0.03 ], "wilcoxon_z": 2.98, "wilcoxon_p": 0.00289, "matched_rank_biserial": 0.243, "paired_dz": 0.245 } }, "seg_entropy": { "control": { "n_pairs": 200, "median_delta_h_minus_ai": -0.001, "median_delta_95ci": [ -0.05, 0.062 ], "wilcoxon_z": 1.18, "wilcoxon_p": 0.23659, "matched_rank_biserial": 0.097, "paired_dz": 0.113 }, "fewshot": { "n_pairs": 200, "median_delta_h_minus_ai": 0.127, "median_delta_95ci": [ 0.083, 0.175 ], "wilcoxon_z": 5.74, "wilcoxon_p": 0.0, "matched_rank_biserial": 0.468, "paired_dz": 0.441 }, "instruction": { "n_pairs": 200, "median_delta_h_minus_ai": -0.018, "median_delta_95ci": [ -0.063, 0.045 ], "wilcoxon_z": -0.54, "wilcoxon_p": 0.59051, "matched_rank_biserial": -0.044, "paired_dz": -0.037 }, "plan": { "n_pairs": 200, "median_delta_h_minus_ai": 0.057, "median_delta_95ci": [ -0.014, 0.102 ], "wilcoxon_z": 1.94, "wilcoxon_p": 0.05222, "matched_rank_biserial": 0.158, "paired_dz": 0.145 }, "self_refine": { "n_pairs": 200, "median_delta_h_minus_ai": 0.029, "median_delta_95ci": [ -0.008, 0.082 ], "wilcoxon_z": 1.22, "wilcoxon_p": 0.2224, "matched_rank_biserial": 0.1, "paired_dz": 0.087 } } }, "consistency_B_n_rank_biserial": { "control": { "it0": 0.608, "it1": 0.637, "it2": 0.613 }, "fewshot": { "it0": 0.745, "it1": 0.724, "it2": 0.725 }, "instruction": { "it0": 0.239, "it1": 0.374, "it2": 0.3 }, "plan": { "it0": 0.32, "it1": 0.26, "it2": 0.466 }, "self_refine": { "it0": 0.286, "it1": 0.176, "it2": 0.255 } }, "note": "Per headline: metric vector averaged across available seeds, then ONE paired Wilcoxon on independent headline pairs (pooling seeds as pairs would pseudo-replicate the human docs). delta = human - model." }, "multiplicity": { "alpha": 0.05, "n_tests": 20, "n_raw_sig": 13, "n_holm": 12, "n_bh": 12, "lost_under_bh": [ "instruction/M (p=0.045)" ], "lost_under_holm": [ "instruction/M (p=0.045)" ], "tests": [ { "metric": "B_n", "mode": "control", "p": 0.0, "rb": 0.673, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "fewshot", "p": 0.0, "rb": 0.795, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "instruction", "p": 3e-05, "rb": 0.343, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "plan", "p": 0.0, "rb": 0.405, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "B_n", "mode": "self_refine", "p": 0.00179, "rb": 0.255, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "control", "p": 0.5202, "rb": -0.052, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "fewshot", "p": 0.00175, "rb": -0.255, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "M", "mode": "instruction", "p": 0.04512, "rb": -0.163, "raw_sig_05": true, "holm_survives": false, "bh_survives": false, "fragile": true }, { "metric": "M", "mode": "plan", "p": 0.427, "rb": -0.065, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "M", "mode": "self_refine", "p": 0.27535, "rb": 0.089, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "gini", "mode": "control", "p": 0.0, "rb": 0.672, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "fewshot", "p": 0.0, "rb": 0.785, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "instruction", "p": 0.00047, "rb": 0.285, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "plan", "p": 2e-05, "rb": 0.351, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "gini", "mode": "self_refine", "p": 0.00289, "rb": 0.243, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "control", "p": 0.23659, "rb": 0.097, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "fewshot", "p": 0.0, "rb": 0.468, "raw_sig_05": true, "holm_survives": true, "bh_survives": true, "fragile": false }, { "metric": "seg_entropy", "mode": "instruction", "p": 0.59051, "rb": -0.044, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "plan", "p": 0.05222, "rb": 0.158, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false }, { "metric": "seg_entropy", "mode": "self_refine", "p": 0.2224, "rb": 0.1, "raw_sig_05": false, "holm_survives": false, "bh_survives": false, "fragile": false } ] } } ] }