#!/usr/bin/env python3 """ LFM2.5-2.6B-GGUF — objective quality sweep. For the F16 reference and every quant in the ladder, measure: * PPL (llama-perplexity, wikitext-2 test, ctx 2048) * KLD (llama-perplexity --kl-divergence vs the F16 reference logits) * Top-1 match ("Same top p" — how often the quant's argmax equals F16's) * Throughput (llama-bench: prompt tok/s @512, gen tok/s @128) Writes metrics/quant-summary-with-kld.json + .csv and reports/-vs-f16-*.json. """ import json, os, re, subprocess, sys, math BIN = os.environ.get("LLAMA_BIN", os.path.expanduser("~/llama.cpp/build-new/bin")) WORK = "/Volumes/Lexar/aq_lfm25" GG = f"{WORK}/gguf" REPO = f"{WORK}/repo" WIKI = f"{WORK}/wikitext-2-raw/wiki.test.raw" KLDAT = f"{WORK}/kld_f16.dat" NGL = os.environ.get("NGL", "99") REF = "F16" LADDER = ["F16", "Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "IQ4_XS", "Q3_K_M"] PPL_CTX, PPL_CHUNKS = 2048, 64 KLD_CTX, KLD_CHUNKS = 512, 32 env = dict(os.environ, DYLD_LIBRARY_PATH=BIN) def run(cmd, log): print("+", " ".join(cmd[:6]), "...", flush=True) with open(log, "w") as f: p = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT, env=env) out = open(log, errors="replace").read() if p.returncode != 0: print(out[-2500:], file=sys.stderr) raise SystemExit(f"FAILED rc={p.returncode}: {' '.join(cmd)}") return out def path_of(q): return f"{GG}/LFM2.5-2.6B-{q}.gguf" def build_base(): if os.path.exists(KLDAT) and os.path.getsize(KLDAT) > 1_000_000: print("kld base already built"); return run([f"{BIN}/llama-perplexity", "-m", path_of(REF), "-f", WIKI, "-c", str(KLD_CTX), "--chunks", str(KLD_CHUNKS), "--kl-divergence-base", KLDAT, "-ngl", NGL], f"{WORK}/logs/kldbase.log") def measure_ppl(q): out = run([f"{BIN}/llama-perplexity", "-m", path_of(q), "-f", WIKI, "-c", str(PPL_CTX), "--chunks", str(PPL_CHUNKS), "-ngl", NGL], f"{WORK}/logs/ppl_{q}.log") m = re.search(r"Final estimate: PPL = ([0-9.]+) \+/- ([0-9.]+)", out) if not m: raise SystemExit(f"no PPL parsed for {q}") return float(m.group(1)), float(m.group(2)) def measure_kld(q): out = run([f"{BIN}/llama-perplexity", "-m", path_of(q), "--kl-divergence", "--kl-divergence-base", KLDAT, "-ngl", NGL], f"{WORK}/logs/kld_{q}.log") g = lambda pat: (float(re.search(pat, out).group(1)) if re.search(pat, out) else None) r = { "kld_nats_mean": g(r"Mean\s+KLD:\s*([\-0-9.]+)"), "kld_nats_mean_unc": g(r"Mean\s+KLD:\s*[\-0-9.]+\s*±\s*([0-9.]+)"), "kld_nats_max": g(r"Maximum KLD:\s*([\-0-9.]+)"), "kld_nats_p999": g(r"99\.9%\s+KLD:\s*([\-0-9.]+)"), "kld_nats_p99": g(r"99\.0%\s+KLD:\s*([\-0-9.]+)"), "kld_nats_p95": g(r"95\.0%\s+KLD:\s*([\-0-9.]+)"), "kld_nats_p90": g(r"90\.0%\s+KLD:\s*([\-0-9.]+)"), "kld_nats_p50": g(r"Median\s+KLD:\s*([\-0-9.]+)"), "kld_nats_min": g(r"Minimum KLD:\s*([\-0-9.]+)"), "delta_p_mean_pct": g(r"Mean\s+Δp\s*:\s*([\-0-9.]+)"), "delta_p_rms_pct": g(r"RMS\s+Δp\s*:\s*([\-0-9.]+)"), "ppl_ratio_vs_f16": g(r"Mean PPL\(Q\)/PPL\(base\)\s*:\s*([0-9.]+)"), "top1_match_pct": g(r"Same top p:\s*([0-9.]+)"), } unc = g(r"Same top p:\s*[0-9.]+\s*±\s*([0-9.]+)") p = (r["top1_match_pct"] or 0) / 100.0 # n recovered from the binomial uncertainty printed alongside the rate if unc and unc > 0: n = int(round(p * (1 - p) / ((unc / 100.0) ** 2) + 1)) else: n = KLD_CHUNKS * (KLD_CTX - 1) r["tokens_evaluated"] = n r["top1_matches"] = int(round(p * n)) r["top1_mismatches"] = n - r["top1_matches"] r["kld_bits_mean"] = (r["kld_nats_mean"] / math.log(2)) if r["kld_nats_mean"] is not None else None r["kld_bits_p95"] = (r["kld_nats_p95"] / math.log(2)) if r["kld_nats_p95"] is not None else None return r def measure_bench(q): out = run([f"{BIN}/llama-bench", "-m", path_of(q), "-p", "512", "-n", "128", "-ngl", NGL, "-r", "3", "-o", "json"], f"{WORK}/logs/bench_{q}.log") j = json.loads(out[out.index("["):out.rindex("]") + 1]) pp = tg = None for e in j: if e.get("n_prompt", 0) == 512: pp = e["avg_ts"] if e.get("n_gen", 0) == 128: tg = e["avg_ts"] return pp, tg def main(): os.makedirs(f"{REPO}/metrics", exist_ok=True) os.makedirs(f"{REPO}/reports", exist_ok=True) os.makedirs(f"{WORK}/logs", exist_ok=True) build_base() rows, ref_ppl = [], None for q in LADDER: p = path_of(q) if not os.path.exists(p): print("MISSING", p); continue print(f"\n########## {q} ##########", flush=True) size = os.path.getsize(p) ppl, ppl_unc = measure_ppl(q) if q == REF: ref_ppl = ppl kld = measure_kld(q) pp, tg = measure_bench(q) row = { "name": f"LFM2.5-2.6B-{q}", "quant": q, "path": os.path.basename(p), "size_bytes": size, "size_gb_decimal": round(size / 1e9, 3), "size_gib": round(size / 2**30, 3), "prompt_tps_p512": round(pp, 2) if pp else None, "gen_tps_n128": round(tg, 2) if tg else None, "ppl_c2048_eval64": round(ppl, 4), "ppl_unc": round(ppl_unc, 4), "ppl_delta_vs_f16": None, } row.update({k: (round(v, 6) if isinstance(v, float) else v) for k, v in kld.items()}) row["top1_match_rate"] = (round(kld["top1_match_pct"] / 100.0, 6) if kld["top1_match_pct"] is not None else None) rows.append(row) json.dump(row, open(f"{REPO}/reports/{q}-vs-f16-kld.json", "w"), indent=2) for r in rows: if ref_ppl: r["ppl_delta_vs_f16"] = round(r["ppl_c2048_eval64"] - ref_ppl, 4) summary = { "model": "LiquidAI/LFM2.5-2.6B", "reference": "LFM2.5-2.6B-F16.gguf (unquantized convert of the bf16 weights)", "ppl_corpus": "wikitext-2-raw test", "ppl_ctx": PPL_CTX, "ppl_chunks": PPL_CHUNKS, "kld_ctx": KLD_CTX, "kld_chunks": KLD_CHUNKS, "kld_tokens": rows[0]["tokens_evaluated"] if rows else None, "quants": rows, } json.dump(summary, open(f"{REPO}/metrics/quant-summary-with-kld.json", "w"), indent=2) cols = ["name", "quant", "path", "size_bytes", "size_gb_decimal", "size_gib", "prompt_tps_p512", "gen_tps_n128", "ppl_c2048_eval64", "ppl_delta_vs_f16", "kld_nats_mean", "kld_nats_p50", "kld_nats_p95", "kld_nats_p99", "kld_nats_max", "kld_bits_mean", "top1_match_rate", "top1_matches", "top1_mismatches", "delta_p_mean_pct", "delta_p_rms_pct", "ppl_ratio_vs_f16", "tokens_evaluated"] with open(f"{REPO}/metrics/quant-summary-with-kld.csv", "w") as f: f.write(",".join(cols) + "\n") for r in rows: f.write(",".join("" if r.get(c) is None else str(r.get(c)) for c in cols) + "\n") print("\nSWEEP_DONE", len(rows), "rows") if __name__ == "__main__": main()