LFM2.5-2.6B-GGUF / scripts /03_quality_sweep.py
KikoCis's picture
Add card, metrics (KLD/PPL/Top-1), charts, reports, scripts, configs, Modelfiles, LICENSE + NOTICE, checksums
b9415fb verified
Raw History Blame Contribute Delete
7.23 kB
#!/usr/bin/env python3
"""
LFM2.5-2.6B-GGUF — objective quality sweep.
For the F16 reference and every quant in the ladder, measure:
* PPL (llama-perplexity, wikitext-2 test, ctx 2048)
* KLD (llama-perplexity --kl-divergence vs the F16 reference logits)
* Top-1 match ("Same top p" — how often the quant's argmax equals F16's)
* Throughput (llama-bench: prompt tok/s @512, gen tok/s @128)
Writes metrics/quant-summary-with-kld.json + .csv and reports/<QUANT>-vs-f16-*.json.
"""
import json, os, re, subprocess, sys, math
BIN = os.environ.get("LLAMA_BIN", os.path.expanduser("~/llama.cpp/build-new/bin"))
WORK = "/Volumes/Lexar/aq_lfm25"
GG = f"{WORK}/gguf"
REPO = f"{WORK}/repo"
WIKI = f"{WORK}/wikitext-2-raw/wiki.test.raw"
KLDAT = f"{WORK}/kld_f16.dat"
NGL = os.environ.get("NGL", "99")
REF = "F16"
LADDER = ["F16", "Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "IQ4_XS", "Q3_K_M"]
PPL_CTX, PPL_CHUNKS = 2048, 64
KLD_CTX, KLD_CHUNKS = 512, 32
env = dict(os.environ, DYLD_LIBRARY_PATH=BIN)
def run(cmd, log):
print("+", " ".join(cmd[:6]), "...", flush=True)
with open(log, "w") as f:
p = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT, env=env)
out = open(log, errors="replace").read()
if p.returncode != 0:
print(out[-2500:], file=sys.stderr)
raise SystemExit(f"FAILED rc={p.returncode}: {' '.join(cmd)}")
return out
def path_of(q):
return f"{GG}/LFM2.5-2.6B-{q}.gguf"
def build_base():
if os.path.exists(KLDAT) and os.path.getsize(KLDAT) > 1_000_000:
print("kld base already built"); return
run([f"{BIN}/llama-perplexity", "-m", path_of(REF), "-f", WIKI,
"-c", str(KLD_CTX), "--chunks", str(KLD_CHUNKS),
"--kl-divergence-base", KLDAT, "-ngl", NGL],
f"{WORK}/logs/kldbase.log")
def measure_ppl(q):
out = run([f"{BIN}/llama-perplexity", "-m", path_of(q), "-f", WIKI,
"-c", str(PPL_CTX), "--chunks", str(PPL_CHUNKS), "-ngl", NGL],
f"{WORK}/logs/ppl_{q}.log")
m = re.search(r"Final estimate: PPL = ([0-9.]+) \+/- ([0-9.]+)", out)
if not m:
raise SystemExit(f"no PPL parsed for {q}")
return float(m.group(1)), float(m.group(2))
def measure_kld(q):
out = run([f"{BIN}/llama-perplexity", "-m", path_of(q),
"--kl-divergence", "--kl-divergence-base", KLDAT, "-ngl", NGL],
f"{WORK}/logs/kld_{q}.log")
g = lambda pat: (float(re.search(pat, out).group(1)) if re.search(pat, out) else None)
r = {
"kld_nats_mean": g(r"Mean\s+KLD:\s*([\-0-9.]+)"),
"kld_nats_mean_unc": g(r"Mean\s+KLD:\s*[\-0-9.]+\s*±\s*([0-9.]+)"),
"kld_nats_max": g(r"Maximum KLD:\s*([\-0-9.]+)"),
"kld_nats_p999": g(r"99\.9%\s+KLD:\s*([\-0-9.]+)"),
"kld_nats_p99": g(r"99\.0%\s+KLD:\s*([\-0-9.]+)"),
"kld_nats_p95": g(r"95\.0%\s+KLD:\s*([\-0-9.]+)"),
"kld_nats_p90": g(r"90\.0%\s+KLD:\s*([\-0-9.]+)"),
"kld_nats_p50": g(r"Median\s+KLD:\s*([\-0-9.]+)"),
"kld_nats_min": g(r"Minimum KLD:\s*([\-0-9.]+)"),
"delta_p_mean_pct": g(r"Mean\s+Δp\s*:\s*([\-0-9.]+)"),
"delta_p_rms_pct": g(r"RMS\s+Δp\s*:\s*([\-0-9.]+)"),
"ppl_ratio_vs_f16": g(r"Mean PPL\(Q\)/PPL\(base\)\s*:\s*([0-9.]+)"),
"top1_match_pct": g(r"Same top p:\s*([0-9.]+)"),
}
unc = g(r"Same top p:\s*[0-9.]+\s*±\s*([0-9.]+)")
p = (r["top1_match_pct"] or 0) / 100.0
# n recovered from the binomial uncertainty printed alongside the rate
if unc and unc > 0:
n = int(round(p * (1 - p) / ((unc / 100.0) ** 2) + 1))
else:
n = KLD_CHUNKS * (KLD_CTX - 1)
r["tokens_evaluated"] = n
r["top1_matches"] = int(round(p * n))
r["top1_mismatches"] = n - r["top1_matches"]
r["kld_bits_mean"] = (r["kld_nats_mean"] / math.log(2)) if r["kld_nats_mean"] is not None else None
r["kld_bits_p95"] = (r["kld_nats_p95"] / math.log(2)) if r["kld_nats_p95"] is not None else None
return r
def measure_bench(q):
out = run([f"{BIN}/llama-bench", "-m", path_of(q), "-p", "512", "-n", "128",
"-ngl", NGL, "-r", "3", "-o", "json"], f"{WORK}/logs/bench_{q}.log")
j = json.loads(out[out.index("["):out.rindex("]") + 1])
pp = tg = None
for e in j:
if e.get("n_prompt", 0) == 512:
pp = e["avg_ts"]
if e.get("n_gen", 0) == 128:
tg = e["avg_ts"]
return pp, tg
def main():
os.makedirs(f"{REPO}/metrics", exist_ok=True)
os.makedirs(f"{REPO}/reports", exist_ok=True)
os.makedirs(f"{WORK}/logs", exist_ok=True)
build_base()
rows, ref_ppl = [], None
for q in LADDER:
p = path_of(q)
if not os.path.exists(p):
print("MISSING", p); continue
print(f"\n########## {q} ##########", flush=True)
size = os.path.getsize(p)
ppl, ppl_unc = measure_ppl(q)
if q == REF:
ref_ppl = ppl
kld = measure_kld(q)
pp, tg = measure_bench(q)
row = {
"name": f"LFM2.5-2.6B-{q}",
"quant": q,
"path": os.path.basename(p),
"size_bytes": size,
"size_gb_decimal": round(size / 1e9, 3),
"size_gib": round(size / 2**30, 3),
"prompt_tps_p512": round(pp, 2) if pp else None,
"gen_tps_n128": round(tg, 2) if tg else None,
"ppl_c2048_eval64": round(ppl, 4),
"ppl_unc": round(ppl_unc, 4),
"ppl_delta_vs_f16": None,
}
row.update({k: (round(v, 6) if isinstance(v, float) else v) for k, v in kld.items()})
row["top1_match_rate"] = (round(kld["top1_match_pct"] / 100.0, 6)
if kld["top1_match_pct"] is not None else None)
rows.append(row)
json.dump(row, open(f"{REPO}/reports/{q}-vs-f16-kld.json", "w"), indent=2)
for r in rows:
if ref_ppl:
r["ppl_delta_vs_f16"] = round(r["ppl_c2048_eval64"] - ref_ppl, 4)
summary = {
"model": "LiquidAI/LFM2.5-2.6B",
"reference": "LFM2.5-2.6B-F16.gguf (unquantized convert of the bf16 weights)",
"ppl_corpus": "wikitext-2-raw test",
"ppl_ctx": PPL_CTX, "ppl_chunks": PPL_CHUNKS,
"kld_ctx": KLD_CTX, "kld_chunks": KLD_CHUNKS,
"kld_tokens": rows[0]["tokens_evaluated"] if rows else None,
"quants": rows,
}
json.dump(summary, open(f"{REPO}/metrics/quant-summary-with-kld.json", "w"), indent=2)
cols = ["name", "quant", "path", "size_bytes", "size_gb_decimal", "size_gib",
"prompt_tps_p512", "gen_tps_n128", "ppl_c2048_eval64", "ppl_delta_vs_f16",
"kld_nats_mean", "kld_nats_p50", "kld_nats_p95", "kld_nats_p99", "kld_nats_max",
"kld_bits_mean", "top1_match_rate", "top1_matches", "top1_mismatches",
"delta_p_mean_pct", "delta_p_rms_pct", "ppl_ratio_vs_f16", "tokens_evaluated"]
with open(f"{REPO}/metrics/quant-summary-with-kld.csv", "w") as f:
f.write(",".join(cols) + "\n")
for r in rows:
f.write(",".join("" if r.get(c) is None else str(r.get(c)) for c in cols) + "\n")
print("\nSWEEP_DONE", len(rows), "rows")
if __name__ == "__main__":
main()