Add unchanged base CLEF to benchmark charts and comparison

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 assets/benchmark-auroc.png filter=lfs diff=lfs merge=lfs -text
+assets/benchmark-pdfs.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -55,18 +55,18 @@ This release achieves **0.9925 English / 0.9744 German full-suite AUROC** and **
 
 **This is a full-parameter update of selected layers, not a LoRA adapter or a standalone 550M model.** Inference requires the pinned public CLEF base (about **9.53B total parameters**). The loader downloads it automatically. Fine-tuning did not shrink the base model. This package uses custom inference code; standard `pipeline()` / `AutoModel.from_pretrained()` loading is not configured for this adapter.
 
-## Benchmarks: Jev and Laya R2a
-
-| Metric | clef-cybersecurity | Jev | Laya R2a |
-|---|---:|---:|---:|
-| Full English (n=510) AUROC | 0.9925 | 0.9800 | 0.9155 |
-| Full German (n=510) AUROC | 0.9744 | 0.9564 | 0.8780 |
-| English skills (n=48) AUROC | 1.0000 | 0.9841 | 0.9277 |
-| German skills (n=48) AUROC | 0.9171 | 0.9603 | 0.9330 |
-| PDF documents (n=730) AUROC | 0.9856 | 0.9785 | 0.8856 |
-| PDF attacks caught / 107 | 84 | 73 | 81 |
-| Clean PDF false alarms / 623 ↓ | 3 | 2 | 0 |
-| Decision threshold (strict >) | 0.5 | 0.5 | 0.95 |
+## Benchmarks: base CLEF, Jev and Laya R2a
+
+| Metric | clef-cybersecurity | CLEF Flash (base) | Jev | Laya R2a |
+|---|---:|---:|---:|---:|
+| Full English (n=510) AUROC | 0.9925 | 0.9588 | 0.9800 | 0.9155 |
+| Full German (n=510) AUROC | 0.9744 | 0.9391 | 0.9564 | 0.8780 |
+| English skills (n=48) AUROC | 1.0000 | 0.9762 | 0.9841 | 0.9277 |
+| German skills (n=48) AUROC | 0.9171 | 0.9048 | 0.9603 | 0.9330 |
+| PDF documents (n=730) AUROC | 0.9856 | 0.8144 | 0.9785 | 0.8856 |
+| PDF attacks caught / 107 | 84 | 6 | 73 | 81 |
+| Clean PDF false alarms / 623 ↓ | 3 | 0 | 2 | 0 |
+| Decision threshold (strict >) | 0.5 | 0.5 | 0.5 | 0.95 |
 
 ![AUROC comparison](assets/benchmark-auroc.png)
 
@@ -74,7 +74,9 @@ This release achieves **0.9925 English / 0.9744 German full-suite AUROC** and **
 
 The full English and German suites each contain 510 cases (256 attacks and 254 clean examples). Skill subsets contain 48 cases each (27 attacks and 21 clean examples), so their estimates are particularly uncertain. The matched PDF cohort contains 107 attacked excerpts and 623 clean documents. The PDF task uses **extracted text**, not a new evaluation of PDF parsing or image/OCR robustness.
 
-**The thresholds differ.** CLEF and Jev use strict `score > 0.5`; Laya R2a uses strict `score > 0.95`. AUROC compares ranking, whereas the detection and false-alarm counts describe those specific operating points. At CLEF's separately predeclared `>0.95` threshold, this selected checkpoint detects **69/107** PDF attacks and flags **2/623** clean documents.
+**CLEF Flash (base)** is the unchanged [Cloudflare/clef-flash](https://huggingface.co/Cloudflare/clef-flash) checkpoint at revision `17f0b0ad64efb65d273590632833508766b2aae6`, evaluated locally on these same cohorts with the native schema head and token-bounded document windows. It is not the separate hosted Cloudflare API evaluation. Fine-tuning improves the point-estimate AUROC on all five displayed cohorts: full English **0.9588 → 0.9925**, full German **0.9391 → 0.9744**, English skills **0.9762 → 1.0000**, German skills **0.9048 → 0.9171**, and PDFs **0.8144 → 0.9856**.
+
+**The thresholds differ.** Both CLEF models and Jev use strict `score > 0.5`; Laya R2a uses strict `score > 0.95`. The base CLEF operating point detects **6/107** PDF attacks with **0/623** clean flags; the fine-tuned release detects **84/107** with **3/623** clean flags. AUROC compares ranking, whereas the detection and false-alarm counts describe those specific operating points. The fine-tuned release also uses validation-fitted temperature calibration, so identical numerical thresholds do not establish equal false-positive rates. At the fine-tuned CLEF's separately predeclared `>0.95` threshold, this selected checkpoint detects **69/107** PDF attacks and flags **2/623** clean documents.
 
 “Laya R2a” refers to our saved r2a fine-tuned checkpoint, not the unchanged public Laya model or the earlier Laya cybersecurity checkpoint. Jev results are saved hosted evaluations; its exact provider-side model revision was not available. Models use different native encoders and window protocols, so this is a detector-system comparison, not a controlled architecture ablation.
 

benchmarks.json

@@ -46,6 +46,51 @@
       },
       "decision_rule": "score > 0.5"
     },
+    {
+      "model": "Cloudflare/clef-flash",
+      "name": "CLEF Flash (base)",
+      "full_en": {
+        "n": 510,
+        "attacks": 256,
+        "clean": 254,
+        "auroc": 0.9587613804133859,
+        "caught": 136,
+        "false_alarms": 0
+      },
+      "full_de": {
+        "n": 510,
+        "attacks": 256,
+        "clean": 254,
+        "auroc": 0.9391455462598425,
+        "caught": 130,
+        "false_alarms": 0
+      },
+      "skills_en": {
+        "n": 48,
+        "attacks": 27,
+        "clean": 21,
+        "auroc": 0.9761904761904762,
+        "caught": 14,
+        "false_alarms": 0
+      },
+      "skills_de": {
+        "n": 48,
+        "attacks": 27,
+        "clean": 21,
+        "auroc": 0.9047619047619048,
+        "caught": 12,
+        "false_alarms": 0
+      },
+      "pdf": {
+        "n": 730,
+        "attacks": 107,
+        "clean": 623,
+        "auroc": 0.8144492281843957,
+        "caught": 6,
+        "false_alarms": 0
+      },
+      "decision_rule": "score > 0.5"
+    },
     {
       "model": "jev",
       "name": "Jev",
@@ -140,7 +185,13 @@
     "clean_pdfs": 623
   },
   "scope": "Previously inspected internal regression sets; not a public leaderboard or fresh blind test.",
-  "threshold_note": "Strict >0.5 for CLEF and Jev; strict >0.95 for Laya R2a. Recall and false alarms are not equal-threshold comparisons.",
+  "threshold_note": "Strict >0.5 for both CLEF models and Jev; strict >0.95 for Laya R2a. Recall and false alarms are not equal-threshold comparisons.",
+  "base_clef_evaluation": {
+    "type": "unchanged_local_base",
+    "model": "Cloudflare/clef-flash",
+    "revision": "17f0b0ad64efb65d273590632833508766b2aae6",
+    "scope": "Unchanged native checkpoint, evaluated locally on the matched cohorts with token-bounded windows. Not the Cloudflare hosted API run."
+  },
   "selected_epoch": 3,
   "epochs_trained": 4,
   "clef_secondary_pdf_at_095": {

release_manifest.json

@@ -9,8 +9,8 @@
   "total_parameters": 9531576564,
   "files": {
     "benchmarks.json": {
-      "bytes": 3561,
-      "sha256": "fb8ece79895f30efd2f20583df50eb4529fc0420bdf8a0050bde4e920382dce0"
+      "bytes": 4893,
+      "sha256": "2c605c591f19ed08a941fcc5d8ae985c943cecd9eadc5cdfd77903aa7b61ec14"
     },
     "clef_detector.json": {
       "bytes": 11115,
@@ -33,8 +33,8 @@
       "sha256": "e9954b5e3ea46a528aeac8643a36aa209d84e9790019307303fc7ed01a794e98"
     },
     "README.md": {
-      "bytes": 9726,
-      "sha256": "6ae456f24bd0674761ff7209412736394885f411538686e556c18262b90b44ba"
+      "bytes": 10754,
+      "sha256": "afc81e0095dcb3b0ddc96a15ea8abe81f7ecbeb60cadffa626d4d6001c1b061d"
     },
     "clef_detector.py": {
       "bytes": 16866,
@@ -45,20 +45,20 @@
       "sha256": "df307b3d52e646186b8fccd278f122c61600a749d91feaff2189208ffa116b17"
     },
     "assets/benchmark-pdfs.svg": {
-      "bytes": 64179,
-      "sha256": "b4433e9f664d377dbfabbe056adad3d5b7d3422807299a3893aafcae658b7a5c"
+      "bytes": 67755,
+      "sha256": "04da9c77d6c8cbfbc2724be4d9d35cf68a2fc63d756401b9096d327c7e313141"
     },
     "assets/benchmark-auroc.svg": {
-      "bytes": 75256,
-      "sha256": "d568614a7245d65746f6b31040fb5259a15b0b16f10d456f0312b61a7392fe79"
+      "bytes": 81391,
+      "sha256": "447b8bd7d654bd662e0e52b57e0c9bd79dbd8f018e129e32efb1cbe68541b036"
     },
     "assets/benchmark-auroc.png": {
-      "bytes": 131047,
-      "sha256": "15336ed2d3f2adb4494d7b60d9a40e4d29a4b2954b9a098a3ee6180f4ff5a888"
+      "bytes": 148204,
+      "sha256": "cb67d379a9905e1b56e184d11b31c5ff5e8bce1b4658a53e2ad38c1af6d5d8ac"
     },
     "assets/benchmark-pdfs.png": {
-      "bytes": 89933,
-      "sha256": "3e97bf36c5a85b6f8287d1204c1c354247e91e95ed2e956135bac126e5cd999b"
+      "bytes": 101107,
+      "sha256": "a27ae1c6ec354cf700e2f04a8a3f94e429ea70e54e3243dbef8b4974135cb68b"
     }
   },
   "benchmark_runtime_sources": {