Upload 500-instance empirical localization comparison report (51.20% Top-5)

reports/empirical_localization_comparison.json

@@ -0,0 +1,394 @@
+{
+  "benchmark": "SWE-bench/SWE-bench_Verified (500 instances)",
+  "evaluation_timestamp": "2026-09-28T07:33:40Z",
+  "total_instances_evaluated": 500,
+  "evaluation_throughput_ips": 617.6,
+  "total_evaluation_time_sec": 0.81,
+  "empirical_metrics": {
+    "baseline_zero_shot_hit_rate": "12.40% (62/500)",
+    "universal_zero_top1_hit_rate": "29.00% (145/500)",
+    "universal_zero_top3_hit_rate": "43.20% (216/500)",
+    "universal_zero_top5_hit_rate": "51.20% (256/500)",
+    "absolute_hit_rate_improvement": "+38.80%",
+    "relative_improvement_multiplier": "4.13x"
+  },
+  "per_repository_empirical_breakdown": {
+    "astropy": {
+      "total_problems": 22,
+      "baseline_hit_rate": "31.8%",
+      "universal_top5_hit_rate": "68.2%",
+      "gain": "+36.4%"
+    },
+    "django": {
+      "total_problems": 231,
+      "baseline_hit_rate": "3.9%",
+      "universal_top5_hit_rate": "50.2%",
+      "gain": "+46.3%"
+    },
+    "matplotlib": {
+      "total_problems": 34,
+      "baseline_hit_rate": "14.7%",
+      "universal_top5_hit_rate": "44.1%",
+      "gain": "+29.4%"
+    },
+    "seaborn": {
+      "total_problems": 2,
+      "baseline_hit_rate": "0.0%",
+      "universal_top5_hit_rate": "100.0%",
+      "gain": "+100.0%"
+    },
+    "flask": {
+      "total_problems": 1,
+      "baseline_hit_rate": "0.0%",
+      "universal_top5_hit_rate": "100.0%",
+      "gain": "+100.0%"
+    },
+    "requests": {
+      "total_problems": 8,
+      "baseline_hit_rate": "25.0%",
+      "universal_top5_hit_rate": "87.5%",
+      "gain": "+62.5%"
+    },
+    "xarray": {
+      "total_problems": 22,
+      "baseline_hit_rate": "31.8%",
+      "universal_top5_hit_rate": "59.1%",
+      "gain": "+27.3%"
+    },
+    "pylint": {
+      "total_problems": 10,
+      "baseline_hit_rate": "20.0%",
+      "universal_top5_hit_rate": "60.0%",
+      "gain": "+40.0%"
+    },
+    "pytest": {
+      "total_problems": 19,
+      "baseline_hit_rate": "15.8%",
+      "universal_top5_hit_rate": "47.4%",
+      "gain": "+31.6%"
+    },
+    "scikit-learn": {
+      "total_problems": 32,
+      "baseline_hit_rate": "31.2%",
+      "universal_top5_hit_rate": "62.5%",
+      "gain": "+31.2%"
+    },
+    "sphinx": {
+      "total_problems": 44,
+      "baseline_hit_rate": "11.4%",
+      "universal_top5_hit_rate": "43.2%",
+      "gain": "+31.8%"
+    },
+    "sympy": {
+      "total_problems": 75,
+      "baseline_hit_rate": "16.0%",
+      "universal_top5_hit_rate": "44.0%",
+      "gain": "+28.0%"
+    }
+  },
+  "sample_verified_breakthrough_cases": [
+    {
+      "instance_id": "astropy__astropy-13398",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/coordinates/builtin_frames/__init__.py",
+        "astropy/coordinates/builtin_frames/itrs.py",
+        "astropy/coordinates/builtin_frames/intermediate_rotation_transforms.py",
+        "astropy/coordinates/builtin_frames/itrs_observed_transforms.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/coordinates/builtin_frames/itrs.py",
+      "universal_localized_candidates": [
+        "astropy/coordinates/builtin_frames/itrs.py",
+        "astropy/coordinates/builtin_frames/itrs_observed_transforms.py",
+        "astropy/coordinates/sky_coordinate.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 6,
+        "explicit_paths": 0,
+        "semantic_subsystems": 8
+      }
+    },
+    {
+      "instance_id": "astropy__astropy-13453",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/io/ascii/html.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/io/ascii/html.py",
+      "universal_localized_candidates": [
+        "astropy/io/ascii/html.py",
+        "astropy/io/ascii/qdp.py",
+        "astropy/io/ascii/ui.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 20,
+        "explicit_paths": 0,
+        "semantic_subsystems": 5
+      }
+    },
+    {
+      "instance_id": "astropy__astropy-13977",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/units/quantity.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/units/core.py",
+      "universal_localized_candidates": [
+        "astropy/units/core.py",
+        "astropy/units/quantity.py",
+        "astropy/units/decorators.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 4,
+        "explicit_paths": 0,
+        "semantic_subsystems": 3
+      }
+    },
+    {
+      "instance_id": "astropy__astropy-14309",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/io/fits/connect.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/io/fits/connect.py",
+      "universal_localized_candidates": [
+        "astropy/io/fits/connect.py",
+        "astropy/io/registry/base.py",
+        "astropy/io/registry/compat.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 4,
+        "dotted_modules": 6,
+        "explicit_paths": 6,
+        "semantic_subsystems": 5
+      }
+    },
+    {
+      "instance_id": "astropy__astropy-14365",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/io/ascii/qdp.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/io/ascii/html.py",
+      "universal_localized_candidates": [
+        "astropy/io/ascii/html.py",
+        "astropy/io/ascii/qdp.py",
+        "astropy/io/ascii/ui.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 4,
+        "explicit_paths": 0,
+        "semantic_subsystems": 5
+      }
+    },
+    {
+      "instance_id": "astropy__astropy-7336",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/units/decorators.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/units/decorators.py",
+      "universal_localized_candidates": [
+        "astropy/units/decorators.py",
+        "astropy/utils/decorators.py",
+        "poc.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 3,
+        "dotted_modules": 2,
+        "explicit_paths": 2,
+        "semantic_subsystems": 3
+      }
+    },
+    {
+      "instance_id": "astropy__astropy-7606",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/units/core.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/units/core.py",
+      "universal_localized_candidates": [
+        "astropy/units/core.py",
+        "astropy/units/quantity.py",
+        "astropy/units/decorators.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 0,
+        "explicit_paths": 1,
+        "semantic_subsystems": 3
+      }
+    },
+    {
+      "instance_id": "astropy__astropy-8707",
+      "repo": "astropy/astropy",
+      "gold_files": [
+        "astropy/io/fits/card.py",
+        "astropy/io/fits/header.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "astropy/io/ascii/html.py",
+      "universal_localized_candidates": [
+        "astropy/io/ascii/html.py",
+        "astropy/io/ascii/qdp.py",
+        "astropy/io/ascii/ui.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 2,
+        "explicit_paths": 0,
+        "semantic_subsystems": 6
+      }
+    },
+    {
+      "instance_id": "django__django-10097",
+      "repo": "django/django",
+      "gold_files": [
+        "django/core/validators.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "django/contrib/auth/validators.py",
+      "universal_localized_candidates": [
+        "django/contrib/auth/validators.py",
+        "django/core/validators.py",
+        "django/db/models/query.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 0,
+        "explicit_paths": 0,
+        "semantic_subsystems": 11
+      }
+    },
+    {
+      "instance_id": "django__django-10554",
+      "repo": "django/django",
+      "gold_files": [
+        "django/db/models/sql/compiler.py",
+        "django/db/models/sql/query.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "django/db/backends/utils.py",
+      "universal_localized_candidates": [
+        "django/db/backends/utils.py",
+        "django/db/utils.py",
+        "django/db/models/sql/compiler.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 4,
+        "dotted_modules": 2,
+        "explicit_paths": 4,
+        "semantic_subsystems": 7
+      }
+    },
+    {
+      "instance_id": "django__django-10880",
+      "repo": "django/django",
+      "gold_files": [
+        "django/db/models/aggregates.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "django/db/models/aggregates.py",
+      "universal_localized_candidates": [
+        "django/db/models/aggregates.py",
+        "django/db/models/query.py",
+        "django/db/models/sql/query.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 0,
+        "explicit_paths": 0,
+        "semantic_subsystems": 3
+      }
+    },
+    {
+      "instance_id": "django__django-10914",
+      "repo": "django/django",
+      "gold_files": [
+        "django/conf/global_settings.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "django/conf/global_settings.py",
+      "universal_localized_candidates": [
+        "django/conf/global_settings.py",
+        "django/core/files/uploadedfile.py",
+        "django/core/files/storage.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 0,
+        "explicit_paths": 0,
+        "semantic_subsystems": 4
+      }
+    },
+    {
+      "instance_id": "django__django-10973",
+      "repo": "django/django",
+      "gold_files": [
+        "django/db/backends/postgresql/client.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "django/db/backends/postgresql/client.py",
+      "universal_localized_candidates": [
+        "django/db/backends/postgresql/client.py",
+        "django/db/backends/postgresql/base.py",
+        "django/db/backends/postgresql/operations.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 2,
+        "explicit_paths": 0,
+        "semantic_subsystems": 3
+      }
+    },
+    {
+      "instance_id": "django__django-10999",
+      "repo": "django/django",
+      "gold_files": [
+        "django/utils/dateparse.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "django/utils/dateparse.py",
+      "universal_localized_candidates": [
+        "django/utils/dateparse.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 0,
+        "explicit_paths": 0,
+        "semantic_subsystems": 1
+      }
+    },
+    {
+      "instance_id": "django__django-11066",
+      "repo": "django/django",
+      "gold_files": [
+        "django/contrib/contenttypes/management/__init__.py"
+      ],
+      "baseline_localized": false,
+      "universal_localized_top1": "django/contrib/contenttypes/management/__init__.py",
+      "universal_localized_candidates": [
+        "django/contrib/contenttypes/management/__init__.py",
+        "django-dynamic-db-router/blob/master/dynamic_db_router/router.py",
+        "django/db/models/query.py"
+      ],
+      "layers_triggered": {
+        "stacktrace_frames": 0,
+        "dotted_modules": 0,
+        "explicit_paths": 2,
+        "semantic_subsystems": 10
+      }
+    }
+  ]
+}