fix(scoring): pass limitations to C1 + judge only C2-owned dimensions

Closes out methodology review findings M-1 and M-2.

M-1: _score_c1_grounding never passed the output's limitations list to
score_facts, even though the scorer documents it as a fallback search
target. A gold-fact value stated only in a caveat ("only 977 students are
reflected in this snapshot") was scored as not found. Now passed through.

M-2: the C2 loop judged all six rubric dimensions per run, then three
(grounding_accuracy, calibration_limitation_handling, consistency) were
discarded and overridden by C1/C3/C4 — 390 wasted judge calls per model at
26 tasks x 5 runs. score_rubric gains a dimensions parameter (full-rubric
weight validation unchanged; unknown names raise before any judge call) and
the dispatcher requests only the three C2-owned dimensions, halving judge
cost. Default score_rubric behavior (all six) is unchanged for existing
callers.

Author: JonathanBechtel

benchmark/rubrics/rubric_scoring.py

@@ -12,8 +12,11 @@
   cannot express cross-field numeric constraints.  Any rubric whose dimension
   weights do not sum to approximately 1.0 (within ``WEIGHT_SUM_TOLERANCE``) is
   rejected with :exc:`ValueError` before any judge calls are made.
-- **Judge calls are capped** at one call per dimension (six total) per task,
-  providing a deterministic upper bound on API usage.
+- **Judge calls are capped** at one call per scored dimension, providing a
+  deterministic upper bound on API usage.  Callers may restrict scoring to a
+  subset of dimensions via the ``dimensions`` parameter; the runner judges
+  only the three C2-owned dimensions, since the other three are computed
+  deterministically by C1/C3/C4.
 - **Temperature 0 and a fixed random seed** are passed through to the judge
   client to ensure reproducible results; see :mod:`benchmark.rubrics.judge_client`
   for the live-judge integration wrapper.
@@ -145,6 +148,7 @@ def score_rubric(
     task: dict[str, Any],
     model_output: dict[str, Any],
     judge_client: JudgeClientProtocol,
+    dimensions: tuple[str, ...] | None = None,
 ) -> dict[str, Any]:
     """Score *model_output* against *task*'s rubric using *judge_client*.
 
@@ -168,21 +172,29 @@ def score_rubric(
         model_output: The normalized model output dict.  Passed through to the
             judge client unchanged.
         judge_client: An object implementing :class:`JudgeClientProtocol`.  The
-            scorer makes exactly one ``judge`` call per dimension (six calls
-            total) with no retries; cap and determinism settings (temperature 0,
-            fixed seed) should be configured on the client itself — see
+            scorer makes exactly one ``judge`` call per scored dimension; cap
+            and determinism settings (temperature 0, fixed seed) should be
+            configured on the client itself — see
             :mod:`benchmark.rubrics.judge_client`.
+        dimensions: Optional subset of :data:`RUBRIC_DIMENSIONS` to judge.
+            When ``None`` (default) all six dimensions are judged.  The
+            dispatcher passes only the C2-owned dimensions
+            (``insight_quality``, ``evidence_linkage``,
+            ``structure_usability``) since the other three are owned by
+            C1/C3/C4 — judging them would be wasted API spend.  Weight
+            validation always runs over the full rubric regardless.
 
     Returns:
         A dict with the following keys:
 
         - ``"task_id"`` (str): Copied from *task*.
         - ``"dimension_scores"`` (dict[str, float]): Per-dimension sub-scores
-          in [0, 1] keyed by dimension name.
+          in [0, 1] keyed by dimension name (scored dimensions only).
         - ``"rubric_weights"`` (dict[str, float]): The per-dimension weights
-          extracted from the task rubric.
+          extracted from the task rubric (scored dimensions only).
         - ``"composite"`` (float): The weighted aggregate score in [0, 1],
-          computed as ``sum(weight_i * score_i)`` over all six dimensions.
+          computed as ``sum(weight_i * score_i)`` over the scored dimensions
+          (partial when a ``dimensions`` subset is requested).
 
     Raises:
         ValueError: If the task rubric weights do not sum to
@@ -202,14 +214,20 @@ def score_rubric(
     rubric: dict[str, Any] = task["rubric"]
     task_id: str = task["task_id"]
 
-    # Validate weight sum before making any judge calls.
+    # Validate weight sum before making any judge calls.  Always validates
+    # the full rubric, even when only a subset of dimensions is judged.
     validate_rubric_weights(rubric)
 
+    scored_dimensions: tuple[str, ...] = dimensions if dimensions is not None else RUBRIC_DIMENSIONS
+    unknown = [d for d in scored_dimensions if d not in RUBRIC_DIMENSIONS]
+    if unknown:
+        raise ValueError(f"Unknown rubric dimension(s) requested: {unknown}")
+
     dimension_scores: dict[str, float] = {}
     rubric_weights: dict[str, float] = {}
     composite: float = 0.0
 
-    for dim in RUBRIC_DIMENSIONS:
+    for dim in scored_dimensions:
         dim_spec: dict[str, Any] = rubric[dim]
         weight: float = float(dim_spec["weight"])
         guidance: str = dim_spec.get("guidance", "")

runner/dispatcher.py

@@ -214,6 +214,10 @@ def judge(
 def _score_c1_grounding(task: dict[str, Any], output: dict[str, Any]) -> float:
     """Run C1 fact scoring and return ``grounding_accuracy`` in [0, 1].
 
+    The output's ``limitations`` list is passed through as a documented
+    fallback search target — models often state caveated numbers ("only 127
+    of the 135 enrolled students attended") in their limitations section.
+
     Args:
         task: Task definition dict.
         output: Normalized model output dict.
@@ -225,22 +229,32 @@ def _score_c1_grounding(task: dict[str, Any], output: dict[str, Any]) -> float:
         gold_facts=task.get("gold_facts", []),
         structured_metrics=output.get("structured_metrics", {}),
         key_findings=output.get("key_findings", []),
+        limitations=output.get("limitations", []),
     )
     return result.grounding_accuracy
 
 
+#: The rubric dimensions whose authoritative score comes from the C2 judge.
+#: The other three (grounding_accuracy, calibration_limitation_handling,
+#: consistency) are owned by C1/C3/C4, so judging them would be wasted API
+#: spend — at 26 tasks × 5 runs that's 390 discarded judge calls per model.
+_C2_OWNED_DIMENSIONS: tuple[str, ...] = (
+    "insight_quality",
+    "evidence_linkage",
+    "structure_usability",
+)
+
+
 def _score_c2_rubric(
     task: dict[str, Any],
     output: dict[str, Any],
     judge_client: Any,
 ) -> dict[str, float]:
     """Run C2 rubric scoring and return per-dimension scores.
 
-    Only the C2-owned dimensions are meaningful here:
-    ``insight_quality``, ``evidence_linkage``, and ``structure_usability``.
-    The ``calibration_limitation_handling`` and ``consistency`` values from
-    C2 are overridden by C3 and C4 respectively; ``grounding_accuracy`` is
-    overridden by C1.
+    Only the C2-owned dimensions are judged: ``insight_quality``,
+    ``evidence_linkage``, and ``structure_usability``.  The remaining three
+    dimensions are owned by C1/C3/C4 and are never sent to the judge.
 
     Args:
         task: Task definition dict.
@@ -249,15 +263,15 @@ def _score_c2_rubric(
             :class:`~benchmark.rubrics.rubric_scoring.JudgeClientProtocol`.
 
     Returns:
-        Dict mapping each of the six dimension names to a float in [0, 1].
-        On error (e.g. malformed rubric), returns zero scores for all
-        dimensions.
+        Dict mapping each C2-owned dimension name to a float in [0, 1].
+        On error (e.g. malformed rubric), returns zero scores for the
+        C2-owned dimensions.
     """
     try:
-        result = score_rubric(task, output, judge_client)
+        result = score_rubric(task, output, judge_client, dimensions=_C2_OWNED_DIMENSIONS)
         return dict(result["dimension_scores"])
     except (ValueError, KeyError):
-        return dict.fromkeys(RUBRIC_DIMENSIONS, 0.0)
+        return dict.fromkeys(_C2_OWNED_DIMENSIONS, 0.0)
 
 
 def _score_c3_calibration(
@@ -407,7 +421,7 @@ def run_task(
         # --- C1: grounding accuracy via fact scoring ---
         c1_score = _score_c1_grounding(task, output)
 
-        # --- C2: rubric scoring (all six dimensions) ---
+        # --- C2: rubric scoring (judged dimensions only) ---
         c2_scores = _score_c2_rubric(task, output, effective_judge)
 
         # --- C3: claim validation → calibration_limitation_handling ---

tests/unit/test_rubric_scoring.py

@@ -252,6 +252,29 @@ def test_judge_called_once_per_dimension(self) -> None:
         score_rubric(_VALID_TASK, _VALID_OUTPUT, mock)
         assert mock.judge.call_count == len(RUBRIC_DIMENSIONS)
 
+    def test_dimensions_subset_judges_only_those(self) -> None:
+        """A dimensions subset must restrict judge calls and returned scores.
+
+        The dispatcher passes only the C2-owned dimensions so judge calls for
+        C1/C3/C4-owned dimensions are never made (methodology finding M-2).
+        """
+        subset = ("insight_quality", "evidence_linkage", "structure_usability")
+        mock = _make_mock_judge(0.5)
+        result = score_rubric(_VALID_TASK, _VALID_OUTPUT, mock, dimensions=subset)
+
+        assert mock.judge.call_count == len(subset)
+        called_dims = {call.kwargs["dimension"] for call in mock.judge.call_args_list}
+        assert called_dims == set(subset)
+        assert set(result["dimension_scores"].keys()) == set(subset)
+        assert set(result["rubric_weights"].keys()) == set(subset)
+
+    def test_dimensions_subset_unknown_dimension_raises(self) -> None:
+        """An unknown dimension in the subset must raise before judge calls."""
+        mock = _make_mock_judge(0.5)
+        with pytest.raises(ValueError, match="Unknown rubric dimension"):
+            score_rubric(_VALID_TASK, _VALID_OUTPUT, mock, dimensions=("not_a_dim",))
+        assert mock.judge.call_count == 0
+
     def test_judge_called_with_correct_dimension_names(self) -> None:
         """Each judge call must receive the correct dimension name as the first arg."""
         mock = _make_mock_judge(0.5)

tests/unit/test_runner_dispatcher.py

@@ -283,6 +283,43 @@ def test_consistency_nonzero_for_multiple_runs(self) -> None:
         # Stub always emits the same first key_finding → consistency close to 1.0.
         assert result.scores.get("consistency", 0.0) > 0.0
 
+    def test_c1_grounding_searches_limitations(self) -> None:
+        """A gold-fact value stated only in limitations must be credited (M-1)."""
+        from runner.dispatcher import _score_c1_grounding
+
+        task = {
+            "task_id": "T-LIM",
+            "gold_facts": [
+                {
+                    "fact_id": "F1",
+                    "claim": "The program enrolls 977 students.",
+                    "source_files": ["students.csv"],
+                    "numeric_value": 977,
+                    "tolerance": 0,
+                }
+            ],
+        }
+        output = {
+            "structured_metrics": {},
+            "key_findings": ["Enrollment is healthy this quarter."],
+            "limitations": ["Note that only 977 students are reflected in this snapshot."],
+        }
+        assert _score_c1_grounding(task, output) == pytest.approx(1.0)
+
+    def test_c2_judge_not_called_for_non_c2_dimensions(self) -> None:
+        """The judge must only be called for C2-owned dimensions (M-2)."""
+        adapter = StubAdapter()
+        mock_judge = MagicMock()
+        mock_judge.judge.return_value = 0.5
+        run_task(_VALID_TASK, adapter, runs=1, judge_client=mock_judge)
+
+        called_dims = {call.kwargs["dimension"] for call in mock_judge.judge.call_args_list}
+        assert "grounding_accuracy" not in called_dims
+        assert "consistency" not in called_dims
+        # C3 may route calibration through the judge as a Stage-2 paraphrase
+        # fallback, but C2 itself must cover exactly the three owned dims.
+        assert {"insight_quality", "evidence_linkage", "structure_usability"} <= called_dims
+
 
 # ---------------------------------------------------------------------------
 # C3 integration tests — calibration_limitation_handling driven by validate_claims