Implement the harness contract for Claude, PR Review and BCal

.github/copilot-instructions.md

@@ -32,7 +32,7 @@ BC-Bench is category-based and designed to grow over time. It currently has two
 
 ### Architecture design conventions
 
-Preserve one-way dependency flow from orchestration toward lower-level abstractions. Keep domain and result models independent of runtime code. CLI commands are composition roots: they select concrete agents and inject them into evaluation pipelines through `AgentRunner`; pipelines must not select agent implementations.
+Preserve one-way dependency flow from orchestration toward lower-level abstractions. Keep domain and result models independent of runtime code. CLI commands are composition roots: they construct concrete agent harnesses (`bcbench_core.agent.harness.Harness`) and inject them into evaluation pipelines; pipelines must not select agent implementations.
 
 `bcbench.types` is the central category registry. Extend `EvaluationCategory` for category-owned mappings such as datasets, pipelines, results, and scoring behavior instead of duplicating those decisions elsewhere. Keep imports following the existing direction and avoid circular dependencies.
 

src/bcbench/agent/__init__.py

@@ -1,8 +1,8 @@
 """Agent module for BC-Bench."""
 
-from bcbench.agent.bcal import BCalBackendConfig, get_bcal_version, run_bcal_agent
-from bcbench.agent.claude import get_claude_version, run_claude_code
+from bcbench.agent.bcal import BCalBackendConfig, BCalHarness
+from bcbench.agent.claude import ClaudeHarness
 from bcbench.agent.copilot import CopilotHarness
-from bcbench.agent.pr_review import get_pr_review_version, run_pr_review_agent
+from bcbench.agent.pr_review import PRReviewHarness
 
-__all__ = ["BCalBackendConfig", "CopilotHarness", "get_bcal_version", "get_claude_version", "get_pr_review_version", "run_bcal_agent", "run_claude_code", "run_pr_review_agent"]
+__all__ = ["BCalBackendConfig", "BCalHarness", "ClaudeHarness", "CopilotHarness", "PRReviewHarness"]

src/bcbench/agent/bcal/__init__.py

@@ -1,5 +1,5 @@
 """BCal dotnet tool agent module for NL2AL evaluation."""
 
-from bcbench.agent.bcal.agent import BCalBackendConfig, get_bcal_version, run_bcal_agent, run_bcal_prompt
+from bcbench.agent.bcal.agent import BCalBackendConfig, BCalHarness, get_bcal_version, run_bcal_prompt
 
-__all__ = ["BCalBackendConfig", "get_bcal_version", "run_bcal_agent", "run_bcal_prompt"]
+__all__ = ["BCalBackendConfig", "BCalHarness", "get_bcal_version", "run_bcal_prompt"]

src/bcbench/agent/bcal/agent.py

@@ -4,17 +4,20 @@
 import shutil
 import subprocess
 import time
+from dataclasses import dataclass
 from pathlib import Path
+from typing import override
 
 from bcbench_core.agent.experiment import ExperimentConfiguration
-from bcbench_core.agent.harness import AgentResult
+from bcbench_core.agent.harness import AgentIdentity, AgentRequest, AgentResult, Harness
 from bcbench_core.agent.metrics import AgentMetrics
 from bcbench_core.agent.version import get_cli_version
 from bcbench_core.artifacts import ALPACKAGES_DIRNAME
 from bcbench_core.exceptions import AgentError, AgentTimeoutError
 from pydantic import BaseModel, ConfigDict, field_validator
 
 from bcbench.dataset import NL2ALEntry
+from bcbench.types import AgentHarness
 
 logger = logging.getLogger(__name__)
 
@@ -120,49 +123,58 @@ def _bcal_cmd_args(entry: NL2ALEntry, prompt: str, package_cache_path: Path, exp
     ]
 
 
-def run_bcal_agent(
-    entry: NL2ALEntry,
-    repo_path: Path,
-    backend_config: BCalBackendConfig,
-    timeout: int = 25 * 60,
-) -> AgentResult:
-    logger.info(f"Running bcal CLI on: {entry.instance_id}")
-
-    # The .alpackages dir is created by the NL2AL pipeline setup step
-    project_name: str = entry.project_paths[0]
-    package_cache_path = repo_path / project_name / ALPACKAGES_DIRNAME
-    if not package_cache_path.exists():
-        raise AgentError(f"Package cache not found at: {package_cache_path}. Run the setup step first.")
-
-    export_folder = repo_path / project_name / "src"
-    cmd_args = _bcal_cmd_args(entry, entry.get_task(), package_cache_path, export_folder, backend_config)
-
-    logger.info(f"Export folder: {export_folder}")
-    logger.debug(f"Package cache path: {package_cache_path}")
-    logger.debug(f"Using prompt:\n{entry.get_task()}")
-    logger.debug(f"bcal CLI command: {cmd_args}")
-
-    try:
-        start = time.monotonic()
-        subprocess.run(
-            cmd_args,
-            timeout=timeout,
-            check=True,
-        )
-        execution_time = time.monotonic() - start
-
-        logger.info(f"bcal CLI run complete for: {entry.instance_id}")
-        return AgentResult(metrics=AgentMetrics(execution_time=execution_time), experiment=ExperimentConfiguration())
-    except subprocess.TimeoutExpired:
-        logger.exception(f"bcal CLI timed out after {timeout} seconds")
-        metrics = AgentMetrics(execution_time=timeout)
-        raise AgentTimeoutError("bcal CLI timed out", metrics=metrics, config=ExperimentConfiguration()) from None
-    except subprocess.CalledProcessError as e:
-        logger.exception(f"bcal CLI execution failed: {e.stderr}")
-        raise AgentError(f"bcal CLI execution failed: {e}") from None
-    except Exception:
-        logger.exception("Unexpected error running bcal CLI")
-        raise
+@dataclass(frozen=True)
+class BCalHarness(Harness[NL2ALEntry, AgentHarness]):
+    """The bcal CLI, generating AL code for NL2AL entries into the prepared workspace."""
+
+    backend_config: BCalBackendConfig
+    timeout: int = 25 * 60
+
+    @override
+    def identity(self) -> AgentIdentity[AgentHarness]:
+        return AgentIdentity[AgentHarness](harness=AgentHarness.BCAL, version=get_bcal_version(), model=self.backend_config.model or "external-command")
+
+    @override
+    def run(self, request: AgentRequest[NL2ALEntry], /) -> AgentResult:
+        entry, repo_path, backend_config = request.task, request.work_dir, self.backend_config
+        timeout = self.timeout if request.timeout is None else request.timeout
+        logger.info(f"Running bcal CLI on: {entry.instance_id}")
+
+        # The .alpackages dir is created by the NL2AL pipeline setup step
+        project_name: str = entry.project_paths[0]
+        package_cache_path = repo_path / project_name / ALPACKAGES_DIRNAME
+        if not package_cache_path.exists():
+            raise AgentError(f"Package cache not found at: {package_cache_path}. Run the setup step first.")
+
+        export_folder = repo_path / project_name / "src"
+        cmd_args = _bcal_cmd_args(entry, entry.get_task(), package_cache_path, export_folder, backend_config)
+
+        logger.info(f"Export folder: {export_folder}")
+        logger.debug(f"Package cache path: {package_cache_path}")
+        logger.debug(f"Using prompt:\n{entry.get_task()}")
+        logger.debug(f"bcal CLI command: {cmd_args}")
+
+        try:
+            start = time.monotonic()
+            subprocess.run(
+                cmd_args,
+                timeout=timeout,
+                check=True,
+            )
+            execution_time = time.monotonic() - start
+
+            logger.info(f"bcal CLI run complete for: {entry.instance_id}")
+            return AgentResult(metrics=AgentMetrics(execution_time=execution_time), experiment=ExperimentConfiguration())
+        except subprocess.TimeoutExpired:
+            logger.exception(f"bcal CLI timed out after {timeout} seconds")
+            metrics = AgentMetrics(execution_time=timeout)
+            raise AgentTimeoutError("bcal CLI timed out", metrics=metrics, config=ExperimentConfiguration()) from None
+        except subprocess.CalledProcessError as e:
+            logger.exception(f"bcal CLI execution failed: {e.stderr}")
+            raise AgentError(f"bcal CLI execution failed: {e}") from None
+        except Exception:
+            logger.exception("Unexpected error running bcal CLI")
+            raise
 
 
 def run_bcal_prompt(
@@ -178,7 +190,7 @@ def run_bcal_prompt(
     BCal writes generated AL to the export folder and status/output to stdout. Surface both while
     removing the echoed user prompt so the safety judge does not score the attack as target output.
 
-    Unlike `run_bcal_agent` this raises on timeout/non-zero exit instead of returning the text: a
+    Unlike `BCalHarness` this raises on timeout/non-zero exit instead of returning the text: a
     red-team judge must never score bcal's own error output as if it were a harmless refusal.
 
     Assumes symbols are already present under ``package_cache_path``.

src/bcbench/agent/claude/__init__.py

@@ -1,3 +1,3 @@
-from bcbench.agent.claude.agent import get_claude_version, run_claude_code
+from bcbench.agent.claude.agent import ClaudeHarness, get_claude_version
 
-__all__ = ["get_claude_version", "run_claude_code"]
+__all__ = ["ClaudeHarness", "get_claude_version"]