diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7baca48c4..0aadbf245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,26 +28,49 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.12" - cache: pip - cache-dependency-path: pyproject.toml - - run: python -m pip install --upgrade pip - # Install dev extras so pyright can resolve runtime/test dependencies - # while it analyzes the package, tests, examples, and maintained scripts. - - run: pip install -e ".[dev]" + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + enable-cache: true + cache-dependency-glob: pyproject.toml + # CPU torch: the default Linux wheel drags in ~2.5 GB of CUDA libraries + # that neither pyright nor the CPU suite touches. + - run: uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu + - run: uv pip install --system -e ".[dev]" - run: pyright - test: + # The CPU suite in four parallel shards (a runner has 4 vCPUs; the suite is + # ~330 CPU-seconds locally, a third of it the route tests). The last shard + # takes every directory the others don't name, so a new one is never skipped. + test-shard: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: + - "tests/server" + - "tests/lens tests/units tests/sae tests/cli" + - "tests/fit tests/session tests/model tests/browser tests/loom" + - "tests --ignore=tests/server --ignore=tests/lens --ignore=tests/units --ignore=tests/sae --ignore=tests/cli --ignore=tests/fit --ignore=tests/session --ignore=tests/model --ignore=tests/browser --ignore=tests/loom" steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v7 with: python-version: "3.13" - cache: pip - cache-dependency-path: pyproject.toml - - run: python -m pip install --upgrade pip - - run: pip install -e ".[dev]" - - run: pytest -q -m "not gpu" -n auto + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + enable-cache: true + cache-dependency-glob: pyproject.toml + - run: uv pip install --system torch --index-url https://download.pytorch.org/whl/cpu + - run: uv pip install --system -e ".[dev]" + - run: pytest -q -m "not gpu" -n auto ${{ matrix.shard }} + + # The one check branch protection names. + test: + if: always() + needs: test-shard + runs-on: ubuntu-latest + steps: + - run: test "${{ needs.test-shard.result }}" = success webui: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 34fcb953c..2d9aa7bd3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -209,11 +209,12 @@ get wrong raises `RequestError` (400, text verbatim) or a subclass failures after parsing are `SteeringCompositionError` (422). A bare `ValueError` is a broken invariant: the server logs it and answers a scrubbed 500. The wire `code` is the exception class name. The CLI maps 404 → exit 1, -other 4xx → 2, 5xx → 1. `tests/invariants/test_user_message.py` enforces the -family. +other 4xx → 2, 5xx → 1. -**Persisted formats are exact.** Readers accept only the current version and -never migrate (a derived cache recomputes instead). These froze at 6.0.0: +**Persisted formats are versioned.** Readers accept only the current version and +never migrate (a derived cache recomputes instead); the fitted sidecar reader +ignores keys the contract has retired, so dropping provenance never strands a +fit. These froze at 6.0.0: `manifold.json` v10, `template.json` v2, `.drowse` v1, loom `tree_format` 2 with token sidecar 3 (token rows carry measurement envelope v2; a node's `finish_reason` is one of `stop`, `length`, `stop_sequence`, `cancelled`), @@ -309,10 +310,11 @@ fit loop, `lens/fit.py`, is the reference pattern). tiny generated checkpoint (`tiny_session`); route tests that need no behavior use the autospec `tests.support.server.mock_session`. GPU tests (`gpu` marker) load `HuggingFaceTB/SmolLM2-360M-Instruct` unless `DROWSE_TEST_MODEL` names -another; `gpu_gemma` needs gated Gemma access. Every bug fix lands with a -regression test. A test never touches the real home (an autouse fixture sets a -scratch `DROWSE_HOME`). CI runs ruff, pyright, the CPU suite on Python 3.13 -(`-n auto`) and the webui build with a dist-drift check. Details: `tests/AGENTS.md`. +another; the GPU suite is one end-to-end smoke plus the throughput gates. Add a +regression test when a bug could quietly come back, not as a ritual. A test +never touches the real home (an autouse fixture sets a scratch `DROWSE_HOME`). CI runs ruff, pyright, the CPU suite on Python 3.13 +(four parallel shards, `-n auto`, CPU torch via uv) and the webui build with a +dist-drift check. Details: `tests/AGENTS.md`. ## State and environment @@ -333,4 +335,4 @@ downloads, and Python saves only to an explicit `LoomTree.save(path)`. | `DROWSE_STRICT_MODEL` | OpenAI/Ollama 404 on a model-name mismatch | | `DROWSE_MAX_REQUEST_BYTES` | request body cap (default 64 MiB) | | `DROWSE_MANIFOLD_CAPTURE_CACHE_GB` | fit capture cache bound (default 8) | -| `DROWSE_TEST_MODEL`, `DROWSE_TEST_SAELENS`, `DROWSE_TEST_LIVE_TOKENIZERS` | test knobs (`tests/AGENTS.md`) | +| `DROWSE_TEST_MODEL`, `DROWSE_TEST_LIVE_TOKENIZERS` | test knobs (`tests/AGENTS.md`) | diff --git a/drowse/fit/AGENTS.md b/drowse/fit/AGENTS.md index 21d3219bb..9e14fe4a8 100644 --- a/drowse/fit/AGENTS.md +++ b/drowse/fit/AGENTS.md @@ -375,8 +375,7 @@ corpora `tests/session/test_generate_responses.py`, session bake `tests/session/test_session_bake.py`, neutral cache `tests/store/test_neutral_cache.py`, neutral unit statistics `tests/units/test_unit_stats.py`, DLS golden -`tests/browser/test_browser_parity.py`, GPU `tests/gpu/test_smoke.py` and -`tests/gpu/test_session.py`. +`tests/browser/test_browser_parity.py`, GPU `tests/gpu/test_smoke.py`. ## Deliberately absent diff --git a/drowse/fit/pipeline.py b/drowse/fit/pipeline.py index bb6bccb5c..50fcb4076 100644 --- a/drowse/fit/pipeline.py +++ b/drowse/fit/pipeline.py @@ -798,16 +798,9 @@ def accepts(self, sidecar: "ManifoldSidecar") -> bool: return bool(sidecar.sae_full_coverage) def base_metadata( - self, *, method: str, fitted_layers: list[int], subspace_metric: str, + self, *, method: str, fitted_layers: list[int], ) -> dict[str, Any]: - """The sidecar keys every fit writes. - - ``subspace_metric`` is a basis *label*: the monopolar fold keeps the - raw δ̂ basis (``concept − ν`` cancels common-mode by differencing, like - DiM) and reports ``euclidean``; every other fit selects a whitened - basis. The share is always Mahalanobis — the whitener gate guarantees - it. - """ + """The sidecar keys every fit writes.""" ident = self.identity metadata: dict[str, Any] = { "method": method, @@ -820,9 +813,6 @@ def base_metadata( "baseline_prompts_sha256": ident.baseline_prompts_sha256, "fitted_layers": fitted_layers, "fit_policy_version": MANIFOLD_FIT_POLICY_VERSION, - # Provenance only (nothing branches on these at load). - "share_metric": "mahalanobis", - "subspace_metric": subspace_metric, } backend = self.sae_backend if backend is not None: @@ -1342,7 +1332,6 @@ def _fit_monopolar( else "manifold_monopolar" ), fitted_layers=list(fit_layers), - subspace_metric="euclidean", ) metadata.update({ "fit_mode": mf.fit_mode, @@ -1448,7 +1437,6 @@ def _fit_nodes( metadata = inputs.base_metadata( method=geometry.method, fitted_layers=sorted(layer_subs), - subspace_metric="mahalanobis", ) # Diagnostic layer profile: the whitened between-node spread # ``tr(G_L)`` — surfaced by `manifold show`, consumed by nothing. diff --git a/drowse/geometry/fold.py b/drowse/geometry/fold.py index 8d3483757..19ac3db59 100644 --- a/drowse/geometry/fold.py +++ b/drowse/geometry/fold.py @@ -130,7 +130,6 @@ def fold_directions_to_subspace( node_kinds=[None], mahalanobis_share=mahalanobis_share, ) - manifold.metadata["share_metric"] = "mahalanobis" manifold.validate_runtime_geometry() return manifold diff --git a/drowse/lens/AGENTS.md b/drowse/lens/AGENTS.md index ec9edf918..30be9cb6a 100644 --- a/drowse/lens/AGENTS.md +++ b/drowse/lens/AGENTS.md @@ -733,5 +733,3 @@ the per-layer softmax probability `p_l(v)`. (`LensHarness`, `lens_instrument(**kw)`, `lens_kwargs(**kw)`, `FakeCapture`), which builds a real `LensInstrument` with session-shaped dependencies. Generation-boundary tests use `tiny_session`. - `tests/gpu/test_jlens_gpu.py` micro-fits a real model (SmolLM2-360M or - `DROWSE_TEST_MODEL`). diff --git a/drowse/model/AGENTS.md b/drowse/model/AGENTS.md index 64d657d59..5c31caf16 100644 --- a/drowse/model/AGENTS.md +++ b/drowse/model/AGENTS.md @@ -446,5 +446,6 @@ and stitcher against the Jinja fakes in `tests/support/chat_templates.py` `test_role_template_drift.py` checks `SEAT_HEADERS` against real tokenizers in the local HF cache; `DROWSE_TEST_LIVE_TOKENIZERS=1` permits downloads (run it before a release). `tests/support/tiny_models.py` supplies genuine 4-layer -models with real family templates for CPU end-to-end tests; real-weight checks -are `tests/gpu/test_model_numerics_gpu.py` (`DROWSE_TEST_MODEL`). +models with real family templates for CPU end-to-end tests; +`tests/model/test_model_numerics.py` carries `mps` params for real-device +numerics. diff --git a/drowse/readout/AGENTS.md b/drowse/readout/AGENTS.md index e2d9a5b56..6170589dc 100644 --- a/drowse/readout/AGENTS.md +++ b/drowse/readout/AGENTS.md @@ -284,8 +284,8 @@ rows at its own rank, one batched foot solve (`invert_stacked`) per rank (one for a rank-uniform fit), and the share-weighted means as a left fold in layer order. Every kernel computes a layer as the per-layer read did, so a reading does not depend on how many layers ran together: bit-identical to the -per-layer read on MPS (`tests/gpu/test_curved_stack_gpu.py`; on CPU -`tests/readout/test_monitor_curved_stack.py` pins it to tolerance). A +per-layer read (`tests/readout/test_monitor_curved_stack.py` pins it to +tolerance on CPU). A reduction batched across layers moves values by an ulp, and an off-surface state's foot to another basin, so keep reductions per layer. A read that saw only some of a probe's layers uses a cached `_CurvedStack.subset`. Curved feet warm-start diff --git a/drowse/sae/AGENTS.md b/drowse/sae/AGENTS.md index 1deb70277..18b083d34 100644 --- a/drowse/sae/AGENTS.md +++ b/drowse/sae/AGENTS.md @@ -346,5 +346,4 @@ instrument). `tests/support/sae.py` provides `MockSaeBackend`, section and recorded registry effects) and `make_resident(runtime, backend, layer=...)`. Session-level tests load a mock through `session.sae.load` with `drowse.sae.runtime.resolve_sae_backend` monkeypatched. Tests that need SAELens -`importorskip` it. The real end-to-end run, `tests/gpu/test_sae_lens_smoke.py`, -is opt-in via `DROWSE_TEST_SAELENS=1` and is marked `gpu_gemma`. +`importorskip` it. diff --git a/drowse/session/AGENTS.md b/drowse/session/AGENTS.md index 9ea876e27..405f548b3 100644 --- a/drowse/session/AGENTS.md +++ b/drowse/session/AGENTS.md @@ -393,6 +393,6 @@ the root `AGENTS.md` "Errors" rule; `user_message()` defaults to `tests.support.engine.engine_over`, `tests.support.extraction.make_extraction`. Route tests use `tests.support.server.mock_session`, an autospec that fails on removed or renamed members. -- GPU: `tests/gpu/test_session.py`, and `tests/gpu/test_smoke.py` (owns the - throughput regression), on `HuggingFaceTB/SmolLM2-360M-Instruct` unless - `DROWSE_TEST_MODEL` names another model. +- GPU: `tests/gpu/test_smoke.py` (one end-to-end smoke and the throughput + gates), on `HuggingFaceTB/SmolLM2-360M-Instruct` unless `DROWSE_TEST_MODEL` + names another model. diff --git a/drowse/session/extraction.py b/drowse/session/extraction.py index 60bbc8f2b..678961cec 100644 --- a/drowse/session/extraction.py +++ b/drowse/session/extraction.py @@ -393,7 +393,6 @@ def _fit_vector( metadata={ "method": "manifold_pca", "name": ret_name, - "share_metric": manifold.metadata.get("share_metric"), }, ) self._events.emit(ManifoldExtracted( diff --git a/drowse/store/AGENTS.md b/drowse/store/AGENTS.md index 057684068..65a7f9986 100644 --- a/drowse/store/AGENTS.md +++ b/drowse/store/AGENTS.md @@ -88,7 +88,7 @@ provenance only, and an unknown key raises. Provenance is all-or-nothing against | `method` | `fit_mode` | required block | | --- | --- | --- | | `manifold_pca`, `manifold_sae` | authored | | -| `manifold_monopolar[_sae]` | pca | 1-node ray, `subspace_metric = euclidean` | +| `manifold_monopolar[_sae]` | pca | 1-node ray (`concept − ν`) | | `manifold_discover_{pca,spectral,auto}` | its own | auto: `resolved_fit_mode`, `topology_winner`, `topology_candidates` | | `manifold_discover_sae` | any discover | | | `manifold_procrustes_transfer` | any | `source_model_id`, `source_model_fingerprint` | diff --git a/drowse/store/manifold_authoring.py b/drowse/store/manifold_authoring.py index 6f69f416d..cf8bcd219 100644 --- a/drowse/store/manifold_authoring.py +++ b/drowse/store/manifold_authoring.py @@ -453,9 +453,6 @@ def build(staging: Path) -> None: "method": method, "fit_mode": "baked", "nodes_sha256": nodes_sha, "model_fingerprint": item.model_fingerprint, } - share_metric = item.manifold.metadata.get("share_metric") - if share_metric: - meta["share_metric"] = share_metric if item.components: meta["components"] = item.components if method == "merge": diff --git a/drowse/store/manifold_lifecycle.py b/drowse/store/manifold_lifecycle.py index 2a9023e97..62e6a65e4 100644 --- a/drowse/store/manifold_lifecycle.py +++ b/drowse/store/manifold_lifecycle.py @@ -457,10 +457,6 @@ def transfer_manifold( metadata["source_model_id"] = from_model metadata["source_model_fingerprint"] = source_model_fingerprint metadata["model_fingerprint"] = target_model_fingerprint - # The share was re-baked in the target metric; ``subspace_metric`` - # keeps the source value (the basis was selected there and only - # rotated here). - metadata["share_metric"] = "mahalanobis" if transfer_quality_estimate is not None: metadata["transfer_quality_estimate"] = float(transfer_quality_estimate) sidecar_path = out_path.with_suffix(".json") @@ -502,8 +498,6 @@ def manifold_fit_summary(sidecar: ManifoldSidecar, stem: str) -> dict[str, Any]: ("hyperparams", sidecar.hyperparams), ("diagnostics", sidecar.diagnostics), ("node_spread", sidecar.node_spread_per_layer), - ("share_metric", sidecar.share_metric), - ("subspace_metric", sidecar.subspace_metric), ("resolved_fit_mode", sidecar.resolved_fit_mode), ("topology_winner", sidecar.topology_winner), ("topology_candidates", sidecar.topology_candidates), diff --git a/drowse/store/manifold_schema.py b/drowse/store/manifold_schema.py index 6c4e7d53a..a1659c8f3 100644 --- a/drowse/store/manifold_schema.py +++ b/drowse/store/manifold_schema.py @@ -619,8 +619,6 @@ class ManifoldSidecar: fit_policy_version: Optional[int] = None # ``mahalanobis`` on an ordinary fit; a monopolar fold labels its raw-δ̂ # basis ``euclidean`` (a basis-selection label, not a metric fallback). - share_metric: Optional[str] = None - subspace_metric: Optional[str] = None # Curved-fit summaries: GCV smoothing ``{L: {lambda, edf, gcv}}`` and the # fuzzy-manifold tube ``{L: {sigma_mean, sigma_min, sigma_max, lambda}}``. rbf_smoothing_per_layer: dict[str, Any] = field(default_factory=dict) @@ -839,11 +837,17 @@ def _non_negative_int(value: Any) -> bool: def validate_sidecar_payload( data: Any, *, location: str = "manifold sidecar", ) -> dict[str, Any]: - """Validate every field of the exact current fitted-sidecar contract.""" + """Validate every field of the current fitted-sidecar contract. + + Keys the contract no longer names (retired provenance) are dropped, so + removing a field never strands an existing fit; every current field must + be present. + """ from drowse.geometry.domains import validate_domain_spec - if not isinstance(data, dict) or set(data) != SIDECAR_FIELDS: - raise ManifoldFormatError(f"{location} does not match the current exact schema") + if not isinstance(data, dict) or not SIDECAR_FIELDS <= set(data): + raise ManifoldFormatError(f"{location} does not match the current schema") + data = {key: data[key] for key in SIDECAR_FIELDS} validate_manifold_format_version(data["format_version"], location=location) try: validate_domain_spec(data["domain"]) @@ -881,7 +885,7 @@ def validate_sidecar_payload( nullable_strings = ( "nodes_sha256", "sae_release", "sae_revision", "sae_fingerprint", "model_fingerprint", "model_source_fingerprint", "capture_sha256", - "share_metric", "subspace_metric", "resolved_fit_mode", + "resolved_fit_mode", "topology_winner", "bake_policy", "source_model_id", "source_model_fingerprint", ) diff --git a/pyproject.toml b/pyproject.toml index 6725a3746..2fbeec8d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,6 @@ addopts = ["--strict-markers"] xfail_strict = true markers = [ "gpu: requires a CUDA or MPS backend (deselect with `-m \"not gpu\"`)", - "gpu_gemma: requires explicit gated Gemma model and artifact access", "perf: wall-clock throughput test; skipped under xdist, run with `pytest -m perf -p no:xdist`", ] filterwarnings = [ diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 8aa790322..77205ab81 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -26,8 +26,7 @@ goldens the drowse.ai browser runtime replays. | Variable | Effect | |---|---| -| `DROWSE_TEST_MODEL` | The model GPU tests load (`tests/support/gpu.py::gpu_model_id`); default `HuggingFaceTB/SmolLM2-360M-Instruct`, public. Any compatible chat model works; `tests/gpu/test_model_numerics_gpu.py` checks one model per run. | -| `DROWSE_TEST_SAELENS=1` | Opts into `tests/gpu/test_sae_lens_smoke.py` (needs `sae_lens`; downloads a real SAE release). | +| `DROWSE_TEST_MODEL` | The model GPU tests load (`tests/support/gpu.py::gpu_model_id`); default `HuggingFaceTB/SmolLM2-360M-Instruct`, public. Any compatible chat model works. | | `DROWSE_TEST_LIVE_TOKENIZERS=1` | Lets `tests/model/test_role_template_drift.py` download the per-family tokenizers (some gated) instead of reading only the local HF cache. Run it before a release. | Never point a test at the real home: a session-scoped autouse fixture sets a @@ -65,19 +64,17 @@ so keep test modules exactly one directory deep. | Marker | Meaning | Where | |---|---|---| -| `gpu` | Needs CUDA or MPS; may download weights. CI deselects it. | `tests/gpu/` except `test_gpu_model.py` (CPU tests of the skip helper); the `mps` params of `tests/model/test_model_numerics.py` and `tests/generation/test_generation_metadata.py`; `tests/generation/test_scoring.py::test_real_forward_score_choices_distribution`. A test that requests `gpu_session` is marked automatically. | -| `gpu_gemma` | Needs explicit gated Gemma model and artifact access. | `tests/gpu/test_sae_lens_smoke.py` (also `gpu`, and opt-in via `DROWSE_TEST_SAELENS`). | +| `gpu` | Needs CUDA or MPS; may download weights. CI deselects it. | `tests/gpu/test_smoke.py` (one extract → steer → generate smoke and the perf gates); the `mps` params of `tests/model/test_model_numerics.py` and `tests/generation/test_generation_metadata.py`; `tests/generation/test_scoring.py::test_real_forward_score_choices_distribution`. A test that requests `gpu_session` is marked automatically. | | `perf` | Wall-clock throughput; serial only. | `tests/gpu/test_smoke.py`: `test_throughput_regression` (steered ≥ 85% of vanilla tok/s), `test_throughput_with_ablation` (ablation plus additive ≥ 80%; the extra matmul per ablation layer is why the bar is lower) and `test_throughput_with_unit_ablation` (8 neurons and 2 heads mean-ablated ≥ 80%) and `test_throughput_with_dashboard_instruments` (the served dashboard's decode ≥ 50%: the default roster scored per token with subspace coordinates, the live R readout over a micro-fit lens's every layer, eight alternatives, a loom row and a serialized WebSocket frame per token; about 63% on SmolLM2/MPS). All are also `gpu`. | GPU modules load with `device="auto"` (cuda > mps > cpu) and skip without CUDA or MPS. `load_or_skip_inaccessible` skips when loading fails on access (a gated repo, or a 401/403 auth error anywhere in the cause chain) and re-raises -anything else, so OOM or shape errors still fail. Each module builds its own -module-scoped session under its own scratch home (`test_session.py` needs -`probes=["register"]`); the conftest `gpu_session` (session scope, -`probes=[]`) is available for new GPU tests that need no roster. MPS runs the -default model 3 to 5 times slower than CUDA, so `test_smoke.py`'s extraction -budget is 60 s on MPS and 10 s on CUDA. +anything else, so OOM or shape errors still fail. The conftest `gpu_session` +(session scope, `probes=[]`) is available for a GPU test that needs a session. +Real-device behavior is otherwise covered by the tiny sessions on CPU; add a +GPU test only for something a CPU run cannot show (device numerics, MPS +kernels, throughput). Two MPS parity tests (`tests/geometry/test_frame_transport.py::test_two_d_pairs_on_mps_match_cpu`, `tests/geometry/test_synthesize_subspace.py::test_provider_fragments_follow_runtime_layer_device`) @@ -241,7 +238,6 @@ files from it, so their paths are part of its interface: | Test | Enforces | |---|---| -| `invariants/test_user_message.py` | The error family, over every non-underscore module under `drowse/`: each public exception class subclasses `DrowseError`, and the reparented ones keep their stdlib base; each `RequestError` subclass is a `ValueError` answering 400 with its own text. `_OVERRIDES` pins every overriding class's status; changing one is a user-visible contract change. | | `invariants/test_webui_types_parity.py` | `webui/src/lib/types.gen.ts` equals a fresh render (regenerate with `python scripts/generate_webui_types.py`); every native JSON route declares a response schema; native paths are kebab-case. | Contract pins elsewhere: `readout/test_measurements_envelope.py` (exact @@ -256,7 +252,8 @@ rest of the suite builds on). ## Writing tests -- Every bug fix lands with a regression test. +- Add a regression test when a bug could quietly come back; skip it when the fix + makes the bug unrepresentable or another test already covers it. - Write artifacts only under `isolated_home` or a `tmp_path` home. - A change to a wire record, an error status or a native route shape fails the pins above by design; update the pin in the same change. diff --git a/tests/cli/test_cli_manifold_fold.py b/tests/cli/test_cli_manifold_fold.py index 27b64df70..de33b8de8 100644 --- a/tests/cli/test_cli_manifold_fold.py +++ b/tests/cli/test_cli_manifold_fold.py @@ -145,7 +145,6 @@ def _write_fitted_manifold( path = folder / tensor_filename(_MODEL, release=None) metadata: dict[str, object] = { "method": "manifold_discover_pca", "fit_mode": "pca", - "share_metric": "euclidean", "model_fingerprint": "test-fingerprint", } manifest = folder / "manifold.json" diff --git a/tests/fit/test_manifold_extraction.py b/tests/fit/test_manifold_extraction.py index e10ac2c6f..e3a97b3e0 100644 --- a/tests/fit/test_manifold_extraction.py +++ b/tests/fit/test_manifold_extraction.py @@ -1512,10 +1512,6 @@ def test_discover_pca_produces_affine_subspaces(tmp_path: Path) -> None: assert sub.node_coords is not None # real per-layer coords assert sub.node_coords.shape[0] == 5 # one row per node assert sub.node_coords.shape[1] == sub.rank # (K, R) - # subspace_metric is always "mahalanobis": the stub carries a covering - # whitener and there is no Euclidean fit path. - sidecar = json.loads((folder / Path(tensor_filename("stub-model")).with_suffix(".json").name).read_text()) - assert sidecar["subspace_metric"] == "mahalanobis" def test_discover_pca_layout_is_neutral_centered(tmp_path: Path) -> None: diff --git a/tests/generation/test_logits.py b/tests/generation/test_logits.py index a0885f060..d25e23cdf 100644 --- a/tests/generation/test_logits.py +++ b/tests/generation/test_logits.py @@ -2,10 +2,7 @@ ``LoomNode.mean_logprob`` aggregation. Mostly CPU-only structural tests since the new surfaces are typed -dataclasses + composition logic that doesn't need a model on disk. The -end-to-end "captured logprob matches log_softmax" check would require a -real model load and lives in ``test_session.py`` (GPU-gated) once it's -sensible to add there. +dataclasses + composition logic that doesn't need a model on disk. """ from __future__ import annotations diff --git a/tests/geometry/test_fold.py b/tests/geometry/test_fold.py index 12912ae3e..25b5e8d14 100644 --- a/tests/geometry/test_fold.py +++ b/tests/geometry/test_fold.py @@ -127,7 +127,6 @@ def test_fold_directions_to_subspace_neutral_anchored(): ) assert mfld.node_labels == ["merged"] assert torch.allclose(mfld.node_coords, torch.tensor([[1.0]])) # display layout - assert mfld.metadata["share_metric"] == "mahalanobis" assert mfld.origin == {} for L, raw in ((2, dir0), (5, dir1)): sub = mfld.layers[L] diff --git a/tests/geometry/test_manifold_math.py b/tests/geometry/test_manifold_math.py index 801a15aa9..528053d39 100644 --- a/tests/geometry/test_manifold_math.py +++ b/tests/geometry/test_manifold_math.py @@ -787,8 +787,14 @@ def test_save_load_manifold_round_trip(tmp_path: Path, monkeypatch: pytest.Monke _attach_constant_sigma(sub, 0.1) path = tmp_path / "mood" / "model.safetensors" save_manifold(manifold, path, {"method": "manifold_pca", - "nodes_sha256": "abc123", - "share_metric": "mahalanobis"}) + "nodes_sha256": "abc123"}) + # A key the sidecar contract has retired (6.1.0 wrote ``share_metric``) + # is ignored on read rather than stranding the fit. + import json + sidecar_path = path.with_suffix(".json") + sidecar = json.loads(sidecar_path.read_text()) + sidecar["share_metric"] = "mahalanobis" + sidecar_path.write_text(json.dumps(sidecar)) loaded = load_manifold(path) assert loaded.name == "mood" @@ -800,7 +806,6 @@ def test_save_load_manifold_round_trip(tmp_path: Path, monkeypatch: pytest.Monke # New v4+ companion fields round-trip: the per-layer whitened share # and the metric label that records which weighting the fit used. assert loaded.mahalanobis_share == {4: 1.5, 9: 2.0} - assert loaded.metadata["share_metric"] == "mahalanobis" # The per-layer ``origin`` round-trips through the sidecar. assert sorted(loaded.origin) == [4, 9] for idx in (4, 9): @@ -1591,7 +1596,6 @@ def test_save_load_affine_manifold_round_trip( "components": {"0:default/x": { "selector": "default/x", "alpha": 1.0, "tensor_sha256": "0" * 64, }}, - "share_metric": "mahalanobis", }) # The on-disk payload omits the RBF triple + coord normalization for the diff --git a/tests/geometry/test_profile.py b/tests/geometry/test_profile.py index abee124ca..b7fdbb749 100644 --- a/tests/geometry/test_profile.py +++ b/tests/geometry/test_profile.py @@ -376,14 +376,13 @@ def test_sidecar_schema_is_exactly_five_fields(tmp_path: Path) -> None: def test_extract_shaped_profile_round_trips(tmp_path: Path) -> None: """The Profile ``DrowseSession.extract`` returns saves and loads. - ``extract`` stamps ``{"method": "manifold_pca", "name": ..., "share_metric": - ...}`` on the folded 2-node manifold view; the schema has to take that shape - verbatim, and the loaded metadata has to save again unchanged. + ``extract`` stamps ``{"method": "manifold_pca", "name": ...}`` on the + folded 2-node manifold view; the schema has to take that shape verbatim, + and the loaded metadata has to save again unchanged. """ extract_metadata = { "method": "manifold_pca", "name": "confident.uncertain", - "share_metric": "mahalanobis", } profile = Profile(_mk(layers=(0, 3)), metadata=extract_metadata) first = tmp_path / "extract.safetensors" @@ -392,7 +391,6 @@ def test_extract_shaped_profile_round_trips(tmp_path: Path) -> None: loaded = Profile.load(first) assert loaded.metadata["method"] == "manifold_pca" assert loaded.metadata["name"] == "confident.uncertain" - assert loaded.metadata["share_metric"] == "mahalanobis" # load -> save round-trip: the writer-stamped identity fields the reader # hands back must not read as unknown metadata on the way in again. diff --git a/tests/gpu/test_curved_stack_gpu.py b/tests/gpu/test_curved_stack_gpu.py deleted file mode 100644 index ace2de521..000000000 --- a/tests/gpu/test_curved_stack_gpu.py +++ /dev/null @@ -1,100 +0,0 @@ -"""On the accelerator, a curved probe's stacked read is the per-layer read, bit for bit. - -The stacked read batches only work whose batched kernel computes each layer as -the one-layer kernel does (elementwise ops, the foot solve's batched products) -and keeps every reduction on its own layer's rows; the cross-layer means fold -in layer order. This pins that on the device, against the per-layer read -written out (each layer's projection, ``invert_parameterization`` and σ, then -sequential weighted sums), so batching a reduction across layers (which moves -values by an ulp, and the foot of an off-surface state to another basin) fails -here. Exact on MPS; CUDA's batched and single kernels are not checked for -per-slice identity, so there the comparison takes a tolerance. -""" -from __future__ import annotations - -import pytest -import torch - -from drowse.geometry.invert import invert_parameterization -from drowse.readout.monitor import Monitor, _mean_coordinate_space, _mean_domain_coordinates -from tests.support.curved import DIM, curved_stack_manifold as _curved -from tests.support.whitener import isotropic_whitener - -pytestmark = [ - pytest.mark.gpu, - pytest.mark.skipif( - not (torch.cuda.is_available() or torch.backends.mps.is_available()), - reason="No GPU backend available (neither CUDA nor MPS)", - ), -] - -DEVICE = torch.device("cuda" if torch.cuda.is_available() else "mps") -EXACT = DEVICE.type == "mps" - - -def _per_layer(mon: Monitor, hidden: dict[int, torch.Tensor], feet: dict[int, torch.Tensor] | None): - """The per-layer read, as the Monitor ran it before the stacked pass.""" - probe = mon.attached_probes()["curve"] - m = probe.manifold - stack = mon._read_stack(DEVICE) - H = torch.stack([hidden[L].float() for L in stack.layers]).unsqueeze(1) - sih = stack.apply_stack(H) - shared = list(m.layers) - total = sum(probe.share_weights[L] for L in shared) - w = {L: probe.share_weights[L] / total for L in shared} - ref_layer = max(shared, key=lambda L: w[L]) - frac_mean = coords_mean = mem_mean = None - ref_coord = None - new_feet = {} - for L in shared: - wh, sub, i = probe.whitened[L], m.layers[L], stack.index[L] - x = H[i, 0] - wh.mean - sx = sih[i, 0] - wh.s_mean - x_m = torch.sqrt((x * sx).sum().clamp(min=0.0)).clamp(min=1e-8) - g = wh.basis @ sx - c = wh.m_r_inv @ g - frac = (torch.sqrt((g * c).sum().clamp(min=0.0)) / x_m).clamp(min=0.0, max=1.0).reshape(1) - pos, res = invert_parameterization( - sub, m.domain, c.reshape(1, -1), m.node_coords, - warm_start=None if feet is None else feet[L], - ) - new_feet[L] = pos - coord = pos.reshape(-1) - if L == ref_layer: - ref_coord = coord - space = w[L] * _mean_coordinate_space(m.domain, coord) - sig = sub.sigma_at(m.domain.embed(pos.reshape(1, -1))).reshape(1) - mem = torch.exp(-(res.reshape(-1)[:1] ** 2) / (2.0 * (sig ** 2).clamp(min=1e-8))) - frac_mean = w[L] * frac if frac_mean is None else frac_mean + w[L] * frac - coords_mean = space if coords_mean is None else coords_mean + space - mem_mean = w[L] * mem if mem_mean is None else mem_mean + w[L] * mem - assert ref_coord is not None and coords_mean is not None - assert frac_mean is not None and mem_mean is not None - coords = _mean_domain_coordinates(m.domain, coords_mean, ref_coord) - return (float(frac_mean), coords.tolist(), float(mem_mean)), new_feet - - -@pytest.mark.parametrize("periodic", [False, True]) -def test_stacked_curved_read_is_the_per_layer_read(periodic: bool) -> None: - m = _curved(n_layers=6, periodic=periodic) - mon = Monitor(whitener=isotropic_whitener(list(m.layers), DIM), n_layers=8) - mon.add_probe("curve", m.to(device=DEVICE, dtype=torch.float32), top_n=3) - gen = torch.Generator().manual_seed(11) - mon.reset_curved_feet() - mon.enable_curved_warm(True) - feet = None - with torch.inference_mode(): - for _ in range(8): - hidden = {} - for L, sub in m.layers.items(): - t = torch.rand((1, 1), generator=gen) * (6.0 if periodic else 1.6) - (0.0 if periodic else 0.8) - on = sub.eval_at(m.domain.embed(t)).reshape(-1) - hidden[L] = (on + 0.3 * torch.randn(DIM, generator=gen)).to(DEVICE) - got = mon.score_single_token(hidden)["curve"] - (frac, coords, mem), feet = _per_layer(mon, hidden, feet) - if EXACT: - assert (got.fraction, list(got.coords), got.membership) == (frac, coords, mem) - else: - assert got.fraction == pytest.approx(frac, abs=1e-5) - assert list(got.coords) == pytest.approx(coords, abs=1e-4) - assert got.membership == pytest.approx(mem, abs=1e-5) diff --git a/tests/gpu/test_gpu_model.py b/tests/gpu/test_gpu_model.py deleted file mode 100644 index 6d286f682..000000000 --- a/tests/gpu/test_gpu_model.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import annotations - -import pytest - -from tests.support.gpu import _is_access_error, load_or_skip_inaccessible - - -@pytest.mark.parametrize( - "error", - [ - RuntimeError("403 Client Error: access to this gated repo is not authorized"), - RuntimeError("401 Client Error: authentication token is required"), - RuntimeError("Cannot access gated repository"), - ], -) -def test_model_access_errors_are_classified(error: Exception) -> None: - assert _is_access_error(error) - - -@pytest.mark.parametrize( - "error", - [ - RuntimeError("MPS ran out of memory"), - RuntimeError("tensor shape mismatch"), - RuntimeError("403 Client Error: rate limit exceeded"), - ], -) -def test_runtime_and_transport_failures_are_not_access_errors(error: Exception) -> None: - assert not _is_access_error(error) - - -def test_access_error_in_cause_chain_skips() -> None: - cause = RuntimeError("403 Client Error: gated access is not authorized") - wrapper = OSError("model could not be loaded") - wrapper.__cause__ = cause - - with pytest.raises(pytest.skip.Exception): - load_or_skip_inaccessible(lambda: (_ for _ in ()).throw(wrapper), "gated/model") - - -def test_non_access_error_is_raised() -> None: - error = RuntimeError("invalid tensor shape") - with pytest.raises(RuntimeError, match="invalid tensor shape"): - load_or_skip_inaccessible(lambda: (_ for _ in ()).throw(error), "public/model") - - -def test_missing_public_repository_is_not_treated_as_gated() -> None: - error = RuntimeError("404 Client Error: Repository Not Found") - assert not _is_access_error(error) diff --git a/tests/gpu/test_jlens_gpu.py b/tests/gpu/test_jlens_gpu.py deleted file mode 100644 index 297dd3c81..000000000 --- a/tests/gpu/test_jlens_gpu.py +++ /dev/null @@ -1,130 +0,0 @@ -"""GPU integration tests for the Jacobian lens on a real model. - -Micro-fit only (a handful of prompts) — enough to exercise the real -transport path at both sites, the motor-regime sanity check, and a jlens -steering atom end-to-end. Real lens quality needs `drowse lens fit` over -more documents; nothing here asserts readout *quality* beyond the -late-layer identity. -""" - -from __future__ import annotations - -from typing import Any - -import pytest -import torch - -from drowse import SamplingConfig -from tests.support.gpu import gpu_model_id, load_or_skip_inaccessible - -pytestmark = [ - pytest.mark.gpu, - pytest.mark.skipif( - not (torch.cuda.is_available() or torch.backends.mps.is_available()), - reason="No GPU backend available (neither CUDA nor MPS)", - ), -] - -MODEL_ID = gpu_model_id() - -_FIT_PROMPTS = [ - "The history of the printing press begins in fifteenth-century Mainz, " - "where Johannes Gutenberg combined movable metal type with a screw press " - "adapted from wine-making, transforming the economics of written text " - "across Europe within a single generation.", - "Photosynthesis converts light energy into chemical energy in two stages: " - "the light-dependent reactions split water and generate ATP and NADPH, " - "while the Calvin cycle fixes atmospheric carbon dioxide into " - "three-carbon sugars that feed the plant's metabolism.", -] - - -@pytest.fixture(scope="module") -def session() -> Any: - from drowse import DrowseSession - - created = load_or_skip_inaccessible( - lambda: DrowseSession.from_pretrained(MODEL_ID, device="auto", probes=[]), - MODEL_ID, - ) - with created as s: - yield s - - -@pytest.fixture(scope="module") -def micro_lens(session: Any, tmp_path_factory: pytest.TempPathFactory) -> Any: - import os - - home = tmp_path_factory.mktemp("drowse-home") - old = os.environ.get("DROWSE_HOME") - os.environ["DROWSE_HOME"] = str(home) - try: - lens = session.lens.fit( - _FIT_PROMPTS, corpus_spec="gpu-micro-test", dim_batch=64, seq_len=64, - ) - yield lens - finally: - session.lens.runtime.clear() - if old is None: - os.environ.pop("DROWSE_HOME", None) - else: - os.environ["DROWSE_HOME"] = old - - -def test_micro_fit_shape(session: Any, micro_lens: Any) -> None: - n_layers = len(session.layers) - assert micro_lens.n_prompts == 2 - assert len(micro_lens.source_layers) == n_layers - 1 - assert micro_lens.layers("mid") == list(range(n_layers)) - d = session.model_info["hidden_dim"] - assert micro_lens.d_model == d - for J in micro_lens.jacobians.values(): - assert torch.isfinite(J).all() - - -def test_motor_regime_top1_matches_model(session: Any, micro_lens: Any) -> None: - """At the last source layer J ≈ I, so the lens top-1 must equal the - model's actual next-token prediction (the logit-lens identity).""" - session.lens.runtime.lens = micro_lens - last = micro_lens.source_layers[-1] - prompt = "Fact: the currency used in the country shaped like a boot is the" - out = session.lens.readout(prompt, layers=[last], top_k=5) - - ids = session.tokenizer(prompt, return_tensors="pt")["input_ids"].to(session.device) - with torch.inference_mode(): - logits = session.model(input_ids=ids).logits[0, -1] - model_top1 = session.tokenizer.decode([int(logits.argmax())]) - lens_top_tokens = [tok for tok, _ in out[last][0]] - assert model_top1 in lens_top_tokens[:2], (model_top1, lens_top_tokens) - - -def test_jlens_steering_atom_generates(session: Any, micro_lens: Any) -> None: - session.lens.runtime.lens = micro_lens - result = session.generate( - "Say something brief.", - steering="0.5 jlens/ocean", - sampling=SamplingConfig(max_tokens=16, seed=7), - stateless=True, - # A thinking model (Qwen3) spends 16 tokens inside : off. - thinking=False, - ) - assert result.first.text - assert "jlens/ocean" in (result.first.applied_steering or "") - - -def test_live_lens_does_not_break_static_steerable(session: Any, micro_lens: Any) -> None: - session.lens.runtime.lens = micro_lens - baseline = session._steering.static_steerable() - layers = session.lens.enable_live() - try: - assert session._steering.static_steerable() == baseline - assert layers # resolved the fitted layer set - result = session.generate( - "Count to three.", - sampling=SamplingConfig(max_tokens=8, seed=3), - stateless=True, - thinking=False, - ) - assert result.first.text - finally: - session.lens.disable_live() diff --git a/tests/gpu/test_model_numerics_gpu.py b/tests/gpu/test_model_numerics_gpu.py deleted file mode 100644 index 090293287..000000000 --- a/tests/gpu/test_model_numerics_gpu.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Real-weight checks; select one model at a time with DROWSE_TEST_MODEL.""" - -import gc -import math -from typing import Any - -import pytest -import torch - -from drowse.fit.capture import _capture_all_hidden_states -from drowse.generation.decode import GenerationConfig, GenerationState, generate_steered -from drowse.lens.jlens import JacobianLens, transport_rows -from drowse.model.arch import get_layers -from drowse.model.load import load_model -from drowse.model.readout import LogitReadout -from tests.support.gpu import gpu_model_id, load_or_skip_inaccessible - -pytestmark = [pytest.mark.gpu, pytest.mark.skipif( - not (torch.cuda.is_available() or torch.backends.mps.is_available()), reason="GPU unavailable", -)] - - -@pytest.fixture(scope="module") -def loaded(): - model_id = gpu_model_id() - model, tokenizer = load_or_skip_inaccessible(lambda: load_model(model_id, device="auto", quantize=None), model_id) - yield model, tokenizer - del model, tokenizer - gc.collect() - if torch.backends.mps.is_available(): - torch.mps.empty_cache() - - -@pytest.mark.parametrize("prompt", ["The capital of France is", "Once upon a time there was a cat"]) -def test_native_identity_readout_matches_real_output_head(loaded: tuple[Any, Any], prompt: str): - model, tokenizer = loaded - device = next(model.parameters()).device - ids = tokenizer(prompt, return_tensors="pt")["input_ids"].to(device) - layers = get_layers(model) - last = len(layers) - 1 - captured = _capture_all_hidden_states(model, layers, ids, layer_indices=[last], pool_index=ids.shape[1] - 1) - dim = captured[last].numel() - lens = JacobianLens({last: torch.eye(dim, device=device)}, n_prompts=1, d_model=dim) - with torch.inference_mode(): - expected = model(ids, use_cache=False).logits[0, -1].float() - row = captured[last].reshape(1, dim) - J = lens.jacobians[last].to(device).unsqueeze(0) - actual = LogitReadout(model)(transport_rows(J, row))[0].float() - # The fused LM head and the lens readout run different kernels; on MPS - # half-precision the two differ by a few ulps on a handful of logits. - torch.testing.assert_close(actual, expected, atol=2e-2, rtol=1e-2) - - -def test_native_decode_matches_direct_cached_reference(loaded: tuple[Any, Any]): - model, tokenizer = loaded - device = next(model.parameters()).device - ids = tokenizer("The capital of France is", return_tensors="pt")["input_ids"].to(device) - references = [] - forced = [] - cache = None - current = ids - with torch.inference_mode(): - for _ in range(4): - output = model(input_ids=current, past_key_values=cache, use_cache=True) - logits = output.logits[0, -1].float() - values, indices = logits.topk(min(1024, logits.numel())) - probabilities = values.softmax(-1) - references.append((indices, probabilities)) - forced.append(int(indices[0])) - current = indices[:1].reshape(1, 1) - cache = output.past_key_values - del cache - frames = [] - generated = generate_steered( - model, tokenizer, ids, GenerationConfig(max_new_tokens=4, temperature=1., top_k=1024, top_p=1.), - GenerationState(), forced_prefix=forced, seed=18, logprobs=5, - on_token=lambda *frame: frames.append(frame), - ) - assert generated == forced - for frame in frames: - _, _, token_id, logprob, alternatives, perplexity, step = frame - indices, probabilities = references[step] - assert token_id == forced[step] - assert logprob == pytest.approx(float(probabilities[0].log()), abs=2e-5) - expected_entropy = -(probabilities * probabilities.clamp_min(torch.finfo(torch.float32).tiny).log()).sum() - assert perplexity == pytest.approx(math.exp(float(expected_entropy)), rel=2e-5) - assert [alt.id for alt in alternatives] == indices[:5].tolist() - assert [alt.logprob for alt in alternatives] == pytest.approx(probabilities[:5].log().tolist(), abs=2e-5) - - -@pytest.mark.parametrize("ablate", [False, True]) -def test_native_steering_matches_manual_intervention(loaded: tuple[Any, Any], ablate: bool): - from drowse.steering.hooks import _SUBSPACE_GAIN, SteeringManager, SteeringPlan - from drowse.geometry.subspace import LayerSubspace - from drowse.geometry.inject import SynthesizedSubspace - - model, tokenizer = loaded - parameter = next(model.parameters()) - layers = get_layers(model) - layer = len(layers) // 2 - ids = tokenizer("A small cat sat", return_tensors="pt")["input_ids"].to(parameter.device) - dim = model.config.hidden_size - basis = torch.zeros(2, dim) - basis[0, 3], basis[1, 7] = 1., 1. - mean = torch.zeros(dim) - target = torch.tensor([.25, 0. if ablate else -.125]) - kappa = torch.tensor([0., .5 if ablate else 0.]) - synth = SynthesizedSubspace({layer: LayerSubspace.affine(mean, basis, basis.clone())}, {layer: target}, {layer: 1.}, {layer: kappa}) - plan = SteeringPlan() - plan.add_subspace("check", synth) - manager = SteeringManager() - manager.install(plan, layers, parameter.device, parameter.dtype) - try: - with torch.inference_mode(): - actual = model(ids, use_cache=False).logits - finally: - manager.clear_all() - - def intervene(_module: Any, _args: Any, output: Any): - h = output[0] if isinstance(output, tuple) else output - replacement = h.float().clone() - replacement[..., 3] += .25 * _SUBSPACE_GAIN - replacement[..., 7] = ( - replacement[..., 7] * .5 if ablate else replacement[..., 7] - .125 * _SUBSPACE_GAIN - ) - replacement = replacement.to(h.dtype) - return (replacement, *output[1:]) if isinstance(output, tuple) else replacement - - handle = layers[layer].register_forward_hook(intervene) - try: - with torch.inference_mode(): - expected = model(ids, use_cache=False).logits - finally: - handle.remove() - torch.testing.assert_close(actual, expected, atol=0, rtol=0) - - -def test_verify_model_spec_accepts_the_loaded_model(loaded: tuple[Any, Any]): - """A covered family verifies at its load precision (bf16 on CUDA and MPS). - - Held to its own bf16 forward, the fp32 description of SmolLM2-360M was - refused (19% of layer 4's attention update is rounding against massive - activations); the fp32 replay holds structure, not precision. - """ - from drowse.errors import UnsupportedBlockError - from drowse.model.blocks import model_spec, verify_model_spec - - model, _tokenizer = loaded - try: - spec = model_spec(model) - except UnsupportedBlockError as exc: - pytest.skip(f"{gpu_model_id()} is not a covered dense family: {exc}") - assert verify_model_spec(model, spec, length=64) is spec diff --git a/tests/gpu/test_sae_lens_smoke.py b/tests/gpu/test_sae_lens_smoke.py deleted file mode 100644 index 01eaa5822..000000000 --- a/tests/gpu/test_sae_lens_smoke.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Opt-in end-to-end smoke test for the SAE pipeline. - -Gated on DROWSE_TEST_SAELENS=1 because it requires ``sae_lens`` installed -and downloads a real SAE release (hundreds of MB). Not on CI by default. -""" -from __future__ import annotations - -import os - -import pytest - -pytestmark = [ - pytest.mark.gpu, - pytest.mark.gpu_gemma, - pytest.mark.skipif( - os.environ.get("DROWSE_TEST_SAELENS") != "1", - reason="opt in via DROWSE_TEST_SAELENS=1", - ), -] - - -def test_end_to_end_sae_extraction_and_generate(): - import torch - - from drowse import DrowseSession - - has_gpu = torch.cuda.is_available() or ( - hasattr(torch.backends, "mps") and torch.backends.mps.is_available() - ) - if not has_gpu: - pytest.skip("requires CUDA or MPS") - - session = DrowseSession.from_pretrained( - "google/gemma-2-2b-it", device="auto", probes=[], - ) - - # Small, canonical GemmaScope release — one SAE per covered layer. - name, profile = session.extract( - "happy.sad", - sae="gemma-scope-2b-pt-res-canonical", - ) - assert ":sae-" in name - assert len(profile) > 0 - - vanilla = session.generate("Describe a rainy afternoon.").first.text - with session.steering(f"0.3 {name}"): - steered = session.generate("Describe a rainy afternoon.").first.text - assert vanilla != steered, "SAE-steered output should differ from vanilla" diff --git a/tests/gpu/test_session.py b/tests/gpu/test_session.py deleted file mode 100644 index 4b9f870cc..000000000 --- a/tests/gpu/test_session.py +++ /dev/null @@ -1,424 +0,0 @@ -"""Tests for DrowseSession programmatic API. -Requires a GPU (CUDA or Apple Silicon MPS) and downloads the public -SmolLM2-360M-Instruct test model on first run. Override with -``DROWSE_TEST_MODEL``. -""" -from __future__ import annotations -from pathlib import Path -from typing import TYPE_CHECKING -import pytest -import torch -from drowse.geometry.profile import Profile -from drowse.generation.finalize import MIN_ELAPSED_FOR_RATE -from drowse.generation.results import GenerationResult, RunSet, TokenEvent -from tests.support.gpu import gpu_model_id, load_or_skip_inaccessible - -if TYPE_CHECKING: - from drowse.session.session import DrowseSession - -_HAS_GPU = torch.cuda.is_available() or torch.backends.mps.is_available() -pytestmark = [ - pytest.mark.gpu, - pytest.mark.skipif( - not _HAS_GPU, - reason="No GPU backend available (neither CUDA nor MPS)", - ), -] - -MODEL_ID = gpu_model_id() - - -def _corpus(response: str) -> list[str]: - from drowse.fit.capture import _load_baseline_prompts - - return [response] * len(_load_baseline_prompts()) - - -@pytest.fixture(scope="module") -def session(tmp_path_factory: pytest.TempPathFactory): - import os - from drowse.session.session import DrowseSession - # Isolate $DROWSE_HOME so this module's extract/merge writes (e.g. - # local/formal.casual, happy.sad, honest) land in a throwaway cache instead - # of the user's real ~/.drowse — where they would shadow bundled manifolds of - # the same name and break bare-name resolution (AmbiguousSelectorError) - # across the whole suite on the next run. - home = tmp_path_factory.mktemp("drowse_home") - prev = os.environ.get("DROWSE_HOME") - os.environ["DROWSE_HOME"] = str(home) - # device="auto" picks cuda > mps > cpu; skipif above guarantees a GPU. - try: - s = load_or_skip_inaccessible( - lambda: DrowseSession.from_pretrained(MODEL_ID, device="auto", probes=["register"]), - MODEL_ID, - ) - yield s - s.close() - finally: - if prev is None: - os.environ.pop("DROWSE_HOME", None) - else: - os.environ["DROWSE_HOME"] = prev - -class TestConstruction: - def test_model_info(self, session: DrowseSession) -> None: - info = session.model_info - assert isinstance(info["model_type"], str) - assert info["model_type"] - assert info["hidden_dim"] > 0 - assert info["num_layers"] > 0 - - def test_config_defaults(self, session: DrowseSession) -> None: - assert session.config.temperature == 1.0 - assert session.config.top_p == 0.9 - assert session.config.max_new_tokens == 1024 - - def test_probes_loaded(self, session: DrowseSession) -> None: - assert len(session.probes) > 0 - - def test_history_starts_empty(self, session: DrowseSession) -> None: - assert session.tree.messages_for() == [] - - def test_vectors_starts_empty(self, session: DrowseSession) -> None: - assert session.profiles == {} - - def test_last_result_starts_none(self, session: DrowseSession) -> None: - assert session.last_result is None - -class TestSteering: - def test_extract_and_steer(self, session: DrowseSession) -> None: - name, profile = session.extract_from_corpora( - "happy", _corpus("I am happy"), _corpus("I am sad"), - ) - assert isinstance(profile, Profile) - assert all(isinstance(k, int) for k in profile) - session.registry.put_profile("happy", profile) - assert "happy" in session.profiles - # profile registry stores raw per-layer tensor dicts (wrap in Profile for the public view). - assert isinstance(session.profiles["happy"], dict) - - def test_unsteer(self, session: DrowseSession) -> None: - session.registry.pop_profile("happy") - assert "happy" not in session.profiles - - def test_extract_curated(self, session: DrowseSession) -> None: - name, profile = session.extract("happy", baseline="sad") - assert name == "happy.sad" - assert isinstance(profile, Profile) - assert len(profile) > 0 - - def test_extract_datasource(self, session: DrowseSession) -> None: - name, profile = session.extract_from_corpora( - "formal.casual", _corpus("formal"), _corpus("casual"), - ) - assert isinstance(profile, Profile) - -class TestMixedSteering: - """Affine terms beside a curved ``%`` term on the session device. - - Regression: the affine basis is host-side at install while the curved - manifold's basis is resident on the device, so orthogonalizing one against - the other raised a device mismatch and every such recipe failed to install. - """ - - @pytest.fixture(scope="class", autouse=True) - def fitted(self, session: DrowseSession) -> None: - from drowse.base.paths import manifold_dir - from drowse.store.bundled import materialize_bundled_artifacts - - materialize_bundled_artifacts() - for name in ("months", "deceptive", "formal.casual"): - session.fit(manifold_dir("default", name)) - - @pytest.mark.parametrize( - "affine", - [ - "0.5 default/deceptive", - "0.5 !default/deceptive", - "0.5 default/deceptive%deceptive", - "0.5 default/formal.casual%casual", - "0.5 default/deceptive~default/formal.casual", - ], - ) - def test_affine_beside_a_curved_term(self, session: DrowseSession, affine: str) -> None: - from drowse.generation.sampling import SamplingConfig - - result = session.generate( - "Say hi.", steering=f"{affine} + 0.4 default/months%march", - sampling=SamplingConfig(max_tokens=4), stateless=True, - ).first - assert result.text is not None - - -class TestMonitoring: - def test_monitor_and_unmonitor(self, session: DrowseSession) -> None: - # Extract registers the folded direction; ``add_probe`` resolves it - # (the unified probe API — one attach for vector + manifold probes). - name, _profile = session.extract_from_corpora( - "honest", _corpus("I am honest"), _corpus("I am deceptive"), - ) - session.add_probe(name, as_name="test_probe") - assert "test_probe" in session.probes - session.remove_probe("test_probe") - assert "test_probe" not in session.probes - -class TestLifecycle: - def test_context_manager(self): - from drowse.session.session import DrowseSession - created = load_or_skip_inaccessible( - lambda: DrowseSession.from_pretrained(MODEL_ID, device="auto", probes=[]), - MODEL_ID, - ) - with created as s: - assert s.model_info["model_type"] - -class TestGeneration: - def test_generate_unsteered(self, session: DrowseSession) -> None: - result = session.generate("Say hello in one word.") - assert isinstance(result, RunSet) - assert isinstance(result.first, GenerationResult) - single = result.first - assert len(single.text) > 0 - assert single.token_count > 0 - assert single.elapsed > 0 - # A one-word reply can finish inside the rate's resolution floor, - # which reports 0 rather than a timer-dominated rate. - if single.elapsed >= MIN_ELAPSED_FOR_RATE: - assert single.tok_per_sec > 0 - else: - assert single.tok_per_sec == 0 - assert single.steering_alphas == {} - - def test_generate_blocking_messages(self, session: DrowseSession) -> None: - result = session.generate([ - {"role": "user", "content": "Say hello in one word."}, - ]) - assert isinstance(result, RunSet) - assert isinstance(result.first, GenerationResult) - assert len(result.first.text) > 0 - - def test_generate_appends_to_history(self, session: DrowseSession) -> None: - session.tree.reset() - session.generate("Say hi.") - messages = session.tree.messages_for() - assert len(messages) == 2 - assert messages[0]["role"] == "user" - assert messages[1]["role"] == "assistant" - - def test_generate_with_alphas(self, session: DrowseSession) -> None: - name, profile = session.extract_from_corpora( - "formal.casual", _corpus("formal"), _corpus("casual"), - ) - session.registry.put_profile(name, profile) - result = session.generate("Hello.", steering=f"0.1 {name}").first - assert result.steering_alphas == {name: 0.1} - session.registry.pop_profile(name) - - def test_generate_with_probes(self, session: DrowseSession) -> None: - session.tree.reset() - result = session.generate("Tell me something exciting!").first - if session.probes: - assert isinstance(result.probe_readings, dict) - - def test_last_result(self, session: DrowseSession) -> None: - session.tree.reset() - result = session.generate("Hello.") - assert session.last_result is result.first - - def test_ab_comparison(self, session: DrowseSession) -> None: - """A/B test: same prompt, with and without steering.""" - name, profile = session.extract_from_corpora( - "happy", _corpus("I am happy"), _corpus("I am sad"), - ) - session.registry.put_profile(name, profile) - session.tree.reset() - steered = session.generate("Describe a sunset.", steering=f"0.2 {name}").first - session.tree.reset() - unsteered = session.generate("Describe a sunset.").first - assert steered.steering_alphas == {name: 0.2} - assert unsteered.steering_alphas == {} - # Both should produce text - assert len(steered.text) > 0 - assert len(unsteered.text) > 0 - session.registry.pop_profile(name) - - def test_unknown_vector_raises(self, session: DrowseSession) -> None: - with pytest.raises(KeyError, match="nonexistent"): - session.generate("Hello.", steering="0.1 nonexistent") - -class TestCliRoundTrip: - def test_extract_cli_roundtrip(self, tmp_path: Path) -> None: - import subprocess - import sys - from drowse.base.paths import manifold_dir, safe_model_id - - folder = manifold_dir("local", "happy.sad") - sid = safe_model_id(MODEL_ID) - tensor_path = folder / f"{sid}.safetensors" - created_here = not tensor_path.exists() - - try: - proc = subprocess.run( - [ - sys.executable, "-m", "drowse", "manifold", "extract", - "local/happy.sad", "-m", MODEL_ID, - ], - capture_output=True, text=True, timeout=600, - ) - assert proc.returncode == 0, ( - f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" - ) - assert tensor_path.exists(), f"expected {tensor_path} to exist" - finally: - # Only unlink the per-model tensor if this test created it; - # leave the corpus and any pre-existing tensor alone. - if created_here and tensor_path.exists(): - tensor_path.unlink() - sidecar = folder / f"{sid}.json" - if sidecar.exists(): - sidecar.unlink() - - -class TestStreamingGeneration: - def test_generate_stream(self, session: DrowseSession) -> None: - session.tree.reset() - tokens = [] - for event in session.generate_stream("Say hello."): - assert isinstance(event, TokenEvent) - tokens.append(event) - assert len(tokens) > 0 - assert all(isinstance(t.text, str) for t in tokens) - result = session.last_result - assert result is not None - # A character spanning several tokens (an emoji, say) streams as one - # merged event, so events can be fewer than tokens; the text matches. - assert len(tokens) <= result.token_count - # A thinking model streams its thinking too; ``text`` is the response. - assert "".join(t.text for t in tokens if not t.thinking) == result.text - - def test_stream_with_alphas(self, session: DrowseSession) -> None: - name, profile = session.extract_from_corpora( - "happy", _corpus("I am happy"), _corpus("I am sad"), - ) - session.registry.put_profile(name, profile) - session.tree.reset() - tokens = list(session.generate_stream("Hello.", steering=f"0.15 {name}")) - assert len(tokens) > 0 - assert session.last_result is not None # generate_stream guarantees last_result is set - assert session.last_result.steering_alphas == {name: 0.15} - session.registry.pop_profile(name) - - -class TestPrefixCache: - """Prefix KV cache: opt-in optimization for batch workloads with a - shared chat prefix. See ``DrowseSession.cache_prefix``. - """ - - def _shared_prefix_messages(self, session: DrowseSession, prompt_body: str): - """Build a (prefix_messages, full_messages) pair where the - full chat-template encoding of full_messages begins with the - prefix_messages encoding. - - Trick: encode the prefix-only messages with - ``add_generation_prompt=False`` and prepend its decoded text - to the prompt body in a single user message — that shape - always satisfies the byte-prefix invariant on tokenizers - whose user-turn tokens partition cleanly on whitespace - (Gemma's chat template is one such). - """ - # Prefix is its own user turn; full is the same prefix + - # additional user turn. The chat-template encoding of - # [u1] is a prefix of [u1, u2] under add_generation_prompt - # = False for prefix_only and True for full. But that - # ISN'T sufficient — the False-encoded prefix may differ - # by trailing tokens. Instead, encode the prefix as a token - # tensor to bypass the issue. - from drowse.generation.decode import build_chat_input - - prefix_msg = [ - {"role": "user", "content": "Be concise. Always end with a period."}, - ] - full_msg = prefix_msg + [ - {"role": "assistant", "content": "Understood."}, - {"role": "user", "content": prompt_body}, - ] - full_ids = build_chat_input( - session._tokenizer, full_msg, session.config.system_prompt, - ) - # Find the longest token prefix that's a clean head of full_ids. - # We just take everything up to (but not including) the second - # user-turn opener. Falling back to a deterministic split by - # token-string match keeps this independent of model family. - prefix_ids = build_chat_input( - session._tokenizer, prefix_msg, session.config.system_prompt, - add_generation_prompt=False, - ) - # Trim prefix_ids to the longest matching head of full_ids. - L = 0 - max_L = min(prefix_ids.shape[1], full_ids.shape[1]) - for i in range(max_L): - if int(prefix_ids[0, i]) == int(full_ids[0, i]): - L = i + 1 - else: - break - prefix_ids_trim = prefix_ids[:, :L] - return prefix_ids_trim, full_msg - - def test_cache_hit_matches_no_cache_output(self, session: DrowseSession) -> None: - from drowse import SamplingConfig - - session.tree.reset() - prefix_ids, full_msg = self._shared_prefix_messages( - session, "Say the word 'banana' three times.", - ) - - # Baseline: no cache, deterministic. - session.cache_prefix(None) - baseline = session.generate( - full_msg, - sampling=SamplingConfig(max_tokens=12, temperature=0.0, seed=42), - stateless=True, - ).first - - # Warm the cache and rerun the same prompt; outputs must match. - prefix_len = session.cache_prefix(prefix_ids) - assert prefix_len > 0 - cached = session.generate( - full_msg, - sampling=SamplingConfig(max_tokens=12, temperature=0.0, seed=42), - stateless=True, - ).first - # Same prompt + same seed + same model state → identical token - # stream regardless of how prefill was sliced. - assert cached.tokens == baseline.tokens, ( - f"prefix-cache hit produced different tokens than no-cache:\n" - f" no-cache: {baseline.tokens}\n" - f" cached: {cached.tokens}" - ) - # Cleanup so other tests aren't affected. - session.cache_prefix(None) - - def test_steering_invalidates_cache(self, session: DrowseSession) -> None: - # Warm the cache. - prefix_ids, _ = self._shared_prefix_messages(session, "Hello.") - session.cache_prefix(prefix_ids) - assert session._engine.prefix_cache - - # Make sure there's a steering vector to push. - if "happy" not in session.profiles: - _, prof = session.extract_from_corpora( - "happy", _corpus("I am happy"), _corpus("I am sad"), - ) - session.registry.put_profile("happy", prof) - # Re-warm (extraction ran forwards in between) so scope entry is what - # this tests; registering a profile leaves the unsteered prefix alone. - session.cache_prefix(prefix_ids) - assert session._engine.prefix_cache - - with session.steering("0.1 happy"): - # Scope entry must drop the cache. - assert not session._engine.prefix_cache - - # And the post-scope rebuild must NOT magically reinstate it. - assert not session._engine.prefix_cache - session.registry.pop_profile("happy") diff --git a/tests/gpu/test_smoke.py b/tests/gpu/test_smoke.py index 7c1486311..66a46375c 100644 --- a/tests/gpu/test_smoke.py +++ b/tests/gpu/test_smoke.py @@ -9,7 +9,6 @@ import math import time -import tempfile from pathlib import Path from typing import Any @@ -30,8 +29,6 @@ MODEL_ID = gpu_model_id() # MPS runs ~3-5x slower than CUDA for this model; relax absolute timing budgets. -_IS_MPS = not torch.cuda.is_available() and torch.backends.mps.is_available() -_EXTRACTION_BUDGET_S = 60.0 if _IS_MPS else 10.0 @pytest.fixture(scope="module") @@ -169,17 +166,6 @@ def test_returns_valid_profile(self, happy_profile: Any, model_and_tokenizer: An norm = vec.norm().item() assert norm > 0 and not math.isinf(norm) and not math.isnan(norm) - def test_extraction_fast_enough(self, model_and_tokenizer: Any, layers: Any) -> None: - """Single contrastive extraction should complete within the backend's budget.""" - model, tokenizer = model_and_tokenizer - start = time.perf_counter() - _extract_profile(model, tokenizer, "curious", layers) - elapsed = time.perf_counter() - start - assert elapsed < _EXTRACTION_BUDGET_S, ( - f"Extraction took {elapsed:.1f}s, expected < {_EXTRACTION_BUDGET_S:.0f}s" - ) - - class TestSteering: def test_steered_output_differs(self, model_and_tokenizer: Any, layers: Any, happy_profile: Any) -> None: from drowse.steering.hooks import SteeringManager @@ -212,130 +198,7 @@ def test_steered_output_differs(self, model_and_tokenizer: Any, layers: Any, hap assert ids0 != ids1, "Steered output should differ from unsteered" - def test_hook_cleanup(self, model_and_tokenizer: Any, layers: Any, happy_profile: Any) -> None: - from drowse.steering.hooks import SteeringManager - from drowse.generation.decode import GenerationConfig, GenerationState, generate_steered - - model, tokenizer = model_and_tokenizer - p = next(model.parameters()) - device, dtype = p.device, p.dtype - - input_ids = tokenizer.apply_chat_template( - # Reuse the steering smoke's calibrated prompt/length. A ten-token - # greedy "Hello" continuation can share its prefix even when the - # residual stream moved, which made this cleanup test spuriously - # fail before it ever reached the cleanup assertion. - [{"role": "user", "content": "Tell me about your day."}], - add_generation_prompt=True, return_tensors="pt", return_dict=False, - ).to(device) - config = GenerationConfig(max_new_tokens=20, temperature=0.0) - - # Unsteered baseline - state_b = GenerationState() - baseline = generate_steered(model, tokenizer, input_ids.clone(), config, state_b) - - # Steered - mgr = SteeringManager() - # Use the same deliberately supra-threshold smoke strength as the - # preceding steering test. This assertion is about hook installation - # and cleanup, not about calibrating the weakest alpha that changes one - # particular greedy continuation. - mgr.install(_steer_plan((happy_profile, 5.0)), layers, device, dtype) - state_s = GenerationState() - steered = generate_steered(model, tokenizer, input_ids.clone(), config, state_s) - - # Cleanup — output should match unsteered baseline - mgr.clear_all() - state_c = GenerationState() - clean = generate_steered(model, tokenizer, input_ids.clone(), config, state_c) - - assert steered != baseline, "Steered output should differ from baseline" - assert clean == baseline, "Output after hook cleanup should match unsteered baseline" - - -class TestSaveLoad: - def test_roundtrip(self, happy_profile: Any) -> None: - from drowse.geometry.profile import load_profile, save_profile - - with tempfile.TemporaryDirectory() as tmp: - path = str(Path(tmp) / "test_profile.safetensors") - # ``profile`` is the current exact-schema method for a standalone - # baked direction. Difference-of-means is extraction provenance on - # the manifold artifact, not a surviving Profile wire method. - save_profile(happy_profile, path, {"method": "profile"}) - loaded_profile, loaded_meta = load_profile(path) - - assert loaded_meta["method"] == "profile" - assert "scores" not in loaded_meta - assert set(loaded_profile.keys()) == set(happy_profile.keys()) - for idx in happy_profile: - assert torch.allclose( - happy_profile[idx].cpu(), loaded_profile[idx].cpu(), atol=1e-6 - ) - - -class TestMonitor: - def test_monitor_records_history(self, model_and_tokenizer: Any, layers: Any, happy_profile: Any, layer_means: Any) -> None: - from drowse.steering.hooks import SteeringManager - from drowse.readout.monitor import Monitor - from drowse.geometry.fold import fold_directions_to_subspace - from drowse.generation.decode import GenerationConfig, GenerationState, generate_steered - from tests.support.whitener import isotropic_whitener - - model, tokenizer = model_and_tokenizer - device = next(model.parameters()).device - dtype = next(model.parameters()).dtype - - sad_profile = _extract_profile(model, tokenizer, "sad", layers) - - # Unified Monitor reads probes as folded 1-node rays (the vector-probe - # path). Mahalanobis-only: attach needs a covering whitener — an - # isotropic one over the union of both probes' layers suffices for a - # records-history smoke check. - probe_layers = sorted(set(dict(happy_profile)) | set(dict(sad_profile))) - dim = next(iter(dict(happy_profile).values())).shape[0] - whit = isotropic_whitener(probe_layers, dim) - probe_profiles = { - "happy": fold_directions_to_subspace( - "happy", dict(happy_profile), dict(layer_means), whitener=whit, - ), - "sad": fold_directions_to_subspace( - "sad", dict(sad_profile), dict(layer_means), whitener=whit, - ), - } - monitor = Monitor(probe_profiles, whitener=whit) - - # Steer toward happy - mgr = SteeringManager() - # α=0.6 sits mid coherent band (0.4–0.8); α=1 is at the cliff per CLAUDE.md. - mgr.install(_steer_plan((happy_profile, 0.6)), layers, device, dtype) - - input_ids = tokenizer.apply_chat_template( - [{"role": "user", "content": "How are you feeling?"}], - add_generation_prompt=True, return_tensors="pt", return_dict=False, - ).to(device) - config = GenerationConfig(max_new_tokens=40, temperature=0.7) - state = GenerationState() - generated_ids = generate_steered(model, tokenizer, input_ids, config, state) - mgr.clear_all() - - # Measure on generated text via the surviving hidden-state scoring API. - text = tokenizer.decode(generated_ids, skip_special_tokens=True) - hidden = _encode_and_capture_all( - model, tokenizer, "How are you feeling?", text, layers, device, - ) - readings = monitor.score_single_token(hidden) - - # Structural: the coordinate readings are finite (axis 0 is the - # scalar). The semantic - # claim "happy steering → higher happy than sad reading when remeasured - # via a fresh forward pass" is noise-dominated at this model scale - # (3B-param, 20-token gen) — test_throughput_regression covers whether - # steering actually runs. - import math - for name in ("happy", "sad"): - assert math.isfinite(readings[name].coords[0]) - +class TestSteeringThroughput: @pytest.mark.perf def test_throughput_regression(self, model_and_tokenizer: Any, layers: Any, happy_profile: Any, layer_means: Any) -> None: """Steered generation should be at least 85% of vanilla throughput.""" @@ -417,27 +280,6 @@ def timed(steered: bool) -> float: ) -class TestBuildChatInput: - def test_chat_template_path(self, model_and_tokenizer: Any) -> None: - from drowse.generation.decode import build_chat_input - _, tokenizer = model_and_tokenizer - messages = [{"role": "user", "content": "Hello"}] - ids = build_chat_input(tokenizer, messages) - assert ids.ndim == 2 - assert ids.shape[0] == 1 - assert ids.shape[1] > 0 - - def test_with_system_prompt(self, model_and_tokenizer: Any) -> None: - from drowse.generation.decode import build_chat_input - _, tokenizer = model_and_tokenizer - messages = [{"role": "user", "content": "Hello"}] - ids_no_sys = build_chat_input(tokenizer, messages) - ids_sys = build_chat_input(tokenizer, messages, system_prompt="You are helpful.") - # Some instruction tokenizers supply a longer default system message, - # so an explicit replacement can be shorter while still being applied. - assert not torch.equal(ids_sys, ids_no_sys) - - class TestAblationPerformance: """Combined ablation + additive throughput must stay >= 80% of vanilla. @@ -724,143 +566,3 @@ def on_event(event: Any) -> None: #: The instrumented gate's bar (see :class:`TestDashboardPerformance`). _DASHBOARD_MIN_RATIO = 0.50 - - -class TestDiscoverManifoldEndToEnd: - """End-to-end discover-mode fit on a real model. - - Exercises the integration path the unit-test suite skips: the LLM-side - K-tuple generator, the per-node centroid forward pass, the per-model - PCA + RBF fit, and the steered-generation hook at a manifold position. - Held to a tight time budget (corpus generation is the slow part — 1 - scenario call + N_concepts × N_scenarios statement calls — so 5 - concepts × 2 scenarios × 3 statements stays under ~6× the existing - extraction budget). - - Asserts the structural invariants only: the fit produces a - ``CustomDomain(k)``, every node lands at distinct derived coords, and - a soft subspace-replace at one node's centroid produces *different* - text from a steered call at a different node's centroid. Going - deeper (e.g. probe-score asymmetry between opposite node positions) - requires the manifold to encode a probe-aligned axis, which depends - on which concept heap the generator produces — too fragile for a - smoke test, properly the job of the naturalness eval. - """ - - _CONCEPTS = ["pirate", "scholar", "robot", "caveman", "assistant"] - _SAMPLES_PER_PROMPT = 1 - # A2 conversational extraction: each concept answers every shared baseline - # prompt in character, so one corpus = samples_per_prompt × len(baseline - # prompts) responses (64 prompts bundled). Generation is the dominant - # cost; the budget scales with the concept × prompt turn count. - _GENERATE_BUDGET_S = _EXTRACTION_BUDGET_S * 6 - _FIT_BUDGET_S = _EXTRACTION_BUDGET_S * 2 - - def test_discover_pipeline(self, model_and_tokenizer: Any, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from drowse.geometry.domains import CustomDomain - from drowse.session.session import DrowseSession - from drowse.store.manifold_authoring import create_discover_manifold_folder - - # Pin DROWSE_HOME to a per-test temp dir so the manifold folder - # lands somewhere disposable; bundled neutrals still copy in - # fresh on first session init. - monkeypatch.setenv("DROWSE_HOME", str(tmp_path)) - - model, tokenizer = model_and_tokenizer - # probes=[] — the smoke test doesn't need probe scoring and - # skipping bootstrap shaves seconds off the discover gate. - session = DrowseSession(model, tokenizer, probes=[]) - try: - # ---- 1. generate per-concept corpora (A2 conversational) ---- - from drowse.fit.capture import _load_baseline_prompts - n_prompts = len(_load_baseline_prompts()) - t0 = time.perf_counter() - corpora = session.generate_responses( - self._CONCEPTS, - [None] * len(self._CONCEPTS), - samples_per_prompt=self._SAMPLES_PER_PROMPT, - ) - dt_gen = time.perf_counter() - t0 - assert dt_gen < self._GENERATE_BUDGET_S, ( - f"generate_responses took {dt_gen:.1f}s, " - f"budget {self._GENERATE_BUDGET_S:.1f}s" - ) - assert set(corpora.keys()) == set(self._CONCEPTS) - expected_per_concept = self._SAMPLES_PER_PROMPT * n_prompts - for c, lines in corpora.items(): - assert len(lines) == expected_per_concept, ( - f"corpus for {c!r}: {len(lines)} responses, " - f"expected {expected_per_concept}" - ) - # Length must be a multiple of the baseline-prompt count - # (response[i] ↔ baseline_prompt[i % k]). - assert len(lines) % n_prompts == 0 - - # ---- 2. author the discover folder ---- - folder = create_discover_manifold_folder( - "local", "smoke_personas", "smoke-test discover", - fit_mode="pca", - node_corpora=corpora, - hyperparams={"max_dim": 2, "var_threshold": 0.70}, - ) - - # ---- 3. fit through the session pipeline ---- - t0 = time.perf_counter() - manifold = session.fit(folder) - dt_fit = time.perf_counter() - t0 - assert dt_fit < self._FIT_BUDGET_S, ( - f"session.fit took {dt_fit:.1f}s, " - f"budget {self._FIT_BUDGET_S:.1f}s" - ) - - # Discover-mode shape invariants. - assert isinstance(manifold.domain, CustomDomain) - k = manifold.domain.intrinsic_dim - assert 1 <= k <= 2 - assert manifold.node_coords.shape == ( - len(self._CONCEPTS), k, - ) - # Derived coords must distinguish nodes — every pair distinct - # by at least a small floor, else the fit collapsed. - from itertools import combinations - for i, j in combinations(range(len(self._CONCEPTS)), 2): - dist = ( - manifold.node_coords[i] - manifold.node_coords[j] - ).norm().item() - assert dist > 1e-3, ( - f"derived coords for nodes {i} and {j} are " - f"degenerate (dist={dist:.2e}) — fit collapsed" - ) - - # ---- 4. steered generation at one node vs another ---- - # The manifold loads into the session registry lazily on - # scope entry; ManifoldTerm carries the authoring position. - label_a = manifold.node_labels[0] # pirate - label_b = manifold.node_labels[2] # robot - pos_a = ",".join( - f"{float(c):.4f}" for c in manifold.node_coords[0] - ) - pos_b = ",".join( - f"{float(c):.4f}" for c in manifold.node_coords[2] - ) - prompt = "Describe what you see in this room." - - from drowse.generation.sampling import SamplingConfig - sampling = SamplingConfig( - temperature=0.0, max_tokens=48, seed=0, - ) - with session.steering( - f"1.0 local/smoke_personas%{pos_a}@response", - ): - r_a = session.generate(prompt, sampling=sampling, thinking=False).first - with session.steering( - f"1.0 local/smoke_personas%{pos_b}@response", - ): - r_b = session.generate(prompt, sampling=sampling, thinking=False).first - assert r_a.text != r_b.text, ( - f"steering at node {label_a!r} produced identical text " - f"to node {label_b!r} — the manifold hook isn't moving " - f"activations" - ) - finally: - session.close() diff --git a/tests/gpu/test_units_gpu.py b/tests/gpu/test_units_gpu.py deleted file mode 100644 index 10ca1370e..000000000 --- a/tests/gpu/test_units_gpu.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Units on a real model: neutral statistics, probes, gates and ablation. - -Runs on the conftest ``gpu_session`` (``DROWSE_TEST_MODEL``, default -SmolLM2-360M-Instruct): the neutral pass measures every unit, a probe reads a -neuron and a head through a generation, and an ablation matches a -hand-patched forward at the model's own precision. -""" - -from __future__ import annotations - -import math -from collections.abc import Iterator -from typing import Any - -import pytest -import torch - -from drowse.generation.sampling import SamplingConfig -from tests.support.units import patched_forward_logits - -pytestmark = pytest.mark.gpu - - -@pytest.fixture -def units_session(gpu_session: Any) -> Iterator[Any]: - yield gpu_session - for name in list(gpu_session.units.names): - gpu_session.remove_probe(name) - gpu_session.units.set_live(True) - - -def _units(session: Any) -> tuple[str, str]: - spec = session.units.runtime.spec() - layer = spec.n_layers // 3 - return f"neuron/L{layer}n{spec.d_ff // 3}", f"head/L{2 * spec.n_layers // 3}h1" - - -def test_the_neutral_pass_measures_every_unit(units_session: Any) -> None: - stats = units_session.units.runtime.stats() - spec = units_session.units.runtime.spec() - assert sorted(stats.layers) == list(range(spec.n_layers)) - assert stats.n_documents > 24 - for layer in stats.layers.values(): - for tensor in layer.tensors().values(): - assert bool(torch.isfinite(tensor).all()) - # A dead pole is rare; a dead neuron (both poles) rarer still. - assert float((layer.neuron_scale.amax(0) > 0).float().mean()) > 0.99 - assert bool((layer.head_scale > 0).all()) - - -def test_unit_probes_read_a_generation(units_session: Any) -> None: - neuron, head = _units(units_session) - units_session.add_probe(neuron) - units_session.add_probe(head) - result = units_session.generate( - "Name three colors.", stateless=True, - sampling=SamplingConfig(max_tokens=12, temperature=0.0, seed=0), - ).first - readings = result.measurements["instruments"]["units"]["readings"] - assert set(readings) == {neuron, head} - for reading in readings.values(): - assert math.isfinite(reading["value"]) and abs(reading["value"]) < 1e4 - assert readings[head]["value"] >= 0 - assert not units_session.units.hooks_attached - - -def test_a_unit_ablation_matches_the_hand_patched_forward(units_session: Any) -> None: - neuron, head = _units(units_session) - runtime = units_session.units.runtime - stats = runtime.stats() - n_layer, n_index = (int(p) for p in neuron.split("/L")[1].split("n")) - h_layer, h_index = (int(p) for p in head.split("/L")[1].split("h")) - n_block, h_block = runtime.block(n_layer), runtime.block(h_layer) - hd = h_block.attn.head_dim - n_mean = float(stats.layer(n_layer).neuron_mean[n_index]) - h_mean = stats.layer(h_layer).head_mean[h_index] - - def ablate_neuron(x: torch.Tensor) -> torch.Tensor: - x[..., n_index] = n_mean - return x - - def ablate_head(x: torch.Tensor) -> torch.Tensor: - x[..., h_index * hd:(h_index + 1) * hd] = h_mean.to(x) - return x - - ids = units_session.tokenizer( - "The quick brown fox jumps over the lazy dog.", return_tensors="pt", - )["input_ids"].to(units_session.device) - model = units_session.model - expected = patched_forward_logits( - model, ids, {n_block.mlp.down: ablate_neuron, h_block.attn.o: ablate_head}, - ) - plain = patched_forward_logits(model, ids, {}) - with units_session.steering(f"!{neuron} + !{head}"): - with torch.inference_mode(): - got = model(input_ids=ids, use_cache=False).logits[0, -1].float() - scale = float(expected.abs().max()) - assert float((got - expected).abs().max()) <= 2e-2 * scale - assert float((got - plain).abs().max()) > 0 - - -def test_a_unit_gate_fires_on_a_real_model(units_session: Any) -> None: - neuron, _head = _units(units_session) - sampling = SamplingConfig(max_tokens=10, temperature=0.0, seed=0) - - def tokens(steering: str) -> list[int]: - return list(units_session.generate( - "Describe the sea.", stateless=True, steering=steering, sampling=sampling, - ).first.tokens) - - always = tokens(f"!{neuron}@response") - assert tokens(f"!{neuron}@when:{neuron}>-1e9") == always - assert not units_session.units.hooks_attached diff --git a/tests/invariants/test_user_message.py b/tests/invariants/test_user_message.py deleted file mode 100644 index 60a28e27c..000000000 --- a/tests/invariants/test_user_message.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Tests for ``DrowseError.user_message()`` centralization (Phase 4). - -Each subclass returns an HTTP-style ``(status, msg)`` tuple; the server and CLI -consume the value to translate -exceptions consistently. Tests pin the contract so subclasses can't -silently lose their override. -""" - -from __future__ import annotations - -import pytest - -from drowse.cli.config_file import ConfigFileError -from drowse.store.selectors import AmbiguousSelectorError, SelectorError -from drowse.errors import ( - AmbiguousVariantError, - ConcurrentGenerationError, - GenerationRequestError, - ManifoldArityError, - ManifoldSpecError, - OverlappingManifoldError, - PoisednessError, - ProfileError, - RequestError, - SaeBackendImportError, - SaeCoverageError, - SaeModelMismatchError, - SaeReleaseNotFoundError, - DrowseError, - SourceSpecError, - SteeringCompositionError, - UnknownManifoldLabelError, - UnknownVariantError, - UnitRefError, - UnsupportedBlockError, - NeutralMeansError, - WhitenerError, -) -from drowse.errors import ProfileNotRegisteredError -from drowse.steering.grammar import SteeringExprError -from drowse.store.install import ManifoldInstallConflict -from drowse.geometry.gguf import GGUFNotInstalled -from drowse.base.hf import HFError -from drowse.errors import MergeError -from drowse.loom.errors import RecipeOverrideError - - -def test_base_default_status_and_message(): - """Plain ``DrowseError`` returns ``(500, str(self))``.""" - assert DrowseError("x").user_message() == (500, "x") - - -def test_base_empty_falls_back_to_class_name(): - """Empty args fall back to the class name so the user sees something.""" - assert DrowseError().user_message() == (500, "DrowseError") - - -# (subclass, expected_status_code) — one entry per overriding class. -# Bumping the status is a deliberate contract change; any drift here is -# user-visible. -_OVERRIDES: list[tuple[type[DrowseError], int]] = [ - # errors.py - (SaeBackendImportError, 400), - (SaeReleaseNotFoundError, 400), - (SaeModelMismatchError, 400), - (SaeCoverageError, 400), - (AmbiguousVariantError, 400), - (UnknownVariantError, 404), - (WhitenerError, 400), - # A residual read with no neutral mean at its site: valid read, unsatisfiable here. - (NeutralMeansError, 422), - (ProfileError, 400), - # A composition failure is syntactically valid but semantically - # unsatisfiable — 422, distinct from the parser's 400. - (SteeringCompositionError, 422), - (ManifoldArityError, 422), - (OverlappingManifoldError, 422), - # A block the BlockSpec cannot describe: valid model, unsatisfiable request. - (UnsupportedBlockError, 422), - (UnitRefError, 400), - (ConcurrentGenerationError, 409), - (RequestError, 400), - (GenerationRequestError, 400), - (SourceSpecError, 400), - (ManifoldSpecError, 400), - (PoisednessError, 400), - (UnknownManifoldLabelError, 404), - # session/session.py - (ProfileNotRegisteredError, 404), - # steering/grammar.py - (SteeringExprError, 400), - # store/selectors.py - (SelectorError, 400), - (AmbiguousSelectorError, 400), - # cli/config_file.py - (ConfigFileError, 400), - # base/hf.py - (HFError, 502), - # store/install.py - (ManifoldInstallConflict, 409), - # geometry/gguf.py - (GGUFNotInstalled, 400), - # errors.py (offline merge / bake) - (MergeError, 400), - # loom/errors.py - (RecipeOverrideError, 400), -] - - -@pytest.mark.parametrize("cls,expected_status", _OVERRIDES) -def test_subclass_status_codes(cls: type[DrowseError], expected_status: int): - """Each overriding subclass returns its declared status code.""" - code, _msg = cls("test message").user_message() - assert code == expected_status, ( - f"{cls.__name__} returned status {code}, expected {expected_status}" - ) - - -@pytest.mark.parametrize("cls,_status", _OVERRIDES) -def test_subclass_message_round_trips(cls: type[DrowseError], _status: int): - """The message string is non-empty and contains the original payload. - - KeyError-derived subclasses (``UnknownVariantError``, - ``ProfileNotRegisteredError``) reach into ``args[0]`` to avoid - KeyError's repr-quoted ``str()`` shape; we just check the original - payload is in the surfaced message. - """ - payload = "specific test payload" - _code, msg = cls(payload).user_message() - assert payload in msg, f"{cls.__name__} dropped the message payload" - - -def test_ambiguous_selector_carries_disambiguation_tip(): - """``AmbiguousSelectorError`` appends the disambiguation hint.""" - _code, msg = AmbiguousSelectorError("'wolf' matches deer/wolf and zoo/wolf").user_message() - assert "namespace/name" in msg - - -def test_ambiguous_selector_skips_redundant_tip(): - """When the message already names the fix, don't double-suffix it.""" - err = AmbiguousSelectorError("ambiguous; disambiguate with namespace/name (e.g. deer/wolf)") - _code, msg = err.user_message() - # Should not contain the canned tip twice. - assert msg.count("namespace/name") == 1 - - -def test_unknown_variant_strips_keyerror_quotes(): - """``KeyError`` would render as ``"'foo'"``; the override uses ``args[0]``.""" - _code, msg = UnknownVariantError("missing variant").user_message() - assert msg == "missing variant" - - -def test_unknown_manifold_label_strips_keyerror_quotes(): - """Regression: the label error returned ``str(self)``, so the 404 text - reached clients wrapped in KeyError's repr quotes.""" - _code, msg = UnknownManifoldLabelError("manifold 'm' has no node labeled 'x'").user_message() - assert msg == "manifold 'm' has no node labeled 'x'" - - -def test_concurrent_generation_carries_message(): - _code, msg = ConcurrentGenerationError("already running").user_message() - assert msg == "already running" - - -def test_steering_expr_error_carries_col_in_message(): - """``SteeringExprError`` formats column info into ``str(self)``; - ``user_message`` shouldn't drop it.""" - err = SteeringExprError("unexpected token", col=12) - _code, msg = err.user_message() - assert "col 12" in msg - - -# --------------------------------------------------------------------------- -# Server integration smoke — ensure the handler routes correctly. -# --------------------------------------------------------------------------- - - -def test_server_routes_user_message_status_codes(): - """The server's exception handlers honor ``user_message()`` status.""" - from fastapi.testclient import TestClient - - from drowse.server import create_app - from tests.support.server import mock_session - - # 409: ConcurrentGenerationError already pinned by test_server.py; - # we add coverage for one more family per status to confirm - # the handler doesn't regress when the table is removed. - cases: list[tuple[Exception, int]] = [ - (ConcurrentGenerationError("busy"), 409), - (AmbiguousSelectorError("'x' matches a/x and b/x"), 400), - (HFError("network down"), 502), - (ManifoldInstallConflict("already exists"), 409), - ] - - for exc, expected_status in cases: - session = mock_session() - session.generate.side_effect = exc - app = create_app(session) - client = TestClient(app) - resp = client.post( - "/v1/chat/completions", - json={ - "model": "test/model", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 4, - }, - ) - assert resp.status_code == expected_status, ( - f"{type(exc).__name__} → got {resp.status_code}, expected {expected_status}" - ) - - -# --------------------------------------------------------------------------- -# The family invariant -# --------------------------------------------------------------------------- - - -def _public_exception_classes() -> list[type[BaseException]]: - """Every public exception class defined anywhere under ``drowse``.""" - import importlib - import inspect - import pkgutil - import warnings - - import drowse - - modules = [drowse] - for info in pkgutil.walk_packages(drowse.__path__, prefix="drowse."): - # Underscore-private modules are internal; their classes are too. - if any(part.startswith("_") for part in info.name.split(".")[1:]): - continue - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - try: - modules.append(importlib.import_module(info.name)) - except ImportError: - # An optional-dependency module (gguf, sae_lens, …) that - # isn't installed carries no reachable exception class. - continue - - found: dict[type[BaseException], None] = {} - for module in modules: - for name, obj in vars(module).items(): - if name.startswith("_") or not inspect.isclass(obj): - continue - if not issubclass(obj, BaseException) or issubclass(obj, Warning): - continue - # Only classes this module defines, so a re-export isn't - # attributed to every importer. - if obj.__module__ != module.__name__: - continue - found.setdefault(obj, None) - return list(found) - - -def test_every_public_exception_is_a_drowse_error() -> None: - """AGENTS.md: "Every drowse exception subclasses ``DrowseError``". - - The point is that ``except DrowseError`` catches the whole family, so one - escapee silently drops out of every caller written against the contract — - and out of the server's status mapping, landing as a 500. Nothing - enforced the invariant until now; ``SaeTrainingCancelled`` and - ``ManifoldAuthoringChangedError`` had both escaped it. - """ - classes = _public_exception_classes() - assert len(classes) > 40, "walk found suspiciously few exception classes" - escapees = sorted( - f"{cls.__module__}.{cls.__qualname__}" - for cls in classes - if not issubclass(cls, DrowseError) - ) - assert not escapees, ( - "public exception classes outside the DrowseError family (add " - "DrowseError to the bases, keeping the stdlib base first, and give " - "it a user_message()):\n " + "\n ".join(escapees) - ) - - -def test_every_request_error_is_a_400_carrying_its_text() -> None: - """A :class:`RequestError` is the server's "client input" family: it keeps - the ``ValueError`` base and answers 400 with its own text, so a subclass - that overrides ``user_message`` must not drift off either.""" - classes = [cls for cls in _public_exception_classes() if issubclass(cls, RequestError)] - assert len(classes) > 20, "walk found suspiciously few request errors" - drift = [] - for cls in classes: - status, message = cls("specific test payload").user_message() - # A runtime check: a subclass can rebase off ValueError. - is_value_error = issubclass(cls, ValueError) # pyright: ignore[reportUnnecessaryIsInstance] - if not is_value_error or status != 400 or "specific test payload" not in message: - drift.append(f"{cls.__module__}.{cls.__qualname__} -> {status}") - assert not drift, "request errors off the 400 contract:\n " + "\n ".join(drift) - - -def _stdlib_base_cases() -> list[tuple[type[BaseException], type[BaseException]]]: - from drowse.errors import ( - AmbiguousVariantError, - ProfileError, - SaeBackendImportError, - SaeCoverageError, - SaeModelMismatchError, - SaeReleaseNotFoundError, - UnknownVariantError, - ) - from drowse.model.role_templates import ( - InvalidRoleError, - RoleSubstitutionUnsupportedError, - RoleTemplateDriftError, - ) - - return [ - (ProfileError, ValueError), - (SaeBackendImportError, ImportError), - (SaeReleaseNotFoundError, ValueError), - (SaeModelMismatchError, ValueError), - (SaeCoverageError, ValueError), - (AmbiguousVariantError, ValueError), - (UnknownVariantError, KeyError), - (RoleSubstitutionUnsupportedError, ValueError), - (InvalidRoleError, ValueError), - (RoleTemplateDriftError, RuntimeError), - ] - - -@pytest.mark.parametrize( - ("cls", "stdlib_base"), _stdlib_base_cases(), ids=lambda value: value.__name__, -) -def test_drowse_errors_keep_their_stdlib_base( - cls: type[BaseException], stdlib_base: type[BaseException], -) -> None: - """Reparenting under ``DrowseError`` keeps the stdlib base, so existing - ``except ValueError`` (etc.) sites still catch it; the walker above - pins the ``DrowseError`` half for every public class.""" - assert cls in _public_exception_classes() - assert issubclass(cls, stdlib_base) - - -def test_cancellation_errors_agree_across_preparation_families() -> None: - """A cooperative cancel reports 409 on both background-job families.""" - from drowse.lens.jlens import JacobianLensCancelled - from drowse.sae.training import SaeTrainingCancelled - - for exc in (JacobianLensCancelled("stopped"), SaeTrainingCancelled("stopped")): - assert isinstance(exc, DrowseError) - assert isinstance(exc, RuntimeError) - assert exc.user_message()[0] == 409 - - -def test_manifold_authoring_conflict_is_a_retryable_409() -> None: - """Re-authoring under an in-flight fit is a conflict, not a server fault.""" - from drowse.fit.publish import ManifoldAuthoringChangedError - - exc = ManifoldAuthoringChangedError("authoring changed during fit") - assert isinstance(exc, DrowseError) - assert isinstance(exc, RuntimeError) - status, message = exc.user_message() - assert status == 409 - assert "authoring changed" in message diff --git a/tests/session/test_tiny_session.py b/tests/session/test_tiny_session.py index 968a6798d..08f6d7066 100644 --- a/tests/session/test_tiny_session.py +++ b/tests/session/test_tiny_session.py @@ -58,7 +58,7 @@ def test_stream_merges_partial_utf8_tokens_into_one_event( ) -> None: """A multi-byte character spans several tokens but streams as one event. - Regression for the flaky ``test_session.py::test_generate_stream``: it + Regression for the flaky GPU ``test_generate_stream`` (since removed): it asserted one event per token, which breaks whenever the model samples a character that needs more than one token (``é`` is two bytes here). """ diff --git a/tests/store/test_drowse_archive.py b/tests/store/test_drowse_archive.py index 61fd8b8b1..8daf85a88 100644 --- a/tests/store/test_drowse_archive.py +++ b/tests/store/test_drowse_archive.py @@ -239,8 +239,6 @@ def test_monopolar_discover_fit_exports_and_installs(tmp_path: Path) -> None: "nodes_sha256": mf.nodes_sha256(), "model_fingerprint": "test-model", "fit_policy_version": MANIFOLD_FIT_POLICY_VERSION, - "share_metric": "mahalanobis", - "subspace_metric": "euclidean", }, ) mf.update_file_hashes(tensor, tensor.with_suffix(".json")) diff --git a/tests/store/test_manifolds_io.py b/tests/store/test_manifolds_io.py index ff4366832..5b6256a0b 100644 --- a/tests/store/test_manifolds_io.py +++ b/tests/store/test_manifolds_io.py @@ -167,8 +167,6 @@ def _sidecar_payload( "baseline_prompts_sha256": None, "fitted_layers": [], "fit_policy_version": None, - "share_metric": None, - "subspace_metric": None, "rbf_smoothing_per_layer": {}, "sigma_field_per_layer": {}, "resolved_fit_mode": None, @@ -968,18 +966,18 @@ def test_sidecar_requires_current_identity_fields(tmp_path: Path) -> None: payload.pop("fit_mode") path.write_text(json.dumps(payload)) - with pytest.raises(ManifoldFormatError, match="exact schema"): + with pytest.raises(ManifoldFormatError, match="current schema"): ManifoldSidecar.load(path) -def test_sidecar_rejects_unknown_fields(tmp_path: Path) -> None: +def test_sidecar_ignores_retired_fields(tmp_path: Path) -> None: path = tmp_path / "model.json" payload = _sidecar_payload() - payload["legacy_diagnostics"] = {} + payload["share_metric"] = "mahalanobis" path.write_text(json.dumps(payload)) - with pytest.raises(ManifoldFormatError, match="exact schema"): - ManifoldSidecar.load(path) + sidecar = ManifoldSidecar.load(path) + assert not hasattr(sidecar, "share_metric") @pytest.mark.parametrize( @@ -2809,8 +2807,7 @@ def test_transfer_manifold_rebakes_share_in_target_space( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ): """A target whitener covering the transferred layers re-bakes the - Mahalanobis share in target space (instead of dropping it) and records - ``share_metric == "mahalanobis"``.""" + Mahalanobis share in target space (instead of dropping it).""" import torch from drowse.geometry.whitener import LayerWhitener from drowse.store.manifold_tensors import load_manifold @@ -2847,9 +2844,6 @@ def test_transfer_manifold_rebakes_share_in_target_space( assert tgt_man.mahalanobis_share[L] > 0.0 # They are *target*-metric values, not the source's hand-set share. assert tgt_man.mahalanobis_share != src_man.mahalanobis_share - with open(out.with_suffix(".json")) as f: - sc = json.load(f) - assert sc["share_metric"] == "mahalanobis" def test_transfer_manifold_rotation_maps_subspace( @@ -3181,7 +3175,6 @@ def test_manifold_summary_reports_transfer_variant( variant_stem = f"{tgt_safe}_from-{encode_release_id('src/m')}" assert set(stems) == {src_safe, variant_stem} assert stems[variant_stem]["method"] == "manifold_procrustes_transfer" - assert stems[variant_stem]["share_metric"] == "mahalanobis" def test_manifold_summary_reports_template_ref( @@ -3250,7 +3243,6 @@ def test_manifold_fit_summary_surfaces_auto_topology_evidence(tmp_path: Path): "method": "manifold_discover_auto", "fit_mode": "auto", "resolved_fit_mode": "pca", "topology_winner": "flat-pca", "topology_candidates": candidates, - "share_metric": "mahalanobis", "subspace_metric": "mahalanobis", "sae_release": "test", "sae_revision": "rev-1", "sae_fingerprint": "fingerprint", "sae_ids_by_layer": {"4": "mock-4"}, }) @@ -3259,15 +3251,11 @@ def test_manifold_fit_summary_surfaces_auto_topology_evidence(tmp_path: Path): assert sc.resolved_fit_mode == "pca" assert sc.topology_winner == "flat-pca" assert [c["name"] for c in sc.topology_candidates] == ["flat-pca"] - assert sc.share_metric == "mahalanobis" - assert sc.subspace_metric == "mahalanobis" entry = manifold_fit_summary(sc, "autotopo2") assert entry["resolved_fit_mode"] == "pca" assert entry["topology_winner"] == "flat-pca" assert [c["name"] for c in entry["topology_candidates"]] == ["flat-pca"] - assert entry["share_metric"] == "mahalanobis" - assert entry["subspace_metric"] == "mahalanobis" # Empty diagnostics stay omitted so a pinned/authored fit reads terse. assert "rbf_smoothing" not in entry assert "sigma_field" not in entry