diff --git a/aspec/commands/convert_to_hf.py b/aspec/commands/convert_to_hf.py index 3004617..62d58df 100644 --- a/aspec/commands/convert_to_hf.py +++ b/aspec/commands/convert_to_hf.py @@ -183,11 +183,18 @@ def _serving_config(raw_config: dict) -> dict: if raw_config.get("mtp_conditioning"): raise NotImplementedError("MTP-conditioned Drafts require native --format aspec export") if raw_config.get("architectures") == ["DominoDraftModel"]: - raise NotImplementedError( - "Domino SGLang export is not validated in ASpec yet; use --format aspec. " - "Upstream SGLang 0.5.21 includes a Domino loader and an end-to-end test, " - "but ASpec has not validated its exported artifact against it" - ) + from aspec.models.draft.domino import official_domino_config + + config = official_domino_config(raw_config) + nested = config.get("dflash_config") or {} + if nested.get("pure_draft_prefix_len") != 1: + raise NotImplementedError( + "SGLang Domino serving requires pure_draft_prefix_len=1; " + "use --format aspec for another label layout" + ) + if not isinstance(nested.get("shift_label"), bool): + raise ValueError("SGLang Domino serving requires a boolean shift_label") + return config if raw_config.get("architectures") == ["MedusaDraftModel"]: raise NotImplementedError( @@ -355,8 +362,10 @@ def _draft_structure_manifest(raw_config: dict) -> dict: "loader_contract": { "runtime": "sglang", "validated_version": "0.5.21", - "speculative_algorithm": ("DFLASH" if algorithm == "dflash2" else - "EAGLE" if algorithm == "mtp" else algorithm.upper()), + "speculative_algorithm": ( + "DFLASH" if algorithm in {"dflash2", "domino"} else + "EAGLE" if algorithm == "mtp" else algorithm.upper() + ), "required_patch_features": ( ["dflash_moe_v1"] if structure["ffn_type"] != "dense" else [] ), diff --git a/aspec/commands/evaluate_draft.py b/aspec/commands/evaluate_draft.py index ba7c6e5..c8dac16 100644 --- a/aspec/commands/evaluate_draft.py +++ b/aspec/commands/evaluate_draft.py @@ -23,6 +23,23 @@ import time from pathlib import Path +_SPECULATIVE_ALGORITHMS = { + "dspark": "DSPARK", + "dflash": "DFLASH", + "dflash2": "DFLASH", + "domino": "DFLASH", + "eagle3": "EAGLE3", + "mtp": "EAGLE", +} + + +def serving_algorithm(name): + """Map an exported draft algorithm to the official SGLang runtime name.""" + try: + return _SPECULATIVE_ALGORITHMS[name] + except KeyError: + raise ValueError(f"unsupported exported draft algorithm: {name!r}") from None + def load_prompts(path, tokenizer, limit, *, enable_thinking=None): """Only user prefixes are evaluated; never prefill held-out assistant answers.""" @@ -127,6 +144,8 @@ def main(): parser.add_argument("--max-new-tokens", type=int, default=256) parser.add_argument("--repeats", type=int, default=3) parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--domino-candidate-pool-size", type=int, default=0, + help="Domino only: 0 scores the full vocabulary") parser.add_argument("--trust-remote-code", action="store_true", help="Explicitly allow code supplied by the Target checkpoint") parser.add_argument("--disable-thinking", action="store_true", help="match a non-thinking training corpus") @@ -135,6 +154,8 @@ def main(): args = parser.parse_args() if args.output.exists() or min(args.samples, args.max_new_tokens, args.repeats, args.tp) < 1: raise ValueError("use positive budgets and a fresh output") + if args.domino_candidate_pool_size < 0: + raise ValueError("Domino candidate pool size must be non-negative") import sglang as sgl import torch from transformers import AutoTokenizer @@ -142,10 +163,13 @@ def main(): from aspec.inference.sglang_v0521 import TrainingEngine manifest = json.loads((args.draft / "export_manifest.json").read_text()) - algorithm = {"dspark": "DSPARK", "dflash": "DFLASH", "dflash2": "DFLASH", - "eagle3": "EAGLE3", "mtp": "EAGLE"}.get(manifest.get("algorithm")) - if algorithm is None or manifest.get("storage_format") != "sglang": + if manifest.get("storage_format") != "sglang": raise ValueError("requires an official SGLang-format draft export") + algorithm = serving_algorithm(manifest.get("algorithm")) + expected = (((manifest.get("draft_structure") or {}).get("loader_contract") or {}) + .get("speculative_algorithm")) + if expected is not None and expected != algorithm: + raise ValueError("export manifest loader contract disagrees with the speculative algorithm") for filename, key in (("model.safetensors", "model_sha256"), ("config.json", "config_sha256")): digest = hashlib.sha256() with (args.draft / filename).open("rb") as stream: @@ -160,6 +184,7 @@ def main(): draft_tokens = speculative_token_budget(config, algorithm) record = {"schema": "aspec.decode-evaluation/v1", "target": args.target, "draft": str(args.draft.resolve()), "algorithm": algorithm, + "draft_algorithm": manifest.get("algorithm"), "export_manifest_sha256": hashlib.sha256((args.draft / "export_manifest.json").read_bytes()).hexdigest(), "data_sha256": hashlib.sha256(args.data.read_bytes()).hexdigest(), "prompt_tokens_sha256": hashlib.sha256(json.dumps(prompts).encode()).hexdigest(), @@ -186,6 +211,8 @@ def persist(): if name == "speculative": kwargs.update(speculative_algorithm=algorithm, speculative_draft_model_path=str(args.draft), speculative_num_draft_tokens=draft_tokens) + if manifest.get("algorithm") == "domino": + kwargs["speculative_domino_candidate_pool_size"] = args.domino_candidate_pool_size if algorithm in {"EAGLE", "EAGLE3"}: kwargs.update(speculative_eagle_topk=1, speculative_num_steps=3) record["runs"][name] = {"requested_engine_args": kwargs, "samples": []} diff --git a/docs/contributing/validation.md b/docs/contributing/validation.md index 72dbc60..954a5ef 100644 --- a/docs/contributing/validation.md +++ b/docs/contributing/validation.md @@ -76,6 +76,39 @@ bash tools/run_in_env.sh python -m tools.recipes export \ --checkpoint outputs/my-run/checkpoints/iter_0000101 --output outputs/my-native-asset ``` +## Domino decode ablation and anchor replay + +Run a paired corrected-versus-backbone decode from one SGLang-format Domino asset. +The tool zeroes only the correction output projection in a copy, so both branches keep +the trained backbone and the same prompt set: + +```bash +CUDA_VISIBLE_DEVICES=0 bash tools/run_in_env.sh python -m tools.validation.domino_acceptance_ablation \ + --corrected outputs/domino-sglang --backbone outputs/domino-backbone \ + --target /models/Qwen3-8B --data held-out.jsonl \ + --output outputs/domino-ablation.json --samples 16 --max-new-tokens 32 --repeats 2 +``` + +The summary rejects mismatched prompt identities and records acceptance length, +verification count and repeat stability for both branches. It is a bounded decode +comparison, not throughput or convergence evidence. + +For matched training comparisons, materialize one canonical anchor stream and verify +it before either framework consumes it: + +```bash +python -m tools.validation.domino_anchor_protocol build \ + --source cache/tokenized_dataset.pt --anchors anchors.json \ + --num-anchors 128 --block-size 16 --seed 42 +python -m tools.validation.domino_anchor_protocol verify \ + --source cache/tokenized_dataset.pt --anchors anchors.json \ + --num-anchors 128 --block-size 16 --seed 42 +``` + +`verify` regenerates the stream and checks that every kept anchor is supervised in +its source sample. Consuming the same file in both trainers is still required before +claiming a matched loss-curve comparison; an identical seed alone is not enough. + ## Algorithm, capacity and fault checks | Scope | Required additional evidence | diff --git a/docs/support.md b/docs/support.md index fe9a9bd..8630251 100644 --- a/docs/support.md +++ b/docs/support.md @@ -12,7 +12,7 @@ and tests do not imply CPU training support. | DSpark / DFlash | Training, exact recovery, evaluation and export; Dense and selected MoE/attention structures | Objectives, feature bindings and assets remain algorithm-specific | | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | -| Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export, strict reload, proposal consumer and a bounded SpecForge comparison are validated; longer convergence/decoding acceptance pending; see [scope](user-guide/domino.md) | +| Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; SGLang/native export, strict reload, proposal consumer, a bounded SpecForge comparison and bounded SGLang decode acceptance are validated; harder data, longer convergence and a formal anchor-replay training mode remain pending; see [scope](user-guide/domino.md) | | Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP, warmstart or candidate-tree acceptance yet; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | diff --git a/docs/user-guide/domino.md b/docs/user-guide/domino.md index 1b18355..c3e26aa 100644 --- a/docs/user-guide/domino.md +++ b/docs/user-guide/domino.md @@ -15,8 +15,8 @@ Target GPU plus two Draft GPUs at 4K (three GPUs total). `block_size=16`, `shift_label=true`, `pure_draft_prefix_len=1`). - Formal recipe: `configs/recipes/domino_qwen3_8b.yaml` (FlexAttention, CP1, non-packing, REPLICATE, Store/TCP transport, held-out eval and the linear - `lambda_base` curriculum). It is catalogued as `domino`; its export boundary is - native-only. `domino_qwen3_8b_4k_tcp.yaml` remains a shorter smoke test. + `lambda_base` curriculum). It is catalogued as `domino`; SGLang and native export + are both validated. `domino_qwen3_8b_4k_tcp.yaml` remains a shorter smoke test. - `training.domino_lambda_base` is the constant fallback. Set both `training.domino_lambda_base_start` and `training.domino_lambda_base_decay_ratio` to apply the reference linear schedule. Exact recovery records the constant, @@ -79,21 +79,41 @@ licensed data, a full learning-rate horizon and the recipe for a real run. ## Export and strict reload -Export a trained checkpoint with the native ASpec artifact format: +Export a trained checkpoint either for SGLang serving (default) or as a native ASpec +artifact: ```bash +aspec export --input-dir outputs/domino-qwen3-8b/checkpoints/iter_0000009 \ + --output-dir outputs/domino-qwen3-8b-sglang + aspec export --input-dir outputs/domino-qwen3-8b/checkpoints/iter_0000009 \ --output-dir outputs/domino-qwen3-8b-asset --format aspec ``` -Domino currently exports native-only. ASpec has not validated its SGLang export: -upstream SGLang 0.5.21 does load `prefix_gru`/`embed_proj` and has an end-to-end -test for the published checkpoint, but a silent DFlash-style export could still -drop or mis-bind the correction head. The native manifest records the Domino -algorithm, trained `block_size`, label layout, Target feature layers and the full -training objective. `aspec.draft_artifact.load_native_draft` validates the checksums and -semantics, then loads every tensor strictly. A real 8-step checkpoint has been -roundtripped through this path (63 tensors, exact values and dtypes). +SGLang export maps Domino to the published layout +(`architectures=["DFlashDraftModel"]`, `model_type="qwen3"`, nested +`dflash_config.projector_type="domino"`) and preserves the trained `block_size`, +`shift_label`, `pure_draft_prefix_len=1`, `emb_dim`, `gru_hidden_dim` and Target +feature layers. Other label layouts fail closed and require the native format. The +manifest records runtime `DFLASH`; SGLang 0.5.21 loads the exported +`prefix_gru`/`embed_proj` tensors. + +The native manifest records the Domino algorithm, trained `block_size`, label layout, +Target feature layers and the full training objective. +`aspec.draft_artifact.load_native_draft` validates the checksums and semantics, then +loads every tensor strictly. A real 8-step checkpoint has been roundtripped through +this path (63 tensors, exact values and dtypes). + +Run real decode acceptance against the SGLang export with `aspec evaluate`. Domino +defaults to the full-vocabulary candidate pool (`--domino-candidate-pool-size 0`), +reports measured acceptance length (including Target tokens) and checks greedy +equality and repeat stability: + +```bash +aspec evaluate --target /path/to/Qwen3-8B \ + --draft outputs/domino-qwen3-8b-sglang \ + --data /path/to/held-out-prompts.jsonl --output outputs/domino-decode.json +``` ### Official checkpoint mapping @@ -135,6 +155,22 @@ Draft GPUs verified 8-step training and step-4 exact resume: losses, model tenso optimizer state and RNG state matched the uninterrupted run. The public CI lane is CPU-only; no throughput or convergence claim is made here. +The SGLang export was loaded by official SGLang 0.5.21 and evaluated with +`aspec evaluate` on 16 held-out prompts (32 new tokens, greedy, two repeats): +speculative output matched target-only greedy output, both repeats were stable, and +the measured acceptance length was 2.016 tokens/verification including Target +tokens. This is a bounded decode integration check, not convergence or throughput +evidence. + +`tools.validation.domino_acceptance_ablation` runs a paired corrected-versus-backbone +decode by zeroing only the correction output projection in a copy of the same SGLang +asset. On the published checkpoint (16 prompts x 32 greedy tokens, two repeats) the +corrected branch measured 2.829 and backbone-only 2.473 tokens/verification +(+0.355). On the 450-step synthetic ASpec checkpoint the same comparison measured +2.016 versus 2.065 (-0.049), so the correction benefit is checkpoint/data-dependent +rather than unconditional. Both runs kept greedy equality and repeat stability; the +tool rejects mismatched prompts. + A bounded comparison against SpecForge at its Domino merge commit used the same published Qwen3-8B Domino checkpoint, 16 train / 16 held-out rows, 8 steps, block 16, 128 anchors, FlexAttention and the same linear `lambda_base` schedule. Both trainers @@ -269,9 +305,12 @@ handling, empty supervision, and frozen Target/embedding behavior. ## Remaining work -Native export with strict reloading works for ASpec FSDP checkpoints, the official -Qwen3-8B Domino config/weight layout maps into ASpec, and the native -`DominoConsumer` produces proposal tokens for both label layouts. Remaining work is -longer convergence/quality evidence and a real decoding-serving acceptance run; -the recipe is catalogued because the native export/strict reload check passes. CPU numerical and Plan checks do not establish -Qwen3-8B training quality, GPU throughput or decoding acceptance. +SGLang and native export with strict reloading work for ASpec FSDP checkpoints, the +official Qwen3-8B Domino config/weight layout maps into ASpec, the native +`DominoConsumer` produces proposal tokens for both label layouts, bounded real SGLang +decode reports corrected and backbone-only acceptance, and +`tools.validation.domino_anchor_protocol` can materialize and verify a hashed +canonical anchor stream. Remaining work is harder/real data, longer convergence and a +formal anchor-replay training mode (the aligned comparison here used a local validation +harness). CPU numerical and Plan checks do not establish Qwen3-8B training quality +or throughput. diff --git a/tests/cpu/test_domino_export.py b/tests/cpu/test_domino_export.py index 3af3e5c..05be94e 100644 --- a/tests/cpu/test_domino_export.py +++ b/tests/cpu/test_domino_export.py @@ -154,9 +154,59 @@ def test_domino_native_export_rejects_block_size_that_disagrees_with_checkpoint( ) -def test_domino_has_no_serving_export(): - with pytest.raises(NotImplementedError, match="--format aspec"): - convert_to_hf._serving_config(_raw_config()) +def test_domino_serving_config_uses_the_published_layout(): + raw = _raw_config() + exported = convert_to_hf._serving_config(raw) + + assert exported["architectures"] == ["DFlashDraftModel"] + assert exported["model_type"] == "qwen3" + assert exported["block_size"] == raw["block_size"] + assert exported["num_target_layers"] == raw["target_num_hidden_layers"] + assert exported["dflash_config"] == { + "projector_type": "domino", + "mask_token_id": raw["mask_token_id"], + "shift_label": True, + "target_layer_ids": raw["target_layer_ids"], + "pure_draft_prefix_len": 1, + "gru_hidden_dim": raw["gru_hidden_dim"], + "emb_dim": raw["emb_dim"], + } + + +@pytest.mark.parametrize("change, message", [ + ({"pure_draft_prefix_len": 0}, "pure_draft_prefix_len=1"), + ({"shift_label": 1}, "boolean shift_label"), +]) +def test_domino_serving_export_rejects_unsupported_layout(change, message): + with pytest.raises((NotImplementedError, ValueError), match=message): + convert_to_hf._serving_config({**_raw_config(), **change}) + + +def test_domino_sglang_export_writes_published_config(tmp_path, monkeypatch): + raw = _raw_config() + model = _model(raw) + _monkeypatch_checkpoint(monkeypatch, model) + source = _prepare_source(tmp_path, raw) + output = tmp_path / "serving" + + manifest = convert_to_hf.convert_checkpoint( + input_dir=source, output_dir=output, artifact_format="sglang" + ) + + assert manifest["storage_format"] == "sglang" + assert manifest["algorithm"] == "domino" + assert manifest["training_contract"]["block_size"] == raw["block_size"] + loader = manifest["draft_structure"]["loader_contract"] + assert loader["runtime"] == "sglang" + assert loader["speculative_algorithm"] == "DFLASH" + exported = json.loads((output / "config.json").read_text()) + assert exported["architectures"] == ["DFlashDraftModel"] + assert exported["model_type"] == "qwen3" + assert exported["block_size"] == raw["block_size"] + assert exported["num_target_layers"] == raw["target_num_hidden_layers"] + assert exported["dflash_config"]["projector_type"] == "domino" + assert exported["dflash_config"]["pure_draft_prefix_len"] == 1 + assert (output / "model.safetensors").is_file() def _write_config(tmp_path, raw): diff --git a/tests/test_decode_evaluation.py b/tests/test_decode_evaluation.py index 5f9ec71..c8545f1 100644 --- a/tests/test_decode_evaluation.py +++ b/tests/test_decode_evaluation.py @@ -17,7 +17,13 @@ import pytest -from tools.evaluate_draft import load_prompts, output_checks, speculative_token_budget, summarize +from tools.evaluate_draft import ( + load_prompts, + output_checks, + serving_algorithm, + speculative_token_budget, + summarize, +) def test_decode_prompt_does_not_contain_held_out_answer(tmp_path): @@ -90,6 +96,13 @@ def test_dspark_verify_budget_includes_the_target_token_not_an_extra_training_sl speculative_token_budget({"block_size": 7, "dflash_config": {"block_size": 8}}, "DSPARK") +def test_domino_uses_dflash_verification_with_the_trained_block_size(): + assert serving_algorithm("domino") == "DFLASH" + assert speculative_token_budget({"block_size": 16}, "DFLASH") == 16 + with pytest.raises(ValueError, match="unsupported exported draft algorithm"): + serving_algorithm("unknown") + + def test_equality_between_frameworks_does_not_hide_repeat_instability(): repeats = [[{"output_ids": [1]}], [{"output_ids": [2]}]] verdict = output_checks(repeats, repeats) diff --git a/tests/test_domino_acceptance_ablation.py b/tests/test_domino_acceptance_ablation.py new file mode 100644 index 0000000..b116ee9 --- /dev/null +++ b/tests/test_domino_acceptance_ablation.py @@ -0,0 +1,104 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import json + +import pytest +import torch +from safetensors.torch import load_file, save_file + +from tools.validation import domino_acceptance_ablation as ablation + + +def _write_asset(root, *, algorithm="domino", storage_format="sglang"): + root.mkdir(parents=True) + tensors = { + "fc.weight": torch.ones(4, 4), + "embed_proj.0.weight": torch.ones(4, 4), + "embed_proj.2.weight": torch.ones(8, 4), + } + save_file(tensors, str(root / "model.safetensors"), metadata={"format": "pt"}) + (root / "config.json").write_text(json.dumps({"model_type": "qwen3"}) + "\n") + manifest = { + "schema_version": "aspec.serving-export/v1", + "storage_format": storage_format, + "algorithm": algorithm, + "model_sha256": ablation._sha256(root / "model.safetensors"), + "config_sha256": hashlib.sha256((root / "config.json").read_bytes()).hexdigest(), + } + (root / "export_manifest.json").write_text(json.dumps(manifest) + "\n") + return tensors + + +def test_backbone_ablation_zeroes_only_the_correction_output(tmp_path): + corrected = tmp_path / "corrected" + original = _write_asset(corrected) + output = tmp_path / "backbone" + + manifest = ablation.build_backbone_only(corrected, output) + + actual = load_file(str(output / "model.safetensors")) + assert torch.count_nonzero(actual["embed_proj.2.weight"]) == 0 + torch.testing.assert_close(actual["fc.weight"], original["fc.weight"]) + torch.testing.assert_close(actual["embed_proj.0.weight"], original["embed_proj.0.weight"]) + assert manifest["ablation"]["kind"] == ablation._ABLATION_KIND + assert manifest["ablation"]["zeroed_tensors"] == ["embed_proj.2.weight"] + assert manifest["model_sha256"] == ablation._sha256(output / "model.safetensors") + assert (output / "config.json").read_bytes() == (corrected / "config.json").read_bytes() + + +@pytest.mark.parametrize("algorithm, storage_format", [ + ("dflash", "sglang"), + ("domino", "aspec_native"), +]) +def test_backbone_ablation_requires_a_sglang_domino_asset(tmp_path, algorithm, storage_format): + corrected = tmp_path / "corrected" + _write_asset(corrected, algorithm=algorithm, storage_format=storage_format) + with pytest.raises(ValueError, match="SGLang-format Domino"): + ablation.build_backbone_only(corrected, tmp_path / "backbone") + + +def test_backbone_ablation_rejects_overlapping_output(tmp_path): + corrected = tmp_path / "assets" / "corrected" + _write_asset(corrected) + for output in (corrected.parent, corrected / "backbone"): + with pytest.raises(ValueError, match="must not overlap"): + ablation.build_backbone_only(corrected, output, force=True) + + +def _record(length, *, data="d", prompt="p", verify=10, target="Qwen3-8B", settings=None): + summary = { + "acceptance_length": length, + "verify_count": verify, + "acceptance_convention": "SGLang completion_tokens / spec_verify_ct; includes target tokens", + } + return { + "schema": "aspec.decode-evaluation/v1", + "algorithm": "DFLASH", + "data_sha256": data, + "prompt_tokens_sha256": prompt, + "target": target, + "settings": settings or { + "samples": 16, "max_new_tokens": 32, "repeats": 2, "tp": 1, + "domino_candidate_pool_size": 0, "disable_thinking": False, + "trust_remote_code": False, "deterministic_inference": False, + }, + "greedy_equal": True, + "target_repeat_stable": True, + "draft_repeat_stable": True, + "runs": {"speculative": {"summary": summary}}, + } + + +def test_ablation_summary_requires_matching_prompts_and_reports_delta(): + result = ablation.summarize(_record(2.4), _record(1.6)) + assert result["corrected"]["acceptance_length"] == 2.4 + assert result["backbone"]["acceptance_length"] == 1.6 + assert result["corrected_minus_backbone"] == pytest.approx(0.8) + with pytest.raises(ValueError, match="prompt_tokens_sha256"): + ablation.summarize(_record(2.4), _record(1.6, prompt="other")) + with pytest.raises(ValueError, match="target differs"): + ablation.summarize(_record(2.4), _record(1.6, target="other")) + with pytest.raises(ValueError, match="decoding settings differ"): + ablation.summarize(_record(2.4), _record(1.6, settings={"samples": 8})) diff --git a/tests/test_domino_anchor_protocol.py b/tests/test_domino_anchor_protocol.py new file mode 100644 index 0000000..3b649e8 --- /dev/null +++ b/tests/test_domino_anchor_protocol.py @@ -0,0 +1,75 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import json +import sys + +import pytest +import torch + +from tools.validation import domino_anchor_protocol as protocol + + +def _samples(): + return [ + {"data_id": "a", "input_ids": torch.arange(40) % 31, + "loss_mask": torch.tensor([0] * 5 + [1] * 30 + [0] * 5)}, + {"data_id": "b", "input_ids": torch.arange(23) % 31, + "loss_mask": torch.tensor([0] * 3 + [1] * 18 + [0] * 2)}, + ] + + +def test_anchor_stream_is_reproducible_and_supervision_valid(): + samples = _samples() + first = protocol.build_anchors(samples, num_anchors=4, block_size=5, seed=7) + second = protocol.build_anchors(samples, num_anchors=4, block_size=5, seed=7) + assert first == second + protocol.verify_anchors(first, samples) + assert first["schema_version"] == protocol._SCHEMA + assert [row["data_id"] for row in first["records"]] == ["a", "b"] + + +def test_changed_anchor_stream_fails_replay_verification(): + samples = _samples() + record = protocol.build_anchors(samples, num_anchors=2, block_size=5, seed=7) + record["records"][0]["anchors"][0] += 1 + with pytest.raises(ValueError, match="anchor replay differs"): + protocol.verify_anchors(record, samples) + + +def test_anchor_protocol_rejects_noncanonical_schema(): + samples = _samples() + record = protocol.build_anchors(samples, num_anchors=2, block_size=5, seed=7) + record["schema_version"] = "other" + with pytest.raises(ValueError, match="not a Domino anchor protocol"): + protocol.verify_anchors(record, samples) + + +def test_anchor_cli_roundtrip_and_source_identity(tmp_path, monkeypatch): + source = tmp_path / "samples.jsonl" + rows = [ + {"data_id": "a", "input_ids": list(range(40)), "loss_mask": [0] * 5 + [1] * 30 + [0] * 5}, + {"data_id": "b", "input_ids": list(range(23)), "loss_mask": [0] * 3 + [1] * 18 + [0] * 2}, + ] + source.write_text("".join(json.dumps(row) + "\n" for row in rows)) + anchors = tmp_path / "anchors.json" + common = ["--source", str(source), "--anchors", str(anchors), + "--num-anchors", "4", "--block-size", "5", "--seed", "7"] + monkeypatch.setattr(sys, "argv", ["domino_anchor_protocol", "build", *common]) + protocol.main() + monkeypatch.setattr(sys, "argv", ["domino_anchor_protocol", "verify", *common]) + protocol.main() + source.write_text(source.read_text() + "\n") + with pytest.raises(ValueError, match="source changed"): + protocol.main() + + +def test_anchor_loader_accepts_jsonl_packed_loss_mask(tmp_path): + source = tmp_path / "samples.jsonl" + source.write_text(json.dumps({ + "data_id": "a", "input_ids": [1, 2, 3, 4, 5], "packed_loss_mask": "2,3", + }) + "\n") + samples = protocol._load_samples(source) + assert samples[0]["loss_mask"].tolist() == [0, 0, 1, 1, 1] + record = protocol.build_anchors(samples, num_anchors=1, block_size=2, seed=1) + protocol.verify_anchors(record, samples) diff --git a/tools/recipe_catalog.json b/tools/recipe_catalog.json index be2254a..7d7e4c6 100644 --- a/tools/recipe_catalog.json +++ b/tools/recipe_catalog.json @@ -41,7 +41,7 @@ "gpus": 3, "target_gpus": 1, "draft_gpus": 2, - "native_only_export": true + "native_only_export": false }, "eagle3": { "configs": [ diff --git a/tools/validation/domino_acceptance_ablation.py b/tools/validation/domino_acceptance_ablation.py new file mode 100644 index 0000000..6e18a02 --- /dev/null +++ b/tools/validation/domino_acceptance_ablation.py @@ -0,0 +1,198 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Paired real-decode ablation: corrected Domino versus its backbone-only branch. + +The backbone-only artifact keeps the trained DFlash weights and zeroes only the +correction head's output projection, so SGLang still loads the same asset schema +and produces the backbone logits exactly. The two artifacts must be evaluated on +the same prompts and decoding settings; the summary rejects mismatched evidence. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import torch +from safetensors.torch import load_file, save_file + +_CORRECTION_OUTPUT = "embed_proj.2." +_ABLATION_KIND = "backbone_only_zero_correction_output" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def build_backbone_only(corrected: Path, output: Path, *, force: bool = False) -> dict: + """Write a SGLang asset whose correction delta is exactly zero.""" + corrected = Path(corrected).resolve() + manifest_path = corrected / "export_manifest.json" + config_path = corrected / "config.json" + model_path = corrected / "model.safetensors" + for path in (manifest_path, config_path, model_path): + if not path.is_file(): + raise FileNotFoundError(f"missing SGLang Domino asset file: {path}") + manifest = json.loads(manifest_path.read_text()) + if manifest.get("algorithm") != "domino" or manifest.get("storage_format") != "sglang": + raise ValueError("requires a SGLang-format Domino export") + + output = Path(output).resolve() + if (output == corrected or corrected.is_relative_to(output) + or output.is_relative_to(corrected)): + raise ValueError("backbone output must not overlap the corrected asset") + if output.exists() and any(output.iterdir()): + if not force: + raise FileExistsError(f"output directory is not empty: {output}") + shutil.rmtree(output) + output.mkdir(parents=True, exist_ok=True) + + state = load_file(str(model_path)) + keys = sorted(key for key in state if key.startswith(_CORRECTION_OUTPUT)) + if not keys: + raise ValueError("Domino asset has no correction output projection") + for key in keys: + state[key] = torch.zeros_like(state[key]) + save_file(state, str(output / "model.safetensors"), metadata={"format": "pt"}) + shutil.copy2(config_path, output / "config.json") + + ablation = { + "kind": _ABLATION_KIND, + "source_manifest_sha256": _sha256(manifest_path), + "zeroed_tensors": keys, + "correction_delta": "exactly zero; backbone logits unchanged", + } + manifest["model_sha256"] = _sha256(output / "model.safetensors") + manifest["ablation"] = ablation + (output / "export_manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n" + ) + return manifest + + +_PAIRED_SETTINGS = ( + "samples", "max_new_tokens", "repeats", "tp", "domino_candidate_pool_size", + "disable_thinking", "trust_remote_code", "deterministic_inference", +) + + +def _paired_settings(record: dict) -> dict: + settings = record.get("settings") or {} + return {key: settings.get(key) for key in _PAIRED_SETTINGS} + + +def summarize(corrected: dict, backbone: dict) -> dict: + """Combine two evaluate records after checking they describe the same prompts.""" + for name, record in (("corrected", corrected), ("backbone", backbone)): + if record.get("schema") != "aspec.decode-evaluation/v1": + raise ValueError(f"{name} is not an ASpec decode evaluation") + if record.get("algorithm") != "DFLASH": + raise ValueError(f"{name} did not run DFLASH verification") + for key in ("data_sha256", "prompt_tokens_sha256"): + if corrected.get(key) != backbone.get(key): + raise ValueError(f"{key} differs between corrected and backbone runs") + if corrected.get("target") != backbone.get("target"): + raise ValueError("target differs between corrected and backbone runs") + if _paired_settings(corrected) != _paired_settings(backbone): + raise ValueError("decoding settings differ between corrected and backbone runs") + corrected_summary = corrected["runs"]["speculative"]["summary"] + backbone_summary = backbone["runs"]["speculative"]["summary"] + corrected_length = corrected_summary.get("acceptance_length") + backbone_length = backbone_summary.get("acceptance_length") + if corrected_length is None or backbone_length is None: + raise ValueError("both runs require measured speculative acceptance") + return { + "schema": "aspec.domino-acceptance-ablation/v1", + "data_sha256": corrected["data_sha256"], + "prompt_tokens_sha256": corrected["prompt_tokens_sha256"], + "corrected": { + "acceptance_length": corrected_length, + "verify_count": corrected_summary["verify_count"], + "greedy_equal": corrected["greedy_equal"], + "target_repeat_stable": corrected["target_repeat_stable"], + "draft_repeat_stable": corrected["draft_repeat_stable"], + }, + "backbone": { + "acceptance_length": backbone_length, + "verify_count": backbone_summary["verify_count"], + "greedy_equal": backbone["greedy_equal"], + "target_repeat_stable": backbone["target_repeat_stable"], + "draft_repeat_stable": backbone["draft_repeat_stable"], + }, + "corrected_minus_backbone": corrected_length - backbone_length, + "acceptance_convention": corrected_summary["acceptance_convention"], + "claim": "bounded paired real decode; not throughput or convergence evidence", + } + + +def _evaluate(draft: Path, output: Path, args) -> dict: + if output.exists(): + raise FileExistsError(f"evaluation output already exists: {output}") + command = [ + sys.executable, "-m", "aspec.commands.evaluate_draft", + "--target", args.target, + "--draft", str(draft), + "--data", args.data, + "--output", str(output), + "--samples", str(args.samples), + "--max-new-tokens", str(args.max_new_tokens), + "--repeats", str(args.repeats), + "--tp", str(args.tp), + "--domino-candidate-pool-size", str(args.candidate_pool_size), + ] + if args.trust_remote_code: + command.append("--trust-remote-code") + if args.disable_thinking: + command.append("--disable-thinking") + subprocess.run(command, check=True) + return json.loads(output.read_text()) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--corrected", type=Path, required=True, + help="SGLang-format Domino export to ablate") + parser.add_argument("--backbone", type=Path, required=True, + help="Output directory for the backbone-only artifact") + parser.add_argument("--target", required=True) + parser.add_argument("--data", required=True, type=Path) + parser.add_argument("--output", type=Path, required=True, + help="Combined ablation summary JSON") + parser.add_argument("--samples", type=int, default=16) + parser.add_argument("--max-new-tokens", type=int, default=32) + parser.add_argument("--repeats", type=int, default=2) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--candidate-pool-size", type=int, default=0) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--disable-thinking", action="store_true") + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + if args.output.exists(): + raise FileExistsError(args.output) + if min(args.samples, args.max_new_tokens, args.repeats, args.tp) < 1: + raise ValueError("use positive sample, token, repeat and TP budgets") + if args.candidate_pool_size < 0: + raise ValueError("candidate pool size must be non-negative") + + build_backbone_only(args.corrected, args.backbone, force=args.force) + corrected_path = args.output.with_suffix(".corrected.json") + backbone_path = args.output.with_suffix(".backbone.json") + corrected = _evaluate(args.corrected, corrected_path, args) + backbone = _evaluate(args.backbone, backbone_path, args) + summary = summarize(corrected, backbone) + args.output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + print(json.dumps(summary, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/domino_anchor_protocol.py b/tools/validation/domino_anchor_protocol.py new file mode 100644 index 0000000..ae95a91 --- /dev/null +++ b/tools/validation/domino_anchor_protocol.py @@ -0,0 +1,178 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Materialize and verify one canonical anchor stream for matched Domino runs. + +ASpec and SpecForge use different anchor samplers. A comparison should consume +the same explicit positions instead of asking two RNGs to agree. This tool turns +tokenized samples into a hashed replay file and proves that regenerating it from +the same seed and supervision is bit-identical. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import torch + +from aspec.data.utils import resolve_loss_mask +from aspec.models.ops.anchors import ( + sample_anchor_positions, + validate_injected_anchors, +) + +_SCHEMA = "aspec.domino-anchor-protocol/v1" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load_samples(path: Path, limit: int | None = None) -> list[dict]: + if path.suffix == ".pt": + payload = torch.load(path, map_location="cpu", weights_only=False) + rows = payload.get("prompts") if isinstance(payload, dict) else payload + if not rows: + raise ValueError("tokenized cache contains no prompts") + else: + rows = [json.loads(line) for line in path.read_text().splitlines() if line] + if limit is not None: + rows = rows[:limit] + samples = [] + for index, row in enumerate(rows): + input_ids = row.get("input_ids") + if input_ids is None: + raise ValueError(f"sample {index} has no input_ids") + input_ids = torch.as_tensor(input_ids).reshape(-1).long() + normalized = dict(row) + normalized["input_ids"] = input_ids + if row.get("packed_loss_mask") is not None: + mask = resolve_loss_mask(normalized) + elif row.get("loss_mask") is not None: + mask = torch.as_tensor(row["loss_mask"]).reshape(-1) + else: + mask = None + if mask is None or mask.shape != input_ids.shape: + raise ValueError(f"sample {index} has no aligned loss_mask") + samples.append({ + "data_id": str(row.get("data_id") or row.get("id") or f"sample_{index}"), + "input_ids": input_ids, + "loss_mask": mask.long(), + }) + if len({sample["data_id"] for sample in samples}) != len(samples): + raise ValueError("duplicate data_id in anchor source") + return samples + + +def build_anchors(samples, *, num_anchors: int, block_size: int, seed: int, + max_gap: int | None = None) -> dict: + """Generate one deterministic anchor stream for the given samples.""" + if min(num_anchors, block_size) < 1: + raise ValueError("num_anchors and block_size must be positive") + generator = torch.Generator(device="cpu").manual_seed(int(seed)) + records = [] + for sample in samples: + input_ids, loss_mask = sample["input_ids"], sample["loss_mask"] + if input_ids.shape != loss_mask.shape or input_ids.ndim != 1: + raise ValueError("input_ids and loss_mask must be aligned 1-D tensors") + anchors, keep = sample_anchor_positions( + int(input_ids.numel()), loss_mask.unsqueeze(0), int(num_anchors), + int(block_size), torch.device("cpu"), max_gap=max_gap, generator=generator, + ) + records.append({ + "data_id": sample["data_id"], + "anchors": anchors[0].tolist(), + "keep_mask": keep[0].tolist(), + }) + return { + "schema_version": _SCHEMA, + "num_anchors": int(num_anchors), + "block_size": int(block_size), + "seed": int(seed), + "max_gap": max_gap, + "sample_count": len(records), + "records": records, + } + + +def _validate_record(record: dict) -> None: + if record.get("schema_version") != _SCHEMA: + raise ValueError("not a Domino anchor protocol file") + records = record.get("records") + if not isinstance(records, list) or not records: + raise ValueError("anchor protocol file has no records") + if len({row["data_id"] for row in records}) != len(records): + raise ValueError("anchor protocol file has duplicate data_id") + + +def verify_anchors(record: dict, samples) -> None: + """Recompute the stream and validate every kept anchor against its supervision.""" + _validate_record(record) + expected = build_anchors( + samples, + num_anchors=record["num_anchors"], + block_size=record["block_size"], + seed=record["seed"], + max_gap=record.get("max_gap"), + ) + if len(expected["records"]) != len(record["records"]): + raise ValueError("anchor protocol sample count changed") + by_id = {sample["data_id"]: sample for sample in samples} + for actual, replay in zip(record["records"], expected["records"], strict=True): + if actual["data_id"] != replay["data_id"]: + raise ValueError("anchor protocol sample order changed") + if actual["anchors"] != replay["anchors"] or actual["keep_mask"] != replay["keep_mask"]: + raise ValueError(f"anchor replay differs for {actual['data_id']}") + sample = by_id[actual["data_id"]] + validate_injected_anchors( + torch.tensor([actual["anchors"]], dtype=torch.long), + torch.tensor([actual["keep_mask"]], dtype=torch.bool), + loss_mask=sample["loss_mask"].unsqueeze(0), + num_anchors=record["num_anchors"], + block_size=record["block_size"], + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("build", "verify")) + parser.add_argument("--source", type=Path, required=True, + help="tokenized .pt cache or JSONL with input_ids/loss_mask") + parser.add_argument("--anchors", type=Path, required=True) + parser.add_argument("--num-anchors", type=int, default=128) + parser.add_argument("--block-size", type=int, default=16) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--max-gap", type=int, default=None) + parser.add_argument("--limit", type=int, default=None) + args = parser.parse_args() + + samples = _load_samples(args.source, args.limit) + if args.action == "build": + if args.anchors.exists(): + raise FileExistsError(args.anchors) + record = build_anchors( + samples, + num_anchors=args.num_anchors, + block_size=args.block_size, + seed=args.seed, + max_gap=args.max_gap, + ) + record["source_sha256"] = _sha256(args.source) + args.anchors.write_text(json.dumps(record, sort_keys=True) + "\n") + else: + record = json.loads(args.anchors.read_text()) + if record.get("source_sha256") != _sha256(args.source): + raise ValueError("anchor source changed; refusing to verify a different dataset") + verify_anchors(record, samples) + print(f"PASS {args.action} {len(samples)} samples -> {args.anchors}") + + +if __name__ == "__main__": + main()