From 8c8b3216c4ee010b79f9966709ebedc14ea51375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=95=86=E6=98=8A=E6=9A=98521021910816?= Date: Mon, 9 Nov 2026 08:54:42 -0800 Subject: [PATCH] v0.8 protocol releases in Bench (B1-B3) B1: requirements.txt pins the hsenv engine (single pin, by the commit SHA of tag v0.8.0, fd38302) and every workflow reads it from there. scripts/research/effects_core.py re-exports hsenv.effects_core. A golden test checks that every community revision with effects.json gives the same problems, warnings and kg digest as the Bench copy it replaces, and that protocol readers never execute protocol text. B2: scripts/check_contributors.py applies one contributor rule to legacy index.json and protocol study.json. scripts/research/protocols.py checks a protocol release with the pinned engine (schema, public-path allowlist, visibility, signed fidelity, no release_hold, protocol hash, current checks report, effects and capabilities, manifest digest). validate_assets gains a protocol branch; validate-studies.yml gains a validate-protocols job. B3: build_catalog adds a protocol block (counts and ids only) to the research_index entry and derives replayable/humanBenchmark from the protocol. studies_index.json and research_index.json stay byte-identical for current studies. Review fixes: a community proposal holding protocol.json is never accepted on the proposal path (always needs_review with requiresMaintainer, no agent review, regardless of self-declared fidelity or checks); shared protocol block fixture for the web reader; public-only docs; limits on the validate-protocols job. Tests use an invented CC0 fixture release. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/maintain-community.yml | 3 + .github/workflows/research-catalog.yml | 6 +- .github/workflows/sync-studies-index.yml | 3 + .github/workflows/validate-studies.yml | 80 +- docs/v080-protocol.md | 123 ++ requirements.txt | 5 + scripts/check_contributors.py | 78 + scripts/maintain_community.py | 8 +- scripts/research/catalog.py | 24 +- scripts/research/effects_core.py | 1280 +---------------- scripts/research/protocols.py | 317 ++++ scripts/research/revisions.py | 64 +- tests/fixtures/effects_core_golden.json | 754 ++++++++++ .../example2026lamps/protocol.json | 174 +++ .../review/checks-latest.json | 74 + .../review/fidelity-review-1.md | 5 + .../example2026lamps/sources/SOURCE.json | 7 + .../example2026lamps/sources/open/survey.txt | 9 + .../example2026lamps/sourcetext/survey.txt | 9 + .../example2026lamps/study.json | 48 + .../research-index-protocol.json | 28 + tests/test_check_contributors.py | 55 + tests/test_hsenv_pin.py | 125 ++ tests/test_protocol_release.py | 313 ++++ 24 files changed, 2277 insertions(+), 1315 deletions(-) create mode 100644 docs/v080-protocol.md create mode 100644 scripts/check_contributors.py create mode 100644 scripts/research/protocols.py create mode 100644 tests/fixtures/effects_core_golden.json create mode 100644 tests/fixtures/protocol_release/example2026lamps/protocol.json create mode 100644 tests/fixtures/protocol_release/example2026lamps/review/checks-latest.json create mode 100644 tests/fixtures/protocol_release/example2026lamps/review/fidelity-review-1.md create mode 100644 tests/fixtures/protocol_release/example2026lamps/sources/SOURCE.json create mode 100644 tests/fixtures/protocol_release/example2026lamps/sources/open/survey.txt create mode 100644 tests/fixtures/protocol_release/example2026lamps/sourcetext/survey.txt create mode 100644 tests/fixtures/protocol_release/example2026lamps/study.json create mode 100644 tests/fixtures/protocol_release/research-index-protocol.json create mode 100644 tests/test_check_contributors.py create mode 100644 tests/test_hsenv_pin.py create mode 100644 tests/test_protocol_release.py diff --git a/.github/workflows/maintain-community.yml b/.github/workflows/maintain-community.yml index 320c0b9..50c8400 100644 --- a/.github/workflows/maintain-community.yml +++ b/.github/workflows/maintain-community.yml @@ -33,6 +33,9 @@ jobs: with: python-version: '3.12' - run: pip install jsonschema==4.26.0 pypdf==6.1.1 + # trusted, pinned engine from main's requirements.txt; never from the proposal + # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + - run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" - name: Read exact proposal commits id: proposal env: diff --git a/.github/workflows/research-catalog.yml b/.github/workflows/research-catalog.yml index dbd2b85..be6cadc 100644 --- a/.github/workflows/research-catalog.yml +++ b/.github/workflows/research-catalog.yml @@ -1,7 +1,7 @@ name: Research catalog checks on: pull_request: - paths: ['scripts/research/**', 'scripts/maintain_community.py', 'scripts/build_studies_index.py', 'catalog/**', 'contracts/human-program.schema.json', 'tests/test_research_catalog.py', 'tests/test_community_worker.py', '.github/workflows/*community*', '.github/workflows/research-catalog.yml'] + paths: ['scripts/research/**', 'scripts/maintain_community.py', 'scripts/build_studies_index.py', 'catalog/**', 'contracts/human-program.schema.json', 'requirements.txt', 'tests/test_research_catalog.py', 'tests/test_community_worker.py', '.github/workflows/*community*', '.github/workflows/research-catalog.yml'] permissions: contents: read jobs: @@ -15,6 +15,8 @@ jobs: with: python-version: '3.12' - run: pip install jsonschema==4.26.0 pytest==8.4.2 - - run: python -m pytest -o addopts='' tests/test_research_catalog.py tests/test_community_worker.py tests/test_build_studies_index.py -q + # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + - run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" + - run: python -m pytest -o addopts='' tests/test_research_catalog.py tests/test_community_worker.py tests/test_build_studies_index.py tests/test_hsenv_pin.py -q - run: python scripts/build_studies_index.py - run: git diff --exit-code -- co_website/data/studies_index.json co_website/data/research_index.json diff --git a/.github/workflows/sync-studies-index.yml b/.github/workflows/sync-studies-index.yml index 5900fd5..ce155a8 100644 --- a/.github/workflows/sync-studies-index.yml +++ b/.github/workflows/sync-studies-index.yml @@ -21,6 +21,9 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" + - name: Install the pinned protocol engine + # effects_core is hsenv's (V080 B1). hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" - name: Build studies index run: python scripts/build_studies_index.py - name: Commit updated index diff --git a/.github/workflows/validate-studies.yml b/.github/workflows/validate-studies.yml index 774a8fc..8d83bee 100644 --- a/.github/workflows/validate-studies.yml +++ b/.github/workflows/validate-studies.yml @@ -6,6 +6,9 @@ on: - "studies/**" - "scripts/build_studies_index.py" - "scripts/verify_study.sh" + - "scripts/check_contributors.py" + - "community/**" + - "scripts/research/protocols.py" jobs: validate: @@ -57,43 +60,50 @@ jobs: exit 0 fi # A failed diff must fail the job, never read as "no studies changed". - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- studies/) - CHANGED_STUDIES=$(printf '%s\n' "$CHANGED_FILES" | grep -oP 'studies/[^/]+' | sort -u || true) + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- studies/ community/) + CHANGED_STUDIES=$(printf '%s\n' "$CHANGED_FILES" | grep -oP '^studies/[^/]+|^community/paper_[a-f0-9]+/revisions/[a-f0-9]{64}' | sort -u || true) if [ -z "$CHANGED_STUDIES" ]; then exit 0 fi - FAILED=0 - for study_path in $CHANGED_STUDIES; do - INDEX="$study_path/index.json" - if [ ! -f "$INDEX" ]; then - continue - fi - # Check if any contributor github field contains the PR author's username - MATCH=$(python3 -c " - import json, sys - with open('$INDEX') as f: - data = json.load(f) - contributors = data.get('contributors', []) - if not contributors: - print('MISSING') - sys.exit(0) - author = '$PR_AUTHOR'.lower() - for c in contributors: - gh = c.get('github', '').lower() - # Match 'username' or 'https://github.com/username' - if gh.rstrip('/').split('/')[-1] == author: - print('MATCH') - sys.exit(0) - print('MISMATCH') - ") - study_id=$(basename "$study_path") - if [ "$MATCH" = "MISSING" ] || [ "$MATCH" = "MISMATCH" ]; then - echo "::error file=$INDEX::Contributor check failed for $study_id. Your GitHub username is '$PR_AUTHOR'. Please add the following to your index.json:%0A%0A\"contributors\": [%0A { \"name\": \"Your Name\", \"github\": \"$PR_AUTHOR\" }%0A]" - FAILED=1 - else - echo "$study_id: Contributor @$PR_AUTHOR verified." - fi + # Same rule for legacy index.json and protocol study.json (V080 B2); a dir with neither is skipped. + python scripts/check_contributors.py --author "$PR_AUTHOR" $CHANGED_STUDIES + + # V080-PLATFORM §2.1: a maintainer release PR adds files under community/**. Every changed revision that + # holds protocol.json is checked with the hsenv engine. This job runs the PR's own checker + # (scripts/research/protocols.py), its tests and its requirements.txt hsenv pin, so a PR can change its own + # checks: it has no secrets, a read-only token and a time limit, and reviewers must look at any change to + # those files. Protocol files themselves are read as data by the engine. + validate-protocols: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install the pinned engine + # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + run: | + pip install jsonschema==4.26.0 pytest==8.4.2 + grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" + - name: Validate changed protocol releases + run: | + # A failed diff must fail the job, never read as "no revisions changed". + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- community/) + REVISIONS=$(printf '%s\n' "$CHANGED_FILES" | grep -oP '^community/paper_[a-f0-9]+/revisions/[a-f0-9]{64}' | sort -u || true) + DIRS="" + for rev in $REVISIONS; do + if [ -f "$rev/protocol.json" ]; then DIRS="$DIRS $rev"; fi done - if [ "$FAILED" -eq 1 ]; then - exit 1 + if [ -z "$DIRS" ]; then + echo "No protocol releases changed." + exit 0 fi + python -m scripts.research.protocols --sessions 300 $DIRS + - name: Test the protocol release rules + run: python -m pytest tests/test_protocol_release.py tests/test_check_contributors.py tests/test_hsenv_pin.py -o addopts='' -q diff --git a/docs/v080-protocol.md b/docs/v080-protocol.md new file mode 100644 index 0000000..670f4bc --- /dev/null +++ b/docs/v080-protocol.md @@ -0,0 +1,123 @@ +# v0.8 protocol releases in HumanStudy-Bench + +This page covers the Bench part of the v0.8 platform design. In v0.8 a study is one folder holding `study.json` and `protocol.json`, and one engine (`hsenv`) +reads it on every platform. Bench publishes a released protocol as a community revision +(`community/paper_/revisions//`). Bench never interprets a protocol itself and never executes +its files: every check calls the pinned `hsenv`. + +## What this change does + +- **B1.** `requirements.txt` pins hsenv by commit SHA; it is the only place the pin is written, and the + sync-studies-index, research-catalog, maintain-community and validate-studies workflows read it from there. + `scripts/research/effects_core.py` re-exports `hsenv.effects_core`, with import paths and the CLI kept. + `tests/test_hsenv_pin.py` checks that every community revision with `effects.json` gives the same problems, + warnings and kg digest as the Bench copy did (`tests/fixtures/effects_core_golden.json`, recorded with the + copy at `b34d214`), and that the protocol readers never execute protocol text. +- **B2.** `scripts/check_contributors.py` (one contributor rule), `scripts/research/protocols.py` (release + checks), a protocol branch in `validate_assets`, and the `validate-protocols` job in `validate-studies.yml`. + The tests use an invented CC0 fixture release. +- **B3.** `build_catalog` adds a `protocol` block to the paper's `research_index.json` entry and computes + `replayable`/`humanBenchmark` from the protocol. `studies_index.json` and `research_index.json` stay + byte-identical for today's studies. + +Not done here: **B4** (`run_protocol.py`, the playground on `kind: protocol`, quiet logs) waits for owner gate G3. +**B5** (the first releases) needs signed reviews (G2) and studies with no `release_hold`. + +## What CI checks on a protocol release + +`python -m scripts.research.protocols ... [--sessions 300]` exits 1 on any error. The steps +follow the order in design §2.1: + +1. **Schema.** `hsenv.schema_check` validates `study.json` (`kind=study`) and `protocol.json` (`kind=protocol`, + including its cross-reference pass). `schema_version` is required: legacy files are not released. +2. **Release rules.** + - Every path must be on the allowlist: `study.json`, `protocol.json`, `effects.json`, `program.json`, + `capabilities.json`, `review/fidelity-review-*.md`, `review/checks-latest.json`, `sources/SOURCE.json`, + `sources/open/**`, `sourcetext/*` (only when `meta.sources[key]` is under `sources/open/`), + `variants/*.json`, `artifacts/**` and `package/**`. + - These are never published: `sources/{researcher,private,quarantine}/**`, `runs/**`, `**/runs/**`, + `**/records.jsonl`, `paper/**` and `bundle.json`. + - `visibility` must be `"public"`. `quality` must be `high_fidelity`, with `fidelity.verdict` HIGH FIDELITY, + a non-empty `fidelity.signed_by` and the review file included (G2). + - No `release_hold` may appear, neither in `study.json` nor on any arm. + - `study.json.protocol.sha256` must equal the canonical hash of `protocol.json`, and `study.json.arms` must + agree with the protocol's arms. + - An image `src` must be a URL, or an `asset:` that points to a file in `sources/open/`. + - `study.json.hsenv` must admit the installed engine. +3. **Checks report.** `review/checks-latest.json` must report `PASS: true` and carry `protocol_sha256` + equal to this protocol's hash. A report without the hash, or with another hash, counts as stale. + `--sessions N` also replays N random sessions through `hsenv.checks` (references and fuzz). +4. **Effects and capabilities.** `effects.json` goes through `hsenv.effects_core.check`, and + `capabilities.json` through the capability rules. +5. **Digest.** The revision directory's name must equal its manifest digest (`revisions.revision_id`). + +Each revision is checked through one of two paths: + +- **Maintainer release PR** (direct files under `community/**`): the `validate-protocols` job in + `validate-studies.yml` runs the checks on each changed revision that holds `protocol.json`. +- **Community proposal** (`maintain-community.yml`): `revisions.validate_assets` sees `protocol.json` and runs + the same checks on the proposal's files. `program.json` becomes optional. The engine comes from main's + pinned requirements and never from the proposal. A proposal that holds `protocol.json` is **never + accepted on this path**: `integrate` always returns `needs_review` with `requiresMaintainer: true`, whatever + the review says (an agent's or anyone's `accept`) and whatever the release declares about itself + (`fidelity.signed_by`, a PASS `checks-latest.json`). No agent review is run for it. A maintainer publishes + it through a release PR. + +**Contributor rule.** `scripts/check_contributors.py --author ...` reads `index.json` for +legacy studies and `study.json` for protocol releases. The rule is unchanged: a `contributors[].github` must +equal the PR author, given as `login` or as a GitHub URL. Admin and maintain users still bypass it in the +workflow. A community proposal gets its PR author appended to `study.json` contributors when missing. + +## research_index.json + +Shared contract fixture: `tests/fixtures/protocol_release/research-index-protocol.json` is the exact +`protocol` block and capabilities Bench writes for the fixture release (`schemaVersion` is the protocol +schema version, `"1"`). The web repo tests its reader against the same JSON; change both together. + +For a current revision with `protocol.json`, the paper entry gains these fields: + +``` +"protocol": {"slug", "schemaVersion", "hsenv", "fidelity": {"verdict", "signed_by"}, + "arms": [{"id", "runnable", "nPaper"}], "runExcluded": [...], "effects": n, "revision"}, +"capabilities": {..., "replayable": any runnable arm without release_hold, "humanBenchmark": any effect with reported values}, +"coverage": "protocol-release" (only when the revision has no program.json) +``` + +The block holds counts and ids only, never protocol text. `studies_index.json` keeps its legacy shape. + +## Test fixture + +`tests/fixtures/protocol_release/example2026lamps/` is an invented study under CC0-1.0. Its material is two +sentences about a lamp and one rating question, and it is no real paper, protocol or participant. Its +`review/checks-latest.json` is a real `hsenv.checks` PASS report (200 sessions, verbatim 3/3 found, keyed sign +correct), with `protocol_sha256` added. To regenerate the fixture, write it again with the same content, run +`python -m hsenv.checks --sessions 200 --source-text survey=/sourcetext/survey.txt`, and store the +output with `protocol_sha256`. + +## Running locally + +```bash +pip install -r requirements.txt # installs hsenv from GitHub at the pinned commit +pytest tests/test_hsenv_pin.py tests/test_protocol_release.py tests/test_check_contributors.py -o addopts='' +python -m scripts.research.protocols community/paper_/revisions/ --sessions 300 +``` + +The hsenv tests skip when hsenv is not installed. + +## Open items (hsenv H1 and maintainer decisions) + +- **hsenv is installed, not vendored.** `requirements.txt` pins hsenv v0.8.0 by its commit SHA + (`fd38302`). The desktop pins the same release (B1 is chained to it). +- **Engine schema vs design.** `study.v1.json` has `additionalProperties: false` and lacks the design's §1.1 + fields (`slug`, `paper`, `contributors`, `visibility`, `hsenv`, `fidelity`, `arms`, `paperId`, `release`). + Bench validates those itself and removes them before calling `schema_check` (`protocols.RELEASE_FIELDS`). + H1 should add them to the schema. +- **H1 must provide:** + - `protocol_sha256` in the `hsenv.checks` report: today it carries none, so CI could not detect a stale report + - `hsenv check --public`: `protocols.replay` is the interim refs and fuzz subset + - `hsenv.release.PUBLIC_PATHS`: a test fails if it differs from Bench's list + - `hsenv.capabilities`: `capabilities.py` stays a Bench copy until then +- **Design statement corrected.** Design §2.1 says `expr.py` has "no eval/exec (verified by grep)". In fact + it calls `compile` and `eval`. It does so on an AST whose node types it has checked against a whitelist, + and it runs with `__builtins__` emptied. The CI test pins that whitelist and a set of escape attempts. +- **Maintainer decisions:** who signs a release (G2), and how B4 keeps public Actions logs quiet (G3). diff --git a/requirements.txt b/requirements.txt index 3a90906..4b18182 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,11 @@ pandas>=1.3.0 # Statistical analysis (used in study configs and evaluation) scipy>=1.7.0 # For statistical tests (ttest, ANOVA, etc.) +# Protocol engine (V080-PLATFORM B1): the one hsenv release Bench, the web runner and the desktop share. +# This line is the single pin: every workflow and test reads it from here (`grep '^hsenv' requirements.txt`, then `pip install -r`). +# Pinned by commit SHA (the commit of tag v0.8.0), not by the mutable tag. +hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + # Data validation jsonschema>=4.0.0 pydantic>=2.0.0 diff --git a/scripts/check_contributors.py b/scripts/check_contributors.py new file mode 100644 index 0000000..5b490e8 --- /dev/null +++ b/scripts/check_contributors.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Contributor attribution for submitted studies (one rule for both doors, V080-PLATFORM §2.1 / B2). + + python scripts/check_contributors.py --author [ ...] + +A legacy study (`studies/study_NNN`) keeps its contributors in index.json; a protocol study keeps them in +study.json. Each directory must list a contributor whose `github` is the PR author ('login' or +'https://github.com/login'). A directory with neither file is skipped, as before. The maintainer bypass +(admin/maintain permission) stays in the workflow, which never calls this script for maintainers. + +This is the inline check that validate-studies.yml carried, moved here unchanged except that study.json +is read when there is no index.json. Exit 1 when any directory fails. +""" +import argparse +import json +import sys +from pathlib import Path + + +def contributors_file(study_dir: Path): + for name in ("index.json", "study.json"): + if (study_dir / name).is_file(): + return study_dir / name + return None + + +def verdict(path: Path, author: str) -> str: + """MATCH, MISMATCH or MISSING, exactly as the former inline check.""" + return verdict_from(json.loads(path.read_text(encoding="utf-8")), author) + + +def verdict_from(data: dict, author: str) -> str: + contributors = data.get("contributors", []) + if not contributors: + return "MISSING" + # Match 'username' or 'https://github.com/username' + for c in contributors: + gh = (c.get("github", "") if isinstance(c, dict) else "").lower() + if gh.rstrip("/").split("/")[-1] == author.lower(): + return "MATCH" + return "MISMATCH" + + +def with_contributor(raw: bytes, login: str, name: str = "") -> bytes: + """study.json bytes with the PR author appended to contributors when absent (community proposals).""" + data = json.loads(raw) + if not isinstance(data, dict) or not isinstance(data.get("contributors", []), list): + return raw # not a study card; the release checks report the malformed study.json + if verdict_from(data, login) == "MATCH": + return raw + data.setdefault("contributors", []).append({"name": name or login, "github": login}) + return (json.dumps(data, ensure_ascii=False, indent=2) + "\n").encode() + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--author", required=True) + ap.add_argument("dirs", nargs="*") + a = ap.parse_args(argv) + failed = False + for d in a.dirs: + study_dir = Path(d) + path = contributors_file(study_dir) + if path is None: + continue + study_id = study_dir.name + if verdict(path, a.author) in ("MISSING", "MISMATCH"): + print(f"::error file={path}::Contributor check failed for {study_id}. Your GitHub username is " + f"'{a.author}'. Please add the following to your {path.name}:%0A%0A\"contributors\": [%0A " + f"{{ \"name\": \"Your Name\", \"github\": \"{a.author}\" }}%0A]") + failed = True + else: + print(f"{study_id}: Contributor @{a.author} verified.") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/maintain_community.py b/scripts/maintain_community.py index 028847b..8b940b1 100644 --- a/scripts/maintain_community.py +++ b/scripts/maintain_community.py @@ -64,6 +64,11 @@ def proposal_from_ref(ref: str, base: str, author: str) -> tuple[Path, dict, dic raise ValueError("Hub submissions require their server-signed contributor identity") elif str(contributor.get("github", "")).casefold() != author.casefold(): raise ValueError("the contributor must match the authenticated GitHub PR author") + # A protocol release credits the same person in study.json (V080 §2.1): append when absent. + login = author if author.casefold() != bot.casefold() else str(contributor.get("github", "")) + if "study.json" in files and "protocol.json" in files and login: + from scripts.check_contributors import with_contributor + files["study.json"] = with_contributor(files["study.json"], login, str(contributor.get("name", ""))) # Existing Hub key encoding remains compatible, but no contribution may write another paper's path. expected = "paper_" + hashlib.sha256(metadata["paperKey"].encode()).hexdigest()[:24] if paper_dir != expected or len(paper_dir) > 300: @@ -109,7 +114,8 @@ def proposal_from_ref(ref: str, base: str, author: str) -> tuple[Path, dict, dic def maintain(ref: str, base: str, author: str, *, agent_review: bool) -> dict: folder, metadata, files = proposal_from_ref(ref, base, author) result = integrate(folder, metadata, files) - if result.get("candidateRevision") and agent_review: + # A protocol release always waits for a maintainer, so no agent review is spent on it. + if result.get("candidateRevision") and agent_review and not result.get("requiresMaintainer"): pointer = folder / "current.json" current = json.loads(pointer.read_text()) if pointer.exists() else None candidate, conflicts = merge_assets(load_revision(folder, metadata.get("baseRevision")), diff --git a/scripts/research/catalog.py b/scripts/research/catalog.py index e1198e2..b21e211 100644 --- a/scripts/research/catalog.py +++ b/scripts/research/catalog.py @@ -235,10 +235,25 @@ def build_catalog(root: Path, legacy: list[dict]) -> dict: file = revision / entry["path"] if not file.resolve().is_relative_to(revision.resolve()) or file.is_symlink() or hashlib.sha256(file.read_bytes()).hexdigest() != entry["sha256"]: raise ValueError(f"community revision content changed: {file}") - program = json.loads((revision / "program.json").read_text()) - projection = project_program(program) + paths = {entry["path"] for entry in manifest["files"]} paper = current["paper"] - if not paper.get("description"): + # A v0.8 protocol release (V080 §3.1) may have no program.json; its block holds counts and ids only. + protocol_extra, capabilities = {}, current["capabilities"] + if "protocol.json" in paths: + from scripts.research.protocols import protocol_capabilities, summary + study_doc = json.loads((revision / "study.json").read_text()) + protocol_doc = json.loads((revision / "protocol.json").read_text()) + protocol_extra = {"protocol": {**summary(study_doc, protocol_doc), "revision": current["revision"]}} + capabilities = {**capabilities, **protocol_capabilities(protocol_doc)} + if "program.json" in paths: + program = json.loads((revision / "program.json").read_text()) + projection = project_program(program) + else: + program = {} + projection = {"studies": [], "issues": [], "effectEstimates": [], + "searchText": "\n".join(strings({k: paper.get(k) for k in ("title", "authors", "year")}) + + strings(protocol_extra.get("protocol", {}).get("slug")))} + if not paper.get("description") and program: paper = {**paper, "description": community_description(revision, manifest, program)} pid = paper["id"] previous = papers.get(pid, {}) @@ -253,7 +268,8 @@ def build_catalog(root: Path, legacy: list[dict]) -> dict: "aliases": sorted(set(previous.get("aliases", []) + paper.get("aliases", []))), "currentRevision": current["revision"], "assetRoot": revision.relative_to(root).as_posix(), "assetFiles": manifest["files"], - "coverage": "human-program", "capabilities": current["capabilities"], + "coverage": "human-program" if program else "protocol-release", "capabilities": capabilities, + **protocol_extra, **({"studyCapabilities": study_caps} if study_caps else {}), "effectEstimates": estimates, **({"graph": graph} if graph else {}), diff --git a/scripts/research/effects_core.py b/scripts/research/effects_core.py index 4c30883..5d80d41 100644 --- a/scripts/research/effects_core.py +++ b/scripts/research/effects_core.py @@ -1,1278 +1,20 @@ #!/usr/bin/env python3 -"""effects.json: the executable link from a paper's hypotheses to its data. +"""effects.json rules and computation: the pinned hsenv engine (V080-PLATFORM B1). -program.json says, in the paper's words, what the study is. effects.json turns its hypotheses into -effects that a program can compute: for each effect, the outcome, the contrast, the predicted -direction, the paper's reported numbers (each one quoted) and how to compute it from sessions. -Human and AI-participant sessions are computed the same way and give the same result block, so -effects from many papers can be pooled later without an agent re-reading any paper. +This file used to be a copy of humanstudy-desktop `pack/skills/export/agent/effects_core.py`. Bench now +reads the one engine every platform reads, `hsenv.effects_core`, at the version pinned in requirements.txt. +Import paths are kept (`scripts.research.effects_core`, and loading this file by path), so callers do not +change. Do not add logic here: a rule change is an hsenv release. - python effects_core.py check validate; exits 1 on a problem - python effects_core.py graph hypothesis -> effect -> program nodes - python effects_core.py kg the paper's whole typed graph (schema 2) - python effects_core.py types the core vocabulary (graph-types.json) - python effects_core.py compute result blocks for every effect - -Raw quantities are kept: a result block holds the cells (n, mean, sd or counts) and the test, never only -a converted effect size. Converting between metrics (d, r, log odds ratio) is left to later tools. -Standard library only, Python 3.8+. Shared by the build skill, the agent export and HumanStudy-Bench. + python scripts/research/effects_core.py check same CLI as hsenv.effects_core """ -import json -import math -import re import sys -from pathlib import Path - -SCHEMA_VERSION = 2 # new files; schema 1 files stay valid (their "other" values included) -SCHEMA_VERSIONS = (1, 2) -VOCAB_VERSION = "2.0.0" # core vocabulary below; bump the minor version when an extension is promoted -METRICS = ( - "mean", "mean_difference", "standardized_mean_difference", "hedges_g", "proportion", "risk_difference", - "risk_ratio", "odds_ratio", "log_odds_ratio", "correlation", "phi", "cramers_v", "chi_square", - "regression_coefficient", "standardized_coefficient", "t", "F", "z", "bayes_factor", "eta_squared", - "partial_eta_squared", "count", "cohens_q", "auc", "ratio", "percent_change", "other") -DESIGNS = ("between", "within", "mixed", "correlational", "other") -OUTCOME_TYPES = ("continuous", "ordinal", "binary", "categorical", "count", "other") -CONTRASTS = ("two_groups", "paired", "association", "interaction", "omnibus", "single_group", "linear", "ratio", - "model_comparison", "other") -# Schema 2 never accepts "other" for these: a new kind is declared as an extension (see EXTENSIONS below). -NO_OTHER = ("contrast", "metric") -TEST_STATISTICS = ("t", "F", "z", "chi_square", "W", "V", "U", "other") -DIRECTIONS = ("positive", "negative", "nonzero", "none", "unspecified") -OPS = ("mean_difference", "paired_difference", "proportion_difference", "contingency", "correlation", "package") -ROLES = ("test", "robustness", "check", "descriptive") -POPULATIONS = ("human", "ai-model", "human-vs-ai-model", "ai-model-vs-ai-model") -UNTESTED = ("not-an-effect", "not-tested-in-paper", "not-computable", "out-of-scope") -# tested: every effect naming it is computed by the replay and they cover its scope; partial: some of its effects -# are computed but others are excluded (capabilities.json run.excluded), or they cover only part of its scope -# (one design or site of a pooled hypothesis, one side of a comparison), with a reason; untested: none computed. -HYPOTHESIS_STATUSES = ("tested", "partial", "untested") -# Metrics that live on a 0-1 scale but are often printed in percentage points: estimates say which (`scale`). -PROPORTION_METRICS = ("proportion", "risk_difference") -SCALES = ("proportion", "percentage_points") -# Metrics without a sign: on a two-group or paired contrast they hide which arm is higher. -UNSIGNED_METRICS = ("phi", "cramers_v", "chi_square", "F", "eta_squared", "partial_eta_squared") -ESTIMATE_TYPES = ("descriptive", "model-based") -# whose numbers `reported` holds, relative to what the replay runs -REPORTED_SCOPES = ("replayed-sample", "pooled", "other-sample") -ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") -NUMBER_FIELDS = ("value", "se", "df", "p", "n", "lower", "upper") -TOLERANCE = "a result is recovered when |computed - reported| <= max(0.05, 0.15 * |reported|)" - - -def load(root): - path = Path(root) / "effects.json" - if not path.is_file(): - return None - return json.loads(path.read_text(encoding="utf-8")) - - -def excluded_effects(root): - """Effect ids capabilities.json run.excluded[].effects lists (the replay does not compute them).""" - try: - caps = json.loads((Path(root) / "capabilities.json").read_text(encoding="utf-8")) - except (OSError, ValueError, TypeError): - return set() - run = caps.get("run") if isinstance(caps, dict) and isinstance(caps.get("run"), dict) else {} - items = run.get("excluded") if isinstance(run.get("excluded"), list) else [] - return {str(x) for item in items if isinstance(item, dict) and isinstance(item.get("effects"), list) - for x in item["effects"]} - - -# ---- numbers in quotes ----------------------------------------------------------------------------- - -_NUM = re.compile(r"(? 1: - P("%s: %s = %s is not on a 0-1 proportion scale; a value printed in percentage points has scale " - "percentage_points" % (at, est.get("metric"), est.get("value"))) - if "estimateType" in est and est["estimateType"] not in ESTIMATE_TYPES: - P("%s: estimateType must be one of %s" % (at, ", ".join(ESTIMATE_TYPES))) - if "scope" in rep and rep["scope"] not in REPORTED_SCOPES: - P("%s: reported.scope must be one of %s" % (where, ", ".join(REPORTED_SCOPES))) - elif rep.get("scope") in ("pooled", "other-sample") and not str(rep.get("scopeNote") or "").strip(): - P("%s: reported.scope %s needs scopeNote (whose numbers these are: \"pooled across 61 countries\")" - % (where, rep["scope"])) - if rep.get("test") and rep["test"].get("statistic") not in TEST_STATISTICS: - P("%s: reported.test.statistic must be t, F, z, chi_square, W, V, U or other" % where) - p_blocks = [("test", rep.get("test") or {})] + [ - ("estimates[%d]" % j, est) for j, est in enumerate(rep.get("estimates") or [])] - for location, block in p_blocks: - p = block.get("p") - if isinstance(p, (int, float)) and not isinstance(p, bool) and p <= 0: - P("%s: reported.%s.p must be greater than zero; a rounded printed zero is not an exact " - "probability. Preserve the printed text with p: null and describe the missing exact p-value; " - "do not invent an unprinted bound" % (where, location)) - if not isinstance(rep.get("missing", []), list): - P("%s: reported.missing must be a list of what the paper does not report" % where) - for loc, value in _numbers(rep): - q = quoted(value, quotes) - if q != "exact" and any(rendered_quote(value, record) for record in quote_records): - W("%s: reported %s = %s uses a rendered-page check for text-layer sign loss" % (where, loc, value)) - if q == "unsigned": # the text layer still prints only the absolute value; keep that visible - W("%s: reported %s = %s: only its absolute value is quoted; check the sign" % (where, loc, value)) - continue - if q is None: - P("%s: reported %s = %s is not written in its cited quotes" % (where, loc, value)) - elif q == "unsigned": - W("%s: reported %s = %s: only its absolute value is quoted; check the sign" % (where, loc, value)) - not_run = excluded_effects(root) - for h in hyps: - if h.get("status") in ("tested", "partial") and h.get("id") not in tested: - P("hypothesis %s is %s but no effect names it" % (h.get("id"), h.get("status"))) - if h.get("status") == "untested" and h.get("id") in tested: - naming = [e.get("id") for e in effs if h.get("id") in (e.get("hypothesisIds") or [])] - # an untested (not-computable) hypothesis may keep its effects when the replay excludes all of them - if h.get("untestedBecause") != "not-computable" or any(x not in not_run for x in naming): - P("hypothesis %s is untested but effect(s) name it" % h.get("id")) - if not effs and not any(h.get("status") == "untested" for h in hyps): - W("no effects") - for i, link in enumerate(effects.get("links") or []): - for msg in link_problems(link, effects, program): - P("links[%d]: %s" % (i, msg)) - reach = graph(effects, program) - for nid in reach["unlinked"]: - W("program %s %s is not reachable from any hypothesis or effect" % (nodes[nid].get("kind"), nid)) - return problems, warnings - - -# ---- the graph -------------------------------------------------------------------------------------- - -LINKED_KINDS = ("hypothesis", "condition", "variable", "analysis", "result") - - -def graph(effects, program): - """Edges from background through hypotheses and effects to the program nodes they bind, extended by - the program's own relations and step conditions. `unlinked` lists hypothesis/condition/variable/ - analysis/result nodes that no hypothesis reaches.""" - nodes = {n["id"]: n for n in program.get("nodes", []) if "id" in n} - edges = [] - add = lambda a, b, t: edges.append({"from": a, "to": b, "type": t}) - for t in effects.get("topics") or []: - add("paper", "topic:" + str(t), "about") - for h in effects.get("hypotheses") or []: - hid = "hypothesis:" + str(h.get("id")) - add(hid, h.get("nodeId"), "states") - for b in h.get("backgroundIds") or []: - add(b, hid, "motivates") - for ph in h.get("phenomena") or []: - add(hid, "phenomenon:" + str(ph), "instance-of") - for e in effects.get("effects") or []: - eid = "effect:" + str(e.get("id")) - for h in e.get("hypothesisIds") or []: - add("hypothesis:" + h, eid, "tested-by") - if (e.get("outcome") or {}).get("variableId"): - add(eid, e["outcome"]["variableId"], "measures") - for c in (e.get("contrast") or {}).get("conditionIds") or []: - add(eid, c, "contrasts") - for c in (e.get("contrast") or {}).get("factorIds") or []: - add(eid, c, "varies") - for key, t in (("analysisIds", "analysed-by"), ("resultIds", "reported-in")): - for x in e.get(key) or []: - add(eid, x, t) - adj = {} - for x in edges: - adj.setdefault(x["from"], set()).add(x["to"]) - for r in program.get("relations", []): # program relations count both ways - adj.setdefault(r.get("from"), set()).add(r.get("to")) - adj.setdefault(r.get("to"), set()).add(r.get("from")) - seen, todo = set(), ["hypothesis:" + str(h.get("id")) for h in effects.get("hypotheses") or []] - todo += ["effect:" + str(e.get("id")) for e in effects.get("effects") or [] if not e.get("hypothesisIds")] - while todo: - x = todo.pop() - if x in seen: - continue - seen.add(x) - todo.extend(adj.get(x, ())) - unlinked = sorted(nid for nid, n in nodes.items() if n.get("kind") in LINKED_KINDS and nid not in seen) - return {"edges": edges, "reached": sorted(x for x in seen if x in nodes), "unlinked": unlinked} - - -# ---- the knowledge graph (schema 2) ------------------------------------------------------------------ -# -# One typed graph per paper: the program's nodes and procedure steps, the effects (estimands), and the -# constructs they are about, joined by edges from a small core vocabulary. Program relations written with -# older names are read through RELATION_ALIASES; anything that fits no core term is reported as a gap, -# never forced into one. The vocabulary grows by extensions (x:name, declared in effects.json with a core -# parent and a definition); an extension used by several papers is promoted into the core and listed in -# PROMOTED, so files that used the old name read as the core term without being rewritten. - -VARLIKE = ["variable", "record", "measure"] -NODE_KINDS = ("paper", "construct", "background", "overview", "hypothesis", "design", "participants", "condition", - "material", "procedure", "step", "variable", "record", "measure", "analysis", "model", "result", "finding", - "limitation", "simulation", "effect") -EDGE_TYPES = { - # claims - "about": (["paper"], ["construct"], "the paper is about this construct or topic"), - "motivates": (["background"], ["hypothesis"], "background that motivates the hypothesis"), - "instance-of": (["hypothesis", "effect"], ["construct"], "tests a case of this phenomenon or construct"), - "predicts": (["hypothesis"], ["construct"], "the hypothesis predicts a change in this construct"), - "tested-by": (["hypothesis"], ["effect", "analysis", "finding"], "evidence that tests the hypothesis"), - "supports": (["result", "finding"], ["hypothesis"], "the paper reads this result as support"), - "contradicts": (["result", "finding"], ["hypothesis"], "the paper reads this result as counter-evidence"), - # the estimand - "measures": (["effect"], VARLIKE, "the effect's outcome variable"), - "contrasts": (["effect"], ["condition"], "a condition the effect compares"), - "varies": (["effect"], VARLIKE, "a predictor the effect varies"), - "estimated-by": (["effect"], ["analysis", "model"], "the analysis that estimates the effect"), - "reported-in": (["effect"], ["result", "finding"], "where the paper reports the effect"), - "moderates": (VARLIKE + ["condition"], ["effect"], "the effect differs across its levels"), - "mediates": (VARLIKE, ["effect"], "the effect runs through it"), - "replicates": (["effect"], ["effect"], "the same effect estimated again, in another sample or paper"), - "compares": (["effect"], ["effect"], "the effect is defined as a difference between these effects"), - # operationalisation and analysis - "operationalizes": (VARLIKE, ["construct"], "the variable measures the construct"), - "manipulates": (["condition"], ["construct"], "the condition manipulates the construct"), - "input-of": (VARLIKE + ["condition"], ["analysis", "model"], "the analysis uses it"), - "reports": (["analysis", "model"], ["result", "finding"], "the analysis produced this reported result"), - "concerns": (["result", "finding"], VARLIKE + ["condition", "hypothesis"], "the reported result is about it"), - "derived-from": (VARLIKE + ["analysis", "model"], VARLIKE + ["analysis", "model"], "computed from it"), - # procedure (from program steps) - "shows": (["step", "condition"], ["*"], "what a participant sees at this step or in this condition"), - "produces": (["step"], ["*"], "what the step records"), - "uses": (["step"], ["*"], "an input of the step"), - "in-condition": (["step"], ["condition"], "the step belongs to this condition"), - "part-of": (["step"], ["step"], "a sub-step"), - "next": (["step"], ["step"], "the step that follows"), - "performed-by": (["step"], ["*"], "who acts at this step"), -} -PROCEDURE_EDGES = ("shows", "produces", "uses", "in-condition", "part-of", "next", "performed-by") -CONTRAST_DEFS = { - "two_groups": "treatment minus reference, two independent groups", - "paired": "treatment minus reference, measured on the same units", - "single_group": "one group's value, or its difference from a fixed constant", - "association": "a correlation or slope between two variables", - "interaction": "a difference between two contrasts (difference of differences, or across samples)", - "omnibus": "an undirected test over more than two levels; not poolable as a signed effect", - "linear": "sum of weight x cell over named cells, plus an optional constant (e.g. 1 vs mean of 2 others)", - "ratio": "treatment divided by reference", - "model_comparison": "evidence for one model of the data over another (e.g. a Bayes factor)", - "other": "schema 1 only", -} -# Older program relation names -> core edge candidates (tried forwards, then reversed, against the kinds). -RELATION_ALIASES = {"uses": ["input-of"], "outcome": ["input-of", "concerns"], "estimates": ["input-of"], - "sees": ["shows"], "analysed-by": ["estimated-by"], "reported-by": ["reports"], - "reports": ["concerns"], "produces": ["reports"]} -# Promoted extensions: {category: {"x:old-name": "core-name"}}. Empty in 2.0.0. -PROMOTED = {"contrast": {}, "metric": {}, "edge": {}, "node": {}} - - -def _kind_ok(kind, allowed): - return "*" in allowed or kind in allowed - - -def _norm(t): - return re.sub(r"[\s_]+", "-", str(t or "").strip().lower()) - - -def map_relation(rtype, from_kind, to_kind): - """(core edge, reversed) for a program relation, or None when no core edge fits the two kinds.""" - t = _norm(rtype) - for cand in ([t] if t in EDGE_TYPES else []) + RELATION_ALIASES.get(t, []): - dom, rng = EDGE_TYPES[cand][0], EDGE_TYPES[cand][1] - if _kind_ok(from_kind, dom) and _kind_ok(to_kind, rng): - return cand, False - if _kind_ok(to_kind, dom) and _kind_ok(from_kind, rng): - return cand, True - return None - - -def construct_id(label): - """A construct keeps the paper's own wording: no case folding, no synonym merging. Merging labels across - papers is a later, reviewed step (a sameAs map), never done here.""" - return "construct:" + str(label).strip() - - -def ref_kind(ref, effects, program): - """The node kind a reference names, or None when it names nothing.""" - ref = str(ref or "") - if ref.startswith("paper:"): - inner = ref.split("/", 1)[1] if "/" in ref else "" - return "effect" if inner.startswith("effect:") else ("external" if inner else None) - if ref.startswith("construct:"): - return "construct" if len(ref) > len("construct:") else None - if ref.startswith("effect:"): - return "effect" if any(e.get("id") == ref[7:] for e in effects.get("effects") or []) else None - for n in program.get("nodes", []): - if n.get("id") == ref: - return n.get("kind") - if any(st.get("id") == ref for st in program.get("steps", [])): - return "step" - return None - - -def link_problems(link, effects, program): - out = [] - t = link.get("type") - core = resolve("edge", t, effects) - if core is None: - return ["type %r is not a core edge or a declared edge extension" % t] - decl = extensions(effects).get(t) if t not in EDGE_TYPES else None - dom, rng = (decl.get("from"), decl.get("to")) if decl else EDGE_TYPES[core][:2] - fk, tk = ref_kind(link.get("from"), effects, program), ref_kind(link.get("to"), effects, program) - if fk is None: - out.append("from %r names no node (program node or step id, effect:, construct: