diff --git a/.github/workflows/maintain-community.yml b/.github/workflows/maintain-community.yml index 320c0b9..50c8400 100644 --- a/.github/workflows/maintain-community.yml +++ b/.github/workflows/maintain-community.yml @@ -33,6 +33,9 @@ jobs: with: python-version: '3.12' - run: pip install jsonschema==4.26.0 pypdf==6.1.1 + # trusted, pinned engine from main's requirements.txt; never from the proposal + # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + - run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" - name: Read exact proposal commits id: proposal env: diff --git a/.github/workflows/research-catalog.yml b/.github/workflows/research-catalog.yml index dbd2b85..be6cadc 100644 --- a/.github/workflows/research-catalog.yml +++ b/.github/workflows/research-catalog.yml @@ -1,7 +1,7 @@ name: Research catalog checks on: pull_request: - paths: ['scripts/research/**', 'scripts/maintain_community.py', 'scripts/build_studies_index.py', 'catalog/**', 'contracts/human-program.schema.json', 'tests/test_research_catalog.py', 'tests/test_community_worker.py', '.github/workflows/*community*', '.github/workflows/research-catalog.yml'] + paths: ['scripts/research/**', 'scripts/maintain_community.py', 'scripts/build_studies_index.py', 'catalog/**', 'contracts/human-program.schema.json', 'requirements.txt', 'tests/test_research_catalog.py', 'tests/test_community_worker.py', '.github/workflows/*community*', '.github/workflows/research-catalog.yml'] permissions: contents: read jobs: @@ -15,6 +15,8 @@ jobs: with: python-version: '3.12' - run: pip install jsonschema==4.26.0 pytest==8.4.2 - - run: python -m pytest -o addopts='' tests/test_research_catalog.py tests/test_community_worker.py tests/test_build_studies_index.py -q + # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + - run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" + - run: python -m pytest -o addopts='' tests/test_research_catalog.py tests/test_community_worker.py tests/test_build_studies_index.py tests/test_hsenv_pin.py -q - run: python scripts/build_studies_index.py - run: git diff --exit-code -- co_website/data/studies_index.json co_website/data/research_index.json diff --git a/.github/workflows/sync-studies-index.yml b/.github/workflows/sync-studies-index.yml index 5900fd5..ce155a8 100644 --- a/.github/workflows/sync-studies-index.yml +++ b/.github/workflows/sync-studies-index.yml @@ -21,6 +21,9 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.11" + - name: Install the pinned protocol engine + # effects_core is hsenv's (V080 B1). hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" - name: Build studies index run: python scripts/build_studies_index.py - name: Commit updated index diff --git a/.github/workflows/validate-studies.yml b/.github/workflows/validate-studies.yml index 774a8fc..8d83bee 100644 --- a/.github/workflows/validate-studies.yml +++ b/.github/workflows/validate-studies.yml @@ -6,6 +6,9 @@ on: - "studies/**" - "scripts/build_studies_index.py" - "scripts/verify_study.sh" + - "scripts/check_contributors.py" + - "community/**" + - "scripts/research/protocols.py" jobs: validate: @@ -57,43 +60,50 @@ jobs: exit 0 fi # A failed diff must fail the job, never read as "no studies changed". - CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- studies/) - CHANGED_STUDIES=$(printf '%s\n' "$CHANGED_FILES" | grep -oP 'studies/[^/]+' | sort -u || true) + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- studies/ community/) + CHANGED_STUDIES=$(printf '%s\n' "$CHANGED_FILES" | grep -oP '^studies/[^/]+|^community/paper_[a-f0-9]+/revisions/[a-f0-9]{64}' | sort -u || true) if [ -z "$CHANGED_STUDIES" ]; then exit 0 fi - FAILED=0 - for study_path in $CHANGED_STUDIES; do - INDEX="$study_path/index.json" - if [ ! -f "$INDEX" ]; then - continue - fi - # Check if any contributor github field contains the PR author's username - MATCH=$(python3 -c " - import json, sys - with open('$INDEX') as f: - data = json.load(f) - contributors = data.get('contributors', []) - if not contributors: - print('MISSING') - sys.exit(0) - author = '$PR_AUTHOR'.lower() - for c in contributors: - gh = c.get('github', '').lower() - # Match 'username' or 'https://github.com/username' - if gh.rstrip('/').split('/')[-1] == author: - print('MATCH') - sys.exit(0) - print('MISMATCH') - ") - study_id=$(basename "$study_path") - if [ "$MATCH" = "MISSING" ] || [ "$MATCH" = "MISMATCH" ]; then - echo "::error file=$INDEX::Contributor check failed for $study_id. Your GitHub username is '$PR_AUTHOR'. Please add the following to your index.json:%0A%0A\"contributors\": [%0A { \"name\": \"Your Name\", \"github\": \"$PR_AUTHOR\" }%0A]" - FAILED=1 - else - echo "$study_id: Contributor @$PR_AUTHOR verified." - fi + # Same rule for legacy index.json and protocol study.json (V080 B2); a dir with neither is skipped. + python scripts/check_contributors.py --author "$PR_AUTHOR" $CHANGED_STUDIES + + # V080-PLATFORM §2.1: a maintainer release PR adds files under community/**. Every changed revision that + # holds protocol.json is checked with the hsenv engine. This job runs the PR's own checker + # (scripts/research/protocols.py), its tests and its requirements.txt hsenv pin, so a PR can change its own + # checks: it has no secrets, a read-only token and a time limit, and reviewers must look at any change to + # those files. Protocol files themselves are read as data by the engine. + validate-protocols: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install the pinned engine + # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + run: | + pip install jsonschema==4.26.0 pytest==8.4.2 + grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt" + - name: Validate changed protocol releases + run: | + # A failed diff must fail the job, never read as "no revisions changed". + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- community/) + REVISIONS=$(printf '%s\n' "$CHANGED_FILES" | grep -oP '^community/paper_[a-f0-9]+/revisions/[a-f0-9]{64}' | sort -u || true) + DIRS="" + for rev in $REVISIONS; do + if [ -f "$rev/protocol.json" ]; then DIRS="$DIRS $rev"; fi done - if [ "$FAILED" -eq 1 ]; then - exit 1 + if [ -z "$DIRS" ]; then + echo "No protocol releases changed." + exit 0 fi + python -m scripts.research.protocols --sessions 300 $DIRS + - name: Test the protocol release rules + run: python -m pytest tests/test_protocol_release.py tests/test_check_contributors.py tests/test_hsenv_pin.py -o addopts='' -q diff --git a/docs/v080-protocol.md b/docs/v080-protocol.md new file mode 100644 index 0000000..670f4bc --- /dev/null +++ b/docs/v080-protocol.md @@ -0,0 +1,123 @@ +# v0.8 protocol releases in HumanStudy-Bench + +This page covers the Bench part of the v0.8 platform design. In v0.8 a study is one folder holding `study.json` and `protocol.json`, and one engine (`hsenv`) +reads it on every platform. Bench publishes a released protocol as a community revision +(`community/paper_/revisions//`). Bench never interprets a protocol itself and never executes +its files: every check calls the pinned `hsenv`. + +## What this change does + +- **B1.** `requirements.txt` pins hsenv by commit SHA; it is the only place the pin is written, and the + sync-studies-index, research-catalog, maintain-community and validate-studies workflows read it from there. + `scripts/research/effects_core.py` re-exports `hsenv.effects_core`, with import paths and the CLI kept. + `tests/test_hsenv_pin.py` checks that every community revision with `effects.json` gives the same problems, + warnings and kg digest as the Bench copy did (`tests/fixtures/effects_core_golden.json`, recorded with the + copy at `b34d214`), and that the protocol readers never execute protocol text. +- **B2.** `scripts/check_contributors.py` (one contributor rule), `scripts/research/protocols.py` (release + checks), a protocol branch in `validate_assets`, and the `validate-protocols` job in `validate-studies.yml`. + The tests use an invented CC0 fixture release. +- **B3.** `build_catalog` adds a `protocol` block to the paper's `research_index.json` entry and computes + `replayable`/`humanBenchmark` from the protocol. `studies_index.json` and `research_index.json` stay + byte-identical for today's studies. + +Not done here: **B4** (`run_protocol.py`, the playground on `kind: protocol`, quiet logs) waits for owner gate G3. +**B5** (the first releases) needs signed reviews (G2) and studies with no `release_hold`. + +## What CI checks on a protocol release + +`python -m scripts.research.protocols ... [--sessions 300]` exits 1 on any error. The steps +follow the order in design §2.1: + +1. **Schema.** `hsenv.schema_check` validates `study.json` (`kind=study`) and `protocol.json` (`kind=protocol`, + including its cross-reference pass). `schema_version` is required: legacy files are not released. +2. **Release rules.** + - Every path must be on the allowlist: `study.json`, `protocol.json`, `effects.json`, `program.json`, + `capabilities.json`, `review/fidelity-review-*.md`, `review/checks-latest.json`, `sources/SOURCE.json`, + `sources/open/**`, `sourcetext/*` (only when `meta.sources[key]` is under `sources/open/`), + `variants/*.json`, `artifacts/**` and `package/**`. + - These are never published: `sources/{researcher,private,quarantine}/**`, `runs/**`, `**/runs/**`, + `**/records.jsonl`, `paper/**` and `bundle.json`. + - `visibility` must be `"public"`. `quality` must be `high_fidelity`, with `fidelity.verdict` HIGH FIDELITY, + a non-empty `fidelity.signed_by` and the review file included (G2). + - No `release_hold` may appear, neither in `study.json` nor on any arm. + - `study.json.protocol.sha256` must equal the canonical hash of `protocol.json`, and `study.json.arms` must + agree with the protocol's arms. + - An image `src` must be a URL, or an `asset:` that points to a file in `sources/open/`. + - `study.json.hsenv` must admit the installed engine. +3. **Checks report.** `review/checks-latest.json` must report `PASS: true` and carry `protocol_sha256` + equal to this protocol's hash. A report without the hash, or with another hash, counts as stale. + `--sessions N` also replays N random sessions through `hsenv.checks` (references and fuzz). +4. **Effects and capabilities.** `effects.json` goes through `hsenv.effects_core.check`, and + `capabilities.json` through the capability rules. +5. **Digest.** The revision directory's name must equal its manifest digest (`revisions.revision_id`). + +Each revision is checked through one of two paths: + +- **Maintainer release PR** (direct files under `community/**`): the `validate-protocols` job in + `validate-studies.yml` runs the checks on each changed revision that holds `protocol.json`. +- **Community proposal** (`maintain-community.yml`): `revisions.validate_assets` sees `protocol.json` and runs + the same checks on the proposal's files. `program.json` becomes optional. The engine comes from main's + pinned requirements and never from the proposal. A proposal that holds `protocol.json` is **never + accepted on this path**: `integrate` always returns `needs_review` with `requiresMaintainer: true`, whatever + the review says (an agent's or anyone's `accept`) and whatever the release declares about itself + (`fidelity.signed_by`, a PASS `checks-latest.json`). No agent review is run for it. A maintainer publishes + it through a release PR. + +**Contributor rule.** `scripts/check_contributors.py --author ...` reads `index.json` for +legacy studies and `study.json` for protocol releases. The rule is unchanged: a `contributors[].github` must +equal the PR author, given as `login` or as a GitHub URL. Admin and maintain users still bypass it in the +workflow. A community proposal gets its PR author appended to `study.json` contributors when missing. + +## research_index.json + +Shared contract fixture: `tests/fixtures/protocol_release/research-index-protocol.json` is the exact +`protocol` block and capabilities Bench writes for the fixture release (`schemaVersion` is the protocol +schema version, `"1"`). The web repo tests its reader against the same JSON; change both together. + +For a current revision with `protocol.json`, the paper entry gains these fields: + +``` +"protocol": {"slug", "schemaVersion", "hsenv", "fidelity": {"verdict", "signed_by"}, + "arms": [{"id", "runnable", "nPaper"}], "runExcluded": [...], "effects": n, "revision"}, +"capabilities": {..., "replayable": any runnable arm without release_hold, "humanBenchmark": any effect with reported values}, +"coverage": "protocol-release" (only when the revision has no program.json) +``` + +The block holds counts and ids only, never protocol text. `studies_index.json` keeps its legacy shape. + +## Test fixture + +`tests/fixtures/protocol_release/example2026lamps/` is an invented study under CC0-1.0. Its material is two +sentences about a lamp and one rating question, and it is no real paper, protocol or participant. Its +`review/checks-latest.json` is a real `hsenv.checks` PASS report (200 sessions, verbatim 3/3 found, keyed sign +correct), with `protocol_sha256` added. To regenerate the fixture, write it again with the same content, run +`python -m hsenv.checks --sessions 200 --source-text survey=/sourcetext/survey.txt`, and store the +output with `protocol_sha256`. + +## Running locally + +```bash +pip install -r requirements.txt # installs hsenv from GitHub at the pinned commit +pytest tests/test_hsenv_pin.py tests/test_protocol_release.py tests/test_check_contributors.py -o addopts='' +python -m scripts.research.protocols community/paper_/revisions/ --sessions 300 +``` + +The hsenv tests skip when hsenv is not installed. + +## Open items (hsenv H1 and maintainer decisions) + +- **hsenv is installed, not vendored.** `requirements.txt` pins hsenv v0.8.0 by its commit SHA + (`fd38302`). The desktop pins the same release (B1 is chained to it). +- **Engine schema vs design.** `study.v1.json` has `additionalProperties: false` and lacks the design's §1.1 + fields (`slug`, `paper`, `contributors`, `visibility`, `hsenv`, `fidelity`, `arms`, `paperId`, `release`). + Bench validates those itself and removes them before calling `schema_check` (`protocols.RELEASE_FIELDS`). + H1 should add them to the schema. +- **H1 must provide:** + - `protocol_sha256` in the `hsenv.checks` report: today it carries none, so CI could not detect a stale report + - `hsenv check --public`: `protocols.replay` is the interim refs and fuzz subset + - `hsenv.release.PUBLIC_PATHS`: a test fails if it differs from Bench's list + - `hsenv.capabilities`: `capabilities.py` stays a Bench copy until then +- **Design statement corrected.** Design §2.1 says `expr.py` has "no eval/exec (verified by grep)". In fact + it calls `compile` and `eval`. It does so on an AST whose node types it has checked against a whitelist, + and it runs with `__builtins__` emptied. The CI test pins that whitelist and a set of escape attempts. +- **Maintainer decisions:** who signs a release (G2), and how B4 keeps public Actions logs quiet (G3). diff --git a/requirements.txt b/requirements.txt index 3a90906..4b18182 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,11 @@ pandas>=1.3.0 # Statistical analysis (used in study configs and evaluation) scipy>=1.7.0 # For statistical tests (ttest, ANOVA, etc.) +# Protocol engine (V080-PLATFORM B1): the one hsenv release Bench, the web runner and the desktop share. +# This line is the single pin: every workflow and test reads it from here (`grep '^hsenv' requirements.txt`, then `pip install -r`). +# Pinned by commit SHA (the commit of tag v0.8.0), not by the mutable tag. +hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662 # v0.8.0 + # Data validation jsonschema>=4.0.0 pydantic>=2.0.0 diff --git a/scripts/check_contributors.py b/scripts/check_contributors.py new file mode 100644 index 0000000..5b490e8 --- /dev/null +++ b/scripts/check_contributors.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Contributor attribution for submitted studies (one rule for both doors, V080-PLATFORM §2.1 / B2). + + python scripts/check_contributors.py --author [ ...] + +A legacy study (`studies/study_NNN`) keeps its contributors in index.json; a protocol study keeps them in +study.json. Each directory must list a contributor whose `github` is the PR author ('login' or +'https://github.com/login'). A directory with neither file is skipped, as before. The maintainer bypass +(admin/maintain permission) stays in the workflow, which never calls this script for maintainers. + +This is the inline check that validate-studies.yml carried, moved here unchanged except that study.json +is read when there is no index.json. Exit 1 when any directory fails. +""" +import argparse +import json +import sys +from pathlib import Path + + +def contributors_file(study_dir: Path): + for name in ("index.json", "study.json"): + if (study_dir / name).is_file(): + return study_dir / name + return None + + +def verdict(path: Path, author: str) -> str: + """MATCH, MISMATCH or MISSING, exactly as the former inline check.""" + return verdict_from(json.loads(path.read_text(encoding="utf-8")), author) + + +def verdict_from(data: dict, author: str) -> str: + contributors = data.get("contributors", []) + if not contributors: + return "MISSING" + # Match 'username' or 'https://github.com/username' + for c in contributors: + gh = (c.get("github", "") if isinstance(c, dict) else "").lower() + if gh.rstrip("/").split("/")[-1] == author.lower(): + return "MATCH" + return "MISMATCH" + + +def with_contributor(raw: bytes, login: str, name: str = "") -> bytes: + """study.json bytes with the PR author appended to contributors when absent (community proposals).""" + data = json.loads(raw) + if not isinstance(data, dict) or not isinstance(data.get("contributors", []), list): + return raw # not a study card; the release checks report the malformed study.json + if verdict_from(data, login) == "MATCH": + return raw + data.setdefault("contributors", []).append({"name": name or login, "github": login}) + return (json.dumps(data, ensure_ascii=False, indent=2) + "\n").encode() + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--author", required=True) + ap.add_argument("dirs", nargs="*") + a = ap.parse_args(argv) + failed = False + for d in a.dirs: + study_dir = Path(d) + path = contributors_file(study_dir) + if path is None: + continue + study_id = study_dir.name + if verdict(path, a.author) in ("MISSING", "MISMATCH"): + print(f"::error file={path}::Contributor check failed for {study_id}. Your GitHub username is " + f"'{a.author}'. Please add the following to your {path.name}:%0A%0A\"contributors\": [%0A " + f"{{ \"name\": \"Your Name\", \"github\": \"{a.author}\" }}%0A]") + failed = True + else: + print(f"{study_id}: Contributor @{a.author} verified.") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/maintain_community.py b/scripts/maintain_community.py index 028847b..8b940b1 100644 --- a/scripts/maintain_community.py +++ b/scripts/maintain_community.py @@ -64,6 +64,11 @@ def proposal_from_ref(ref: str, base: str, author: str) -> tuple[Path, dict, dic raise ValueError("Hub submissions require their server-signed contributor identity") elif str(contributor.get("github", "")).casefold() != author.casefold(): raise ValueError("the contributor must match the authenticated GitHub PR author") + # A protocol release credits the same person in study.json (V080 §2.1): append when absent. + login = author if author.casefold() != bot.casefold() else str(contributor.get("github", "")) + if "study.json" in files and "protocol.json" in files and login: + from scripts.check_contributors import with_contributor + files["study.json"] = with_contributor(files["study.json"], login, str(contributor.get("name", ""))) # Existing Hub key encoding remains compatible, but no contribution may write another paper's path. expected = "paper_" + hashlib.sha256(metadata["paperKey"].encode()).hexdigest()[:24] if paper_dir != expected or len(paper_dir) > 300: @@ -109,7 +114,8 @@ def proposal_from_ref(ref: str, base: str, author: str) -> tuple[Path, dict, dic def maintain(ref: str, base: str, author: str, *, agent_review: bool) -> dict: folder, metadata, files = proposal_from_ref(ref, base, author) result = integrate(folder, metadata, files) - if result.get("candidateRevision") and agent_review: + # A protocol release always waits for a maintainer, so no agent review is spent on it. + if result.get("candidateRevision") and agent_review and not result.get("requiresMaintainer"): pointer = folder / "current.json" current = json.loads(pointer.read_text()) if pointer.exists() else None candidate, conflicts = merge_assets(load_revision(folder, metadata.get("baseRevision")), diff --git a/scripts/research/catalog.py b/scripts/research/catalog.py index e1198e2..b21e211 100644 --- a/scripts/research/catalog.py +++ b/scripts/research/catalog.py @@ -235,10 +235,25 @@ def build_catalog(root: Path, legacy: list[dict]) -> dict: file = revision / entry["path"] if not file.resolve().is_relative_to(revision.resolve()) or file.is_symlink() or hashlib.sha256(file.read_bytes()).hexdigest() != entry["sha256"]: raise ValueError(f"community revision content changed: {file}") - program = json.loads((revision / "program.json").read_text()) - projection = project_program(program) + paths = {entry["path"] for entry in manifest["files"]} paper = current["paper"] - if not paper.get("description"): + # A v0.8 protocol release (V080 §3.1) may have no program.json; its block holds counts and ids only. + protocol_extra, capabilities = {}, current["capabilities"] + if "protocol.json" in paths: + from scripts.research.protocols import protocol_capabilities, summary + study_doc = json.loads((revision / "study.json").read_text()) + protocol_doc = json.loads((revision / "protocol.json").read_text()) + protocol_extra = {"protocol": {**summary(study_doc, protocol_doc), "revision": current["revision"]}} + capabilities = {**capabilities, **protocol_capabilities(protocol_doc)} + if "program.json" in paths: + program = json.loads((revision / "program.json").read_text()) + projection = project_program(program) + else: + program = {} + projection = {"studies": [], "issues": [], "effectEstimates": [], + "searchText": "\n".join(strings({k: paper.get(k) for k in ("title", "authors", "year")}) + + strings(protocol_extra.get("protocol", {}).get("slug")))} + if not paper.get("description") and program: paper = {**paper, "description": community_description(revision, manifest, program)} pid = paper["id"] previous = papers.get(pid, {}) @@ -253,7 +268,8 @@ def build_catalog(root: Path, legacy: list[dict]) -> dict: "aliases": sorted(set(previous.get("aliases", []) + paper.get("aliases", []))), "currentRevision": current["revision"], "assetRoot": revision.relative_to(root).as_posix(), "assetFiles": manifest["files"], - "coverage": "human-program", "capabilities": current["capabilities"], + "coverage": "human-program" if program else "protocol-release", "capabilities": capabilities, + **protocol_extra, **({"studyCapabilities": study_caps} if study_caps else {}), "effectEstimates": estimates, **({"graph": graph} if graph else {}), diff --git a/scripts/research/effects_core.py b/scripts/research/effects_core.py index 4c30883..5d80d41 100644 --- a/scripts/research/effects_core.py +++ b/scripts/research/effects_core.py @@ -1,1278 +1,20 @@ #!/usr/bin/env python3 -"""effects.json: the executable link from a paper's hypotheses to its data. +"""effects.json rules and computation: the pinned hsenv engine (V080-PLATFORM B1). -program.json says, in the paper's words, what the study is. effects.json turns its hypotheses into -effects that a program can compute: for each effect, the outcome, the contrast, the predicted -direction, the paper's reported numbers (each one quoted) and how to compute it from sessions. -Human and AI-participant sessions are computed the same way and give the same result block, so -effects from many papers can be pooled later without an agent re-reading any paper. +This file used to be a copy of humanstudy-desktop `pack/skills/export/agent/effects_core.py`. Bench now +reads the one engine every platform reads, `hsenv.effects_core`, at the version pinned in requirements.txt. +Import paths are kept (`scripts.research.effects_core`, and loading this file by path), so callers do not +change. Do not add logic here: a rule change is an hsenv release. - python effects_core.py check validate; exits 1 on a problem - python effects_core.py graph hypothesis -> effect -> program nodes - python effects_core.py kg the paper's whole typed graph (schema 2) - python effects_core.py types the core vocabulary (graph-types.json) - python effects_core.py compute result blocks for every effect - -Raw quantities are kept: a result block holds the cells (n, mean, sd or counts) and the test, never only -a converted effect size. Converting between metrics (d, r, log odds ratio) is left to later tools. -Standard library only, Python 3.8+. Shared by the build skill, the agent export and HumanStudy-Bench. + python scripts/research/effects_core.py check same CLI as hsenv.effects_core """ -import json -import math -import re import sys -from pathlib import Path - -SCHEMA_VERSION = 2 # new files; schema 1 files stay valid (their "other" values included) -SCHEMA_VERSIONS = (1, 2) -VOCAB_VERSION = "2.0.0" # core vocabulary below; bump the minor version when an extension is promoted -METRICS = ( - "mean", "mean_difference", "standardized_mean_difference", "hedges_g", "proportion", "risk_difference", - "risk_ratio", "odds_ratio", "log_odds_ratio", "correlation", "phi", "cramers_v", "chi_square", - "regression_coefficient", "standardized_coefficient", "t", "F", "z", "bayes_factor", "eta_squared", - "partial_eta_squared", "count", "cohens_q", "auc", "ratio", "percent_change", "other") -DESIGNS = ("between", "within", "mixed", "correlational", "other") -OUTCOME_TYPES = ("continuous", "ordinal", "binary", "categorical", "count", "other") -CONTRASTS = ("two_groups", "paired", "association", "interaction", "omnibus", "single_group", "linear", "ratio", - "model_comparison", "other") -# Schema 2 never accepts "other" for these: a new kind is declared as an extension (see EXTENSIONS below). -NO_OTHER = ("contrast", "metric") -TEST_STATISTICS = ("t", "F", "z", "chi_square", "W", "V", "U", "other") -DIRECTIONS = ("positive", "negative", "nonzero", "none", "unspecified") -OPS = ("mean_difference", "paired_difference", "proportion_difference", "contingency", "correlation", "package") -ROLES = ("test", "robustness", "check", "descriptive") -POPULATIONS = ("human", "ai-model", "human-vs-ai-model", "ai-model-vs-ai-model") -UNTESTED = ("not-an-effect", "not-tested-in-paper", "not-computable", "out-of-scope") -# tested: every effect naming it is computed by the replay and they cover its scope; partial: some of its effects -# are computed but others are excluded (capabilities.json run.excluded), or they cover only part of its scope -# (one design or site of a pooled hypothesis, one side of a comparison), with a reason; untested: none computed. -HYPOTHESIS_STATUSES = ("tested", "partial", "untested") -# Metrics that live on a 0-1 scale but are often printed in percentage points: estimates say which (`scale`). -PROPORTION_METRICS = ("proportion", "risk_difference") -SCALES = ("proportion", "percentage_points") -# Metrics without a sign: on a two-group or paired contrast they hide which arm is higher. -UNSIGNED_METRICS = ("phi", "cramers_v", "chi_square", "F", "eta_squared", "partial_eta_squared") -ESTIMATE_TYPES = ("descriptive", "model-based") -# whose numbers `reported` holds, relative to what the replay runs -REPORTED_SCOPES = ("replayed-sample", "pooled", "other-sample") -ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") -NUMBER_FIELDS = ("value", "se", "df", "p", "n", "lower", "upper") -TOLERANCE = "a result is recovered when |computed - reported| <= max(0.05, 0.15 * |reported|)" - - -def load(root): - path = Path(root) / "effects.json" - if not path.is_file(): - return None - return json.loads(path.read_text(encoding="utf-8")) - - -def excluded_effects(root): - """Effect ids capabilities.json run.excluded[].effects lists (the replay does not compute them).""" - try: - caps = json.loads((Path(root) / "capabilities.json").read_text(encoding="utf-8")) - except (OSError, ValueError, TypeError): - return set() - run = caps.get("run") if isinstance(caps, dict) and isinstance(caps.get("run"), dict) else {} - items = run.get("excluded") if isinstance(run.get("excluded"), list) else [] - return {str(x) for item in items if isinstance(item, dict) and isinstance(item.get("effects"), list) - for x in item["effects"]} - - -# ---- numbers in quotes ----------------------------------------------------------------------------- - -_NUM = re.compile(r"(? 1: - P("%s: %s = %s is not on a 0-1 proportion scale; a value printed in percentage points has scale " - "percentage_points" % (at, est.get("metric"), est.get("value"))) - if "estimateType" in est and est["estimateType"] not in ESTIMATE_TYPES: - P("%s: estimateType must be one of %s" % (at, ", ".join(ESTIMATE_TYPES))) - if "scope" in rep and rep["scope"] not in REPORTED_SCOPES: - P("%s: reported.scope must be one of %s" % (where, ", ".join(REPORTED_SCOPES))) - elif rep.get("scope") in ("pooled", "other-sample") and not str(rep.get("scopeNote") or "").strip(): - P("%s: reported.scope %s needs scopeNote (whose numbers these are: \"pooled across 61 countries\")" - % (where, rep["scope"])) - if rep.get("test") and rep["test"].get("statistic") not in TEST_STATISTICS: - P("%s: reported.test.statistic must be t, F, z, chi_square, W, V, U or other" % where) - p_blocks = [("test", rep.get("test") or {})] + [ - ("estimates[%d]" % j, est) for j, est in enumerate(rep.get("estimates") or [])] - for location, block in p_blocks: - p = block.get("p") - if isinstance(p, (int, float)) and not isinstance(p, bool) and p <= 0: - P("%s: reported.%s.p must be greater than zero; a rounded printed zero is not an exact " - "probability. Preserve the printed text with p: null and describe the missing exact p-value; " - "do not invent an unprinted bound" % (where, location)) - if not isinstance(rep.get("missing", []), list): - P("%s: reported.missing must be a list of what the paper does not report" % where) - for loc, value in _numbers(rep): - q = quoted(value, quotes) - if q != "exact" and any(rendered_quote(value, record) for record in quote_records): - W("%s: reported %s = %s uses a rendered-page check for text-layer sign loss" % (where, loc, value)) - if q == "unsigned": # the text layer still prints only the absolute value; keep that visible - W("%s: reported %s = %s: only its absolute value is quoted; check the sign" % (where, loc, value)) - continue - if q is None: - P("%s: reported %s = %s is not written in its cited quotes" % (where, loc, value)) - elif q == "unsigned": - W("%s: reported %s = %s: only its absolute value is quoted; check the sign" % (where, loc, value)) - not_run = excluded_effects(root) - for h in hyps: - if h.get("status") in ("tested", "partial") and h.get("id") not in tested: - P("hypothesis %s is %s but no effect names it" % (h.get("id"), h.get("status"))) - if h.get("status") == "untested" and h.get("id") in tested: - naming = [e.get("id") for e in effs if h.get("id") in (e.get("hypothesisIds") or [])] - # an untested (not-computable) hypothesis may keep its effects when the replay excludes all of them - if h.get("untestedBecause") != "not-computable" or any(x not in not_run for x in naming): - P("hypothesis %s is untested but effect(s) name it" % h.get("id")) - if not effs and not any(h.get("status") == "untested" for h in hyps): - W("no effects") - for i, link in enumerate(effects.get("links") or []): - for msg in link_problems(link, effects, program): - P("links[%d]: %s" % (i, msg)) - reach = graph(effects, program) - for nid in reach["unlinked"]: - W("program %s %s is not reachable from any hypothesis or effect" % (nodes[nid].get("kind"), nid)) - return problems, warnings - - -# ---- the graph -------------------------------------------------------------------------------------- - -LINKED_KINDS = ("hypothesis", "condition", "variable", "analysis", "result") - - -def graph(effects, program): - """Edges from background through hypotheses and effects to the program nodes they bind, extended by - the program's own relations and step conditions. `unlinked` lists hypothesis/condition/variable/ - analysis/result nodes that no hypothesis reaches.""" - nodes = {n["id"]: n for n in program.get("nodes", []) if "id" in n} - edges = [] - add = lambda a, b, t: edges.append({"from": a, "to": b, "type": t}) - for t in effects.get("topics") or []: - add("paper", "topic:" + str(t), "about") - for h in effects.get("hypotheses") or []: - hid = "hypothesis:" + str(h.get("id")) - add(hid, h.get("nodeId"), "states") - for b in h.get("backgroundIds") or []: - add(b, hid, "motivates") - for ph in h.get("phenomena") or []: - add(hid, "phenomenon:" + str(ph), "instance-of") - for e in effects.get("effects") or []: - eid = "effect:" + str(e.get("id")) - for h in e.get("hypothesisIds") or []: - add("hypothesis:" + h, eid, "tested-by") - if (e.get("outcome") or {}).get("variableId"): - add(eid, e["outcome"]["variableId"], "measures") - for c in (e.get("contrast") or {}).get("conditionIds") or []: - add(eid, c, "contrasts") - for c in (e.get("contrast") or {}).get("factorIds") or []: - add(eid, c, "varies") - for key, t in (("analysisIds", "analysed-by"), ("resultIds", "reported-in")): - for x in e.get(key) or []: - add(eid, x, t) - adj = {} - for x in edges: - adj.setdefault(x["from"], set()).add(x["to"]) - for r in program.get("relations", []): # program relations count both ways - adj.setdefault(r.get("from"), set()).add(r.get("to")) - adj.setdefault(r.get("to"), set()).add(r.get("from")) - seen, todo = set(), ["hypothesis:" + str(h.get("id")) for h in effects.get("hypotheses") or []] - todo += ["effect:" + str(e.get("id")) for e in effects.get("effects") or [] if not e.get("hypothesisIds")] - while todo: - x = todo.pop() - if x in seen: - continue - seen.add(x) - todo.extend(adj.get(x, ())) - unlinked = sorted(nid for nid, n in nodes.items() if n.get("kind") in LINKED_KINDS and nid not in seen) - return {"edges": edges, "reached": sorted(x for x in seen if x in nodes), "unlinked": unlinked} - - -# ---- the knowledge graph (schema 2) ------------------------------------------------------------------ -# -# One typed graph per paper: the program's nodes and procedure steps, the effects (estimands), and the -# constructs they are about, joined by edges from a small core vocabulary. Program relations written with -# older names are read through RELATION_ALIASES; anything that fits no core term is reported as a gap, -# never forced into one. The vocabulary grows by extensions (x:name, declared in effects.json with a core -# parent and a definition); an extension used by several papers is promoted into the core and listed in -# PROMOTED, so files that used the old name read as the core term without being rewritten. - -VARLIKE = ["variable", "record", "measure"] -NODE_KINDS = ("paper", "construct", "background", "overview", "hypothesis", "design", "participants", "condition", - "material", "procedure", "step", "variable", "record", "measure", "analysis", "model", "result", "finding", - "limitation", "simulation", "effect") -EDGE_TYPES = { - # claims - "about": (["paper"], ["construct"], "the paper is about this construct or topic"), - "motivates": (["background"], ["hypothesis"], "background that motivates the hypothesis"), - "instance-of": (["hypothesis", "effect"], ["construct"], "tests a case of this phenomenon or construct"), - "predicts": (["hypothesis"], ["construct"], "the hypothesis predicts a change in this construct"), - "tested-by": (["hypothesis"], ["effect", "analysis", "finding"], "evidence that tests the hypothesis"), - "supports": (["result", "finding"], ["hypothesis"], "the paper reads this result as support"), - "contradicts": (["result", "finding"], ["hypothesis"], "the paper reads this result as counter-evidence"), - # the estimand - "measures": (["effect"], VARLIKE, "the effect's outcome variable"), - "contrasts": (["effect"], ["condition"], "a condition the effect compares"), - "varies": (["effect"], VARLIKE, "a predictor the effect varies"), - "estimated-by": (["effect"], ["analysis", "model"], "the analysis that estimates the effect"), - "reported-in": (["effect"], ["result", "finding"], "where the paper reports the effect"), - "moderates": (VARLIKE + ["condition"], ["effect"], "the effect differs across its levels"), - "mediates": (VARLIKE, ["effect"], "the effect runs through it"), - "replicates": (["effect"], ["effect"], "the same effect estimated again, in another sample or paper"), - "compares": (["effect"], ["effect"], "the effect is defined as a difference between these effects"), - # operationalisation and analysis - "operationalizes": (VARLIKE, ["construct"], "the variable measures the construct"), - "manipulates": (["condition"], ["construct"], "the condition manipulates the construct"), - "input-of": (VARLIKE + ["condition"], ["analysis", "model"], "the analysis uses it"), - "reports": (["analysis", "model"], ["result", "finding"], "the analysis produced this reported result"), - "concerns": (["result", "finding"], VARLIKE + ["condition", "hypothesis"], "the reported result is about it"), - "derived-from": (VARLIKE + ["analysis", "model"], VARLIKE + ["analysis", "model"], "computed from it"), - # procedure (from program steps) - "shows": (["step", "condition"], ["*"], "what a participant sees at this step or in this condition"), - "produces": (["step"], ["*"], "what the step records"), - "uses": (["step"], ["*"], "an input of the step"), - "in-condition": (["step"], ["condition"], "the step belongs to this condition"), - "part-of": (["step"], ["step"], "a sub-step"), - "next": (["step"], ["step"], "the step that follows"), - "performed-by": (["step"], ["*"], "who acts at this step"), -} -PROCEDURE_EDGES = ("shows", "produces", "uses", "in-condition", "part-of", "next", "performed-by") -CONTRAST_DEFS = { - "two_groups": "treatment minus reference, two independent groups", - "paired": "treatment minus reference, measured on the same units", - "single_group": "one group's value, or its difference from a fixed constant", - "association": "a correlation or slope between two variables", - "interaction": "a difference between two contrasts (difference of differences, or across samples)", - "omnibus": "an undirected test over more than two levels; not poolable as a signed effect", - "linear": "sum of weight x cell over named cells, plus an optional constant (e.g. 1 vs mean of 2 others)", - "ratio": "treatment divided by reference", - "model_comparison": "evidence for one model of the data over another (e.g. a Bayes factor)", - "other": "schema 1 only", -} -# Older program relation names -> core edge candidates (tried forwards, then reversed, against the kinds). -RELATION_ALIASES = {"uses": ["input-of"], "outcome": ["input-of", "concerns"], "estimates": ["input-of"], - "sees": ["shows"], "analysed-by": ["estimated-by"], "reported-by": ["reports"], - "reports": ["concerns"], "produces": ["reports"]} -# Promoted extensions: {category: {"x:old-name": "core-name"}}. Empty in 2.0.0. -PROMOTED = {"contrast": {}, "metric": {}, "edge": {}, "node": {}} - - -def _kind_ok(kind, allowed): - return "*" in allowed or kind in allowed - - -def _norm(t): - return re.sub(r"[\s_]+", "-", str(t or "").strip().lower()) - - -def map_relation(rtype, from_kind, to_kind): - """(core edge, reversed) for a program relation, or None when no core edge fits the two kinds.""" - t = _norm(rtype) - for cand in ([t] if t in EDGE_TYPES else []) + RELATION_ALIASES.get(t, []): - dom, rng = EDGE_TYPES[cand][0], EDGE_TYPES[cand][1] - if _kind_ok(from_kind, dom) and _kind_ok(to_kind, rng): - return cand, False - if _kind_ok(to_kind, dom) and _kind_ok(from_kind, rng): - return cand, True - return None - - -def construct_id(label): - """A construct keeps the paper's own wording: no case folding, no synonym merging. Merging labels across - papers is a later, reviewed step (a sameAs map), never done here.""" - return "construct:" + str(label).strip() - - -def ref_kind(ref, effects, program): - """The node kind a reference names, or None when it names nothing.""" - ref = str(ref or "") - if ref.startswith("paper:"): - inner = ref.split("/", 1)[1] if "/" in ref else "" - return "effect" if inner.startswith("effect:") else ("external" if inner else None) - if ref.startswith("construct:"): - return "construct" if len(ref) > len("construct:") else None - if ref.startswith("effect:"): - return "effect" if any(e.get("id") == ref[7:] for e in effects.get("effects") or []) else None - for n in program.get("nodes", []): - if n.get("id") == ref: - return n.get("kind") - if any(st.get("id") == ref for st in program.get("steps", [])): - return "step" - return None - - -def link_problems(link, effects, program): - out = [] - t = link.get("type") - core = resolve("edge", t, effects) - if core is None: - return ["type %r is not a core edge or a declared edge extension" % t] - decl = extensions(effects).get(t) if t not in EDGE_TYPES else None - dom, rng = (decl.get("from"), decl.get("to")) if decl else EDGE_TYPES[core][:2] - fk, tk = ref_kind(link.get("from"), effects, program), ref_kind(link.get("to"), effects, program) - if fk is None: - out.append("from %r names no node (program node or step id, effect:, construct: