v0.8: protocol releases — pin hsenv v0.8.0, validate protocols, research_index protocol block (B1–B3)

.github/workflows/maintain-community.yml

@@ -33,6 +33,9 @@ jobs:
         with:
           python-version: '3.12'
       - run: pip install jsonschema==4.26.0 pypdf==6.1.1
+      # trusted, pinned engine from main's requirements.txt; never from the proposal
+      # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662  # v0.8.0
+      - run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt"
       - name: Read exact proposal commits
         id: proposal
         env:

.github/workflows/research-catalog.yml

@@ -1,7 +1,7 @@
 name: Research catalog checks
 on:
   pull_request:
-    paths: ['scripts/research/**', 'scripts/maintain_community.py', 'scripts/build_studies_index.py', 'catalog/**', 'contracts/human-program.schema.json', 'tests/test_research_catalog.py', 'tests/test_community_worker.py', '.github/workflows/*community*', '.github/workflows/research-catalog.yml']
+    paths: ['scripts/research/**', 'scripts/maintain_community.py', 'scripts/build_studies_index.py', 'catalog/**', 'contracts/human-program.schema.json', 'requirements.txt', 'tests/test_research_catalog.py', 'tests/test_community_worker.py', '.github/workflows/*community*', '.github/workflows/research-catalog.yml']
 permissions:
   contents: read
 jobs:
@@ -15,6 +15,8 @@ jobs:
         with:
           python-version: '3.12'
       - run: pip install jsonschema==4.26.0 pytest==8.4.2
-      - run: python -m pytest -o addopts='' tests/test_research_catalog.py tests/test_community_worker.py tests/test_build_studies_index.py -q
+      # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662  # v0.8.0
+      - run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt"
+      - run: python -m pytest -o addopts='' tests/test_research_catalog.py tests/test_community_worker.py tests/test_build_studies_index.py tests/test_hsenv_pin.py -q
       - run: python scripts/build_studies_index.py
       - run: git diff --exit-code -- co_website/data/studies_index.json co_website/data/research_index.json

.github/workflows/sync-studies-index.yml

@@ -21,6 +21,9 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
+      - name: Install the pinned protocol engine
+        # effects_core is hsenv's (V080 B1). hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662  # v0.8.0
+        run: grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt"
       - name: Build studies index
         run: python scripts/build_studies_index.py
       - name: Commit updated index

.github/workflows/validate-studies.yml

@@ -6,6 +6,9 @@ on:
       - "studies/**"
       - "scripts/build_studies_index.py"
       - "scripts/verify_study.sh"
+      - "scripts/check_contributors.py"
+      - "community/**"
+      - "scripts/research/protocols.py"
 
 jobs:
   validate:
@@ -57,43 +60,50 @@ jobs:
             exit 0
           fi
           # A failed diff must fail the job, never read as "no studies changed".
-          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- studies/)
-          CHANGED_STUDIES=$(printf '%s\n' "$CHANGED_FILES" | grep -oP 'studies/[^/]+' | sort -u || true)
+          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- studies/ community/)
+          CHANGED_STUDIES=$(printf '%s\n' "$CHANGED_FILES" | grep -oP '^studies/[^/]+|^community/paper_[a-f0-9]+/revisions/[a-f0-9]{64}' | sort -u || true)
           if [ -z "$CHANGED_STUDIES" ]; then
             exit 0
           fi
-          FAILED=0
-          for study_path in $CHANGED_STUDIES; do
-            INDEX="$study_path/index.json"
-            if [ ! -f "$INDEX" ]; then
-              continue
-            fi
-            # Check if any contributor github field contains the PR author's username
-            MATCH=$(python3 -c "
-          import json, sys
-          with open('$INDEX') as f:
-              data = json.load(f)
-          contributors = data.get('contributors', [])
-          if not contributors:
-              print('MISSING')
-              sys.exit(0)
-          author = '$PR_AUTHOR'.lower()
-          for c in contributors:
-              gh = c.get('github', '').lower()
-              # Match 'username' or 'https://github.com/username'
-              if gh.rstrip('/').split('/')[-1] == author:
-                  print('MATCH')
-                  sys.exit(0)
-          print('MISMATCH')
-          ")
-            study_id=$(basename "$study_path")
-            if [ "$MATCH" = "MISSING" ] || [ "$MATCH" = "MISMATCH" ]; then
-              echo "::error file=$INDEX::Contributor check failed for $study_id. Your GitHub username is '$PR_AUTHOR'. Please add the following to your index.json:%0A%0A\"contributors\": [%0A  { \"name\": \"Your Name\", \"github\": \"$PR_AUTHOR\" }%0A]"
-              FAILED=1
-            else
-              echo "$study_id: Contributor @$PR_AUTHOR verified."
-            fi
+          # Same rule for legacy index.json and protocol study.json (V080 B2); a dir with neither is skipped.
+          python scripts/check_contributors.py --author "$PR_AUTHOR" $CHANGED_STUDIES
+
+  # V080-PLATFORM §2.1: a maintainer release PR adds files under community/**. Every changed revision that
+  # holds protocol.json is checked with the hsenv engine. This job runs the PR's own checker
+  # (scripts/research/protocols.py), its tests and its requirements.txt hsenv pin, so a PR can change its own
+  # checks: it has no secrets, a read-only token and a time limit, and reviewers must look at any change to
+  # those files. Protocol files themselves are read as data by the engine.
+  validate-protocols:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install the pinned engine
+        # hsenv pin lives only in requirements.txt: hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662  # v0.8.0
+        run: |
+          pip install jsonschema==4.26.0 pytest==8.4.2
+          grep '^hsenv' requirements.txt > "$RUNNER_TEMP/hsenv-pin.txt" && pip install -r "$RUNNER_TEMP/hsenv-pin.txt"
+      - name: Validate changed protocol releases
+        run: |
+          # A failed diff must fail the job, never read as "no revisions changed".
+          CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- community/)
+          REVISIONS=$(printf '%s\n' "$CHANGED_FILES" | grep -oP '^community/paper_[a-f0-9]+/revisions/[a-f0-9]{64}' | sort -u || true)
+          DIRS=""
+          for rev in $REVISIONS; do
+            if [ -f "$rev/protocol.json" ]; then DIRS="$DIRS $rev"; fi
           done
-          if [ "$FAILED" -eq 1 ]; then
-            exit 1
+          if [ -z "$DIRS" ]; then
+            echo "No protocol releases changed."
+            exit 0
           fi
+          python -m scripts.research.protocols --sessions 300 $DIRS
+      - name: Test the protocol release rules
+        run: python -m pytest tests/test_protocol_release.py tests/test_check_contributors.py tests/test_hsenv_pin.py -o addopts='' -q

docs/v080-protocol.md

@@ -0,0 +1,123 @@
+# v0.8 protocol releases in HumanStudy-Bench
+
+This page covers the Bench part of the v0.8 platform design. In v0.8 a study is one folder holding `study.json` and `protocol.json`, and one engine (`hsenv`)
+reads it on every platform. Bench publishes a released protocol as a community revision
+(`community/paper_<id>/revisions/<sha256>/`). Bench never interprets a protocol itself and never executes
+its files: every check calls the pinned `hsenv`.
+
+## What this change does
+
+- **B1.** `requirements.txt` pins hsenv by commit SHA; it is the only place the pin is written, and the
+  sync-studies-index, research-catalog, maintain-community and validate-studies workflows read it from there.
+  `scripts/research/effects_core.py` re-exports `hsenv.effects_core`, with import paths and the CLI kept.
+  `tests/test_hsenv_pin.py` checks that every community revision with `effects.json` gives the same problems,
+  warnings and kg digest as the Bench copy did (`tests/fixtures/effects_core_golden.json`, recorded with the
+  copy at `b34d214`), and that the protocol readers never execute protocol text.
+- **B2.** `scripts/check_contributors.py` (one contributor rule), `scripts/research/protocols.py` (release
+  checks), a protocol branch in `validate_assets`, and the `validate-protocols` job in `validate-studies.yml`.
+  The tests use an invented CC0 fixture release.
+- **B3.** `build_catalog` adds a `protocol` block to the paper's `research_index.json` entry and computes
+  `replayable`/`humanBenchmark` from the protocol. `studies_index.json` and `research_index.json` stay
+  byte-identical for today's studies.
+
+Not done here: **B4** (`run_protocol.py`, the playground on `kind: protocol`, quiet logs) waits for owner gate G3.
+**B5** (the first releases) needs signed reviews (G2) and studies with no `release_hold`.
+
+## What CI checks on a protocol release
+
+`python -m scripts.research.protocols <revision-dir>... [--sessions 300]` exits 1 on any error. The steps
+follow the order in design §2.1:
+
+1. **Schema.** `hsenv.schema_check` validates `study.json` (`kind=study`) and `protocol.json` (`kind=protocol`,
+   including its cross-reference pass). `schema_version` is required: legacy files are not released.
+2. **Release rules.**
+   - Every path must be on the allowlist: `study.json`, `protocol.json`, `effects.json`, `program.json`,
+     `capabilities.json`, `review/fidelity-review-*.md`, `review/checks-latest.json`, `sources/SOURCE.json`,
+     `sources/open/**`, `sourcetext/<key>*` (only when `meta.sources[key]` is under `sources/open/`),
+     `variants/*.json`, `artifacts/**` and `package/**`.
+   - These are never published: `sources/{researcher,private,quarantine}/**`, `runs/**`, `**/runs/**`,
+     `**/records.jsonl`, `paper/**` and `bundle.json`.
+   - `visibility` must be `"public"`. `quality` must be `high_fidelity`, with `fidelity.verdict` HIGH FIDELITY,
+     a non-empty `fidelity.signed_by` and the review file included (G2).
+   - No `release_hold` may appear, neither in `study.json` nor on any arm.
+   - `study.json.protocol.sha256` must equal the canonical hash of `protocol.json`, and `study.json.arms` must
+     agree with the protocol's arms.
+   - An image `src` must be a URL, or an `asset:<sha16>` that points to a file in `sources/open/`.
+   - `study.json.hsenv` must admit the installed engine.
+3. **Checks report.** `review/checks-latest.json` must report `PASS: true` and carry `protocol_sha256`
+   equal to this protocol's hash. A report without the hash, or with another hash, counts as stale.
+   `--sessions N` also replays N random sessions through `hsenv.checks` (references and fuzz).
+4. **Effects and capabilities.** `effects.json` goes through `hsenv.effects_core.check`, and
+   `capabilities.json` through the capability rules.
+5. **Digest.** The revision directory's name must equal its manifest digest (`revisions.revision_id`).
+
+Each revision is checked through one of two paths:
+
+- **Maintainer release PR** (direct files under `community/**`): the `validate-protocols` job in
+  `validate-studies.yml` runs the checks on each changed revision that holds `protocol.json`.
+- **Community proposal** (`maintain-community.yml`): `revisions.validate_assets` sees `protocol.json` and runs
+  the same checks on the proposal's files. `program.json` becomes optional. The engine comes from main's
+  pinned requirements and never from the proposal. A proposal that holds `protocol.json` is **never
+  accepted on this path**: `integrate` always returns `needs_review` with `requiresMaintainer: true`, whatever
+  the review says (an agent's or anyone's `accept`) and whatever the release declares about itself
+  (`fidelity.signed_by`, a PASS `checks-latest.json`). No agent review is run for it. A maintainer publishes
+  it through a release PR.
+
+**Contributor rule.** `scripts/check_contributors.py --author <login> <dir>...` reads `index.json` for
+legacy studies and `study.json` for protocol releases. The rule is unchanged: a `contributors[].github` must
+equal the PR author, given as `login` or as a GitHub URL. Admin and maintain users still bypass it in the
+workflow. A community proposal gets its PR author appended to `study.json` contributors when missing.
+
+## research_index.json
+
+Shared contract fixture: `tests/fixtures/protocol_release/research-index-protocol.json` is the exact
+`protocol` block and capabilities Bench writes for the fixture release (`schemaVersion` is the protocol
+schema version, `"1"`). The web repo tests its reader against the same JSON; change both together.
+
+For a current revision with `protocol.json`, the paper entry gains these fields:
+
+```
+"protocol": {"slug", "schemaVersion", "hsenv", "fidelity": {"verdict", "signed_by"},
+             "arms": [{"id", "runnable", "nPaper"}], "runExcluded": [...], "effects": n, "revision"},
+"capabilities": {..., "replayable": any runnable arm without release_hold, "humanBenchmark": any effect with reported values},
+"coverage": "protocol-release"        (only when the revision has no program.json)
+```
+
+The block holds counts and ids only, never protocol text. `studies_index.json` keeps its legacy shape.
+
+## Test fixture
+
+`tests/fixtures/protocol_release/example2026lamps/` is an invented study under CC0-1.0. Its material is two
+sentences about a lamp and one rating question, and it is no real paper, protocol or participant. Its
+`review/checks-latest.json` is a real `hsenv.checks` PASS report (200 sessions, verbatim 3/3 found, keyed sign
+correct), with `protocol_sha256` added. To regenerate the fixture, write it again with the same content, run
+`python -m hsenv.checks <dir> --sessions 200 --source-text survey=<dir>/sourcetext/survey.txt`, and store the
+output with `protocol_sha256`.
+
+## Running locally
+
+```bash
+pip install -r requirements.txt          # installs hsenv from GitHub at the pinned commit
+pytest tests/test_hsenv_pin.py tests/test_protocol_release.py tests/test_check_contributors.py -o addopts=''
+python -m scripts.research.protocols community/paper_<id>/revisions/<sha> --sessions 300
+```
+
+The hsenv tests skip when hsenv is not installed.
+
+## Open items (hsenv H1 and maintainer decisions)
+
+- **hsenv is installed, not vendored.** `requirements.txt` pins hsenv v0.8.0 by its commit SHA
+  (`fd38302`). The desktop pins the same release (B1 is chained to it).
+- **Engine schema vs design.** `study.v1.json` has `additionalProperties: false` and lacks the design's §1.1
+  fields (`slug`, `paper`, `contributors`, `visibility`, `hsenv`, `fidelity`, `arms`, `paperId`, `release`).
+  Bench validates those itself and removes them before calling `schema_check` (`protocols.RELEASE_FIELDS`).
+  H1 should add them to the schema.
+- **H1 must provide:**
+  - `protocol_sha256` in the `hsenv.checks` report: today it carries none, so CI could not detect a stale report
+  - `hsenv check --public`: `protocols.replay` is the interim refs and fuzz subset
+  - `hsenv.release.PUBLIC_PATHS`: a test fails if it differs from Bench's list
+  - `hsenv.capabilities`: `capabilities.py` stays a Bench copy until then
+- **Design statement corrected.** Design §2.1 says `expr.py` has "no eval/exec (verified by grep)". In fact
+  it calls `compile` and `eval`. It does so on an AST whose node types it has checked against a whitelist,
+  and it runs with `__builtins__` emptied. The CI test pins that whitelist and a set of escape attempts.
+- **Maintainer decisions:** who signs a release (G2), and how B4 keeps public Actions logs quiet (G3).

requirements.txt

@@ -5,6 +5,11 @@ pandas>=1.3.0
 # Statistical analysis (used in study configs and evaluation)
 scipy>=1.7.0  # For statistical tests (ttest, ANOVA, etc.)
 
+# Protocol engine (V080-PLATFORM B1): the one hsenv release Bench, the web runner and the desktop share.
+# This line is the single pin: every workflow and test reads it from here (`grep '^hsenv' requirements.txt`, then `pip install -r`).
+# Pinned by commit SHA (the commit of tag v0.8.0), not by the mutable tag.
+hsenv @ git+https://github.com/HumanStudy-Hub/hsenv@fd38302574260b5876cfeaa6741540c5163ba662  # v0.8.0
+
 # Data validation
 jsonschema>=4.0.0
 pydantic>=2.0.0

scripts/check_contributors.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Contributor attribution for submitted studies (one rule for both doors, V080-PLATFORM §2.1 / B2).
+
+    python scripts/check_contributors.py --author <github-login> <study-dir> [<study-dir> ...]
+
+A legacy study (`studies/study_NNN`) keeps its contributors in index.json; a protocol study keeps them in
+study.json. Each directory must list a contributor whose `github` is the PR author ('login' or
+'https://github.com/login'). A directory with neither file is skipped, as before. The maintainer bypass
+(admin/maintain permission) stays in the workflow, which never calls this script for maintainers.
+
+This is the inline check that validate-studies.yml carried, moved here unchanged except that study.json
+is read when there is no index.json. Exit 1 when any directory fails.
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def contributors_file(study_dir: Path):
+    for name in ("index.json", "study.json"):
+        if (study_dir / name).is_file():
+            return study_dir / name
+    return None
+
+
+def verdict(path: Path, author: str) -> str:
+    """MATCH, MISMATCH or MISSING, exactly as the former inline check."""
+    return verdict_from(json.loads(path.read_text(encoding="utf-8")), author)
+
+
+def verdict_from(data: dict, author: str) -> str:
+    contributors = data.get("contributors", [])
+    if not contributors:
+        return "MISSING"
+    # Match 'username' or 'https://github.com/username'
+    for c in contributors:
+        gh = (c.get("github", "") if isinstance(c, dict) else "").lower()
+        if gh.rstrip("/").split("/")[-1] == author.lower():
+            return "MATCH"
+    return "MISMATCH"
+
+
+def with_contributor(raw: bytes, login: str, name: str = "") -> bytes:
+    """study.json bytes with the PR author appended to contributors when absent (community proposals)."""
+    data = json.loads(raw)
+    if not isinstance(data, dict) or not isinstance(data.get("contributors", []), list):
+        return raw   # not a study card; the release checks report the malformed study.json
+    if verdict_from(data, login) == "MATCH":
+        return raw
+    data.setdefault("contributors", []).append({"name": name or login, "github": login})
+    return (json.dumps(data, ensure_ascii=False, indent=2) + "\n").encode()
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    ap.add_argument("--author", required=True)
+    ap.add_argument("dirs", nargs="*")
+    a = ap.parse_args(argv)
+    failed = False
+    for d in a.dirs:
+        study_dir = Path(d)
+        path = contributors_file(study_dir)
+        if path is None:
+            continue
+        study_id = study_dir.name
+        if verdict(path, a.author) in ("MISSING", "MISMATCH"):
+            print(f"::error file={path}::Contributor check failed for {study_id}. Your GitHub username is "
+                  f"'{a.author}'. Please add the following to your {path.name}:%0A%0A\"contributors\": [%0A  "
+                  f"{{ \"name\": \"Your Name\", \"github\": \"{a.author}\" }}%0A]")
+            failed = True
+        else:
+            print(f"{study_id}: Contributor @{a.author} verified.")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())