Sync effects_core and capabilities from desktop #84

scripts/research/capabilities.py

@@ -16,6 +16,16 @@
 - profiles      the runner does not sample participant profiles yet
 
 `--check` validates an existing capabilities.json and artifacts/failures.json and exits 1 on a problem.
+
+run.excluded entries: {"what", "reason", "arms", "affectsArms", "effects", "field"}.
+- `arms`        arms that cannot be run at all; tools refuse them (study_core.require_arms). An arm the package
+                runs (task/task.json conditions without status "blocked") is never one of them.
+- `affectsArms` arms that still run, with this part left out (a picture, a timer, a questionnaire, demographics).
+- `effects`     effects.json effects the replay cannot compute; their results are not_ready. An effect whose arms
+                are blocked (task.json status "blocked" or listed in some entry's `arms`) is listed here.
+With task.json conditions, --check fails an `arms` entry naming an arm the package runs (above all one that would
+leave no arm runnable) and a blocked effect missing from `effects`. These rules came with build contract 3:
+for a package whose study.json says "buildContract" below 3 they print as WARN and do not fail.
 Standard library only, Python 3.8+. A copy of humanstudy-desktop pack/skills/build/capabilities.py
 (cross-repo invariant: change both together). HumanStudy-Bench uses validate() for community revisions
 and build() for the maintainer backfill.
@@ -28,6 +38,7 @@
 
 STATUS = ("yes", "partial", "no", "unknown")
 REPRO = ("pass", "partial", "fail", None)
+RUN_RULES_CONTRACT = 3     # study.json buildContract from which the run.excluded arm rules fail --check
 PATTERNS = ("non-text-stimuli", "materials-not-public", "missing-information", "interaction",
             "sequential-dependence", "modelling", "non-human-participants", "unsimulable-measure",
             "author-code-unusable", "author-code-not-reproducing", "human-data-incomplete", "other")
@@ -73,6 +84,120 @@ def _human_source(package):
     return None
 
 
+def _names(values):
+    return [str(x) for x in values if isinstance(x, (str, int)) and not isinstance(x, bool)] \
+        if isinstance(values, list) else []
+
+
+def package_arms(root):
+    """{"runnable": [...], "blocked": [...], "by": {program condition id or arm id: arm}, "study": {arm: study}}
+    from package/<slug>/task/task.json `conditions` (a list of {id|arm, status, study, programConditionIds} or a
+    mapping keyed by arm); None when the package names no conditions."""
+    package = _package(Path(root))
+    task = _json(package / "task/task.json") if package else None
+    conds = task.get("conditions") if isinstance(task, dict) else None
+    items = []
+    if isinstance(conds, dict):
+        items = [dict(v, id=k) if isinstance(v, dict) else {"id": k} for k, v in conds.items()]
+    elif isinstance(conds, list):
+        items = [c if isinstance(c, dict) else {"id": c} for c in conds if isinstance(c, (dict, str))]
+    out = {"runnable": [], "blocked": [], "by": {}, "study": {}}
+    for c in items:
+        arm = c.get("id") or c.get("arm") or c.get("name")
+        if not isinstance(arm, str) or not arm:
+            continue
+        blocked = c.get("status") == "blocked" or c.get("runnable") is False
+        out["blocked" if blocked else "runnable"].append(arm)
+        for alias in [arm, c.get("arm")] + _names(c.get("programConditionIds")):
+            if isinstance(alias, str) and alias:
+                out["by"].setdefault(alias, arm)
+        if c.get("study") is not None:
+            out["study"][arm] = str(c["study"])
+    return out if out["runnable"] or out["blocked"] else None
+
+
+def contract_level(root):
+    """The highest study.json buildContract of the project's package (0 when none says)."""
+    package = _package(Path(root))
+    study = _json(package / "study.json") if package else None
+    level = study.get("buildContract") if isinstance(study, dict) else None
+    return level if isinstance(level, (int, float)) and not isinstance(level, bool) else 0
+
+
+def effect_arms(effect, arms):
+    """The package arms an effects.json effect contrasts (treatment, reference, conditionIds, groups)."""
+    con = effect.get("contrast") if isinstance(effect.get("contrast"), dict) else {}
+    names = [con.get("treatment"), con.get("reference")] + _names(con.get("conditionIds")) + _names(con.get("groups"))
+    return sorted({arms["by"][n] for n in names if isinstance(n, str) and n in arms["by"]})
+
+
+def blocked_effects(caps, effects, arms):
+    """{effect id: why} for effects whose arms cannot run: every arm it contrasts is blocked (task.json status
+    "blocked" or named in a run.excluded entry's `arms`), or every arm of its study is."""
+    if not isinstance(effects, dict) or not arms:
+        return {}
+    refused = {a for item in ((caps or {}).get("run") or {}).get("excluded") or [] if isinstance(item, dict)
+               for a in _names(item.get("arms"))}
+    blocked = (set(arms["blocked"]) | refused) - set(arms["runnable"]) if arms["runnable"] else set(arms["blocked"]) | refused
+    out = {}
+    for e in effects.get("effects") or []:
+        if not isinstance(e, dict) or not isinstance(e.get("id"), str):
+            continue
+        mine = effect_arms(e, arms)
+        study_arms = [a for a, st in arms["study"].items() if st == str(e.get("studyId"))]
+        if mine and any(a in blocked for a in mine):
+            out[e["id"]] = "arm %s cannot be run" % ", ".join(a for a in mine if a in blocked)
+        elif not mine and study_arms and all(a in blocked for a in study_arms):
+            out[e["id"]] = "every arm of study %s cannot be run" % e.get("studyId")
+    return out
+
+
+def run_rules(caps, root):
+    """(problems, warnings) for run.excluded against the package's arms and effects.json (see the docstring).
+    Problems only when the package's study.json says buildContract >= RUN_RULES_CONTRACT."""
+    found = []
+    run = (caps or {}).get("run") if isinstance((caps or {}).get("run"), dict) else {}
+    items = [x for x in run.get("excluded") or [] if isinstance(x, dict)]
+    arms = package_arms(root)
+    effects = _json(Path(root) / "effects.json")
+    for i, item in enumerate(items):
+        for key in ("arms", "affectsArms", "effects"):
+            if key in item and not (isinstance(item[key], list) and all(isinstance(x, str) for x in item[key])):
+                found.append(("problem", "run.excluded[%d].%s must be a list of names" % (i, key)))
+    if arms:
+        known = set(arms["runnable"]) | set(arms["blocked"])
+        refused_all = set()
+        for i, item in enumerate(items):
+            listed = _names(item.get("arms"))
+            refused_all |= set(listed)
+            runs = [a for a in listed if a in arms["runnable"]]
+            if runs:
+                found.append(("rule", "run.excluded[%d] (%s): `arms` lists %s, which the package runs (task.json); "
+                              "`arms` means the arm cannot be run and is refused. For a part left out inside arms "
+                              "that still run, list them in `affectsArms`; for effects the replay cannot compute, "
+                              "in `effects`" % (i, str(item.get("what"))[:60], ", ".join(runs))))
+            for a in _names(item.get("arms")) + _names(item.get("affectsArms")):
+                if a not in known and a not in arms["by"]:
+                    found.append(("warning", "run.excluded[%d] names arm %r, which task.json does not list" % (i, a)))
+        if arms["runnable"] and set(arms["runnable"]) <= refused_all and run.get("status") != "no":
+            found.append(("rule", "run.excluded `arms` together list every arm the package runs (%s), so every "
+                          "named-arm run is refused although run is %s; an exclusion of part of the content belongs "
+                          "in `affectsArms`" % (", ".join(arms["runnable"]), run.get("status"))))
+    if isinstance(effects, dict):
+        ids = {e.get("id") for e in effects.get("effects") or [] if isinstance(e, dict)}
+        listed = {x for item in items for x in _names(item.get("effects"))}
+        for x in sorted(listed - ids):
+            found.append(("warning", "run.excluded lists effect %r, which effects.json does not have" % x))
+        for eid, why in sorted(blocked_effects(caps, effects, arms).items()):
+            if eid not in listed:
+                found.append(("rule", "effect %s cannot be computed (%s) but no run.excluded entry lists it in "
+                              "`effects`; add it so its result is not_ready with this reason" % (eid, why)))
+    strict = contract_level(root) >= RUN_RULES_CONTRACT
+    problems = [m for k, m in found if k == "problem" or (k == "rule" and strict)]
+    warnings = [m for k, m in found if k == "warning" or (k == "rule" and not strict)]
+    return problems, warnings
+
+
 def build(root, previous=None):
     root = Path(root)
     program = _json(root / "program.json")
@@ -197,6 +322,13 @@ def main():
             print("FAIL artifacts/failures.json must be a list")
             return 1
         bad = validate(caps, failures) if caps is not None else ["capabilities.json is missing or invalid"]
+        warn = []
+        if caps is not None and not bad:
+            more, warn = run_rules(caps, root)
+            bad += more
+        for line in warn:
+            print("WARN " + line + (" (older build: blocking from buildContract %d)" % RUN_RULES_CONTRACT
+                                    if contract_level(root) < RUN_RULES_CONTRACT else ""))
         for line in bad:
             print("FAIL " + line)
         print("PASS capabilities" if not bad else "")