test(site): run Core against device-simulator, short set on PRs, full set nightly

Makefile

@@ -12,7 +12,7 @@
 #   make clean                — remove all build artifacts
 
 .PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 release release-linux release-linux-arm64 release-linux-amd64 \
-        run-sim sim-ocpp dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \
+        run-sim sim-ocpp dev fmt vet clean e2e siterig siterig-full siterig-random siterig-coverage siterig-judge ci ci-ui ci-hw-pi docs \
 		verify verify-all install-hooks driver-repository-validate driver-versions protocol-boundary cloudlink-boundary \
         drivers drivers-present driver-versions-across-pin
 
@@ -121,6 +121,32 @@ release-workflow-test:
 e2e: drivers-present
 	cd go && FTW_E2E=1 go test ./test/e2e -v -timeout 180s
 
+# FTW against device-simulator's house with a real fuse
+# (go/test/site/README.md). SITERIG_SET is short, full or scenario names;
+# SITERIG_LANES runs that many at once on ports 100 apart.
+SITERIG_SET ?= short
+SITERIG_LANES ?= 2
+siterig: drivers-present
+	cd go && FTW_SITE=$(SITERIG_SET) FTW_SITE_LANES=$(SITERIG_LANES) \
+		go test -count=1 -timeout 300m -run 'TestSite$$' -v ./test/site/
+
+siterig-full:
+	$(MAKE) siterig SITERIG_SET=full
+
+# Random cases: SEED=n replays one, COUNT=m runs m seeds from SEED.
+SEED ?= 1
+COUNT ?= 1
+siterig-random:
+	$(MAKE) siterig SITERIG_SET=random:$(SEED):$(COUNT)
+
+# Which fault-catalog entries have a scenario: CATALOG=path/to/catalog.yaml
+siterig-coverage:
+	cd go && FTW_SITE_CATALOG=$(CATALOG) go test -count=1 -run TestCatalogCoverage -v ./test/site/
+
+# Judge a recorded run again with the current criteria: DIR=results directory
+siterig-judge:
+	cd go && FTW_SITE_JUDGE=$(DIR) go test -count=1 -run 'TestJudge$$' -v ./test/site/
+
 driver-repository-validate: drivers-present
 	cd go && go run ./cmd/ftw-driver-repository publish -unsigned -drivers ../drivers -output ../dist/driver-repository -base-url https://example.invalid/releases/download/drivers-local -repository https://github.com/srcfl/ftw
 

go/test/site/README.md

@@ -0,0 +1,100 @@
+# Site test: FTW against a simulated house
+
+`TestSite` runs the real `ftw` binary against a house from
+[srcfl/device-simulator](https://github.com/srcfl/device-simulator): its
+headless run (`simulator run`, docs/HEADLESS_RUN.md there) builds a Sungrow
+hybrid whose battery follows FTW's EMS writes after 2 s, an SDM630 site
+meter behind an IEC gG fuse on each phase, and OCPP 1.6 chargers with the
+car on them. FTW talks to it over Modbus and OCPP as on a real site. Each
+scenario starts a fresh house and a fresh `state.db`, drives the house and
+FTW, and judges the run against fixed criteria from what flowed through the
+fuse (the house's record) and what FTW said (its API, once a second). FTW
+runs `planner_self` with the Core planner and SE3 prices.
+
+The house lives in device-simulator; this directory keeps only what is
+FTW's: the scenarios, FTW's configuration, the criteria and the pinned
+simulator commit in [`devsim.version`](devsim.version).
+
+## Run it
+
+```bash
+make siterig                          # the short set, two lanes
+make siterig-full                     # every scenario
+make siterig SITERIG_SET=s1-25,s9-25  # named scenarios
+make siterig-coverage CATALOG=../reports/felkatalog/catalog.yaml
+make siterig-judge DIR=/tmp/ftw-site-123  # judge a recorded run again
+```
+
+| Variable | Meaning |
+|---|---|
+| `FTW_SITE` | `short`, `full`, or names. Unset skips the test. |
+| `FTW_SITE_LANES` | scenarios at once (`SITERIG_LANES`, default 2), each lane on its own 100 ports |
+| `FTW_SITE_PORT_BASE` | first port on 127.0.0.1 (default 18700): the house on base+0..59, FTW on base+90 and +91 |
+| `FTW_SITE_SHARD` | `k/n`: shard k of n, for several machines |
+| `FTW_SITE_OUT` | run directories, `summary.md` and `summary.json` (default: a new temporary directory) |
+| `FTW_SITE_DEVSIM_SRC` | a device-simulator checkout at the pinned commit; without it the test clones the pin into the user cache |
+| `FTW_SITE_FTW_BIN`, `FTW_SITE_DEVSIM_BIN` | prebuilt binaries |
+| `FTW_SITE_CATALOG` | the fault catalog, for `TestCatalogCoverage` |
+| `FTW_SITE_JUDGE` | a results directory: `TestJudge` judges its runs again with the criteria as they are now, and rewrites their `result.json` and the summary |
+
+Each run directory keeps `house/plant.jsonl` (the house once a second, with
+every event and mark), `house/ocpp-<charger>.jsonl` (every OCPP frame the
+charger read or wrote), `ftw.jsonl` (FTW's `/api/status` and
+`/api/loadpoints` once a second), `ftw.log`, `house.log`, `config.yaml`,
+`house.json` and `result.json`.
+
+A run whose record has a hole over 15 s fails as "host paused": a sleeping
+laptop stretches every wait and proves nothing.
+
+## Where it runs
+
+Not in GitHub Actions. `siterig-runner` ([runner/](runner/README.md)) runs
+on a machine of its own, watches `master` and open pull requests, and posts
+the result on the exact commit as the status `siterig/short` (pull requests
+that touch control, loadpoint, OCPP or telemetry code), `siterig/full`
+(each new `master` commit) or `siterig/random`. A beta needs
+`siterig/full` on its commit.
+
+## Known failures
+
+A criterion named in a scenario's `known` map fails on master for the
+finding it names. The run passes while it keeps failing, and fails once it
+passes: a fix deletes its line in the same pull request.
+
+## The fault catalog
+
+Each scenario names the entries of FTW's field fault catalog it covers and
+its sources (`provenance` in [catalog_test.go](catalog_test.go)).
+`make siterig-coverage` lists, per area, which faults have a scenario.
+
+## Scenarios
+
+"Over the limit" is the worst phase above the fuse − 1 A, measured through
+the fuse. A device's response time counts as 2 s, the meter and control
+tick as 2 s more.
+
+| Scenario | House and steps | Criteria | Short |
+|---|---|---|---|
+| `s0-25` Idle charger | 25 A, 3 A per phase, a charger with no car, heartbeat 60 s, 300 s | Home use `measured` in 95 % of samples; no OCPP disconnect | yes |
+| `s0b-25` | as s0, Core's heartbeat | as s0 | |
+| `s1-16`, `s1-25` Charge now | 3 A per phase, Charge now 16 A for 180 s | the room the fuse gives (25 A: 11 040 W) within 60 s; held 95 %; fuse holds; on 25 A no `fuse_limit` | yes |
+| `s2-16` Charge now during export | 16 A, 1 A house, 7 kW PV, full battery: 9.1 A export per phase | car starts within 60 s (**known**: export counted as import); fuse holds | |
+| `s3-16`, `s3-25` Load step | car and a 3 kW battery hold charging, then 13 A (16 A) or 21 A (25 A) per phase for 180 s | fuse holds; over the limit at most 7 s after the step | yes |
+| `s4-25`, `s4b-25` Saved unlock | `surplus_unlock_bat_soc` 0.9 saved, not armed; Charge now 3p, and bare (s4b) | no stops; no offer drops to 0 W; full power over the last minute | |
+| `s5-25`, `s5s-25` Surplus only | 25 A, 9.3 kW surplus, `auto`; s5 without steps, s5s with Easee's | no offer above 16 A; fuse holds; only 6–16 A; ≥ 4 140 W 90 % of the time (**known** in both) | |
+| `s5t-25` Surplus with a goal | as s5s with a goal, 3.8 kW of export, a charger that switches phases | no offer above 16 A; fuse holds; ≥ 6 A on one phase 90 % of the time | |
+| `s6-25` Inverter offline | 1.5 kW PV, the inverter silent for 300 s | Home use shown and marked estimated throughout; measured before and after; never estimated while every driver reads | |
+| `s7-25` Restart | Charge now, 15 min, Core down 20 s, 420 s after | the day's Home use is kept; Home use back within 30 s; Charge now never below 4 140 W; full power within 420 s | |
+| `s7s-25` Short restart | as s7, restarted after 60 s | as s7 without the day total, which needs the long run (10 % rule) | yes |
+| `s8-60-25`, `s8-120-25` Slow charger | 15 A per phase, battery at its 5 % floor, profiles applied 60 or 120 s late | fuse holds; over the limit at most the delay + 7 s | 60 s |
+| `s9-25` 3p to 1p | 10 A per phase, a 32 A charger that switches phases; Charge now 7 360 W on 3p, after 90 s on 1p | fuse holds; over the limit at most 3 s before and after the switch | |
+| `s9r-25` 1p to 3p beside a loaded L2 | house 2/20/2 A, the 32 A charger; Charge now 7 360 W on 1p, after 90 s on 3p | as s9 | |
+| `s10-25` Two chargers | `carport` 60 s late; cut 6 A and raise 16 A together | fuse holds; over the limit at most 3 s after the step | |
+| `s11-25` Pause, then restart | a charger applying 5 s late; Charge now 16 A, 0 W for 25 s, then 6 A | an offer within 60 s of the restart; over the limit at most 3 s; fuse holds | |
+| `s12-20` One-phase night | 3 × 20 A, house 1.6/1.7/2.3 A, a charger applying 40 s late, the field's step table on `auto`, a plan for 6.6 kWh by the first quarter hour 40 min out | fuse holds; planned charging never orders one phase; no order cut while every phase has room, unless the plan paused; 90 % of the plan by the ready time | |
+| `s13-25` Socket reconnect | Charge now 16 A, the charger's connection gone for 10 s | Charge now goes on after the charger returns (**known**: #1788); no unplug seen | |
+| `s15-25` One-phase charger, Charge now | 25 A, 3 A per phase, an Easee wired on L1 (6-16 A, steps to 3 680 W, no phase mode); Charge now on the phases `/api/loadpoints` reports, as the app sends it, 180 s | 16 A on L1 within 60 s; held 95 %; fuse holds; no `fuse_limit` | yes |
+| `s15b-25`, `s15c-25` | as s15, phase mode `1p` and `auto` | as s15 | |
+| `s15bat-16` One-phase charger, battery rising | 16 A, 3 A per phase, the s15 charger, battery at 50 % following its EMS 8 s late; PV 2.1 kW rising every 30 s (+2.5 kW, then 1.5 kW) so Core charges the battery in steps; Charge now 6 s after the second step, while the raise is on its way | the battery order rose over 1 150 W in the 20 s before (setup check); car charging within 10 s; fuse holds; over the limit at most 3 s | |
+| `s15p-25` One-phase charger, planned | as s15, a plan for 600 Wh by the first quarter hour 15 min out | fuse holds; 90 % of the plan by the ready time; never called not following | |
+| `s14-25` One-phase car | an 11 kW charger with a one-phase car, a plan it cannot meet, 20 min | 3.4–3.8 kW over the last 10 min; never called not following | |

go/test/site/catalog_test.go

@@ -0,0 +1,171 @@
+//go:build unix
+
+package site
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+// provenance says where each scenario comes from: the entries of FTW's
+// field fault catalog it covers (reports/felkatalog/catalog.yaml in the FTW
+// stack workspace) and its sources.
+var provenance = map[string]struct {
+	catalog []string
+	sources []string
+}{
+	"s0-25":     {[]string{"home-070", "ocpp-011", "ocpp-013", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1779", "https://github.com/srcfl/ftw/pull/1790"}},
+	"s0b-25":    {[]string{"home-070", "ocpp-011", "ocpp-013", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1790"}},
+	"s1-16":     {nil, []string{"simulator test before beta 0.155.0, 21 Oct 2026"}},
+	"s1-25":     {nil, []string{"simulator test before beta 0.155.0, 21 Oct 2026"}},
+	"s2-16":     {[]string{"ev-078"}, []string{"https://github.com/srcfl/ftw/pull/1782"}},
+	"s3-16":     {[]string{"ev-100"}, []string{"https://github.com/srcfl/ftw/pull/1776"}},
+	"s3-25":     {[]string{"ev-100"}, []string{"https://github.com/srcfl/ftw/pull/1776"}},
+	"s4-25":     {[]string{"ev-093"}, []string{"https://github.com/srcfl/ftw/pull/1774"}},
+	"s4b-25":    {[]string{"ev-093"}, []string{"https://github.com/srcfl/ftw/pull/1774"}},
+	"s5-25":     {[]string{"ev-089", "ev-110"}, []string{"https://github.com/srcfl/ftw/pull/1774"}},
+	"s5s-25":    {[]string{"ev-089", "ev-102"}, []string{"https://github.com/srcfl/ftw/pull/1791"}},
+	"s6-25":     {[]string{"home-061", "ocpp-010", "tel-035", "tel-037", "tel-038", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1780"}},
+	"s7-25":     {[]string{"dev-100", "ev-085", "ev-091", "home-061", "home-074", "hw-061"}, []string{"https://github.com/srcfl/ftw/pull/1778", "https://github.com/srcfl/ftw/pull/1787"}},
+	"s7s-25":    {[]string{"ev-085", "ev-091"}, []string{"https://github.com/srcfl/ftw/pull/1778"}},
+	"s8-60-25":  {[]string{"dev-076", "ev-099", "ev-107"}, []string{"https://github.com/srcfl/ftw/pull/1776"}},
+	"s8-120-25": {[]string{"dev-076", "ev-099", "ev-107"}, []string{"https://github.com/srcfl/ftw/pull/1776"}},
+	"s9-25":     {[]string{"ev-094"}, []string{"https://github.com/srcfl/ftw/pull/1776"}},
+	"s9r-25":    {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (probe: a held 1p order raised to 3p beside L2 at 20 A)"}},
+	"s10-25":    {[]string{"dev-076", "ev-105"}, []string{"https://github.com/srcfl/ftw/pull/1776"}},
+	"s11-25":    {[]string{"ev-104"}, []string{"https://github.com/srcfl/ftw/pull/1763", "https://github.com/srcfl/ftw/pull/1793"}},
+	"s5t-25":    {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (probe: surplus-only with a goal)", "https://github.com/srcfl/ftw/pull/1799"}},
+	"s12-20":    {[]string{"ev-097"}, []string{"https://github.com/srcfl/ftw/pull/1799", "field report, 22 Oct 2026, v0.155.0-beta.1"}},
+	"s13-25":    {[]string{"ocpp-009", "dev-119"}, []string{"https://github.com/srcfl/ftw/issues/1788"}},
+	"s14-25":    {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (help1022 car1p, fix D)"}},
+	"s15-25":    {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now"}},
+	"s15b-25":   {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now (phase mode 1p)"}},
+	"s15c-25":   {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now (phase mode auto)"}},
+	"s15bat-16": {nil, []string{"https://github.com/srcfl/ftw/pull/1805", "release blocker 22 Oct 2026: a one-phase charger never starts while Core raises its battery"}},
+	"s15p-25":   {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger, planned charging"}},
+}
+
+func TestEveryScenarioSaysWhereItComesFrom(t *testing.T) {
+	for _, sc := range scenarios {
+		p, ok := provenance[sc.name]
+		if !ok || len(p.catalog)+len(p.sources) == 0 {
+			t.Errorf("%s has no catalog entry or source", sc.name)
+		}
+	}
+	for name := range provenance {
+		found := false
+		for _, sc := range scenarios {
+			found = found || sc.name == name
+		}
+		if !found {
+			t.Errorf("provenance for %s, which is no scenario", name)
+		}
+	}
+}
+
+type catalogEntry struct {
+	ID      string   `yaml:"id"`
+	Kind    string   `yaml:"kind"`
+	Area    string   `yaml:"area"`
+	Title   string   `yaml:"title"`
+	Status  string   `yaml:"status"`
+	Sources []string `yaml:"sources"`
+}
+
+// TestCatalogCoverage reads the fault catalog at FTW_SITE_CATALOG and writes
+// coverage.md: per area, which faults have a scenario here and which do not.
+// It fails when a scenario names an entry the catalog does not have.
+func TestCatalogCoverage(t *testing.T) {
+	path := os.Getenv("FTW_SITE_CATALOG")
+	if path == "" {
+		t.Skip("set FTW_SITE_CATALOG to the fault catalog (catalog.yaml)")
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var cat struct {
+		Entries []catalogEntry `yaml:"entries"`
+	}
+	if err := yaml.Unmarshal(b, &cat); err != nil {
+		t.Fatal(err)
+	}
+	byID := map[string]catalogEntry{}
+	for _, e := range cat.Entries {
+		byID[e.ID] = e
+	}
+	covered := map[string][]string{}
+	for name, p := range provenance {
+		for _, id := range p.catalog {
+			if _, ok := byID[id]; !ok {
+				t.Errorf("%s names catalog entry %s, which the catalog does not have", name, id)
+			}
+			covered[id] = append(covered[id], name)
+		}
+	}
+	type tally struct{ faults, open, covered, openCovered int }
+	areas := map[string]*tally{}
+	var md strings.Builder
+	md.WriteString("# Site-test coverage of the fault catalog\n\n")
+	md.WriteString("| Entry | Status | Title | Scenarios |\n|---|---|---|---|\n")
+	ids := make([]string, 0, len(byID))
+	for id := range byID {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	for _, id := range ids {
+		e := byID[id]
+		if e.Kind != "fault" {
+			if len(covered[id]) > 0 {
+				sort.Strings(covered[id])
+				fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", id, e.Kind, e.Title, strings.Join(covered[id], ", "))
+			}
+			continue
+		}
+		a := areas[e.Area]
+		if a == nil {
+			a = &tally{}
+			areas[e.Area] = a
+		}
+		open := e.Status == "open" || e.Status == "partly_fixed" || e.Status == "fix_unmerged"
+		a.faults++
+		if open {
+			a.open++
+		}
+		if len(covered[id]) > 0 {
+			a.covered++
+			if open {
+				a.openCovered++
+			}
+			sort.Strings(covered[id])
+			fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", id, e.Status, e.Title, strings.Join(covered[id], ", "))
+		}
+	}
+	md.WriteString("\n| Area | Faults | With a scenario | Open | Open with a scenario |\n|---|---:|---:|---:|---:|\n")
+	names := make([]string, 0, len(areas))
+	for a := range areas {
+		names = append(names, a)
+	}
+	sort.Strings(names)
+	for _, a := range names {
+		x := areas[a]
+		fmt.Fprintf(&md, "| %s | %d | %d | %d | %d |\n", a, x.faults, x.covered, x.open, x.openCovered)
+	}
+	out := os.Getenv("FTW_SITE_OUT")
+	if out == "" {
+		// No place to keep it: the log is the report.
+		t.Logf("%d catalog entries, %d with a scenario\n\n%s", len(byID), len(covered), md.String())
+		return
+	}
+	dst := filepath.Join(out, "coverage.md")
+	if err := os.WriteFile(dst, []byte(md.String()), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("%d catalog entries, %d with a scenario; wrote %s", len(byID), len(covered), dst)
+}

go/test/site/devsim.version

@@ -0,0 +1 @@
+a3030d3ff2d4b547d0aa596022ca85daa69904eb