diff --git a/Makefile b/Makefile index afe4082a..c24d24bb 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ # make clean — remove all build artifacts .PHONY: help test compose-migration-test container-boundary-test release-workflow-test build build-arm64 build-amd64 release release-linux release-linux-arm64 release-linux-amd64 \ - run-sim sim-ocpp dev fmt vet clean e2e ci ci-ui ci-hw-pi docs \ + run-sim sim-ocpp dev fmt vet clean e2e siterig siterig-full siterig-random siterig-coverage siterig-judge ci ci-ui ci-hw-pi docs \ verify verify-all install-hooks driver-repository-validate driver-versions protocol-boundary cloudlink-boundary \ drivers drivers-present driver-versions-across-pin @@ -121,6 +121,32 @@ release-workflow-test: e2e: drivers-present cd go && FTW_E2E=1 go test ./test/e2e -v -timeout 180s +# FTW against device-simulator's house with a real fuse +# (go/test/site/README.md). SITERIG_SET is short, full or scenario names; +# SITERIG_LANES runs that many at once on ports 100 apart. +SITERIG_SET ?= short +SITERIG_LANES ?= 2 +siterig: drivers-present + cd go && FTW_SITE=$(SITERIG_SET) FTW_SITE_LANES=$(SITERIG_LANES) \ + go test -count=1 -timeout 300m -run 'TestSite$$' -v ./test/site/ + +siterig-full: + $(MAKE) siterig SITERIG_SET=full + +# Random cases: SEED=n replays one, COUNT=m runs m seeds from SEED. +SEED ?= 1 +COUNT ?= 1 +siterig-random: + $(MAKE) siterig SITERIG_SET=random:$(SEED):$(COUNT) + +# Which fault-catalog entries have a scenario: CATALOG=path/to/catalog.yaml +siterig-coverage: + cd go && FTW_SITE_CATALOG=$(CATALOG) go test -count=1 -run TestCatalogCoverage -v ./test/site/ + +# Judge a recorded run again with the current criteria: DIR=results directory +siterig-judge: + cd go && FTW_SITE_JUDGE=$(DIR) go test -count=1 -run 'TestJudge$$' -v ./test/site/ + driver-repository-validate: drivers-present cd go && go run ./cmd/ftw-driver-repository publish -unsigned -drivers ../drivers -output ../dist/driver-repository -base-url https://example.invalid/releases/download/drivers-local -repository https://github.com/srcfl/ftw diff --git a/go/test/site/README.md b/go/test/site/README.md new file mode 100644 index 00000000..3289356f --- /dev/null +++ b/go/test/site/README.md @@ -0,0 +1,100 @@ +# Site test: FTW against a simulated house + +`TestSite` runs the real `ftw` binary against a house from +[srcfl/device-simulator](https://github.com/srcfl/device-simulator): its +headless run (`simulator run`, docs/HEADLESS_RUN.md there) builds a Sungrow +hybrid whose battery follows FTW's EMS writes after 2 s, an SDM630 site +meter behind an IEC gG fuse on each phase, and OCPP 1.6 chargers with the +car on them. FTW talks to it over Modbus and OCPP as on a real site. Each +scenario starts a fresh house and a fresh `state.db`, drives the house and +FTW, and judges the run against fixed criteria from what flowed through the +fuse (the house's record) and what FTW said (its API, once a second). FTW +runs `planner_self` with the Core planner and SE3 prices. + +The house lives in device-simulator; this directory keeps only what is +FTW's: the scenarios, FTW's configuration, the criteria and the pinned +simulator commit in [`devsim.version`](devsim.version). + +## Run it + +```bash +make siterig # the short set, two lanes +make siterig-full # every scenario +make siterig SITERIG_SET=s1-25,s9-25 # named scenarios +make siterig-coverage CATALOG=../reports/felkatalog/catalog.yaml +make siterig-judge DIR=/tmp/ftw-site-123 # judge a recorded run again +``` + +| Variable | Meaning | +|---|---| +| `FTW_SITE` | `short`, `full`, or names. Unset skips the test. | +| `FTW_SITE_LANES` | scenarios at once (`SITERIG_LANES`, default 2), each lane on its own 100 ports | +| `FTW_SITE_PORT_BASE` | first port on 127.0.0.1 (default 18700): the house on base+0..59, FTW on base+90 and +91 | +| `FTW_SITE_SHARD` | `k/n`: shard k of n, for several machines | +| `FTW_SITE_OUT` | run directories, `summary.md` and `summary.json` (default: a new temporary directory) | +| `FTW_SITE_DEVSIM_SRC` | a device-simulator checkout at the pinned commit; without it the test clones the pin into the user cache | +| `FTW_SITE_FTW_BIN`, `FTW_SITE_DEVSIM_BIN` | prebuilt binaries | +| `FTW_SITE_CATALOG` | the fault catalog, for `TestCatalogCoverage` | +| `FTW_SITE_JUDGE` | a results directory: `TestJudge` judges its runs again with the criteria as they are now, and rewrites their `result.json` and the summary | + +Each run directory keeps `house/plant.jsonl` (the house once a second, with +every event and mark), `house/ocpp-.jsonl` (every OCPP frame the +charger read or wrote), `ftw.jsonl` (FTW's `/api/status` and +`/api/loadpoints` once a second), `ftw.log`, `house.log`, `config.yaml`, +`house.json` and `result.json`. + +A run whose record has a hole over 15 s fails as "host paused": a sleeping +laptop stretches every wait and proves nothing. + +## Where it runs + +Not in GitHub Actions. `siterig-runner` ([runner/](runner/README.md)) runs +on a machine of its own, watches `master` and open pull requests, and posts +the result on the exact commit as the status `siterig/short` (pull requests +that touch control, loadpoint, OCPP or telemetry code), `siterig/full` +(each new `master` commit) or `siterig/random`. A beta needs +`siterig/full` on its commit. + +## Known failures + +A criterion named in a scenario's `known` map fails on master for the +finding it names. The run passes while it keeps failing, and fails once it +passes: a fix deletes its line in the same pull request. + +## The fault catalog + +Each scenario names the entries of FTW's field fault catalog it covers and +its sources (`provenance` in [catalog_test.go](catalog_test.go)). +`make siterig-coverage` lists, per area, which faults have a scenario. + +## Scenarios + +"Over the limit" is the worst phase above the fuse − 1 A, measured through +the fuse. A device's response time counts as 2 s, the meter and control +tick as 2 s more. + +| Scenario | House and steps | Criteria | Short | +|---|---|---|---| +| `s0-25` Idle charger | 25 A, 3 A per phase, a charger with no car, heartbeat 60 s, 300 s | Home use `measured` in 95 % of samples; no OCPP disconnect | yes | +| `s0b-25` | as s0, Core's heartbeat | as s0 | | +| `s1-16`, `s1-25` Charge now | 3 A per phase, Charge now 16 A for 180 s | the room the fuse gives (25 A: 11 040 W) within 60 s; held 95 %; fuse holds; on 25 A no `fuse_limit` | yes | +| `s2-16` Charge now during export | 16 A, 1 A house, 7 kW PV, full battery: 9.1 A export per phase | car starts within 60 s (**known**: export counted as import); fuse holds | | +| `s3-16`, `s3-25` Load step | car and a 3 kW battery hold charging, then 13 A (16 A) or 21 A (25 A) per phase for 180 s | fuse holds; over the limit at most 7 s after the step | yes | +| `s4-25`, `s4b-25` Saved unlock | `surplus_unlock_bat_soc` 0.9 saved, not armed; Charge now 3p, and bare (s4b) | no stops; no offer drops to 0 W; full power over the last minute | | +| `s5-25`, `s5s-25` Surplus only | 25 A, 9.3 kW surplus, `auto`; s5 without steps, s5s with Easee's | no offer above 16 A; fuse holds; only 6–16 A; ≥ 4 140 W 90 % of the time (**known** in both) | | +| `s5t-25` Surplus with a goal | as s5s with a goal, 3.8 kW of export, a charger that switches phases | no offer above 16 A; fuse holds; ≥ 6 A on one phase 90 % of the time | | +| `s6-25` Inverter offline | 1.5 kW PV, the inverter silent for 300 s | Home use shown and marked estimated throughout; measured before and after; never estimated while every driver reads | | +| `s7-25` Restart | Charge now, 15 min, Core down 20 s, 420 s after | the day's Home use is kept; Home use back within 30 s; Charge now never below 4 140 W; full power within 420 s | | +| `s7s-25` Short restart | as s7, restarted after 60 s | as s7 without the day total, which needs the long run (10 % rule) | yes | +| `s8-60-25`, `s8-120-25` Slow charger | 15 A per phase, battery at its 5 % floor, profiles applied 60 or 120 s late | fuse holds; over the limit at most the delay + 7 s | 60 s | +| `s9-25` 3p to 1p | 10 A per phase, a 32 A charger that switches phases; Charge now 7 360 W on 3p, after 90 s on 1p | fuse holds; over the limit at most 3 s before and after the switch | | +| `s9r-25` 1p to 3p beside a loaded L2 | house 2/20/2 A, the 32 A charger; Charge now 7 360 W on 1p, after 90 s on 3p | as s9 | | +| `s10-25` Two chargers | `carport` 60 s late; cut 6 A and raise 16 A together | fuse holds; over the limit at most 3 s after the step | | +| `s11-25` Pause, then restart | a charger applying 5 s late; Charge now 16 A, 0 W for 25 s, then 6 A | an offer within 60 s of the restart; over the limit at most 3 s; fuse holds | | +| `s12-20` One-phase night | 3 × 20 A, house 1.6/1.7/2.3 A, a charger applying 40 s late, the field's step table on `auto`, a plan for 6.6 kWh by the first quarter hour 40 min out | fuse holds; planned charging never orders one phase; no order cut while every phase has room, unless the plan paused; 90 % of the plan by the ready time | | +| `s13-25` Socket reconnect | Charge now 16 A, the charger's connection gone for 10 s | Charge now goes on after the charger returns (**known**: #1788); no unplug seen | | +| `s15-25` One-phase charger, Charge now | 25 A, 3 A per phase, an Easee wired on L1 (6-16 A, steps to 3 680 W, no phase mode); Charge now on the phases `/api/loadpoints` reports, as the app sends it, 180 s | 16 A on L1 within 60 s; held 95 %; fuse holds; no `fuse_limit` | yes | +| `s15b-25`, `s15c-25` | as s15, phase mode `1p` and `auto` | as s15 | | +| `s15bat-16` One-phase charger, battery rising | 16 A, 3 A per phase, the s15 charger, battery at 50 % following its EMS 8 s late; PV 2.1 kW rising every 30 s (+2.5 kW, then 1.5 kW) so Core charges the battery in steps; Charge now 6 s after the second step, while the raise is on its way | the battery order rose over 1 150 W in the 20 s before (setup check); car charging within 10 s; fuse holds; over the limit at most 3 s | | +| `s15p-25` One-phase charger, planned | as s15, a plan for 600 Wh by the first quarter hour 15 min out | fuse holds; 90 % of the plan by the ready time; never called not following | | +| `s14-25` One-phase car | an 11 kW charger with a one-phase car, a plan it cannot meet, 20 min | 3.4–3.8 kW over the last 10 min; never called not following | | diff --git a/go/test/site/catalog_test.go b/go/test/site/catalog_test.go new file mode 100644 index 00000000..ab7a5519 --- /dev/null +++ b/go/test/site/catalog_test.go @@ -0,0 +1,171 @@ +//go:build unix + +package site + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// provenance says where each scenario comes from: the entries of FTW's +// field fault catalog it covers (reports/felkatalog/catalog.yaml in the FTW +// stack workspace) and its sources. +var provenance = map[string]struct { + catalog []string + sources []string +}{ + "s0-25": {[]string{"home-070", "ocpp-011", "ocpp-013", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1779", "https://github.com/srcfl/ftw/pull/1790"}}, + "s0b-25": {[]string{"home-070", "ocpp-011", "ocpp-013", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1790"}}, + "s1-16": {nil, []string{"simulator test before beta 0.155.0, 21 Oct 2026"}}, + "s1-25": {nil, []string{"simulator test before beta 0.155.0, 21 Oct 2026"}}, + "s2-16": {[]string{"ev-078"}, []string{"https://github.com/srcfl/ftw/pull/1782"}}, + "s3-16": {[]string{"ev-100"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s3-25": {[]string{"ev-100"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s4-25": {[]string{"ev-093"}, []string{"https://github.com/srcfl/ftw/pull/1774"}}, + "s4b-25": {[]string{"ev-093"}, []string{"https://github.com/srcfl/ftw/pull/1774"}}, + "s5-25": {[]string{"ev-089", "ev-110"}, []string{"https://github.com/srcfl/ftw/pull/1774"}}, + "s5s-25": {[]string{"ev-089", "ev-102"}, []string{"https://github.com/srcfl/ftw/pull/1791"}}, + "s6-25": {[]string{"home-061", "ocpp-010", "tel-035", "tel-037", "tel-038", "ui-052"}, []string{"https://github.com/srcfl/ftw/pull/1780"}}, + "s7-25": {[]string{"dev-100", "ev-085", "ev-091", "home-061", "home-074", "hw-061"}, []string{"https://github.com/srcfl/ftw/pull/1778", "https://github.com/srcfl/ftw/pull/1787"}}, + "s7s-25": {[]string{"ev-085", "ev-091"}, []string{"https://github.com/srcfl/ftw/pull/1778"}}, + "s8-60-25": {[]string{"dev-076", "ev-099", "ev-107"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s8-120-25": {[]string{"dev-076", "ev-099", "ev-107"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s9-25": {[]string{"ev-094"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s9r-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (probe: a held 1p order raised to 3p beside L2 at 20 A)"}}, + "s10-25": {[]string{"dev-076", "ev-105"}, []string{"https://github.com/srcfl/ftw/pull/1776"}}, + "s11-25": {[]string{"ev-104"}, []string{"https://github.com/srcfl/ftw/pull/1763", "https://github.com/srcfl/ftw/pull/1793"}}, + "s5t-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (probe: surplus-only with a goal)", "https://github.com/srcfl/ftw/pull/1799"}}, + "s12-20": {[]string{"ev-097"}, []string{"https://github.com/srcfl/ftw/pull/1799", "field report, 22 Oct 2026, v0.155.0-beta.1"}}, + "s13-25": {[]string{"ocpp-009", "dev-119"}, []string{"https://github.com/srcfl/ftw/issues/1788"}}, + "s14-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1800 (help1022 car1p, fix D)"}}, + "s15-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now"}}, + "s15b-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now (phase mode 1p)"}}, + "s15c-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger gets 0 W under Charge now (phase mode auto)"}}, + "s15bat-16": {nil, []string{"https://github.com/srcfl/ftw/pull/1805", "release blocker 22 Oct 2026: a one-phase charger never starts while Core raises its battery"}}, + "s15p-25": {nil, []string{"https://github.com/srcfl/ftw/pull/1804", "release blocker 22 Oct 2026: a one-phase charger, planned charging"}}, +} + +func TestEveryScenarioSaysWhereItComesFrom(t *testing.T) { + for _, sc := range scenarios { + p, ok := provenance[sc.name] + if !ok || len(p.catalog)+len(p.sources) == 0 { + t.Errorf("%s has no catalog entry or source", sc.name) + } + } + for name := range provenance { + found := false + for _, sc := range scenarios { + found = found || sc.name == name + } + if !found { + t.Errorf("provenance for %s, which is no scenario", name) + } + } +} + +type catalogEntry struct { + ID string `yaml:"id"` + Kind string `yaml:"kind"` + Area string `yaml:"area"` + Title string `yaml:"title"` + Status string `yaml:"status"` + Sources []string `yaml:"sources"` +} + +// TestCatalogCoverage reads the fault catalog at FTW_SITE_CATALOG and writes +// coverage.md: per area, which faults have a scenario here and which do not. +// It fails when a scenario names an entry the catalog does not have. +func TestCatalogCoverage(t *testing.T) { + path := os.Getenv("FTW_SITE_CATALOG") + if path == "" { + t.Skip("set FTW_SITE_CATALOG to the fault catalog (catalog.yaml)") + } + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var cat struct { + Entries []catalogEntry `yaml:"entries"` + } + if err := yaml.Unmarshal(b, &cat); err != nil { + t.Fatal(err) + } + byID := map[string]catalogEntry{} + for _, e := range cat.Entries { + byID[e.ID] = e + } + covered := map[string][]string{} + for name, p := range provenance { + for _, id := range p.catalog { + if _, ok := byID[id]; !ok { + t.Errorf("%s names catalog entry %s, which the catalog does not have", name, id) + } + covered[id] = append(covered[id], name) + } + } + type tally struct{ faults, open, covered, openCovered int } + areas := map[string]*tally{} + var md strings.Builder + md.WriteString("# Site-test coverage of the fault catalog\n\n") + md.WriteString("| Entry | Status | Title | Scenarios |\n|---|---|---|---|\n") + ids := make([]string, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + e := byID[id] + if e.Kind != "fault" { + if len(covered[id]) > 0 { + sort.Strings(covered[id]) + fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", id, e.Kind, e.Title, strings.Join(covered[id], ", ")) + } + continue + } + a := areas[e.Area] + if a == nil { + a = &tally{} + areas[e.Area] = a + } + open := e.Status == "open" || e.Status == "partly_fixed" || e.Status == "fix_unmerged" + a.faults++ + if open { + a.open++ + } + if len(covered[id]) > 0 { + a.covered++ + if open { + a.openCovered++ + } + sort.Strings(covered[id]) + fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", id, e.Status, e.Title, strings.Join(covered[id], ", ")) + } + } + md.WriteString("\n| Area | Faults | With a scenario | Open | Open with a scenario |\n|---|---:|---:|---:|---:|\n") + names := make([]string, 0, len(areas)) + for a := range areas { + names = append(names, a) + } + sort.Strings(names) + for _, a := range names { + x := areas[a] + fmt.Fprintf(&md, "| %s | %d | %d | %d | %d |\n", a, x.faults, x.covered, x.open, x.openCovered) + } + out := os.Getenv("FTW_SITE_OUT") + if out == "" { + // No place to keep it: the log is the report. + t.Logf("%d catalog entries, %d with a scenario\n\n%s", len(byID), len(covered), md.String()) + return + } + dst := filepath.Join(out, "coverage.md") + if err := os.WriteFile(dst, []byte(md.String()), 0o644); err != nil { + t.Fatal(err) + } + t.Logf("%d catalog entries, %d with a scenario; wrote %s", len(byID), len(covered), dst) +} diff --git a/go/test/site/devsim.version b/go/test/site/devsim.version new file mode 100644 index 00000000..c0ca38fa --- /dev/null +++ b/go/test/site/devsim.version @@ -0,0 +1 @@ +a3030d3ff2d4b547d0aa596022ca85daa69904eb diff --git a/go/test/site/evaluate_test.go b/go/test/site/evaluate_test.go new file mode 100644 index 00000000..83b34ff2 --- /dev/null +++ b/go/test/site/evaluate_test.go @@ -0,0 +1,682 @@ +//go:build unix + +package site + +import ( + "bufio" + "encoding/json" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + volt = 230.0 + maxW = 11040.0 // the rig charger, 16 A on three phases + minW = 4140.0 // the loadpoints' minimum, 6 A on three phases +) + +// ---- what the house and the recorder wrote ---- + +// plantLine is a line of the house's plant.jsonl: a sample a second, an +// event or a mark. +type plantLine struct { + T float64 `json:"t"` + Mark string `json:"mark"` + Event json.RawMessage `json:"event"` + PhaseA [3]float64 `json:"phase_a"` // through each fuse, + into the site + Fuse *plantFuse `json:"fuse"` + Devices map[string]plantDevice `json:"devices"` +} + +type plantFuse struct { + Damage [3]float64 `json:"damage"` + Blown [3]bool `json:"blown"` +} + +type plantDevice struct { + W float64 `json:"w"` // a charger's draw + BatteryW float64 `json:"battery_w"` // an inverter's battery, + charging + SoCPct float64 `json:"soc_pct"` + EMSCmd float64 `json:"ems_cmd"` // 170 charge, 187 discharge, 204 stop + EMSSetW float64 `json:"ems_setpoint_w"` + Connected bool `json:"connected"` +} + +type ftwStatus struct { + Drivers map[string]struct { + Status string `json:"status"` + } `json:"drivers"` + LoadQuality *string `json:"load_quality"` + LoadReason *string `json:"load_reason"` + LoadW *float64 `json:"load_w"` + Energy struct { + Today struct { + LoadWh *float64 `json:"load_wh"` + HistoryQuality *string `json:"history_quality"` + } `json:"today"` + } `json:"energy"` +} + +type ftwLoadpoint struct { + ID string `json:"id"` + Plugged bool `json:"plugged_in"` + Phases int `json:"phases"` + Charger struct { + Known bool `json:"known"` + } `json:"charger"` + CommandedReason string `json:"commanded_reason"` + ControlFeedback struct { + Status string `json:"status"` // following, not_following, limited... + Severity string `json:"severity"` + Reason string `json:"reason"` + } `json:"control_feedback"` +} + +type ftwLine struct { + T float64 `json:"t"` + Status *ftwStatus `json:"status"` + LPs *struct { + Loadpoints []ftwLoadpoint `json:"loadpoints"` + } `json:"lps"` +} + +// sample is one second: the house's truth and the last FTW reading at or +// before it. +type sample struct { + T float64 + PhaseA [3]float64 + Damage [3]float64 + Blown [3]bool + CarW []float64 // by loadpoint, in lpIDs order + BatW float64 + Bat plantDevice // the inverter + Status *ftwStatus + LPs []ftwLoadpoint +} + +type frame struct { + T float64 `json:"t"` + Dir string `json:"dir"` + Frame json.RawMessage `json:"frame"` +} + +// houseEventLine is a change the house made, as its record has it. +type houseEventLine struct { + T float64 + Event houseEvent +} + +// runData is one finished scenario's record. +type runData struct { + dir string + samples []*sample + marks map[string]float64 + allMarks []markLine // every mark, in order, also repeated ones + events []houseEventLine + frames []frame // garage's charger + meta map[string]any +} + +type markLine struct { + T float64 + Label string +} + +func loadRun(dir string, meta map[string]any) (*runData, error) { + d := &runData{dir: dir, marks: map[string]float64{}, meta: meta} + var ftw []ftwLine + if err := readLines(filepath.Join(dir, "ftw.jsonl"), func(b []byte) { + var l ftwLine + if json.Unmarshal(b, &l) == nil { + ftw = append(ftw, l) + } + }); err != nil { + return nil, err + } + next := 0 + var last *ftwLine + err := readLines(filepath.Join(dir, "house", "plant.jsonl"), func(b []byte) { + var l plantLine + if json.Unmarshal(b, &l) != nil { + return + } + switch { + case l.Mark != "": + if _, ok := d.marks[l.Mark]; !ok { + d.marks[l.Mark] = l.T + } + d.allMarks = append(d.allMarks, markLine{l.T, l.Mark}) + return + case l.Event != nil: + var ev houseEvent + if json.Unmarshal(l.Event, &ev) == nil { + d.events = append(d.events, houseEventLine{l.T, ev}) + } + return + case l.Fuse == nil: + return + } + for next < len(ftw) && ftw[next].T <= l.T { + last = &ftw[next] + next++ + } + s := &sample{T: l.T, PhaseA: l.PhaseA, Damage: l.Fuse.Damage, Blown: l.Fuse.Blown, + BatW: l.Devices["inverter"].BatteryW, Bat: l.Devices["inverter"]} + for _, id := range lpIDs { + if dev, ok := l.Devices[id]; ok { + s.CarW = append(s.CarW, dev.W) + } + } + // A reading older than 3 s is no reading: FTW was down or slow. + if last != nil && l.T-last.T <= 3 { + s.Status = last.Status + if last.LPs != nil { + s.LPs = last.LPs.Loadpoints + } + } + d.samples = append(d.samples, s) + }) + if err != nil { + return nil, err + } + d.frames = d.loadFrames(0) + return d, nil +} + +// loadFrames is the OCPP frames charger i read and wrote, from its own log. +func (d *runData) loadFrames(i int) []frame { + var out []frame + readLines(filepath.Join(d.dir, "house", "ocpp-"+lpIDs[i]+".jsonl"), func(b []byte) { + var f frame + if json.Unmarshal(b, &f) == nil { + out = append(out, f) + } + }) + return out +} + +func readLines(path string, fn func([]byte)) error { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 64<<20) + for sc.Scan() { + fn(sc.Bytes()) + } + return sc.Err() +} + +func (d *runData) metaF(k string) float64 { + v, _ := d.meta[k].(float64) + if i, ok := d.meta[k].(int); ok { + v = float64(i) + } + return v +} + +// longestGap is the longest time between two samples, and when it began. +func (d *runData) longestGap() (gap, at float64) { + for i := 1; i < len(d.samples); i++ { + if g := d.samples[i].T - d.samples[i-1].T; g > gap { + gap, at = g, d.samples[i-1].T-d.samples[0].T + } + } + return gap, at +} + +// ---- per-sample readings ---- + +func carW(r *sample, i int) float64 { + if i < len(r.CarW) { + return r.CarW[i] + } + return 0 +} + +// phaseI is the current through each phase's fuse, + into the site. +func phaseI(r *sample) [3]float64 { return r.PhaseA } + +func maxAbsI(r *sample) float64 { + m := 0.0 + for _, i := range r.PhaseA { + m = math.Max(m, math.Abs(i)) + } + return m +} + +func damage(r *sample) float64 { return math.Max(r.Damage[0], math.Max(r.Damage[1], r.Damage[2])) } + +func blown(r *sample) bool { return r.Blown[0] || r.Blown[1] || r.Blown[2] } + +func firstLP(r *sample) ftwLoadpoint { + if len(r.LPs) > 0 { + return r.LPs[0] + } + return ftwLoadpoint{} +} + +func reasons(r *sample) []string { + lp := firstLP(r) + set := map[string]bool{} + for _, k := range []string{lp.CommandedReason, lp.ControlFeedback.Reason} { + if k != "" { + set[k] = true + } + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + return out +} + +func status(r *sample) ftwStatus { + if r.Status != nil { + return *r.Status + } + return ftwStatus{} +} + +func str(p *string) string { + if p == nil { + return "null" + } + return *p +} + +func allRead(r *sample) bool { + st := status(r) + if len(st.Drivers) == 0 { + return false + } + for _, d := range st.Drivers { + if d.Status != "ok" { + return false + } + } + return true +} + +func batSigned(r *sample) float64 { return r.BatW } + +// ---- windows ---- + +func (d *runData) window(t0, t1 float64) []*sample { + var out []*sample + for _, r := range d.samples { + if t0 <= r.T && r.T < t1 { + out = append(out, r) + } + } + return out +} + +// firstT is the seconds from t0 to the first sample from t0 on that +// satisfies pred, or nil. +func (d *runData) firstT(t0 float64, pred func(*sample) bool) *float64 { + for _, r := range d.samples { + if r.T >= t0 && pred(r) { + v := r.T - t0 + return &v + } + } + return nil +} + +// runsOver is the longest contiguous stretch and the total seconds with the +// worst phase above limit. +func (d *runData) runsOver(t0, t1, limit float64) (longest, total float64) { + var curStart, prevT *float64 + for _, r := range d.window(t0, t1) { + t := r.T + if maxAbsI(r) > limit { + if curStart == nil { + curStart = &t + } + if prevT != nil { + total += t - *prevT + } + } else { + if curStart != nil { + longest = math.Max(longest, t-*curStart) + } + curStart = nil + } + prevT = &t + } + if curStart != nil && prevT != nil { + longest = math.Max(longest, *prevT-*curStart) + } + return round(longest, 1), round(total, 1) +} + +// i2Over is the seconds the worst phase's I² filter (tau 60 s) is above +// limit, and the filter's peak. +func (d *runData) i2Over(t0, t1, limit float64) (over, peak float64) { + const tau = 60.0 + var y, prev float64 + started := false + for _, r := range d.window(t0, t1) { + x := math.Pow(maxAbsI(r), 2) + if !started { + y, started = x, true + } else { + dt := r.T - prev + y += (1 - math.Exp(-dt/tau)) * (x - y) + if math.Sqrt(y) > limit { + over += dt + } + } + prev = r.T + peak = math.Max(peak, math.Sqrt(y)) + } + return round(over, 1), round(peak, 2) +} + +func mean(xs []float64) float64 { + if len(xs) == 0 { + return 0 + } + s := 0.0 + for _, x := range xs { + s += x + } + return s / float64(len(xs)) +} + +func meanOf(ws []*sample, f func(*sample) float64) float64 { + xs := make([]float64, len(ws)) + for i, r := range ws { + xs[i] = f(r) + } + return mean(xs) +} + +func maxOf(ws []*sample, f func(*sample) float64) float64 { + m := 0.0 + for _, r := range ws { + m = math.Max(m, f(r)) + } + return m +} + +func round(x float64, n int) float64 { + p := math.Pow(10, float64(n)) + return math.Round(x*p) / p +} + +func roundp(p *float64, n int) *float64 { + if p == nil { + return nil + } + v := round(*p, n) + return &v +} + +// counter counts strings and lists the commonest first. +type counter map[string]int + +func (c counter) top(n int) map[string]int { + type kv struct { + k string + v int + } + var all []kv + for k, v := range c { + all = append(all, kv{k, v}) + } + sort.Slice(all, func(i, j int) bool { + if all[i].v != all[j].v { + return all[i].v > all[j].v + } + return all[i].k < all[j].k + }) + out := map[string]int{} + for i, x := range all { + if i >= n { + break + } + out[x.k] = x.v + } + return out +} + +// ---- OCPP profiles FTW sent ---- + +type profile struct { + T float64 `json:"t"` + Conn int `json:"conn"` + Purpose string `json:"purpose"` + Unit string `json:"unit"` + Limit float64 `json:"limit"` + Phases *float64 `json:"phases"` +} + +func (p profile) phasesOr3() float64 { + if p.Phases == nil || *p.Phases == 0 { + return 3 + } + return *p.Phases +} + +func (p profile) amps() bool { return strings.HasPrefix(strings.ToUpper(p.Unit), "A") } + +func (p profile) offerW() float64 { + if p.amps() { + return p.Limit * volt * p.phasesOr3() + } + return p.Limit +} + +// perPhaseA is the offer in amperes per phase. +func (p profile) perPhaseA() float64 { + if p.amps() { + return p.Limit + } + return p.Limit / (volt * p.phasesOr3()) +} + +// profiles lists the SetChargingProfile CALLs FTW sent in [t0, t1), timed +// from t0. +func profiles(frames []frame, t0, t1 float64) []profile { + var out []profile + for _, f := range frames { + if f.T < t0 || f.T >= t1 || f.Dir != "cs->cp" { + continue + } + var arr []json.RawMessage + if json.Unmarshal(f.Frame, &arr) != nil || len(arr) < 4 { + continue + } + var typ int + var action string + json.Unmarshal(arr[0], &typ) + json.Unmarshal(arr[2], &action) + if typ != 2 || action != "SetChargingProfile" { + continue + } + var req struct { + ConnectorID int `json:"connectorId"` + Profiles struct { + Purpose string `json:"chargingProfilePurpose"` + Schedule struct { + Unit string `json:"chargingRateUnit"` + Periods []struct { + Limit float64 `json:"limit"` + Phases *float64 `json:"numberPhases"` + } `json:"chargingSchedulePeriod"` + } `json:"chargingSchedule"` + } `json:"csChargingProfiles"` + } + json.Unmarshal(arr[3], &req) + p := profile{T: round(f.T-t0, 1), Conn: req.ConnectorID, Purpose: req.Profiles.Purpose, Unit: req.Profiles.Schedule.Unit} + if per := req.Profiles.Schedule.Periods; len(per) > 0 { + p.Limit, p.Phases = per[0].Limit, per[0].Phases + } + out = append(out, p) + } + return out +} + +func connector1(ps []profile) []profile { + var out []profile + for _, p := range ps { + if p.Conn == 1 { + out = append(out, p) + } + } + return out +} + +// ---- verdicts ---- + +// metrics are a run's numbers, written to result.json. +type metrics map[string]any + +// criterion is one pass condition. A scenario's known map names the +// criteria that fail on master and why. +type criterion struct { + Name string `json:"name"` + OK bool `json:"ok"` +} + +func crit(name string, ok bool) criterion { return criterion{Name: name, OK: ok} } + +// ---- shared summaries ---- + +func (d *runData) carSummary(t0, t1, targetW float64) metrics { + w := d.window(t0, t1) + ts := d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.95*minW }) + tm := d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.98*targetW }) + var hold *float64 + if tm != nil { + var after []*sample + for _, r := range w { + if r.T >= t0+*tm { + after = append(after, r) + } + } + n := 0 + for _, r := range after { + if carW(r, 0) >= 0.95*targetW { + n++ + } + } + h := round(float64(n)/math.Max(1, float64(len(after))), 3) + hold = &h + } + drops, on := 0, false + for _, r := range w { + c := carW(r, 0) + if c >= 0.9*minW { + on = true + } else if c < 500 && on { + drops++ + on = false + } + } + rs := counter{} + for _, r := range w { + for _, x := range reasons(r) { + rs[x]++ + } + } + fuseLimit := 0 + for k, v := range rs { + if strings.Contains(k, "fuse") { + fuseLimit += v + } + } + var last60 []float64 + for _, r := range w { + if r.T >= t1-60 { + last60 = append(last60, carW(r, 0)) + } + } + var minAfter *float64 + if ts != nil { + m := math.Inf(1) + for _, r := range w { + if r.T >= t0+*ts { + m = math.Min(m, carW(r, 0)) + } + } + m = math.Round(m) + minAfter = &m + } + return metrics{ + "time_to_start_s": roundp(ts, 1), + "time_to_target_s": roundp(tm, 1), + "hold_fraction": hold, + "stops_after_start": drops, + "car_w_last60_mean": math.Round(mean(last60)), + "car_w_min_after_start": minAfter, + "fuse_limit_samples": fuseLimit, + "reasons": rs.top(6), + "max_abs_i": round(maxOf(w, maxAbsI), 2), + "fuse_damage_max": round(maxOf(w, damage), 4), + "fuse_blown": anyBlown(w), + } +} + +func anyBlown(ws []*sample) bool { + for _, r := range ws { + if blown(r) { + return true + } + } + return false +} + +func le(p *float64, limit float64) bool { return p != nil && *p <= limit } + +func (d *runData) overage(t0, t1, fuse float64) metrics { + w := d.window(t0, t1) + m := metrics{"max_i": round(maxOf(w, maxAbsI), 2)} + m["over_limit_longest_s"], m["over_limit_total_s"] = d.runsOver(t0, t1, fuse-1) + m["over_fuse_longest_s"], _ = d.runsOver(t0, t1, fuse) + m["fuse_damage_max"] = round(maxOf(w, damage), 4) + m["fuse_blown"] = anyBlown(w) + return m +} + +func (d *runData) phaseMean(t0, t1 float64) [3]float64 { + w := d.window(t0, t1) + var out [3]float64 + for k := 0; k < 3; k++ { + out[k] = round(meanOf(w, func(r *sample) float64 { return math.Abs(phaseI(r)[k]) }), 1) + } + return out +} + +// disconnects lists "OCPP charger disconnected" lines in FTW's log between +// t0 and t1, in seconds from t0. +func (d *runData) disconnects(t0, t1 float64) []float64 { + out := []float64{} + readLines(filepath.Join(d.dir, "ftw.log"), func(b []byte) { + line := string(b) + if !strings.Contains(line, "OCPP charger disconnected") || !strings.HasPrefix(line, "time=") { + return + } + ts, _, _ := strings.Cut(strings.TrimPrefix(line, "time="), " ") + t, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return + } + at := float64(t.UnixMicro()) / 1e6 + if t0 <= at && at <= t1 { + out = append(out, round(at-t0, 1)) + } + }) + return out +} diff --git a/go/test/site/procattr_linux_test.go b/go/test/site/procattr_linux_test.go new file mode 100644 index 00000000..7fa4fff1 --- /dev/null +++ b/go/test/site/procattr_linux_test.go @@ -0,0 +1,9 @@ +package site + +import "syscall" + +// childProcAttr puts a child in its own process group and kills it if the +// test binary dies, as when go test's -timeout fires. +func childProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL} +} diff --git a/go/test/site/procattr_other_test.go b/go/test/site/procattr_other_test.go new file mode 100644 index 00000000..c38c4e54 --- /dev/null +++ b/go/test/site/procattr_other_test.go @@ -0,0 +1,11 @@ +//go:build unix && !linux + +package site + +import "syscall" + +// childProcAttr puts a child in its own process group. Without Pdeathsig a +// child outlives a test binary that dies; stop it by its run directory. +func childProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Setpgid: true} +} diff --git a/go/test/site/random_test.go b/go/test/site/random_test.go new file mode 100644 index 00000000..aeaa2b10 --- /dev/null +++ b/go/test/site/random_test.go @@ -0,0 +1,549 @@ +//go:build unix + +package site + +import ( + "encoding/json" + "fmt" + "math" + "math/rand" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// A random case draws FTW's side from a seed: the fuse, the house, the +// loadpoint's settings and FTW's requests (Charge now, a pause, a plan, a +// Core restart, an import ceiling) at random times. The house draws its own +// timeline from the same seed (device-simulator's random block): load steps +// and spikes, PV clouds, a battery at its floor, a car taking less, slow +// chargers, silent devices, reconnects, a late or glitched meter, signed +// currents. FTW's invariants are checked every second. The house writes +// the timeline it drew, so a failing seed can be replayed and kept. + +// randomDurationS is how long a random case runs. +const randomDurationS = 1200.0 + +// randomTolerated are invariants a seed may break for a finding still open +// on master. A fix deletes its line here. +var randomTolerated = map[string]string{ + "no discharge asked at the battery's floor": "ev-099: the fuse guard drives the battery below its floor; fix unmerged", +} + +// ftwAction is one of FTW's requests in a random case. +type ftwAction struct { + AtS float64 `json:"at_s"` + Do string `json:"do"` // charge_now, pause, clear, plan, restart, ceiling + W float64 `json:"w,omitempty"` + PhaseMode string `json:"phase_mode,omitempty"` + DownS float64 `json:"down_s,omitempty"` + CeilingW float64 `json:"ceiling_w,omitempty"` + SoCPct float64 `json:"soc_pct,omitempty"` + GoalPct float64 `json:"goal_pct,omitempty"` + ReadyMin float64 `json:"ready_min,omitempty"` +} + +// randomPlan is everything a random case does, so it can be saved and run +// again: House is set once the house has drawn its timeline. +type randomPlan struct { + Seed int64 `json:"seed"` + FuseA float64 `json:"fuse_a"` + HouseA [3]float64 `json:"house_a"` + PVW float64 `json:"pv_w"` + SoCPct float64 `json:"soc_pct"` + LPExtra map[string]any `json:"lp_extra,omitempty"` + Actions []ftwAction `json:"actions"` + DurationS float64 `json:"duration_s"` + House *houseSpec `json:"house,omitempty"` + Found *foundNote `json:"found,omitempty"` +} + +// foundNote says what a saved case failed and when it was found. +type foundNote struct { + Invariant string `json:"invariant"` + On string `json:"on"` // the date + Commit string `json:"commit,omitempty"` +} + +func drawPlan(seed int64) randomPlan { + rng := rand.New(rand.NewSource(seed)) + pick := func(xs ...float64) float64 { return xs[rng.Intn(len(xs))] } + between := func(lo, hi float64) float64 { return math.Round((lo+rng.Float64()*(hi-lo))*10) / 10 } + p := randomPlan{Seed: seed, DurationS: randomDurationS, + FuseA: pick(16, 20, 25), PVW: pick(0, 0, 2000, 5000, 8000), SoCPct: pick(6, 15, 40, 70, 95)} + for k := range p.HouseA { + p.HouseA[k] = between(0.5, p.FuseA/2) + } + p.LPExtra = map[string]any{} + switch rng.Intn(3) { + case 1: + p.LPExtra["phase_mode"] = "auto" + case 2: + p.LPExtra["phase_mode"] = "3p" + } + if rng.Intn(2) == 0 { + p.LPExtra["allowed_steps_w"] = easeeSteps + } + for t := 40 + between(0, 60); t < p.DurationS-60; t += between(60, 300) { + a := ftwAction{AtS: t} + switch rng.Intn(10) { + case 0, 1, 2: + a.Do, a.W, a.PhaseMode = "charge_now", pick(4140, 7360, 11040), "3p" + case 3: + a.Do, a.W, a.PhaseMode = "charge_now", pick(1380, 3680), "1p" + case 4: + a.Do = "pause" + case 5: + a.Do = "clear" + case 6: + a.Do, a.SoCPct, a.GoalPct, a.ReadyMin = "plan", between(20, 50), between(60, 90), pick(30, 60, 240) + case 7: + a.Do, a.DownS = "restart", between(5, 30) + case 8: + a.Do, a.CeilingW = "ceiling", pick(0, 3000, 5000, 8000) + default: + continue + } + p.Actions = append(p.Actions, a) + } + return p +} + +// randomScenario runs a plan drawn from seed, or a saved one. +func randomScenario(p randomPlan) scenario { + name, dir := fmt.Sprintf("r%d", p.Seed), fmt.Sprintf("random-%d", p.Seed) + sc := scenario{name: name, dir: dir, minutes: p.DurationS/60 + 1, + run: func(c *runCtx) (map[string]any, error) { return runPlan(c, p) }, + judge: judgeInvariants, + tolerate: randomTolerated} + if p.Found != nil { + sc.name, sc.dir = "found-"+fmt.Sprint(p.Seed), "found-"+fmt.Sprint(p.Seed) + sc.known = map[string]string{p.Found.Invariant: fmt.Sprintf("found by random seed %d on %s; triage it", p.Seed, p.Found.On)} + } + return sc +} + +func runPlan(c *runCtx, p randomPlan) (map[string]any, error) { + loads := [3]float64{p.HouseA[0] * volt, p.HouseA[1] * volt, p.HouseA[2] * volt} + spec := stackSpec{fuseA: p.FuseA, loadsW: loads, pvW: p.PVW, socPct: p.SoCPct, lpExtra: p.LPExtra} + if p.House != nil { + spec.house = p.House + } else { + spec.random = &houseRandom{Seed: p.Seed, DurationS: p.DurationS, GapS: 45} + } + plan, _ := json.MarshalIndent(p, "", " ") + err := with(c, spec, func(st *stack) error { + os.WriteFile(filepath.Join(st.rundir, "plan.json"), plan, 0o644) + if err := st.plug(0); err != nil { + return err + } + t0 := time.Now() + st.mark("start") + for _, a := range p.Actions { + sleepS(time.Until(t0.Add(time.Duration(a.AtS * float64(time.Second)))).Seconds()) + st.mark("ftw:" + a.Do) + var err error + switch a.Do { + case "charge_now": + err = st.chargeNow("garage", 0, a.PhaseMode, a.W) + case "pause": + err = st.chargeNow("garage", 0, "3p", -1) + case "clear": + _, err = call("DELETE", st.ftw+"/api/loadpoints/garage/manual_hold", nil, nil) + case "plan": + _, err = st.planTo("garage", a.SoCPct, a.GoalPct, time.Duration(a.ReadyMin*float64(time.Minute))) + case "restart": + st.stopFTW() + sleepS(a.DownS) + err = st.startFTW() + st.mark("ftw:up") + case "ceiling": + err = st.post("/api/peak_import_ceiling", map[string]any{"peak_import_ceiling_w": a.CeilingW}, nil) + } + if err != nil { + st.mark("ftw:error") + } + } + sleepS(time.Until(t0.Add(time.Duration(p.DurationS * float64(time.Second)))).Seconds()) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "random", "seed": p.Seed, "fuse": p.FuseA}, err +} + +// ---- invariants, every second ---- + +// houseTiming is what the house picked for the charger, from the spec it +// ran (timeline.json for a random house). +type houseTiming struct { + commandS, startupS float64 + floorPct float64 +} + +func (d *runData) timing() houseTiming { + h := houseTiming{floorPct: 5} + for _, name := range []string{"timeline.json", "../house.json"} { + b, err := os.ReadFile(filepath.Join(d.dir, "house", name)) + if err != nil { + continue + } + var spec houseSpec + if json.Unmarshal(b, &spec) != nil { + continue + } + for _, dev := range spec.Devices { + if dev.ID == "garage" { + if dev.CommandResponseS != nil { + h.commandS = *dev.CommandResponseS + } + if dev.StartupDelayS != nil { + h.startupS = *dev.StartupDelayS + } + } + if dev.ID == "inverter" && dev.MinSoCPct != nil { + h.floorPct = *dev.MinSoCPct + } + } + break + } + return h +} + +// eventWindow is when the house changed on its own, and how long FTW may +// take to answer it. +type eventWindow struct{ from, to float64 } + +func judgeInvariants(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + limit := fuse - 1 + h := d.timing() + t0, t1 := d.marks["start"], d.marks["end"] + run := d.window(t0, t1) + m := metrics{"charger_command_s": h.commandS, "charger_startup_s": h.startupS, "events": len(d.events)} + + // Windows in which an over-limit second is the house's doing. + var windows []eventWindow + faultUntil := map[float64]float64{} // a fault's start and end + for _, ev := range d.events { + batteryUsable := true + if r := d.at(ev.T); r != nil && r.Bat.SoCPct <= h.floorPct+1 { + batteryUsable = false + } + // The fastest lever: the battery answers in 2 s, the charger in its + // own time, never under 2 s; then the meter and tick, 2 s, and 3 s. + lever := math.Max(2, h.commandS) + if batteryUsable { + lever = 2 + } + allow := lever + 2 + 3 + switch ev.Event.Do { + case "meter_late", "silence", "offline", "reconnect": + allow += ev.Event.ForS + faultUntil[ev.T] = ev.T + ev.Event.ForS + case "meter_glitch": + faultUntil[ev.T] = ev.T + 5 + } + windows = append(windows, eventWindow{ev.T, ev.T + allow}) + } + for _, mk := range d.allMarks { + if mk.Label == "ftw:restart" { + windows = append(windows, eventWindow{mk.T, mk.T + 60}) + } + } + explained := func(t float64) (float64, bool) { + best, ok := 0.0, false + for _, w := range windows { + if t >= w.from && t <= w.to { + best, ok = math.Max(best, w.to), true + } + } + return best, ok + } + + // 1, 2: the fuse holds; over the limit only as long as an outside step + // allows, and at most 3 s after FTW's own raise. + blownAny := anyBlown(run) + unexplained, longestOwn := 0.0, 0.0 + var firstBad float64 + var start *sample + for i, r := range run { + over := maxAbsI(r) > limit + if over && start == nil { + start = r + } + if (!over || i == len(run)-1) && start != nil { + end := r.T + if to, ok := explained(start.T); ok { + if end > to { + unexplained = math.Max(unexplained, end-to) + if firstBad == 0 { + firstBad = to + } + } + } else if end-start.T > 3 { + longestOwn = math.Max(longestOwn, end-start.T) + if firstBad == 0 { + firstBad = start.T + 3 + } + } + start = nil + } + } + m["over_limit_after_steps_s"], m["over_limit_own_s"] = round(unexplained, 1), round(longestOwn, 1) + + // 3: with every device reading and no fault, Home use is measured. + inFault := func(t float64) bool { + for from, to := range faultUntil { + if t >= from && t <= to+30 { + return true + } + } + return false + } + restartAt := []float64{t0} + for _, mk := range d.allMarks { + if mk.Label == "ftw:up" { + restartAt = append(restartAt, mk.T) + } + } + nearRestart := func(t float64) bool { + for _, r := range restartAt { + if t >= r && t <= r+60 { + return true + } + } + return false + } + first := map[string]float64{} + note := func(name string, t float64) { + if _, ok := first[name]; !ok { + first[name] = t + } + } + if firstBad > 0 { + note("limit", firstBad) + } + for _, r := range run { + if blown(r) { + note("fuse", r.T) + break + } + } + dishonest, run3 := 0.0, 0.0 + for i, r := range run { + bad := r.Status != nil && allRead(r) && !inFault(r.T) && !nearRestart(r.T) && str(status(r).LoadQuality) != "measured" + if bad && i > 0 { + run3 += r.T - run[i-1].T + dishonest = math.Max(dishonest, run3) + if dishonest > 10 { + note("home use", r.T) + } + } else { + run3 = 0 + } + } + m["home_use_estimated_while_reading_s"] = round(dishonest, 1) + + // 4: during Charge now the car charges when there is room for it. + var holds []eventWindow + var holdFrom float64 + for _, mk := range d.allMarks { + switch mk.Label { + case "ftw:charge_now": + if holdFrom == 0 { + holdFrom = mk.T + } + case "ftw:pause", "ftw:clear", "ftw:plan", "end": + if holdFrom > 0 { + holds = append(holds, eventWindow{holdFrom, mk.T}) + holdFrom = 0 + } + } + } + grace := 60 + h.commandS + h.startupS + 10 + idle, run4 := 0.0, 0.0 + for _, hw := range holds { + run4 = 0 + for i, r := range d.window(hw.from+grace, hw.to) { + roomA := limit - (maxAbsI(r) - carW(r, 0)/(3*volt)) + bad := carW(r, 0) < 0.5*1380 && roomA >= 7 && !inFault(r.T) && !nearRestart(r.T) + if bad && i > 0 { + run4++ + idle = math.Max(idle, run4) + if idle > 30 { + note("charge now", r.T) + } + } else { + run4 = 0 + } + } + } + m["charge_now_idle_with_room_s"] = idle + + // 5: no order above the charger's 16 A per phase. + over := 0 + for _, p := range connector1(profiles(d.frames, t0, t1)) { + if p.Limit > 0 && p.perPhaseA() > 16+1e-6 { + over++ + note("rating", t0+p.T) + } + } + m["offers_over_rating"] = over + + // 6: FTW does not ask the battery to discharge at its floor. + floorS := 0.0 + for _, r := range run { + if r.Bat.EMSCmd == 187 && r.Bat.EMSSetW > 50 && r.Bat.SoCPct <= h.floorPct+0.5 { + floorS++ + if floorS > 10 { + note("floor", r.T) + } + } + } + m["discharge_at_floor_s"] = floorS + + // 7: a stale meter means no raise. + raises := 0 + prs := connector1(profiles(d.frames, t0, t1)) + for _, ev := range d.events { + if ev.Event.Do != "meter_late" { + continue + } + for i := 1; i < len(prs); i++ { + at := t0 + prs[i].T + if at > ev.T+2 && at <= ev.T+ev.Event.ForS && prs[i].offerW() > prs[i-1].offerW()+1 { + raises++ + note("stale meter", at) + } + } + } + m["raises_on_a_stale_meter"] = raises + // The first second an invariant broke, from the start, per invariant: + // a kept seed is cut a minute after the one it failed on. + firsts := map[string]float64{} + for k, t := range first { + firsts[k] = round(t-t0, 1) + } + m["first_violation_s"] = firsts + return m, []criterion{ + crit("fuse holds", !blownAny), + crit("over the limit no longer than an outside step allows", unexplained == 0), + crit("over the limit at most 3 s after FTW's own raise", longestOwn == 0), + crit("Home use measured while every device reads", dishonest <= 10), + crit("Charge now charges when there is room", idle <= 30), + crit("no offer above the charger's rating", over == 0), + crit("no discharge asked at the battery's floor", floorS <= 10), + crit("no raise on a stale meter", raises == 0), + } +} + +// ---- seeds, replays and saved cases ---- + +// randomSet parses FTW_SITE=random:: (or random:). +func randomSet(which string) ([]scenario, bool) { + if !strings.HasPrefix(which, "random:") { + return nil, false + } + var first, count int64 = 0, 1 + parts := strings.Split(strings.TrimPrefix(which, "random:"), ":") + fmt.Sscan(parts[0], &first) + if len(parts) > 1 { + fmt.Sscan(parts[1], &count) + } + var out []scenario + for s := first; s < first+count; s++ { + out = append(out, randomScenario(drawPlan(s))) + } + return out, true +} + +// foundScenarios are saved failing seeds (testdata/found/*.json): fixed +// cases in the full set, known until someone triages them. +func foundScenarios(root string) []scenario { + files, _ := filepath.Glob(filepath.Join(root, "go", "test", "site", "testdata", "found", "*.json")) + sort.Strings(files) + var out []scenario + for _, f := range files { + b, err := os.ReadFile(f) + if err != nil { + continue + } + var p randomPlan + if json.Unmarshal(b, &p) == nil && p.House != nil && p.Found != nil { + out = append(out, randomScenario(p)) + } + } + return out +} + +// invariantKey ties each criterion to its first_violation_s entry. +var invariantKey = map[string]string{ + "fuse holds": "fuse", + "over the limit no longer than an outside step allows": "limit", + "over the limit at most 3 s after FTW's own raise": "limit", + "Home use measured while every device reads": "home use", + "Charge now charges when there is room": "charge now", + "no offer above the charger's rating": "rating", + "no discharge asked at the battery's floor": "floor", + "no raise on a stale meter": "stale meter", +} + +// keepFailure writes a failing random run as a case to keep: its plan with +// the house's timeline, both cut a minute after the first violation. +func keepFailure(rundir, out string, o outcome) string { + b, err := os.ReadFile(filepath.Join(rundir, "plan.json")) + if err != nil { + return "" + } + var p randomPlan + if json.Unmarshal(b, &p) != nil { + return "" + } + var h houseSpec + if hb, err := os.ReadFile(filepath.Join(rundir, "house", "timeline.json")); err == nil && json.Unmarshal(hb, &h) == nil { + p.House = &h + } else if p.House == nil { + return "" + } + failed := "" + for _, c := range o.Criteria { + if _, tolerated := randomTolerated[c.Name]; !c.OK && !tolerated { + failed = c.Name + break + } + } + var res struct { + Metrics map[string]any `json:"metrics"` + } + if rb, err := os.ReadFile(filepath.Join(rundir, "result.json")); err == nil && json.Unmarshal(rb, &res) == nil { + firsts, _ := res.Metrics["first_violation_s"].(map[string]any) + at, ok := firsts[invariantKey[failed]].(float64) + if ok && at+60 < p.DurationS { + p.DurationS = math.Ceil(at + 60) + var evs []houseEvent + for _, ev := range p.House.Timeline { + if ev.AtS <= p.DurationS { + evs = append(evs, ev) + } + } + p.House.Timeline = evs + var acts []ftwAction + for _, a := range p.Actions { + if a.AtS <= p.DurationS { + acts = append(acts, a) + } + } + p.Actions = acts + } + } + p.Found = &foundNote{Invariant: failed, On: time.Now().Format("2006-01-02")} + dst := filepath.Join(out, "found", fmt.Sprintf("%d.json", p.Seed)) + os.MkdirAll(filepath.Dir(dst), 0o755) + nb, _ := json.MarshalIndent(p, "", " ") + if os.WriteFile(dst, nb, 0o644) != nil { + return "" + } + return dst +} diff --git a/go/test/site/rig_test.go b/go/test/site/rig_test.go new file mode 100644 index 00000000..35184ffb --- /dev/null +++ b/go/test/site/rig_test.go @@ -0,0 +1,413 @@ +//go:build unix + +// Package site runs the real FTW binary against srcfl/device-simulator: a +// Sungrow hybrid over Modbus, an SDM630 site meter behind an IEC gG fuse on +// each phase, and OCPP 1.6 chargers. Each scenario builds a fresh house and +// a fresh state.db, drives it, records the simulator and FTW once a second +// and judges the run. See README.md for the scenarios and how to run them. +package site + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// rigEnv is what every scenario shares. +type rigEnv struct { + root string // repository root: drivers/ and web/ + ftwBin string + devsimBin string + out string +} + +func repoRoot(t *testing.T) string { + _, file, _, _ := runtime.Caller(0) + dir := filepath.Dir(file) + for i := 0; i < 6; i++ { + if _, err := os.Stat(filepath.Join(dir, "Makefile")); err == nil { + return dir + } + dir = filepath.Dir(dir) + } + t.Fatal("could not locate the repository root") + return "" +} + +// devsimCommit is the device-simulator commit the rig runs, from +// devsim.version. +func devsimCommit(t *testing.T, root string) string { + b, err := os.ReadFile(filepath.Join(root, "go", "test", "site", "devsim.version")) + if err != nil { + t.Fatal(err) + } + c := strings.TrimSpace(string(b)) + if len(c) != 40 { + t.Fatalf("devsim.version: want a full commit hash, got %q", c) + } + return c +} + +func mustRun(t *testing.T, dir string, name string, args ...string) { + t.Helper() + cmd := exec.Command(name, args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) + } +} + +// buildDevsim builds the pinned device-simulator, from FTW_SITE_DEVSIM_SRC +// when set (it must be at the pinned commit) or from a clone in the cache. +func buildDevsim(t *testing.T, root, bin string) { + commit := devsimCommit(t, root) + src := os.Getenv("FTW_SITE_DEVSIM_SRC") + if src == "" { + cache, err := os.UserCacheDir() + if err != nil { + t.Fatal(err) + } + src = filepath.Join(cache, "ftw-site", "device-simulator-"+commit[:12]) + if _, err := os.Stat(filepath.Join(src, "go.mod")); err != nil { + os.RemoveAll(src) + mustRun(t, "", "git", "init", "-q", src) + mustRun(t, src, "git", "fetch", "-q", "--depth=1", "https://github.com/srcfl/device-simulator", commit) + mustRun(t, src, "git", "checkout", "-q", "FETCH_HEAD") + } + } + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = src + head, err := cmd.Output() + if err != nil || strings.TrimSpace(string(head)) != commit { + t.Fatalf("device-simulator in %s is at %s, not the pinned %s", src, strings.TrimSpace(string(head)), commit) + } + cmd = exec.Command("go", "build", "-o", bin, "./cmd/simulator/") + cmd.Dir = src + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build device-simulator: %v\n%s", err, out) + } +} + +func setup(t *testing.T) *rigEnv { + root := repoRoot(t) + for _, f := range []string{"sdm630.lua", "sungrow.lua"} { + if _, err := os.Stat(filepath.Join(root, "drivers", f)); err != nil { + t.Fatalf("drivers/%s missing: run make drivers first", f) + } + } + out := os.Getenv("FTW_SITE_OUT") + if out == "" { + var err error + if out, err = os.MkdirTemp("", "ftw-site-"); err != nil { + t.Fatal(err) + } + } + env := &rigEnv{root: root, out: out, + ftwBin: os.Getenv("FTW_SITE_FTW_BIN"), devsimBin: os.Getenv("FTW_SITE_DEVSIM_BIN")} + bin := filepath.Join(out, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + if env.ftwBin == "" { + env.ftwBin = filepath.Join(bin, "ftw") + mustRun(t, filepath.Join(root, "go"), "go", "build", "-o", env.ftwBin, "./cmd/ftw") + } + if env.devsimBin == "" { + env.devsimBin = filepath.Join(bin, "devsim") + buildDevsim(t, root, env.devsimBin) + } + t.Logf("results in %s", out) + return env +} + +// selected resolves FTW_SITE: "short" (the pull-request set), "full" (every +// scenario and every saved random case), "random::", or +// a comma-separated list of names. +func selected(t *testing.T, which string) []scenario { + if rs, ok := randomSet(which); ok { + return rs + } + var out []scenario + switch which { + case "full", "all": + for _, sc := range scenarios { + if sc.name != "s7s-25" { // the full set runs the long s7 + out = append(out, sc) + } + } + out = append(out, foundScenarios(repoRoot(t))...) + case "short": + for _, sc := range scenarios { + if sc.short { + out = append(out, sc) + } + } + default: + byName := map[string]scenario{} + for _, sc := range scenarios { + byName[sc.name] = sc + } + for _, n := range strings.Split(which, ",") { + sc, ok := byName[strings.TrimSpace(n)] + if !ok { + t.Fatalf("unknown scenario %q", n) + } + out = append(out, sc) + } + } + return out +} + +// shard picks shard k of n (FTW_SITE_SHARD "k/n"). +func shard(t *testing.T, scs []scenario, spec string) []scenario { + if spec == "" { + return scs + } + k, n := 0, 0 + if _, err := fmt.Sscanf(spec, "%d/%d", &k, &n); err != nil || n < 1 || k < 1 || k > n { + t.Fatalf("FTW_SITE_SHARD %q: want k/n", spec) + } + return split(scs, n)[k-1] +} + +// split deals scenarios into n groups of about equal time: longest first, +// each to the group with the least time so far. +func split(scs []scenario, n int) [][]scenario { + sorted := append([]scenario(nil), scs...) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].minutes > sorted[j].minutes }) + load := make([]float64, n) + out := make([][]scenario, n) + for _, sc := range sorted { + best := 0 + for i := range load { + if load[i] < load[best] { + best = i + } + } + load[best] += sc.minutes + out[best] = append(out[best], sc) + } + return out +} + +// outcome is one scenario's line in summary.json. +type outcome struct { + Name string `json:"name"` + Dir string `json:"dir"` + Verdict string `json:"verdict"` // pass, known, fail + DurationS float64 `json:"duration_s"` + Criteria []criterion `json:"criteria"` + Known []string `json:"known,omitempty"` + Failures []string `json:"failures,omitempty"` + Error string `json:"error,omitempty"` +} + +// TestSite runs the scenarios FTW_SITE names. FTW_SITE_LANES runs that many +// at once on ports 100 apart from FTW_SITE_PORT_BASE (default 18700); +// FTW_SITE_SHARD splits a set over machines; FTW_SITE_OUT keeps the run +// directories. +func TestSite(t *testing.T) { + which := os.Getenv("FTW_SITE") + if which == "" { + t.Skip("set FTW_SITE=short, full or a list of scenarios; see go/test/site/README.md") + } + scs := shard(t, selected(t, which), os.Getenv("FTW_SITE_SHARD")) + base := 18700 + if s := os.Getenv("FTW_SITE_PORT_BASE"); s != "" { + var err error + if base, err = strconv.Atoi(s); err != nil { + t.Fatalf("FTW_SITE_PORT_BASE: %v", err) + } + } + lanes := 1 + if s := os.Getenv("FTW_SITE_LANES"); s != "" { + var err error + if lanes, err = strconv.Atoi(s); err != nil || lanes < 1 { + t.Fatalf("FTW_SITE_LANES %q", s) + } + } + env := setup(t) + var mu sync.Mutex + var outcomes []outcome + t.Run("lanes", func(t *testing.T) { + for k, group := range split(scs, lanes) { + k, group := k, group + t.Run(fmt.Sprintf("lane%d", k+1), func(t *testing.T) { + if lanes > 1 { + t.Parallel() + } + for _, sc := range group { + sc := sc + t.Run(sc.name, func(t *testing.T) { + o := runScenario(env, lane(base+100*k), sc) + mu.Lock() + outcomes = append(outcomes, o) + mu.Unlock() + if strings.HasPrefix(sc.dir, "random-") && o.Verdict == "fail" && o.Error == "" { + if kept := keepFailure(filepath.Join(env.out, sc.dir), env.out, o); kept != "" { + t.Logf("kept as %s; copy it to go/test/site/testdata/found/ to run it in the full set", kept) + } + } + t.Logf("%s in %.0f s: %v", o.Verdict, o.DurationS, o.Criteria) + for _, k := range o.Known { + t.Logf("fails as known: %s", k) + } + if o.Error != "" { + t.Fatal(o.Error) + } + for _, f := range o.Failures { + t.Error(f) + } + }) + } + }) + } + }) + sort.Slice(outcomes, func(i, j int) bool { return outcomes[i].Name < outcomes[j].Name }) + writeSummary(t, env.out, outcomes) +} + +func runScenario(env *rigEnv, l lane, sc scenario) outcome { + o := outcome{Name: sc.name, Dir: sc.dir, Verdict: "fail"} + rundir := filepath.Join(env.out, sc.dir) + t0 := time.Now() + meta, err := sc.run(&runCtx{env: env, lane: l, rundir: rundir}) + o.DurationS = time.Since(t0).Round(time.Second).Seconds() + if meta != nil { + b, _ := json.MarshalIndent(meta, "", " ") + os.WriteFile(filepath.Join(rundir, "meta.json"), b, 0o644) + } + if err != nil { + o.Error = err.Error() + return o + } + return judgeRun(sc, rundir, meta, o) +} + +// judgeRun judges a recorded run against the scenario's criteria and its +// known and tolerated lines, and writes result.json. +func judgeRun(sc scenario, rundir string, meta map[string]any, o outcome) outcome { + // Numbers go through JSON, as a rerun from the run directory sees them. + var m map[string]any + b, _ := json.Marshal(meta) + json.Unmarshal(b, &m) + d, err := loadRun(rundir, m) + if err != nil { + o.Error = err.Error() + return o + } + // The criteria count seconds. A host that sleeps or stalls (a laptop + // lid) stretches the run and leaves holes in the record. A Core restart + // costs a sample or two at most: FTW refuses at once or times out in 5 s. + if gap, at := d.longestGap(); gap > 15 { + o.Error = fmt.Sprintf("the recorder missed %.0f s at %.0f s into the run: the host paused, so the timing proves nothing", gap, at) + return o + } + numbers, criteria := sc.judge(d) + o.Criteria = criteria + for _, c := range criteria { + if finding, ok := sc.tolerate[c.Name]; ok { + if !c.OK { + o.Known = append(o.Known, fmt.Sprintf("%s: %s (tolerated)", c.Name, finding)) + } + continue + } + finding, known := sc.known[c.Name] + switch { + case known && c.OK: + o.Failures = append(o.Failures, fmt.Sprintf("%q passes now: delete its known line (%s)", c.Name, finding)) + case known: + o.Known = append(o.Known, fmt.Sprintf("%s: %s", c.Name, finding)) + case !c.OK: + o.Failures = append(o.Failures, fmt.Sprintf("%q fails", c.Name)) + } + } + for name := range sc.known { + found := false + for _, c := range criteria { + found = found || c.Name == name + } + if !found { + o.Failures = append(o.Failures, fmt.Sprintf("known line names no criterion: %q", name)) + } + } + switch { + case len(o.Failures) > 0: + o.Verdict = "fail" + case len(o.Known) > 0: + o.Verdict = "known" + default: + o.Verdict = "pass" + } + res, _ := json.MarshalIndent(map[string]any{"verdict": o.Verdict, "criteria": criteria, "known": o.Known, + "failures": o.Failures, "duration_s": o.DurationS, "metrics": numbers, "meta": meta, "samples": len(d.samples)}, "", " ") + os.WriteFile(filepath.Join(rundir, "result.json"), res, 0o644) + return o +} + +// TestJudge judges the recorded runs in FTW_SITE_JUDGE again with the +// criteria as they are now, without running anything: a changed criterion +// or known line is checked against a long run in seconds. +func TestJudge(t *testing.T) { + dir := os.Getenv("FTW_SITE_JUDGE") + if dir == "" { + t.Skip("set FTW_SITE_JUDGE to a results directory of an earlier run") + } + var outs []outcome + for _, sc := range selected(t, "full") { + rundir := filepath.Join(dir, sc.dir) + b, err := os.ReadFile(filepath.Join(rundir, "meta.json")) + if err != nil { + continue + } + var meta map[string]any + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatalf("%s: %v", rundir, err) + } + o := outcome{Name: sc.name, Dir: sc.dir, Verdict: "fail"} + if b, err := os.ReadFile(filepath.Join(rundir, "result.json")); err == nil { + var prev struct { + DurationS float64 `json:"duration_s"` + } + json.Unmarshal(b, &prev) + o.DurationS = prev.DurationS + } + o = judgeRun(sc, rundir, meta, o) + t.Logf("%s: %s %v", sc.name, o.Verdict, append(o.Failures, o.Known...)) + outs = append(outs, o) + } + if len(outs) == 0 { + t.Fatalf("no recorded run of a scenario in %s", dir) + } + writeSummary(t, dir, outs) +} + +func writeSummary(t *testing.T, out string, outs []outcome) { + if len(outs) == 0 { + return + } + b, _ := json.MarshalIndent(outs, "", " ") + os.WriteFile(filepath.Join(out, "summary.json"), b, 0o644) + var md strings.Builder + md.WriteString("| Scenario | Verdict | Time | Failed or known |\n|---|---|---|---|\n") + for _, o := range outs { + notes := append(append([]string{}, o.Failures...), o.Known...) + if o.Error != "" { + notes = append([]string{o.Error}, notes...) + } + fmt.Fprintf(&md, "| %s | %s | %s | %s |\n", o.Name, o.Verdict, + (time.Duration(o.DurationS) * time.Second).String(), strings.ReplaceAll(strings.Join(notes, "; "), "|", "/")) + } + os.WriteFile(filepath.Join(out, "summary.md"), []byte(md.String()), 0o644) + t.Logf("summary:\n%s", md.String()) +} diff --git a/go/test/site/rig_unit_test.go b/go/test/site/rig_unit_test.go new file mode 100644 index 00000000..7e921c2a --- /dev/null +++ b/go/test/site/rig_unit_test.go @@ -0,0 +1,141 @@ +//go:build unix + +package site + +import ( + "encoding/json" + "testing" +) + +// These run in every go test, without a simulator. + +func TestShardsCoverEveryScenarioOnce(t *testing.T) { + for _, set := range []string{"short", "full"} { + all := selected(t, set) + for n := 1; n <= 7; n++ { + seen := map[string]int{} + for k := 1; k <= n; k++ { + for _, sc := range shard(t, all, fmtShard(k, n)) { + seen[sc.name]++ + } + } + for _, sc := range all { + if seen[sc.name] != 1 { + t.Fatalf("%s in %d shards: %s runs %d times", set, n, sc.name, seen[sc.name]) + } + } + } + } +} + +func fmtShard(k, n int) string { return string(rune('0'+k)) + "/" + string(rune('0'+n)) } + +func TestFullSetRunsTheLongRestart(t *testing.T) { + names := map[string]bool{} + for _, sc := range selected(t, "full") { + names[sc.name] = true + } + if !names["s7-25"] || names["s7s-25"] || len(names) != len(scenarios)-1 { + t.Fatalf("full set %v", names) + } +} + +// A known line must name a criterion its judge reports. +func TestKnownLinesNameCriteria(t *testing.T) { + for _, sc := range scenarios { + _, criteria := sc.judge(&runData{marks: map[string]float64{}, meta: map[string]any{"fuse": 25.0}}) + have := map[string]bool{} + for _, c := range criteria { + have[c.Name] = true + } + for name := range sc.known { + if !have[name] { + t.Errorf("%s: known line names no criterion: %q", sc.name, name) + } + } + } + _, criteria := judgeInvariants(&runData{marks: map[string]float64{}, meta: map[string]any{"fuse": 25.0}}) + have := map[string]bool{} + for _, c := range criteria { + have[c.Name] = true + if invariantKey[c.Name] == "" { + t.Errorf("invariant %q has no first_violation_s key", c.Name) + } + } + for name := range randomTolerated { + if !have[name] { + t.Errorf("tolerated line names no invariant: %q", name) + } + } +} + +func TestRunsOver(t *testing.T) { + d := &runData{} + for i, a := range []float64{20, 25, 26, 26, 20, 25, 20} { + d.samples = append(d.samples, &sample{T: float64(i), PhaseA: [3]float64{a, 0, 0}}) + } + // As the Python rig counted: a run lasts until the first sample under + // the limit, and each sample over it adds the time since the last one. + longest, total := d.runsOver(0, 10, 24) + if longest != 3 || total != 4 { + t.Fatalf("longest %v total %v, want 3 and 4", longest, total) + } +} + +func TestRandomPlansAreSeeded(t *testing.T) { + a, b, c := drawPlan(3), drawPlan(3), drawPlan(4) + ja, _ := json.Marshal(a) + jb, _ := json.Marshal(b) + jc, _ := json.Marshal(c) + if string(ja) != string(jb) || string(ja) == string(jc) { + t.Fatal("seed 3 twice differs, or seed 4 is the same") + } + for seed := int64(1); seed <= 30; seed++ { + p := drawPlan(seed) + for _, a := range p.Actions { + if a.AtS < 40 || a.AtS > p.DurationS-60 { + t.Fatalf("seed %d: %s at %.0f s", seed, a.Do, a.AtS) + } + } + for _, x := range p.HouseA { + if x > p.FuseA/2 { + t.Fatalf("seed %d: house %.1f A on %.0f A", seed, x, p.FuseA) + } + } + } +} + +// An over-limit stretch right after an outside step is the house's; the +// same stretch with no step before it is FTW's own and fails after 3 s. +func TestInvariantsTellStepsFromOwnRaises(t *testing.T) { + mk := func(withStep bool) *runData { + d := &runData{marks: map[string]float64{"start": 0, "end": 100}, meta: map[string]any{"fuse": 25.0}} + for i := 0; i <= 100; i++ { + a := 10.0 + if i >= 50 && i < 56 { + a = 26 // 6 s over the 24 A limit + } + d.samples = append(d.samples, &sample{T: float64(i), PhaseA: [3]float64{a, a, a}, Bat: plantDevice{SoCPct: 50}}) + } + if withStep { + d.events = append(d.events, houseEventLine{T: 49.5, Event: houseEvent{Do: "loads"}}) + } + return d + } + failed := func(cs []criterion) []string { + var out []string + for _, c := range cs { + if !c.OK { + out = append(out, c.Name) + } + } + return out + } + if _, cs := judgeInvariants(mk(true)); len(failed(cs)) != 0 { + t.Fatalf("after a step: %v", failed(cs)) + } + _, cs := judgeInvariants(mk(false)) + if f := failed(cs); len(f) != 1 || f[0] != "over the limit at most 3 s after FTW's own raise" { + t.Fatalf("without a step: %v", f) + } +} diff --git a/go/test/site/runner/README.md b/go/test/site/runner/README.md new file mode 100644 index 00000000..346d7747 --- /dev/null +++ b/go/test/site/runner/README.md @@ -0,0 +1,78 @@ +# siterig-runner + +Runs FTW's site test (`go/test/site`) on a machine of its own and publishes +the result as a GitHub commit status on the exact commit it tested: + +| Status | When | What runs | +|---|---|---| +| `siterig/full` | each new commit on `master` | every scenario | +| `siterig/short` | the head of each open pull request from this repository that touches `go/internal/{control,loadpoint,ocpp,telemetry}` or `go/test/site` | the short set | +| `siterig/random` | once a night on `master` | that night's random seeds | + +A commit that already has the status is not run again. To run one again, +post a new `pending` status or remove the result and wait a day. No GitHub +Actions workflow runs the rig; `native-release.yml` only reads +`siterig/full`. + +The description says how it went, for example `16 pass, 3 known, 0 fail of +19 in 24m`. The run directories stay in `~/siterig/results` for 14 days. + +## The machine + +Any Mac or Linux host with Go, git, make and the `gh` CLI that can stay +awake. A laptop works if it does not sleep: a sleeping host stretches the +rig's timing, and the rig then fails the run as "host paused". Not the home +energy box: the rig needs the CPU, and the box is production. + +The rig uses 100 ports per lane from `-port-base` (default 18700) on +127.0.0.1. Leave them free. + +Pull requests from forks are skipped unless `-allow-forks` is set: the rig +builds and runs the pull request's code on this machine. + +## Install + +```bash +gh auth login # an account that may write commit statuses on srcfl/ftw +mkdir -p ~/siterig/bin +git clone https://github.com/srcfl/ftw ~/siterig/src +cd ~/siterig/src/go && go build -o ~/siterig/bin/siterig-runner ./test/site/runner +~/siterig/bin/siterig-runner -once # one pass in the foreground +``` + +macOS (launchd): + +```bash +sed "s/USER/$USER/g" ~/siterig/src/go/test/site/runner/energy.ftw.siterig-runner.plist \ + > ~/Library/LaunchAgents/energy.ftw.siterig-runner.plist +launchctl load ~/Library/LaunchAgents/energy.ftw.siterig-runner.plist +``` + +Linux (systemd): edit `siterig-runner.service`, copy it to +`/etc/systemd/system/` and run `sudo systemctl enable --now siterig-runner`. + +The log is `~/siterig/runner.log` (launchd) or `journalctl -u +siterig-runner` (systemd). Rebuild the binary after a change to this +directory; the scenarios themselves always come from the commit under test. + +## How long it takes + +On an M5 MacBook (10 cores), 22 October 2026: the 23 scenarios of that +morning took 72 minutes in three lanes. With the one-phase charger cases the +set adds up to about 270 minutes, so three lanes need about 90 and four +about 70; `s12-20` alone takes up to 56, so more than four do not help. +Each lane is one `ftw` and one `simulator run`, both mostly idle. + +## Flags + +| Flag | Default | | +|---|---|---| +| `-repo` | `srcfl/ftw` | | +| `-work` | `~/siterig` | clone, worktrees, results, `last-random` | +| `-every` | `5m` | how often to look for new commits | +| `-lanes` | `3` | scenarios at once | +| `-port-base` | `18700` | first port | +| `-random-at`, `-random-seeds` | `2`, `10` | local hour and count of the nightly random seeds; `-random-at -1` turns them off | +| `-drafts` | `true` | also run draft pull requests | +| `-allow-forks` | `false` | also run forks' pull requests | +| `-once` | | one pass, then exit | diff --git a/go/test/site/runner/energy.ftw.siterig-runner.plist b/go/test/site/runner/energy.ftw.siterig-runner.plist new file mode 100644 index 00000000..10f4604c --- /dev/null +++ b/go/test/site/runner/energy.ftw.siterig-runner.plist @@ -0,0 +1,24 @@ + + + + + + Labelenergy.ftw.siterig-runner + ProgramArguments + + /Users/USER/siterig/bin/siterig-runner + -work/Users/USER/siterig + -lanes3 + + EnvironmentVariables + + PATH/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin + HOME/Users/USER + + RunAtLoad + KeepAlive + StandardOutPath/Users/USER/siterig/runner.log + StandardErrorPath/Users/USER/siterig/runner.log + + diff --git a/go/test/site/runner/main.go b/go/test/site/runner/main.go new file mode 100644 index 00000000..6878257c --- /dev/null +++ b/go/test/site/runner/main.go @@ -0,0 +1,232 @@ +// siterig-runner runs FTW's site test (go/test/site) on a machine of its own +// and publishes the result as a GitHub commit status on the exact commit: +// siterig/full on each new master commit, siterig/short on the head of each +// open pull request that touches control, loadpoint, OCPP or telemetry code, +// and siterig/random for a night's random seeds. native-release.yml reads +// siterig/full before it publishes a beta. See README.md. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" +) + +func main() { + var cfg config + flag.StringVar(&cfg.Repo, "repo", "srcfl/ftw", "GitHub repository") + work := flag.String("work", filepath.Join(os.Getenv("HOME"), "siterig"), "clone, worktrees and results") + every := flag.Duration("every", 5*time.Minute, "how often to look for new commits") + lanes := flag.Int("lanes", 3, "scenarios run at once") + portBase := flag.Int("port-base", 18700, "first port; lanes use 100 each") + keep := flag.Duration("keep", 14*24*time.Hour, "how long to keep results") + flag.BoolVar(&cfg.AllowForks, "allow-forks", false, "also run pull requests from forks (their code runs here)") + flag.BoolVar(&cfg.Drafts, "drafts", true, "also run draft pull requests") + flag.IntVar(&cfg.RandomAt, "random-at", 2, "local hour of the nightly random seeds; -1 never") + flag.IntVar(&cfg.RandomSeeds, "random-seeds", 10, "random seeds a night") + once := flag.Bool("once", false, "run what is due and exit") + flag.Parse() + + r := &runner{cfg: cfg, gh: ghCLI{repo: cfg.Repo}, work: *work, lanes: *lanes, portBase: *portBase, keep: *keep} + if err := r.prepare(); err != nil { + log.Fatal(err) + } + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + for { + r.tick() + if *once { + return + } + select { + case <-stop: + return + case <-time.After(*every): + } + } +} + +type runner struct { + cfg config + gh github + work string + lanes int + portBase int + keep time.Duration +} + +func (r *runner) clone() string { return filepath.Join(r.work, "ftw") } +func (r *runner) results() string { return filepath.Join(r.work, "results") } + +// prepare clones the repository once. +func (r *runner) prepare() error { + if err := os.MkdirAll(r.results(), 0o755); err != nil { + return err + } + if _, err := os.Stat(filepath.Join(r.clone(), ".git")); err == nil { + return nil + } + return run(r.work, "git", "clone", "--quiet", "https://github.com/"+r.cfg.Repo, r.clone()) +} + +// tick runs everything that is due, one job at a time. +func (r *runner) tick() { + stateFile := filepath.Join(r.work, "last-random") + last, _ := os.ReadFile(stateFile) + jobs, err := plan(r.gh, r.cfg, time.Now(), strings.TrimSpace(string(last))) + if err != nil { + log.Printf("plan: %v", err) + return + } + for _, j := range jobs { + log.Printf("start %s", j) + state, desc := r.runJob(j) + log.Printf("done %s: %s, %s", j, state, desc) + if err := r.gh.SetStatus(j.SHA, j.Context, state, desc); err != nil { + log.Printf("status: %v", err) + } + if j.Context == ctxRandom { + os.WriteFile(stateFile, []byte(time.Now().Format("2006-01-02")), 0o644) + } + } + r.prune() +} + +// runJob checks out the commit in a worktree of its own and runs the rig. +func (r *runner) runJob(j job) (state, desc string) { + if err := r.gh.SetStatus(j.SHA, j.Context, "pending", "running on "+hostname()); err != nil { + log.Printf("status: %v", err) + } + stamp := time.Now().Format("20060102-150405") + out := filepath.Join(r.results(), fmt.Sprintf("%s-%s-%s", stamp, short(j.SHA), strings.TrimPrefix(j.Context, "siterig/"))) + wt := filepath.Join(r.work, "wt-"+short(j.SHA)+"-"+stamp) + if err := os.MkdirAll(out, 0o755); err != nil { + return "error", err.Error() + } + defer func() { + run(r.clone(), "git", "worktree", "remove", "--force", wt) + run(r.clone(), "git", "worktree", "prune") + }() + fetch := []string{"fetch", "--quiet", "origin", "master"} + if j.PR > 0 { + fetch = append(fetch, fmt.Sprintf("refs/pull/%d/head", j.PR)) + } + if err := run(r.clone(), "git", fetch...); err != nil { + return "error", "git fetch failed" + } + if err := run(r.clone(), "git", "worktree", "add", "--detach", wt, j.SHA); err != nil { + return "error", "commit not found" + } + log1, _ := os.Create(filepath.Join(out, "rig.log")) + defer log1.Close() + mk := exec.Command("make", "drivers") + mk.Dir, mk.Stdout, mk.Stderr = wt, log1, log1 + if err := mk.Run(); err != nil { + return "error", "make drivers failed" + } + t0 := time.Now() + test := exec.Command("go", "test", "-count=1", "-timeout", "300m", "-run", "TestSite$", "-v", "./test/site/") + test.Dir, test.Stdout, test.Stderr = filepath.Join(wt, "go"), log1, log1 + test.Env = append(os.Environ(), "FTW_SITE="+j.Set, fmt.Sprintf("FTW_SITE_LANES=%d", r.lanes), + fmt.Sprintf("FTW_SITE_PORT_BASE=%d", r.portBase), "FTW_SITE_OUT="+out) + test.Run() // the summary, not the exit code, says what passed + summary, _ := os.ReadFile(filepath.Join(out, "summary.json")) + state, desc = describe(summary, time.Since(t0)) + if state == "success" && test.ProcessState != nil && !test.ProcessState.Success() { + state, desc = "failure", "the rig failed outside the scenarios: "+desc + } + return state, desc +} + +// prune drops results older than keep. +func (r *runner) prune() { + entries, _ := os.ReadDir(r.results()) + for _, e := range entries { + if info, err := e.Info(); err == nil && time.Since(info.ModTime()) > r.keep { + os.RemoveAll(filepath.Join(r.results(), e.Name())) + } + } +} + +func run(dir, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("%s %s: %v: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return nil +} + +func hostname() string { + h, _ := os.Hostname() + return h +} + +// ghCLI asks GitHub through the gh CLI, as the machine's logged-in user. +type ghCLI struct{ repo string } + +func (g ghCLI) api(out any, args ...string) error { + b, err := exec.Command("gh", append([]string{"api"}, args...)...).Output() + if err != nil { + return fmt.Errorf("gh api %s: %w", strings.Join(args, " "), err) + } + if out == nil { + return nil + } + return json.Unmarshal(b, out) +} + +func (g ghCLI) MasterSHA() (string, error) { + var c struct { + SHA string `json:"sha"` + } + err := g.api(&c, "repos/"+g.repo+"/commits/master") + return c.SHA, err +} + +func (g ghCLI) Pulls() ([]pull, error) { + var ps []pull + err := g.api(&ps, "repos/"+g.repo+"/pulls?state=open&per_page=100") + return ps, err +} + +func (g ghCLI) Files(pr int) ([]string, error) { + var fs []struct { + Filename string `json:"filename"` + } + if err := g.api(&fs, fmt.Sprintf("repos/%s/pulls/%d/files?per_page=100", g.repo, pr)); err != nil { + return nil, err + } + out := make([]string, len(fs)) + for i, f := range fs { + out[i] = f.Filename + } + return out, nil +} + +func (g ghCLI) Contexts(sha string) (map[string]bool, error) { + var ss []struct { + Context string `json:"context"` + } + if err := g.api(&ss, "repos/"+g.repo+"/commits/"+sha+"/statuses?per_page=100"); err != nil { + return nil, err + } + have := map[string]bool{} + for _, s := range ss { + have[s.Context] = true + } + return have, nil +} + +func (g ghCLI) SetStatus(sha, context, state, desc string) error { + return g.api(nil, "-X", "POST", "repos/"+g.repo+"/statuses/"+sha, + "-f", "state="+state, "-f", "context="+context, "-f", "description="+desc) +} diff --git a/go/test/site/runner/runner.go b/go/test/site/runner/runner.go new file mode 100644 index 00000000..1a0def0c --- /dev/null +++ b/go/test/site/runner/runner.go @@ -0,0 +1,173 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "time" +) + +// Status contexts the runner publishes on a commit. +const ( + ctxShort = "siterig/short" + ctxFull = "siterig/full" + ctxRandom = "siterig/random" +) + +// watchedPaths are the parts of the tree whose changes get the short set on +// a pull request. +var watchedPaths = []string{ + "go/internal/control/", "go/internal/loadpoint/", "go/internal/ocpp/", "go/internal/telemetry/", + "go/test/site/", +} + +// job is one run of the rig on one commit. +type job struct { + Context string // which status it publishes + SHA string + Set string // FTW_SITE: short, full or random:: + PR int // 0 for master +} + +func (j job) String() string { + if j.PR > 0 { + return fmt.Sprintf("%s on #%d %s", j.Context, j.PR, short(j.SHA)) + } + return fmt.Sprintf("%s on master %s", j.Context, short(j.SHA)) +} + +func short(sha string) string { + if len(sha) > 8 { + return sha[:8] + } + return sha +} + +// pull is the part of a pull request the runner needs. +type pull struct { + Number int `json:"number"` + Draft bool `json:"draft"` + Head struct { + SHA string `json:"sha"` + Repo struct { + FullName string `json:"full_name"` + } `json:"repo"` + } `json:"head"` +} + +// github is what the runner asks GitHub; gh api serves it in production. +type github interface { + MasterSHA() (string, error) + Pulls() ([]pull, error) + Files(pr int) ([]string, error) + Contexts(sha string) (map[string]bool, error) // status contexts already on a commit + SetStatus(sha, context, state, description string) error +} + +// config is the runner's flags. +type config struct { + Repo string + AllowForks bool // a fork's code runs on this machine; off by default + Drafts bool + RandomAt int // local hour of the nightly random run; -1 never + RandomSeeds int +} + +// plan lists what to run now, master first. A commit that already carries +// the context, in any state, is not run again. Random seeds run once a day +// (lastRandom is the day they last ran) on whatever master is then. +func plan(gh github, cfg config, now time.Time, lastRandom string) ([]job, error) { + var jobs []job + master, err := gh.MasterSHA() + if err != nil { + return nil, err + } + have, err := gh.Contexts(master) + if err != nil { + return nil, err + } + if !have[ctxFull] { + jobs = append(jobs, job{Context: ctxFull, SHA: master, Set: "full"}) + } + pulls, err := gh.Pulls() + if err != nil { + return nil, err + } + for _, p := range pulls { + if p.Draft && !cfg.Drafts { + continue + } + if p.Head.Repo.FullName != cfg.Repo && !cfg.AllowForks { + continue + } + have, err := gh.Contexts(p.Head.SHA) + if err != nil { + return nil, err + } + if have[ctxShort] { + continue + } + files, err := gh.Files(p.Number) + if err != nil { + return nil, err + } + if touchesWatched(files) { + jobs = append(jobs, job{Context: ctxShort, SHA: p.Head.SHA, Set: "short", PR: p.Number}) + } + } + day := now.Format("2006-01-02") + if cfg.RandomAt >= 0 && cfg.RandomSeeds > 0 && now.Hour() >= cfg.RandomAt && lastRandom != day { + // One seed range per day, so each night tries new timelines. + first := int(now.Unix()/86400) * cfg.RandomSeeds + jobs = append(jobs, job{Context: ctxRandom, SHA: master, Set: fmt.Sprintf("random:%d:%d", first, cfg.RandomSeeds)}) + } + return jobs, nil +} + +func touchesWatched(files []string) bool { + for _, f := range files { + for _, p := range watchedPaths { + if strings.HasPrefix(f, p) { + return true + } + } + } + return false +} + +// outcome is one line of the rig's summary.json. +type outcome struct { + Name string `json:"name"` + Verdict string `json:"verdict"` // pass, known, fail + DurationS float64 `json:"duration_s"` + Error string `json:"error"` +} + +// describe turns summary.json into a status: success when nothing failed, +// with counts that fit GitHub's 140 characters. +func describe(summary []byte, took time.Duration) (state, desc string) { + var outs []outcome + if err := json.Unmarshal(summary, &outs); err != nil || len(outs) == 0 { + return "error", "no summary: the rig did not finish (see the runner's log)" + } + n := map[string]int{} + var failed []string + for _, o := range outs { + n[o.Verdict]++ + if o.Verdict == "fail" { + failed = append(failed, o.Name) + } + } + state = "success" + if len(failed) > 0 { + state = "failure" + } + desc = fmt.Sprintf("%d pass, %d known, %d fail of %d in %s", n["pass"], n["known"], n["fail"], len(outs), took.Round(time.Minute)) + if len(failed) > 0 { + desc += ": " + strings.Join(failed, " ") + } + if len(desc) > 140 { + desc = desc[:137] + "..." + } + return state, desc +} diff --git a/go/test/site/runner/runner_test.go b/go/test/site/runner/runner_test.go new file mode 100644 index 00000000..d05a1dd3 --- /dev/null +++ b/go/test/site/runner/runner_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +type fakeGitHub struct { + master string + pulls []pull + files map[int][]string + contexts map[string]map[string]bool +} + +func (f fakeGitHub) MasterSHA() (string, error) { return f.master, nil } +func (f fakeGitHub) Pulls() ([]pull, error) { return f.pulls, nil } +func (f fakeGitHub) Files(pr int) ([]string, error) { return f.files[pr], nil } +func (f fakeGitHub) Contexts(sha string) (map[string]bool, error) { + return f.contexts[sha], nil +} +func (f fakeGitHub) SetStatus(sha, context, state, description string) error { return nil } + +func pr(n int, sha, repo string, draft bool) pull { + p := pull{Number: n, Draft: draft} + p.Head.SHA, p.Head.Repo.FullName = sha, repo + return p +} + +func TestPlanRunsWhatIsDue(t *testing.T) { + gh := fakeGitHub{ + master: "m1", + pulls: []pull{ + pr(1, "a1", "srcfl/ftw", false), // control code: short set + pr(2, "b1", "srcfl/ftw", false), // docs only: nothing + pr(3, "c1", "someone/ftw", false), // a fork: never by default + pr(4, "d1", "srcfl/ftw", false), // already has its status + }, + files: map[int][]string{ + 1: {"go/internal/control/dispatch.go"}, 2: {"docs/x.md"}, + 3: {"go/internal/ocpp/handlers.go"}, 4: {"go/internal/loadpoint/x.go"}, + }, + contexts: map[string]map[string]bool{"d1": {ctxShort: true}}, + } + cfg := config{Repo: "srcfl/ftw", Drafts: true, RandomAt: 2, RandomSeeds: 10} + night := time.Date(2026, 10, 23, 3, 0, 0, 0, time.Local) + jobs, err := plan(gh, cfg, night, "") + if err != nil { + t.Fatal(err) + } + var got []string + for _, j := range jobs { + got = append(got, j.String()+" "+j.Set) + } + want := []string{"siterig/full on master m1 full", "siterig/short on #1 a1 short", "siterig/random on master m1 random:"} + if len(got) != len(want) { + t.Fatalf("jobs %v, want %v", got, want) + } + for i := range want { + if !strings.HasPrefix(got[i], want[i]) { + t.Fatalf("job %d: %q, want %q", i, got[i], want[i]) + } + } + + // Master already tested, random already run today, before the hour. + gh.contexts["m1"] = map[string]bool{ctxFull: true} + jobs, _ = plan(gh, cfg, night, "2026-10-23") + if len(jobs) != 1 || jobs[0].PR != 1 { + t.Fatalf("second look: %v", jobs) + } + cfg.AllowForks = true + jobs, _ = plan(gh, cfg, night.Add(-2*time.Hour), "") + if len(jobs) != 2 || jobs[1].PR != 3 { + t.Fatalf("with forks allowed, before the random hour: %v", jobs) + } +} + +func TestDescribe(t *testing.T) { + state, desc := describe([]byte(`[{"name":"s1-25","verdict":"pass"},{"name":"s2-16","verdict":"known"},{"name":"s9-25","verdict":"fail"}]`), 24*time.Minute) + if state != "failure" || desc != "1 pass, 1 known, 1 fail of 3 in 24m0s: s9-25" { + t.Fatalf("%s %q", state, desc) + } + state, _ = describe([]byte(`[{"name":"s1-25","verdict":"pass"},{"name":"s2-16","verdict":"known"}]`), time.Minute) + if state != "success" { + t.Fatalf("known only: %s", state) + } + if state, _ := describe(nil, time.Minute); state != "error" { + t.Fatalf("no summary: %s", state) + } + long := `[` + strings.Repeat(`{"name":"a-very-long-scenario-name","verdict":"fail"},`, 9) + `{"name":"x","verdict":"fail"}]` + if _, desc := describe([]byte(long), time.Minute); len(desc) > 140 { + t.Fatalf("%d characters", len(desc)) + } +} diff --git a/go/test/site/runner/siterig-runner.service b/go/test/site/runner/siterig-runner.service new file mode 100644 index 00000000..f287a597 --- /dev/null +++ b/go/test/site/runner/siterig-runner.service @@ -0,0 +1,18 @@ +# systemd unit for siterig-runner (README.md). Replace USER and the paths, +# then: sudo cp siterig-runner.service /etc/systemd/system/ && +# sudo systemctl enable --now siterig-runner +[Unit] +Description=FTW site test runner (siterig/short, siterig/full commit statuses) +After=network-online.target +Wants=network-online.target + +[Service] +User=USER +Environment=HOME=/home/USER +Environment=PATH=/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin +ExecStart=/home/USER/siterig/bin/siterig-runner -work /home/USER/siterig -lanes 3 +Restart=always +RestartSec=60 + +[Install] +WantedBy=multi-user.target diff --git a/go/test/site/scenarios_test.go b/go/test/site/scenarios_test.go new file mode 100644 index 00000000..4c42c9c6 --- /dev/null +++ b/go/test/site/scenarios_test.go @@ -0,0 +1,1026 @@ +//go:build unix + +package site + +import ( + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "time" +) + +// scenario is one house and what happens in it. known names the criteria +// that fail on master and the finding behind each: such a criterion must +// keep failing, and once a fix makes it pass the run fails until the fix +// deletes its line. Criteria come from reports/simulatorprov-20261021. +type scenario struct { + name string // what FTW_SITE names it by + dir string // its run directory + minutes float64 // its run time on a GitHub runner, for sharding + short bool // in the pull-request set + run func(c *runCtx) (map[string]any, error) + judge func(d *runData) (metrics, []criterion) + known map[string]string + // tolerate names criteria that may fail for a known finding but need + // not: random cases, where a seed may or may not reach it. + tolerate map[string]string +} + +var scenarios = []scenario{ + {name: "s0-25", dir: "s0-idle-charger-25A", minutes: 5.1, short: true, + run: func(c *runCtx) (map[string]any, error) { return s0(c, 25, 60) }, judge: judgeS0}, + {name: "s0b-25", dir: "s0b-idle-charger-default-hb-25A", minutes: 5.1, + run: func(c *runCtx) (map[string]any, error) { return s0(c, 25, 0) }, judge: judgeS0}, + {name: "s1-16", dir: "s1-chargenow-16A", minutes: 3.6, short: true, + run: func(c *runCtx) (map[string]any, error) { return s1(c, 16) }, judge: judgeS1}, + {name: "s1-25", dir: "s1-chargenow-25A", minutes: 3.6, short: true, + run: func(c *runCtx) (map[string]any, error) { return s1(c, 25) }, judge: judgeS1}, + {name: "s2-16", dir: "s2-export-16A", minutes: 3.1, + run: func(c *runCtx) (map[string]any, error) { return s2(c, 16) }, judge: judgeS2, + known: map[string]string{"car starts within 60 s": "export counted as import (review fuse finding 1), next beta"}}, + {name: "s3-16", dir: "s3-step-16A", minutes: 6.7, short: true, + run: func(c *runCtx) (map[string]any, error) { return s3(c, 16) }, judge: judgeS3}, + {name: "s3-25", dir: "s3-step-25A", minutes: 6.7, short: true, + run: func(c *runCtx) (map[string]any, error) { return s3(c, 25) }, judge: judgeS3}, + {name: "s4-25", dir: "s4-unlock-25A", minutes: 3.6, + run: func(c *runCtx) (map[string]any, error) { return s4(c, 25, "3p") }, judge: judgeS4}, + {name: "s4b-25", dir: "s4b-unlock-nophase-25A", minutes: 3.6, + run: func(c *runCtx) (map[string]any, error) { return s4(c, 25, "") }, judge: judgeS4}, + {name: "s5-25", dir: "s5-surplus-auto-25A", minutes: 5.2, + run: func(c *runCtx) (map[string]any, error) { return s5(c, 25, nil) }, judge: judgeS5, + known: map[string]string{"charges at least the minimum 90 % of the time": "surplus without allowed_steps_w never starts, loadpoint/controller.go surplus3PhaseSteps; next beta"}}, + {name: "s5s-25", dir: "s5s-surplus-auto-steps-25A", minutes: 5.2, + run: func(c *runCtx) (map[string]any, error) { return s5(c, 25, easeeSteps) }, judge: judgeS5, + known: map[string]string{"charges at least the minimum 90 % of the time": "phase mode auto stays on 1p under 9 kW surplus (R1); next beta"}}, + {name: "s6-25", dir: "s6-inverter-offline-25A", minutes: 7.7, + run: func(c *runCtx) (map[string]any, error) { return s6(c, 25) }, judge: judgeS6}, + {name: "s7-25", dir: "s7-restart-25A", minutes: 23, + run: func(c *runCtx) (map[string]any, error) { return s7(c, 25, false) }, judge: judgeS7}, + // The full s7 runs 15 min before the restart, because the day's Home + // use may have at most 10 % unknown time. The short one restarts after + // a minute and leaves the day total out. + {name: "s7s-25", dir: "s7s-restart-short-25A", minutes: 9, short: true, + run: func(c *runCtx) (map[string]any, error) { return s7(c, 25, true) }, judge: judgeS7}, + {name: "s8-60-25", dir: "s8-slow60-25A", minutes: 7.5, short: true, + run: func(c *runCtx) (map[string]any, error) { return s8(c, 25, 60) }, judge: judgeS8}, + {name: "s8-120-25", dir: "s8-slow120-25A", minutes: 12.5, + run: func(c *runCtx) (map[string]any, error) { return s8(c, 25, 120) }, judge: judgeS8}, + {name: "s9-25", dir: "s9-phase-switch-25A", minutes: 4.7, + run: func(c *runCtx) (map[string]any, error) { return s9(c, 25, 7360) }, judge: judgeS9}, + {name: "s10-25", dir: "s10-two-chargers-25A", minutes: 7.9, + run: func(c *runCtx) (map[string]any, error) { return s10(c, 25, 60) }, judge: judgeS10}, + {name: "s11-25", dir: "s11-pause-restart-25A", minutes: 5.1, + run: func(c *runCtx) (map[string]any, error) { return s11(c, 25) }, judge: judgeS11}, + {name: "s9r-25", dir: "s9r-house-on-l2-25A", minutes: 5, + run: func(c *runCtx) (map[string]any, error) { return s9r(c, 25, 7360) }, judge: judgeS9}, + {name: "s5t-25", dir: "s5t-surplus-goal-25A", minutes: 6, + run: func(c *runCtx) (map[string]any, error) { return s5t(c, 25) }, judge: judgeS5t}, + {name: "s12-20", dir: "s12-one-phase-night-20A", minutes: 57, + run: func(c *runCtx) (map[string]any, error) { return s12(c, 20) }, judge: judgeS12}, + {name: "s13-25", dir: "s13-socket-reconnect-25A", minutes: 6, + run: func(c *runCtx) (map[string]any, error) { return s13(c, 25) }, judge: judgeS13, + known: map[string]string{"Charge now goes on after the charger returns": "a socket reconnect turns Charge now into a 0 W pause; srcfl/ftw#1788"}}, + {name: "s15-25", dir: "s15-one-phase-charger-chargenow-25A", minutes: 4.6, short: true, + run: func(c *runCtx) (map[string]any, error) { return s15(c, 25) }, judge: judgeS15}, + {name: "s15b-25", dir: "s15b-one-phase-charger-1p-25A", minutes: 4.6, + run: func(c *runCtx) (map[string]any, error) { return s15mode(c, 25, "1p") }, judge: judgeS15}, + {name: "s15c-25", dir: "s15c-one-phase-charger-auto-25A", minutes: 4.6, + run: func(c *runCtx) (map[string]any, error) { return s15mode(c, 25, "auto") }, judge: judgeS15}, + {name: "s15bat-16", dir: "s15bat-one-phase-charger-battery-rising-16A", minutes: 5.5, + run: func(c *runCtx) (map[string]any, error) { return s15bat(c, 16) }, judge: judgeS15bat}, + {name: "s15p-25", dir: "s15p-one-phase-charger-plan-25A", minutes: 32, + run: func(c *runCtx) (map[string]any, error) { return s15p(c, 25) }, judge: judgeS15p}, + {name: "s14-25", dir: "s14-one-phase-car-25A", minutes: 22, + run: func(c *runCtx) (map[string]any, error) { return s14(c, 25) }, judge: judgeS14}, +} + +// runCtx is where one scenario runs. +type runCtx struct { + env *rigEnv + lane lane + rundir string +} + +func (c *runCtx) stack(spec stackSpec) *stack { return newStack(c.env, c.lane, c.rundir, spec) } + +func houseA(a float64) [3]float64 { return [3]float64{a * volt, a * volt, a * volt} } + +// ---- what a client does ---- + +type loadpointView struct { + ID string `json:"id"` + Plugged bool `json:"plugged_in"` + Charger struct { + Known bool `json:"known"` + } `json:"charger"` +} + +func (st *stack) loadpoint(id string) (loadpointView, error) { + var j struct { + Loadpoints []loadpointView `json:"loadpoints"` + } + if err := st.get("/api/loadpoints", &j); err != nil { + return loadpointView{}, err + } + for _, x := range j.Loadpoints { + if x.ID == id { + return x, nil + } + } + if len(j.Loadpoints) > 0 { + return j.Loadpoints[0], nil + } + return loadpointView{}, fmt.Errorf("no loadpoints") +} + +// chargeNow is what the web, PWA and iOS clients send. An empty phaseMode +// is a bare API call. powerW 0 means amps on three phases, below 0 a pause. +func (st *stack) chargeNow(lpID string, amps float64, phaseMode string, powerW float64) error { + switch { + case powerW < 0: // a pause + powerW = 0 + case powerW == 0: + powerW = amps * volt * 3 + } + body := map[string]any{"power_w": powerW, "hold_s": 0} + if phaseMode != "" { + body["phase_mode"] = phaseMode + } + return st.post("/api/loadpoints/"+lpID+"/manual_hold", body, nil) +} + +// chargerOnline waits until charger i talks to FTW. +func (st *stack) chargerOnline(i int) error { + serial := st.serial(i) + ok := waitUntil(90*time.Second, time.Second, func() bool { + var s struct { + Drivers map[string]json.RawMessage `json:"drivers"` + } + if st.get("/api/status", &s) != nil { + return false + } + _, ok := s.Drivers[serial] + return ok + }) + if !ok { + return fmt.Errorf("charger %s never connected to FTW", serial) + } + return nil +} + +// plug plugs a car into charger i once the charger is online, and waits +// for FTW to see it. +func (st *stack) plug(i int) error { + if err := st.chargerOnline(i); err != nil { + return err + } + time.Sleep(2 * time.Second) + label := "plug" + if i > 0 { + label = fmt.Sprintf("plug%d", i) + } + st.mark(label) + on := true + if err := st.event(houseEvent{Do: "plug", Device: lpIDs[i], On: &on}); err != nil { + return err + } + ok := waitUntil(90*time.Second, time.Second, func() bool { + lp, err := st.loadpoint(lpIDs[i]) + return err == nil && lp.Plugged && lp.Charger.Known + }) + if !ok { + return fmt.Errorf("loadpoint %s never saw the car plugged in", lpIDs[i]) + } + return nil +} + +func sleepS(s float64) { time.Sleep(time.Duration(s * float64(time.Second))) } + +// with brings a stack up, runs body and always takes the stack down. +func with(c *runCtx, spec stackSpec, body func(st *stack) error) error { + st := c.stack(spec) + defer st.down() + if err := st.up(); err != nil { + return err + } + if err := body(st); err != nil { + return err + } + return st.markErr +} + +// ---- 0. Adopted OCPP charger, no car: is Home use measured? ---- + +func s0(c *runCtx, fuse float64, heartbeatS int) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, heartbeatS: heartbeatS}, func(st *stack) error { + if err := st.chargerOnline(0); err != nil { + return err + } + st.mark("connected") + // Several heartbeats: #1779 keeps the idle 0 W fresh with them. + sleepS(300) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s0", "fuse": fuse, "house_a": 3, "heartbeat_s": heartbeatS}, err +} + +// ---- 1. Charge now, 3 A house load per phase ---- + +func s1(c *runCtx, fuse float64) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s1", "fuse": fuse, "house_a": 3}, err +} + +// ---- 2. Charge now during PV export of 9 A per phase, 16 A fuse ---- + +func s2(c *runCtx, fuse float64) (map[string]any, error) { + // Full battery so it cannot soak up the export. PV 7 kW, house 1 A: + // 6.3 kW export, 9.1 A per phase out of the site. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(1), pvW: 7000, socPct: 100}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(150) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s2", "fuse": fuse, "house_a": 1, "pv_w": 7000, "soc": 100}, err +} + +// ---- 3. House load steps to near the fuse while car and battery charge ---- + +func s3(c *runCtx, fuse float64) (map[string]any, error) { + stepA := map[float64]float64{16: 13, 25: 21}[fuse] + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 40}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("battery_hold") + if err := st.post("/api/battery/manual_hold", map[string]any{"direction": "charge", "power_w": 3000, "hold_s": 1800}, nil); err != nil { + return err + } + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(120) + st.mark("step") + if err := st.setLoadsA(stepA); err != nil { + return err + } + sleepS(180) + st.mark("step_back") + if err := st.setLoadsA(3); err != nil { + return err + } + sleepS(60) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s3", "fuse": fuse, "house_a": 3, "step_a": stepA, "battery_hold_w": 3000, "charger_delay_s": 0}, err +} + +// ---- 4. surplus_unlock_bat_soc saved but not armed, Charge now ---- + +func s4(c *runCtx, fuse float64, phaseMode string) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + st.mark("schedule") + // Only the unlock, as in #1774's R2 test: a SoC goal would make the + // plan charge at once and hide the fault. + var resp json.RawMessage + if err := st.post("/api/loadpoints/garage/target", map[string]any{"schedule": map[string]any{ + "soc": 0, "time_of_day_min": 7 * 60, "time_of_day_min_utc": 5 * 60, + "recurring": false, "days": 0, "surplus_unlock_bat_soc": 0.9}}, &resp); err != nil { + return err + } + os.WriteFile(filepath.Join(st.rundir, "target-response.json"), resp, 0o644) + sleepS(10) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, phaseMode, 0); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s4", "fuse": fuse, "house_a": 3, "unlock": 0.9, "soc": 50, "phase_mode": phaseMode}, err +} + +// ---- 5. Surplus only, phase mode auto, ~9 kW surplus ---- + +// easeeSteps are Easee's whole-amp offers at 230 V, as in #1774's tests: +// 6-16 A on one phase, then 6-16 A on three. +var easeeSteps = []float64{0, 1380, 1610, 1840, 2070, 2300, 2530, 2760, 2990, 3220, 3450, 3680, + 4140, 4830, 5520, 6210, 6900, 7590, 8280, 8970, 9660, 10350, 11040} + +func s5(c *runCtx, fuse float64, steps []float64) (map[string]any, error) { + extra := map[string]any{"surplus_only": true, "phase_mode": "auto"} + if steps != nil { + extra["allowed_steps_w"] = steps + } + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(1), pvW: 10000, socPct: 100, lpExtra: extra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + st.mark("plugged") + sleepS(300) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s5", "fuse": fuse, "house_a": 1, "pv_w": 10000, "soc": 100, "phase_mode": "auto", "steps": steps != nil}, err +} + +// ---- 6. PV inverter offline for 5 minutes ---- + +func s6(c *runCtx, fuse float64) (map[string]any, error) { + // A car plugged in, so that case 0 does not interfere. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), pvW: 1500, socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(60) + st.mark("offline") + if err := st.event(houseEvent{Do: "offline", Device: "inverter"}); err != nil { + return err + } + sleepS(300) + st.mark("online") + if err := st.event(houseEvent{Do: "online", Device: "inverter"}); err != nil { + return err + } + sleepS(90) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s6", "fuse": fuse, "house_a": 3, "pv_w": 1500, "offline_s": 300}, err +} + +// ---- 7. Core restart of ~20 s while charging ---- + +func s7(c *runCtx, fuse float64, short bool) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + if short { + sleepS(60) + } else { + // A new box has minutes of history; unknown time may be at most + // 10 % of it (history_dashboard.go). Wait until the day total is + // known, then long enough that the restart's ~110 s of unknown + // time stays below the rule, as on a box that has run since morning. + if waitUntil(900*time.Second, 5*time.Second, func() bool { + var s ftwStatus + return st.get("/api/status", &s) == nil && s.Energy.Today.LoadWh != nil + }) { + st.mark("today_known") + } + sleepS(900) + } + st.mark("ftw_stop") + st.stopFTW() + sleepS(20) + st.mark("ftw_start") + if err := st.startFTW(); err != nil { + return err + } + st.mark("ftw_up") + // Long enough to see whether the car gets back to full power. + sleepS(420) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s7", "fuse": fuse, "house_a": 3, "down_s": 20, "short": short}, err +} + +// ---- 8. Slow charger: profiles apply 60 s or 120 s late ---- + +func s8(c *runCtx, fuse, delayS float64) (map[string]any, error) { + // Battery at the rig's 5 % floor: it cannot cover the house. At 8 % FTW + // forced it to discharge for the fuse and the meter showed room that + // vanished when it ran empty, which hid the stacking (see the report). + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(15), socPct: 5, chargers: []charger{{delayS: delayS}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + // The charger draws its maximum until FTW's first profile lands, + // delayS after plug-in. Start from there, at 0 W. + st.mark("plug_wait") + sleepS(delayS + 20) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + // Long enough for the last rise and the cut after it to land. + sleepS(max(300, 4*delayS+120)) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s8", "fuse": fuse, "house_a": 15, "charger_delay_s": delayS, "soc_pct": 5}, err +} + +// ---- 9. Charge now switched from 3p to 1p at the same watts ---- + +func s9(c *runCtx, fuse, watts float64) (map[string]any, error) { + // A 32 A charger that can switch phases. 7 360 W is 10.7 A on three + // phases and 32 A on one. House 10 A per phase, empty battery: one phase + // has 14 A of room below FTW's 24 A limit. + spec := stackSpec{fuseA: fuse, loadsW: houseA(10), socPct: 5, chargers: []charger{{kw: 22.08, phaseSwitch: true}}, + lpExtra: map[string]any{"min_charge_w": 1380, "max_charge_w": 22080, "phase_mode": "auto"}} + err := with(c, spec, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 0, "3p", watts); err != nil { + return err + } + sleepS(90) + st.mark("switch") + if err := st.chargeNow("garage", 0, "1p", watts); err != nil { + return err + } + sleepS(150) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s9", "fuse": fuse, "house_a": 10, "watts": watts, "charger_a": 32}, err +} + +// ---- 10. Two chargers: one raised while the other has just been cut ---- + +func s10(c *runCtx, fuse, delayS float64) (map[string]any, error) { + // garage answers at once; carport is a cloud charger that applies a + // profile delayS late. House 3 A per phase, empty battery. Both charge + // at 10 A (23 A per phase). Then carport is cut to 6 A and garage raised + // to 16 A in the same second: the room carport frees is not there until + // its cut lands. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 5, chargers: []charger{{}, {delayS: delayS}}}, func(st *stack) error { + if err := st.plug(1); err != nil { + return err + } + // The slow charger draws its maximum until FTW's first profile lands. + st.mark("plug_wait") + sleepS(delayS + 20) + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + st.mark("charge_now") + if err := st.chargeNow("garage", 10, "3p", 0); err != nil { + return err + } + if err := st.chargeNow("carport", 10, "3p", 0); err != nil { + return err + } + sleepS(delayS + 60) + st.mark("step") + if err := st.chargeNow("carport", 6, "3p", 0); err != nil { + return err + } + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(2*delayS + 120) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s10", "fuse": fuse, "house_a": 3, "charger_delay_s": []float64{0, delayS}}, err +} + +// ---- 11. Pause and restart under a long declared response time (#1793) ---- + +func s11(c *runCtx, fuse float64) (map[string]any, error) { + // OCPP declares no response time, so Core reserves 180 s for each order, + // as for a charger that declares 180 s. Charge now 16 A, pause (0 W) for + // 25 s, then start at 6 A: the cut must end the old order's reservation. + // The charger applies a profile 5 s late (ev-104's plant). + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, chargers: []charger{{delayS: 5}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(60) + st.mark("pause") + if err := st.chargeNow("garage", 0, "3p", -1); err != nil { + return err + } + sleepS(25) + st.mark("restart") + if err := st.chargeNow("garage", 6, "3p", 0); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s11", "fuse": fuse, "house_a": 3, "pause_s": 25, "charger_delay_s": 5}, err +} + +// ---- planned charging ---- + +// planTo tells FTW the car is at socPct and must reach goalPct by the first +// quarter hour at least readyIn from now. It returns the ready time. +func (st *stack) planTo(lp string, socPct, goalPct float64, readyIn time.Duration) (time.Time, error) { + if err := st.post("/api/loadpoints/"+lp+"/soc", map[string]any{"soc_pct": socPct}, nil); err != nil { + return time.Time{}, err + } + ready := time.Now().Add(readyIn).Truncate(15 * time.Minute).Add(15 * time.Minute) + if err := st.post("/api/loadpoints/"+lp+"/target", map[string]any{"schedule": map[string]any{ + "soc": goalPct / 100, "time_of_day_min": ready.Hour()*60 + ready.Minute(), "recurring": false, "days": 0}}, nil); err != nil { + return time.Time{}, err + } + return ready, nil +} + +// ---- 9r. s9 reversed: a 1p order raised to 3p beside a loaded L2 ---- + +func s9r(c *runCtx, fuse, watts float64) (map[string]any, error) { + // House 2/20/2 A on 25 A: L2 has 4 A left under FTW's 24 A limit. A + // 32 A charger that switches phases gets Charge now 7 360 W on 1p + // (L1 has room), then the same watts on 3p, which would put 10.7 A on + // L2 too. + spec := stackSpec{fuseA: fuse, loadsW: [3]float64{2 * volt, 20 * volt, 2 * volt}, socPct: 5, + chargers: []charger{{kw: 22.08, phaseSwitch: true}}, + lpExtra: map[string]any{"min_charge_w": 1380, "max_charge_w": 22080, "phase_mode": "auto"}} + err := with(c, spec, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 0, "1p", watts); err != nil { + return err + } + sleepS(90) + st.mark("switch") + if err := st.chargeNow("garage", 0, "3p", watts); err != nil { + return err + } + sleepS(150) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s9", "fuse": fuse, "house_a": []float64{2, 20, 2}, "watts": watts, "charger_a": 32, "order": "1p then 3p"}, err +} + +// ---- 5t. s5s with a goal: surplus below the three-phase minimum ---- + +func s5t(c *runCtx, fuse float64) (map[string]any, error) { + // PV 4.5 kW, house 1 A per phase, full battery: about 3.8 kW of surplus, + // under the 4 140 W three-phase minimum. With a goal saved, a + // surplus-only loadpoint must still charge, on one phase: the charger + // switches phases, as an Easee does. + extra := map[string]any{"surplus_only": true, "phase_mode": "auto", "allowed_steps_w": easeeSteps} + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(1), pvW: 4500, socPct: 100, lpExtra: extra, + chargers: []charger{{phaseSwitch: true}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + if _, err := st.planTo("garage", 40, 80, 8*time.Hour); err != nil { + return err + } + st.mark("plugged") + sleepS(300) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s5t", "fuse": fuse, "house_a": 1, "pv_w": 4500, "soc": 100}, err +} + +func judgeS5t(d *runData) (metrics, []criterion) { + m, cs := judgeS5(d) + t0, t1 := d.marks["plugged"], d.marks["end"] + w := d.window(t0+60, t1) + n := 0 + for _, r := range w { + if carW(r, 0) >= 0.95*1380 { + n++ + } + } + frac := round(float64(n)/math.Max(1, float64(len(w))), 3) + ones := 0 + prs := connector1(profiles(d.frames, t0, t1)) + for _, p := range prs { + if p.Limit > 0 && p.phasesOr3() == 1 { + ones++ + } + } + m["charging_fraction_ge_1p_min"], m["one_phase_offers"] = frac, ones + out := []criterion{} + for _, c := range cs { + if c.Name == "no offer above the charger's rating" || c.Name == "fuse holds" { + out = append(out, c) + } + } + return m, append(out, crit("charges at least 6 A on one phase 90 % of the time", frac >= 0.9)) +} + +// ---- 12. One-phase night: planned charging on auto with Easee's steps ---- + +// fieldSteps is the field site's step table (22 Oct): one-phase levels +// 1 600-3 800 W and three-phase levels 4 800-11 100 W. +var fieldSteps = []float64{0, 1600, 1840, 2070, 2300, 2530, 2760, 2990, 3220, 3450, 3680, 3800, + 4800, 5520, 6210, 6900, 7590, 8280, 8970, 9660, 10350, 11100} + +func s12(c *runCtx, fuse float64) (map[string]any, error) { + // 3 × 20 A, house 1.6/1.7/2.3 A, an Easee-like charger applying in + // 40 s, a three-phase car, and a plan for 6.6 kWh by the first quarter + // hour 40 min out: 7-10 kW on three phases. One phase cannot deliver + // that, so this tests the cuts and the shortfall of the field night, + // not its one-phase choice (#1799 cause 1, which needs a lighter plan). + extra := map[string]any{"phase_mode": "auto", "allowed_steps_w": fieldSteps, "min_charge_w": 1600, "max_charge_w": 11100} + var ready time.Time + err := with(c, stackSpec{fuseA: fuse, loadsW: [3]float64{1.6 * volt, 1.7 * volt, 2.3 * volt}, socPct: 5, + chargers: []charger{{delayS: 40}}, lpExtra: extra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + var err error + if ready, err = st.planTo("garage", 40, 51, 40*time.Minute); err != nil { + return err + } + st.mark("plan") + sleepS(time.Until(ready).Seconds()) + st.mark("ready") + sleepS(30) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s12", "fuse": fuse, "plan_wh": 0.11 * 60000, "charger_delay_s": 40}, err +} + +func judgeS12(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + t0, tReady := d.marks["plan"], d.marks["ready"] + m := metrics{} + // Energy the car took between the plan and the ready time. + wh := 0.0 + w := d.window(t0, tReady) + for i := 1; i < len(w); i++ { + wh += carW(w[i], 0) * (w[i].T - w[i-1].T) / 3600 + } + plan := d.metaF("plan_wh") + m["delivered_wh"], m["plan_wh"] = math.Round(wh), plan + prs := connector1(profiles(d.frames, t0, tReady)) + ones, cuts := 0, 0 + for i, p := range prs { + if p.Limit > 0 && p.phasesOr3() == 1 { + ones++ + } + // A lower order while every phase has room is a cut on room alone, + // unless the plan itself asked for less (a pause between slots). + if i > 0 && p.offerW() < prs[i-1].offerW()-1 { + if r := d.at(t0 + p.T); r != nil && maxAbsI(r) < fuse-1-0.5 { + if after := d.at(t0 + p.T + 2); after == nil || firstLP(after).CommandedReason != "plan" { + cuts++ + } + } + } + } + // Seconds FTW gave fuse_limit while the worst phase had 6 A of room: + // recorded, not judged, since a slow charger's pending order may hold it. + limited := 0 + for _, r := range w { + if r.LPs != nil && firstLP(r).CommandedReason == "fuse_limit" && maxAbsI(r) < fuse-1-6 { + limited++ + } + } + m["one_phase_orders"], m["cuts_with_room"], m["profiles"], m["fuse_limit_with_room_s"] = ones, cuts, len(prs), limited + isBlown := anyBlown(d.window(t0, d.marks["end"])) + return m, []criterion{ + crit("fuse holds", !isBlown), + crit("planned charging never orders one phase", ones == 0), + crit("no order cut while every phase has room", cuts == 0), + crit("at least 90 % of the plan by the ready time", wh >= 0.9*plan), + } +} + +// at is the last sample at or before t. +func (d *runData) at(t float64) *sample { + var last *sample + for _, r := range d.samples { + if r.T > t { + break + } + last = r + } + return last +} + +// ---- 13. Charge now across a charger socket reconnect ---- + +func s13(c *runCtx, fuse float64) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + if err := st.chargeNow("garage", 16, "3p", 0); err != nil { + return err + } + sleepS(60) + st.mark("drop") + if err := st.event(houseEvent{Do: "reconnect", Device: "garage", ForS: 10}); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s13", "fuse": fuse, "house_a": 3, "drop_s": 10}, err +} + +func judgeS13(d *runData) (metrics, []criterion) { + tDrop, t1 := d.marks["drop"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + minAfter := minOf(d.window(tDrop, t1), car) + plugged := 0 + for _, r := range d.window(tDrop, t1) { + if lp := firstLP(r); r.LPs != nil && !lp.Plugged { + plugged++ + } + } + m := metrics{"car_min_after_drop": math.Round(minAfter), "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), car)), + "unplugged_samples": plugged} + return m, []criterion{ + crit("Charge now goes on after the charger returns", minAfter >= 0.95*minW), + crit("no unplug seen", plugged == 0), + } +} + +// ---- 14. A one-phase car under a three-phase order ---- + +func s14(c *runCtx, fuse float64) (map[string]any, error) { + // A one-phase car on an 11 kW charger draws on L1 whatever FTW offers. + // A plan asks for more than it can take, so FTW should plan its 16 A + // on one phase (3.5-3.7 kW) and not call the charger not following. + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 5, chargers: []charger{{carPhases: 1}}}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + // 24 kWh in about 45 min: more than even 11 kW gives, so FTW must + // charge from the start. + if _, err := st.planTo("garage", 40, 80, 30*time.Minute); err != nil { + return err + } + st.mark("plan") + sleepS(20 * 60) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s14", "fuse": fuse, "house_a": 3, "car_phases": 1}, err +} + +func judgeS14(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["plan"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + last := meanOf(d.window(t1-600, t1), car) + notFollowing := 0 + for _, r := range d.window(t0+180, t1) { + if firstLP(r).ControlFeedback.Status == "not_following" { + notFollowing++ + } + } + m := metrics{"car_w_last10min": math.Round(last), "not_following_samples": notFollowing} + return m, []criterion{ + crit("delivers 3.4-3.8 kW over the last 10 min", last >= 3400 && last <= 3800), + crit("never called not following", notFollowing == 0), + } +} + +// ---- 15. A charger wired on one phase: Charge now and planned charging ---- + +// onePhaseCharger is an Easee wired on L1: 6-16 A there, nothing on three +// phases. Its loadpoint names no phase mode, as most do. +var onePhaseCharger = charger{kw: 3.68, wiredPhases: 1} + +var onePhaseExtra = map[string]any{"min_charge_w": 1380, "max_charge_w": 3680, + "allowed_steps_w": easeeSteps[:12]} // 0, 1 380 ... 3 680 W + +// chargeNowAsTheApp sends Charge now at 16 A on the phases /api/loadpoints +// reports, as the app does. +func (st *stack) chargeNowAsTheApp(lpID string) (int, error) { + var lps struct { + Loadpoints []ftwLoadpoint `json:"loadpoints"` + } + if err := st.get("/api/loadpoints", &lps); err != nil { + return 0, err + } + phases := 3 + for _, lp := range lps.Loadpoints { + if lp.ID == lpID && lp.Phases > 0 { + phases = lp.Phases + } + } + mode := "3p" + if phases == 1 { + mode = "1p" + } + return phases, st.chargeNow(lpID, 0, mode, 16*volt*float64(phases)) +} + +func s15(c *runCtx, fuse float64) (map[string]any, error) { return s15mode(c, fuse, "") } + +// s15mode is s15 with the loadpoint's phase mode set ("" leaves it out). +func s15mode(c *runCtx, fuse float64, mode string) (map[string]any, error) { + extra := map[string]any{} + for k, v := range onePhaseExtra { + extra[k] = v + } + if mode != "" { + extra["phase_mode"] = mode + } + phases := 0 + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, chargers: []charger{onePhaseCharger}, + lpExtra: extra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + st.mark("charge_now") + var err error + if phases, err = st.chargeNowAsTheApp("garage"); err != nil { + return err + } + sleepS(180) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s15", "fuse": fuse, "house_a": 3, "api_phases": phases, "phase_mode": mode}, err +} + +func judgeS15(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + const target = 16 * volt // all a one-phase charger gives + m := d.carSummary(t0, t1, target) + m["target_w"], m["api_phases"] = target, d.meta["api_phases"] + hold, _ := m["hold_fraction"].(*float64) + return m, []criterion{ + crit("16 A on L1 within 60 s", le(m["time_to_target_s"].(*float64), 60)), + crit("holds 95 % of the time", hold != nil && *hold >= 0.95), + crit("fuse holds", !m["fuse_blown"].(bool)), + crit("no fuse_limit with room to spare", m["fuse_limit_samples"].(int) == 0), + } +} + +func s15p(c *runCtx, fuse float64) (map[string]any, error) { + // 600 Wh by the first quarter hour 15 min out: about 10 min at 3.68 kW. + var ready time.Time + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 5, chargers: []charger{onePhaseCharger}, + lpExtra: onePhaseExtra}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(20) + var err error + if ready, err = st.planTo("garage", 40, 41, 15*time.Minute); err != nil { + return err + } + st.mark("plan") + sleepS(time.Until(ready).Seconds()) + st.mark("ready") + sleepS(30) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s15p", "fuse": fuse, "house_a": 3, "plan_wh": 0.01 * 60000}, err +} + +func judgeS15p(d *runData) (metrics, []criterion) { + t0, tReady := d.marks["plan"], d.marks["ready"] + wh := 0.0 + w := d.window(t0, tReady) + for i := 1; i < len(w); i++ { + wh += carW(w[i], 0) * (w[i].T - w[i-1].T) / 3600 + } + plan := d.metaF("plan_wh") + notFollowing := 0 + for _, r := range w { + if r.LPs != nil && firstLP(r).ControlFeedback.Status == "not_following" { + notFollowing++ + } + } + m := metrics{"delivered_wh": math.Round(wh), "plan_wh": plan, "not_following_samples": notFollowing, + "car_w_max": math.Round(maxOf(w, func(r *sample) float64 { return carW(r, 0) }))} + return m, []criterion{ + crit("fuse holds", !anyBlown(d.window(t0, d.marks["end"]))), + crit("at least 90 % of the plan by the ready time", wh >= 0.9*plan), + crit("never called not following", notFollowing == 0), + } +} + +// s15bat is Charge now on a one-phase charger while Core is raising its own +// battery order: PV climbs every 30 s (+2.5 kW for the second step, 1.5 kW +// for the others) and self-consumption charges the battery with it. The +// battery follows 8 s after each write, and Charge now goes 6 s after the +// second step, while Core's raise is on its way. On +// 16 A with 3 A of house load a raise of more than +// (15 - 3) x 230 - (1 380 + 230) = 1 150 W on its way once kept the car from +// starting at all (#1805). With the battery's 2 s the car started 25 s late +// when Charge now caught a raise in flight, and at once when it did not. +func s15bat(c *runCtx, fuse float64) (map[string]any, error) { + err := with(c, stackSpec{fuseA: fuse, loadsW: houseA(3), socPct: 50, chargers: []charger{onePhaseCharger}, + lpExtra: onePhaseExtra, batteryDelayS: 8}, func(st *stack) error { + if err := st.plug(0); err != nil { + return err + } + sleepS(30) + for i, pv := range []float64{2100, 4600, 6100, 7600, 9100} { + if err := st.event(houseEvent{Do: "pv", Device: "inverter", PVW: f64(pv)}); err != nil { + return err + } + if i == 1 { + sleepS(6) + st.mark("charge_now") + if _, err := st.chargeNowAsTheApp("garage"); err != nil { + return err + } + sleepS(24) + continue + } + sleepS(30) + } + sleepS(90) + st.mark("end") + return nil + }) + return map[string]any{"scenario": "s15bat", "fuse": fuse, "house_a": 3, "battery_delay_s": 8}, err +} + +func judgeS15bat(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + fuse := d.metaF("fuse") + // Core's battery order: the EMS command and setpoint it wrote. + order := func(r *sample) float64 { + switch r.Bat.EMSCmd { + case 170: + return r.Bat.EMSSetW + case 187: + return -r.Bat.EMSSetW + } + return 0 + } + rise, low := 0.0, math.Inf(1) + for _, r := range d.window(t0-20, t0) { + low = math.Min(low, order(r)) + rise = math.Max(rise, order(r)-low) + } + start := d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.95*1380 }) + over, longest := 0.0, 0.0 + for _, r := range d.window(t0, t1) { + if maxAbsI(r) > fuse-1 { + over++ + longest = math.Max(longest, over) + } else { + over = 0 + } + } + m := metrics{"battery_rise_w_20s_before": math.Round(rise), "car_start_s": roundp(start, 1), + "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), func(r *sample) float64 { return carW(r, 0) })), + "over_fuse_minus1_longest_s": longest, "max_abs_i": round(maxOf(d.window(t0, t1), maxAbsI), 1)} + return m, []criterion{ + crit("the battery order rose over 1 150 W in the 20 s before Charge now", rise > 1150), // the setup + crit("car charging within 10 s of Charge now", le(start, 10)), + crit("fuse holds", !anyBlown(d.window(t0, t1))), + crit("over fuse - 1 A at most 3 s", longest <= 3), + } +} diff --git a/go/test/site/stack_test.go b/go/test/site/stack_test.go new file mode 100644 index 00000000..aa4b156b --- /dev/null +++ b/go/test/site/stack_test.go @@ -0,0 +1,537 @@ +//go:build unix + +package site + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "sync" + "syscall" + "time" +) + +// ocppPass is the shared OCPP credential the rig gives FTW and the chargers. +const ocppPass = "sim-ocpp-rig-20261021" + +// lane is the first of the 100 ports one rig uses on 127.0.0.1. The house +// (device-simulator's headless run) takes base+0 to base+59; FTW takes +// base+90 (API) and base+91 (OCPP). +type lane int + +func (l lane) simHTTP() int { return int(l) } +func (l lane) ftwAPI() int { return int(l) + 90 } +func (l lane) ftwOCPP() int { return int(l) + 91 } + +// ---- processes ---- + +type proc struct { + cmd *exec.Cmd + done chan struct{} +} + +// procs runs each child in its own process group, so a stop reaches +// everything it started. +type procs struct { + mu sync.Mutex + m map[string]*proc +} + +func (ps *procs) start(name string, argv []string, dir string, env []string, logPath string) error { + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return err + } + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = append(append(os.Environ(), "GOMAXPROCS=2"), env...) + cmd.Stdout, cmd.Stderr = f, f + cmd.SysProcAttr = childProcAttr() + if err := cmd.Start(); err != nil { + f.Close() + return fmt.Errorf("start %s: %w", name, err) + } + p := &proc{cmd: cmd, done: make(chan struct{})} + go func() { cmd.Wait(); f.Close(); close(p.done) }() + ps.mu.Lock() + if ps.m == nil { + ps.m = map[string]*proc{} + } + ps.m[name] = p + ps.mu.Unlock() + return nil +} + +func (ps *procs) stop(name string, sig syscall.Signal, wait time.Duration) { + ps.mu.Lock() + p := ps.m[name] + delete(ps.m, name) + ps.mu.Unlock() + if p == nil { + return + } + syscall.Kill(-p.cmd.Process.Pid, sig) + select { + case <-p.done: + case <-time.After(wait): + syscall.Kill(-p.cmd.Process.Pid, syscall.SIGKILL) + <-p.done + } +} + +func (ps *procs) stopAll() { + ps.mu.Lock() + names := make([]string, 0, len(ps.m)) + for n := range ps.m { + names = append(names, n) + } + ps.mu.Unlock() + for _, n := range names { + ps.stop(n, syscall.SIGTERM, 10*time.Second) + } +} + +// ---- HTTP ---- + +var client = &http.Client{Timeout: 5 * time.Second} + +// call sends body as JSON and decodes the answer into out when out is not +// nil. It returns the raw answer. +func call(method, url string, body, out any) ([]byte, error) { + var rd io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return nil, err + } + rd = bytes.NewReader(b) + } + req, err := http.NewRequest(method, url, rd) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode >= 300 { + return raw, fmt.Errorf("%s %s: %s: %s", method, url, resp.Status, bytes.TrimSpace(raw)) + } + if out != nil && len(raw) > 0 { + if err := json.Unmarshal(raw, out); err != nil { + return raw, fmt.Errorf("%s %s: %w", method, url, err) + } + } + return raw, nil +} + +func waitHTTP(url string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + if _, last = call("GET", url, nil, nil); last == nil { + return nil + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("%s did not answer: %v", url, last) +} + +// waitUntil polls pred every step until it holds or the timeout passes. +func waitUntil(timeout, step time.Duration, pred func() bool) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if pred() { + return true + } + time.Sleep(step) + } + return false +} + +func now() float64 { return float64(time.Now().UnixMicro()) / 1e6 } + +// ---- the house: device-simulator's headless run ---- + +// houseSpec is device-simulator's run spec (docs/HEADLESS_RUN.md there). +type houseSpec struct { + Name string `json:"name"` + Site houseSite `json:"site"` + Devices []houseDevice `json:"devices"` + Timeline []houseEvent `json:"timeline,omitempty"` + Random *houseRandom `json:"random,omitempty"` +} + +// houseRandom asks the house to draw its own timeline from a seed. +type houseRandom struct { + Seed int64 `json:"seed"` + DurationS float64 `json:"duration_s"` + GapS float64 `json:"gap_s,omitempty"` +} + +type houseSite struct { + FuseA int `json:"fuse_a"` + FuseProfile string `json:"fuse_profile,omitempty"` + HouseW [3]float64 `json:"house_w"` + SignedPhaseCurrents *bool `json:"signed_phase_currents,omitempty"` +} + +type houseDevice struct { + ID string `json:"id"` + Driver string `json:"driver"` + CommandResponseS *float64 `json:"command_response_s,omitempty"` + StartupDelayS *float64 `json:"startup_delay_s,omitempty"` + LoadBalancerA *float64 `json:"load_balancer_a,omitempty"` + BatteryKWh float64 `json:"battery_kwh,omitempty"` + SoCPct *float64 `json:"soc_pct,omitempty"` + MinSoCPct *float64 `json:"min_soc_pct,omitempty"` + PVW float64 `json:"pv_w,omitempty"` + OCPPURL string `json:"ocpp_url,omitempty"` + MaxKW float64 `json:"max_kw,omitempty"` + Vendor string `json:"vendor,omitempty"` + Model string `json:"model,omitempty"` + PhaseSwitch bool `json:"phase_switch,omitempty"` + OCPPConfig map[string]string `json:"ocpp_config,omitempty"` + CarMaxA float64 `json:"car_max_a,omitempty"` + CarPhases int `json:"car_phases,omitempty"` + WiredPhases int `json:"wired_phases,omitempty"` +} + +// houseEvent is one change to the house (device-simulator's runEvent). +type houseEvent struct { + AtS float64 `json:"at_s,omitempty"` + Do string `json:"do"` + Device string `json:"device,omitempty"` + Label string `json:"label,omitempty"` + W *[3]float64 `json:"w,omitempty"` + A *[3]float64 `json:"a,omitempty"` + PVW *float64 `json:"pv_w,omitempty"` + Pct *float64 `json:"pct,omitempty"` + On *bool `json:"on,omitempty"` + MaxA *float64 `json:"max_a,omitempty"` + Phases *int `json:"phases,omitempty"` + ForS float64 `json:"for_s,omitempty"` +} + +type housePorts struct { + HTTP int `json:"http"` + Devices map[string]struct { + Driver string `json:"driver"` + Modbus int `json:"modbus_port"` + UnitID int `json:"unit_id"` + Serial string `json:"serial"` + } `json:"devices"` +} + +// ---- one run ---- + +// charger is one OCPP charger and the car on it. FTW's loadpoint has the +// charger's id. +type charger struct { + kw float64 // three-phase rating; default 11.04 + phaseSwitch bool + delayS float64 // applies a profile this late, like a cloud charger + carPhases int // 0: as many as offered + wiredPhases int // 1: wired on L1 only, kw its rating there + carMaxA float64 +} + +type stackSpec struct { + // house replays a spec the house wrote (timeline.json); random asks it + // to draw a timeline. Without either the house has no timeline. + house *houseSpec + random *houseRandom + + fuseA float64 + loadsW [3]float64 + pvW float64 + socPct float64 + chargers []charger // nil: one 11 kW charger + lpExtra map[string]any + heartbeatS int // FTW's OCPP heartbeat; 0 leaves Core's default + // batteryDelayS is how long the battery takes to follow an EMS write; + // 0 leaves the model's (2 s for sungrow). + batteryDelayS float64 +} + +type stack struct { + env *rigEnv + lane lane + rundir string + spec stackSpec + ps procs + ports housePorts + ftw string // FTW's API base URL + sim string // the house's API base URL + rec *recorder + cfg string + // markErr is the first mark the house did not take; the run cannot be + // judged without it. + markErr error +} + +func newStack(env *rigEnv, l lane, rundir string, spec stackSpec) *stack { + if spec.chargers == nil { + spec.chargers = []charger{{}} + } + return &stack{env: env, lane: l, rundir: rundir, spec: spec, + ftw: fmt.Sprintf("http://127.0.0.1:%d", l.ftwAPI()), + sim: fmt.Sprintf("http://127.0.0.1:%d", l.simHTTP())} +} + +func f64(v float64) *float64 { return &v } + +func (st *stack) houseSpec() houseSpec { + if st.spec.house != nil { + // A replay: the same house on this lane's ports. + h := *st.spec.house + h.Name = filepath.Base(st.rundir) + h.Random = nil + h.Devices = append([]houseDevice(nil), h.Devices...) + for i := range h.Devices { + if h.Devices[i].Driver == "ocpp" { + h.Devices[i].OCPPURL = fmt.Sprintf("ws://ftw:%s@127.0.0.1:%d/", ocppPass, st.lane.ftwOCPP()) + } + } + return h + } + h := houseSpec{Name: filepath.Base(st.rundir), Random: st.spec.random, + Site: houseSite{FuseA: int(st.spec.fuseA), FuseProfile: "iec_gg", HouseW: st.spec.loadsW}, + Devices: []houseDevice{{ID: "inverter", Driver: "sungrow", BatteryKWh: 10, + SoCPct: f64(st.spec.socPct), MinSoCPct: f64(5), PVW: st.spec.pvW}}} + if st.spec.batteryDelayS > 0 { + h.Devices[0].CommandResponseS = f64(st.spec.batteryDelayS) + } + for i, c := range st.spec.chargers { + var delay *float64 + if st.spec.random == nil { + delay = f64(c.delayS) // a random house draws its own + } + kw := c.kw + if kw == 0 { + kw = 11.04 + } + h.Devices = append(h.Devices, houseDevice{ID: lpIDs[i], Driver: "ocpp", + OCPPURL: fmt.Sprintf("ws://ftw:%s@127.0.0.1:%d/", ocppPass, st.lane.ftwOCPP()), + MaxKW: kw, Vendor: "RigCo", Model: fmt.Sprintf("Rig%.0f", kw), PhaseSwitch: c.phaseSwitch, CommandResponseS: delay, + CarPhases: c.carPhases, CarMaxA: c.carMaxA, WiredPhases: c.wiredPhases, + OCPPConfig: map[string]string{"HeartbeatInterval": "60", "MeterValueSampleInterval": "10"}}) + } + return h +} + +func (st *stack) up() error { + if err := os.RemoveAll(st.rundir); err != nil { + return err + } + if err := os.MkdirAll(st.rundir, 0o755); err != nil { + return err + } + spec, _ := json.MarshalIndent(st.houseSpec(), "", " ") + specPath := filepath.Join(st.rundir, "house.json") + if err := os.WriteFile(specPath, spec, 0o644); err != nil { + return err + } + houseDir := filepath.Join(st.rundir, "house") + if err := st.ps.start("house", []string{st.env.devsimBin, "run", "-spec", specPath, + "-port-base", fmt.Sprint(int(st.lane)), "-out", houseDir}, st.rundir, nil, + filepath.Join(st.rundir, "house.log")); err != nil { + return err + } + if !waitUntil(30*time.Second, 200*time.Millisecond, func() bool { + b, err := os.ReadFile(filepath.Join(houseDir, "ports.json")) + return err == nil && json.Unmarshal(b, &st.ports) == nil && len(st.ports.Devices) > 0 + }) { + return fmt.Errorf("the house never wrote ports.json (see house.log)") + } + if err := waitHTTP(st.sim+"/api/run", 10*time.Second); err != nil { + return err + } + var err error + if st.cfg, err = st.ftwConfig(); err != nil { + return err + } + if err := st.startFTW(); err != nil { + return err + } + if _, err := call("POST", st.ftw+"/api/mode", map[string]any{"mode": "planner_self"}, nil); err != nil { + return err + } + st.rec = &recorder{ftw: st.ftw, path: filepath.Join(st.rundir, "ftw.jsonl")} + st.rec.start() + return nil +} + +func (st *stack) startFTW() error { + if err := st.ps.start("ftw", []string{st.env.ftwBin, "-config", st.cfg, + "-web", filepath.Join(st.env.root, "web"), "-drivers", filepath.Join(st.env.root, "drivers")}, + st.rundir, []string{"GOMAXPROCS=4"}, filepath.Join(st.rundir, "ftw.log")); err != nil { + return err + } + return waitHTTP(st.ftw+"/api/status", 90*time.Second) +} + +func (st *stack) stopFTW() { st.ps.stop("ftw", syscall.SIGTERM, 30*time.Second) } + +func (st *stack) down() { + if st.rec != nil { + st.rec.close() + } + st.ps.stop("ftw", syscall.SIGTERM, 30*time.Second) + st.ps.stop("house", syscall.SIGTERM, 15*time.Second) + st.ps.stopAll() +} + +// event changes the house now. +func (st *stack) event(ev houseEvent) error { + _, err := call("POST", st.sim+"/api/run/event", ev, nil) + return err +} + +// mark names this moment in the house's record. +func (st *stack) mark(label string) { + if _, err := call("POST", st.sim+"/api/run/mark", map[string]string{"label": label}, nil); err != nil && st.markErr == nil { + st.markErr = fmt.Errorf("mark %s: %w", label, err) + } +} + +func (st *stack) get(path string, out any) error { + _, err := call("GET", st.ftw+path, nil, out) + return err +} + +func (st *stack) post(path string, body, out any) error { + _, err := call("POST", st.ftw+path, body, out) + return err +} + +func (st *stack) setLoadsA(a float64) error { + return st.event(houseEvent{Do: "loads", A: &[3]float64{a, a, a}}) +} + +func (st *stack) serial(i int) string { return st.ports.Devices[lpIDs[i]].Serial } + +// ---- FTW's API, once a second ---- + +type recorder struct { + ftw string + path string + mu sync.Mutex + stop chan struct{} + done chan struct{} +} + +func (r *recorder) start() { + r.stop, r.done = make(chan struct{}), make(chan struct{}) + go r.run() +} + +func (r *recorder) close() { + if r.stop == nil { + return + } + close(r.stop) + <-r.done +} + +func (r *recorder) append(rec map[string]any) { + b, _ := json.Marshal(rec) + r.mu.Lock() + defer r.mu.Unlock() + f, err := os.OpenFile(r.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + return + } + f.Write(append(b, '\n')) + f.Close() +} + +func (r *recorder) run() { + defer close(r.done) + for { + t0 := time.Now() + rec := map[string]any{"t": now()} + for key, path := range map[string]string{"status": "/api/status", "lps": "/api/loadpoints"} { + if b, err := call("GET", r.ftw+path, nil, nil); err != nil { + rec[key+"_err"] = err.Error() + } else { + rec[key] = json.RawMessage(b) + } + } + r.append(rec) + select { + case <-r.stop: + return + case <-time.After(time.Second - time.Since(t0)): + } + } +} + +// ---- FTW's configuration ---- + +var lpIDs = []string{"garage", "carport"} + +func (st *stack) ftwConfig() (string, error) { + var lps []map[string]any + for i := range st.spec.chargers { + lp := map[string]any{ + "id": lpIDs[i], "driver_name": st.serial(i), + "min_charge_w": 4140, "max_charge_w": 11040, "vehicle_capacity_wh": 60000, + } + for k, v := range st.spec.lpExtra { + lp[k] = v + } + lps = append(lps, lp) + } + ocpp := map[string]any{"enabled": true, "bind": "127.0.0.1", "port": st.lane.ftwOCPP(), "path": "/", + "username": "ftw", "password": ocppPass} + // 60 s is what every box configured from the old example has; 0 leaves + // it unset so Core's default applies. + if st.spec.heartbeatS > 0 { + ocpp["heartbeat_interval_s"] = st.spec.heartbeatS + } + modbus := func(id string) map[string]any { + d := st.ports.Devices[id] + return map[string]any{"modbus": map[string]any{"host": "127.0.0.1", "port": d.Modbus, "unit_id": max(d.UnitID, 1)}} + } + cfg := map[string]any{ + "site": map[string]any{"name": "rig", "control_interval_s": 2, "grid_target_w": 0, + "grid_tolerance_w": 42, "watchdog_timeout_s": 60}, + "fuse": map[string]any{"max_amps": st.spec.fuseA, "phases": 3, "voltage": 230}, + "drivers": []map[string]any{ + {"name": "meter", "lua": "drivers/sdm630.lua", "is_site_meter": true, "capabilities": modbus("meter")}, + {"name": "sungrow", "lua": "drivers/sungrow.lua", "battery_capacity_wh": 10000, "capabilities": modbus("inverter")}, + }, + "api": map[string]any{"port": st.lane.ftwAPI()}, + "ocpp": ocpp, + "loadpoints": lps, + "state": map[string]any{"path": filepath.Join(st.rundir, "state.db")}, + "app_link": map[string]any{"enabled": false}, + "fleet_ping": map[string]any{"enabled": false}, + "device_repository": map[string]any{"enabled": false}, + // A real box plans. A box without a planner runs Self, where the + // battery chases the raw meter and covers the car (dispatch.go), + // which hides the stress on the fuse. + "planner": map[string]any{"enabled": true, "engine": "core", "mode": "self_consumption"}, + "price": map[string]any{"provider": "elprisetjustnu", "zone": "SE3"}, + } + path := filepath.Join(st.rundir, "config.yaml") + b, err := json.MarshalIndent(cfg, "", " ") // JSON is valid YAML + if err != nil { + return "", err + } + return path, os.WriteFile(path, b, 0o644) +} diff --git a/go/test/site/testdata/found/.gitkeep b/go/test/site/testdata/found/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/go/test/site/verdicts_test.go b/go/test/site/verdicts_test.go new file mode 100644 index 00000000..9cb180db --- /dev/null +++ b/go/test/site/verdicts_test.go @@ -0,0 +1,553 @@ +//go:build unix + +package site + +import ( + "math" + "sort" +) + +// Each judge turns a finished run into numbers and pass criteria. The +// numbers and limits are the ones reports/simulatorprov-20261021 used. + +func judgeS0(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["connected"], d.marks["end"] + w := d.window(t0+20, t1) + lq, lr, today := counter{}, counter{}, counter{} + nAllRead := 0 + for _, r := range w { + st := status(r) + lq[str(st.LoadQuality)]++ + lr[str(st.LoadReason)]++ + today[str(st.Energy.Today.HistoryQuality)]++ + if allRead(r) { + nAllRead++ + } + } + var last *float64 + if len(w) > 0 { + last = status(w[len(w)-1]).Energy.Today.LoadWh + } + disc := d.disconnects(t0, t1-2) // the rig stops FTW at the end + m := metrics{"load_quality": lq, "load_reason": lr, "today_history_quality": today, + "all_read_samples": nAllRead, "samples_window": len(w), "today_load_wh_last": last, + "disconnects_s": disc} + return m, []criterion{ + crit("Home use measured in 95 % of samples", float64(lq["measured"]) >= 0.95*float64(len(w))), + crit("no OCPP disconnects", len(disc) == 0), + } +} + +func judgeS1(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + fuse := d.metaF("fuse") + target := maxW + if fuse < 25 { + // Room on a 16 A fuse: fuse - 1 A margin - house, in 1 A steps. + target = math.Min(maxW, math.Floor(fuse-1-d.metaF("house_a"))*volt*3) + } + m := d.carSummary(t0, t1, target) + m["target_w"] = target + hold, _ := m["hold_fraction"].(*float64) + cs := []criterion{ + crit("target within 60 s", le(m["time_to_target_s"].(*float64), 60)), + crit("holds 95 % of the time", hold != nil && *hold >= 0.95), + crit("fuse holds", !m["fuse_blown"].(bool)), + } + if fuse >= 25 { + cs = append(cs, crit("no fuse_limit with room to spare", m["fuse_limit_samples"].(int) == 0)) + } + return m, cs +} + +func judgeS2(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + pre := d.window(t0-15, t0) + var exp [3]float64 + for k := 0; k < 3; k++ { + exp[k] = round(meanOf(pre, func(r *sample) float64 { return phaseI(r)[k] }), 2) + } + m := d.carSummary(t0, t1, maxW) + m["export_a_before"] = exp + return m, []criterion{ + crit("car starts within 60 s", le(m["time_to_start_s"].(*float64), 60)), + crit("fuse holds", !m["fuse_blown"].(bool)), + } +} + +func judgeS3(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tStep, tBack, t1 := d.marks["charge_now"], d.marks["step"], d.marks["step_back"], d.marks["end"] + pre := d.window(tStep-30, tStep) + m := metrics{ + "pre_step_car_w": math.Round(meanOf(pre, func(r *sample) float64 { return carW(r, 0) })), + "pre_step_bat_w": math.Round(meanOf(pre, batSigned)), + "pre_step_max_i": round(maxOf(pre, maxAbsI), 2), + } + m["post_step_max_i"] = round(maxOf(d.window(tStep, tBack), maxAbsI), 2) + m["over_fuse_longest_s"], m["over_fuse_total_s"] = d.runsOver(tStep, tBack, fuse) + overLongest, overTotal := d.runsOver(tStep, tBack, fuse-1) + m["over_fuse_minus1_longest_s"], m["over_fuse_minus1_total_s"] = overLongest, overTotal + m["i2_over_fuse_minus1_s"], m["i2_peak_a"] = d.i2Over(tCN, t1, fuse-1) + m["back_under_fuse_minus1_s"] = roundp(d.firstT(tStep, func(r *sample) bool { return maxAbsI(r) <= fuse-1 && r.T > tStep+1 }), 1) + last := d.window(tBack-60, tBack) + m["post_step_car_w_last60"] = math.Round(meanOf(last, func(r *sample) float64 { return carW(r, 0) })) + m["post_step_bat_w_last60"] = math.Round(meanOf(last, batSigned)) + var after []*sample + for _, r := range d.samples { + if r.T >= tCN { + after = append(after, r) + } + } + m["fuse_damage_max"] = round(maxOf(after, damage), 4) + isBlown := anyBlown(d.samples) + m["fuse_blown"] = isBlown + // Device response + meter and tick (2 + 2 s) + 3 s. + allowed := d.metaF("charger_delay_s") + 2 + 2 + 3 + m["allowed_over_s"] = allowed + m["run_over_fuse_minus1_longest_s"], _ = d.runsOver(tCN, t1, fuse-1) + m["run_max_i"] = round(maxOf(after, maxAbsI), 2) + // Before the step, layers with different margins may hold a phase + // between fuse - 1 A and fuse - 0.5 A; reported, not judged. + m["pre_step_over_fuse_minus1_longest_s"], _ = d.runsOver(tCN, tStep, fuse-1) + m["pre_step_over_fuse_minus_half_longest_s"], _ = d.runsOver(tCN, tStep, fuse-0.5) + return m, []criterion{ + crit("fuse holds", !isBlown), + crit("over fuse - 1 A no longer than the response time + 3 s after the step", overLongest <= allowed), + } +} + +func judgeS4(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["charge_now"], d.marks["end"] + m := d.carSummary(t0, t1, maxW) + tx := connector1(profiles(d.frames, t0, t1)) + m["profiles_sent"] = len(tx) + offers := map[float64]bool{} + for _, p := range tx { + offers[math.Round(p.offerW())] = true + } + m["distinct_offers_w"] = sortedKeys(offers) + zeros := 0 + for i := 1; i < len(tx); i++ { + if tx[i-1].offerW() > 0 && tx[i].offerW() == 0 { + zeros++ + } + } + m["offer_drops_to_zero"] = zeros + return m, []criterion{ + crit("no stops", m["stops_after_start"].(int) == 0), + crit("no offer drops to 0 W", zeros == 0), + crit("car starts", m["time_to_start_s"].(*float64) != nil), + crit("full power over the last minute", m["car_w_last60_mean"].(float64) >= 0.95*maxW), + } +} + +func sortedKeys(set map[float64]bool) []float64 { + out := make([]float64, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Float64s(out) + return out +} + +func judgeS5(d *runData) (metrics, []criterion) { + t0, t1 := d.marks["plugged"], d.marks["end"] + prs := connector1(profiles(d.frames, t0, t1)) + ratingA := maxW / (3 * volt) + var bad []profile + over, maxOffer := 0, 0.0 + offers := map[float64]bool{} + type ap struct { + A float64 + Phases float64 + } + offersAP := map[ap]bool{} + for _, p := range prs { + offers[math.Round(p.offerW())] = true + offersAP[ap{round(p.Limit, 2), p.phasesOr3()}] = true + maxOffer = math.Max(maxOffer, p.perPhaseA()) + if p.Limit <= 0 { + continue + } + a := p.perPhaseA() + // A valid step is 6-16 A per phase: the charger is rated 16 A. + if a < 6-1e-6 || a > ratingA+1e-6 { + bad = append(bad, p) + } + if a > ratingA+1e-6 { + over++ + } + } + w := d.window(t0+60, t1) + pre := d.window(t0-10, t0+5) + surplus := meanOf(pre, func(r *sample) float64 { + i := phaseI(r) + return -(i[0] + i[1] + i[2]) * volt + }) + carMean := math.Round(meanOf(w, func(r *sample) float64 { return carW(r, 0) })) + charging := 0 + for _, r := range w { + if carW(r, 0) >= 0.95*minW { + charging++ + } + } + frac := round(float64(charging)/math.Max(1, float64(len(w))), 3) + rs := counter{} + for _, r := range w { + for _, x := range reasons(r) { + rs[x]++ + } + } + all := d.window(t0, t1) + isBlown := anyBlown(all) + if len(bad) > 5 { + bad = bad[:5] + } + m := metrics{ + "surplus_w_at_plug": math.Round(surplus), + "profiles_sent": len(prs), + "invalid_examples": bad, + "offers_w": firstN(sortedKeys(offers), 12), + "car_w_mean_after_60s": carMean, + "charging_fraction_ge_min": frac, + "time_to_start_s": roundp(d.firstT(t0, func(r *sample) bool { return carW(r, 0) >= 0.95*minW }), 1), + "reasons": rs.top(6), + "offers_over_rating": over, + "max_offer_a": round(maxOffer, 2), + "fuse_blown": isBlown, + "fuse_damage_max": round(maxOf(all, damage), 4), + "surplus_used_fraction": round(carMean/math.Max(1, math.Round(surplus)), 3), + } + nInvalid := 0 + for _, p := range prs { + if a := p.perPhaseA(); p.Limit > 0 && (a < 6-1e-6 || a > ratingA+1e-6) { + nInvalid++ + } + } + m["invalid_steps"] = nInvalid + return m, []criterion{ + // The beta criteria (Fredrik, 21 Oct): no offer above the charger's + // rating per phase, and the fuse never trips. + crit("no offer above the charger's rating", over == 0), + crit("fuse holds", !isBlown), + crit("only valid steps (6-16 A)", nInvalid == 0), + crit("charges at least the minimum 90 % of the time", frac >= 0.9), + } +} + +func firstN(xs []float64, n int) []float64 { + if len(xs) > n { + return xs[:n] + } + return xs +} + +type qualitySummary struct { + Samples int `json:"samples"` + LoadWShown int `json:"load_w_shown"` + LoadQuality map[string]int `json:"load_quality"` + TodayFirst *float64 `json:"today_load_wh_first"` + TodayLast *float64 `json:"today_load_wh_last"` +} + +func summarize(ws []*sample) qualitySummary { + q := qualitySummary{Samples: len(ws), LoadQuality: counter{}} + for i, r := range ws { + st := status(r) + q.LoadQuality[str(st.LoadQuality)]++ + if st.LoadW != nil { + q.LoadWShown++ + } + if i == 0 { + q.TodayFirst = st.Energy.Today.LoadWh + } + q.TodayLast = st.Energy.Today.LoadWh + } + return q +} + +func judgeS6(d *runData) (metrics, []criterion) { + tOff, tOn, t1 := d.marks["offline"], d.marks["online"], d.marks["end"] + before := summarize(d.window(tOff-40, tOff)) + during := summarize(d.window(tOff+15, tOn)) + after := summarize(d.window(tOn+30, t1)) + // From 20 s after plug-in: FTW's start and the charger's first + // MeterValues after plug-in are reported apart (plug_in_approx_s). + plugT, ok := d.marks["plug"] + if !ok && len(d.samples) > 0 { + plugT = d.samples[0].T + } + var viol []*sample + for _, r := range d.samples { + if r.T >= plugT+20 && allRead(r) && str(status(r).LoadQuality) != "measured" && !(tOff <= r.T && r.T < tOn+30) { + viol = append(viol, r) + } + } + plugIn := 0 + for _, r := range d.window(plugT, plugT+20) { + if allRead(r) && str(status(r).LoadQuality) != "measured" { + plugIn++ + } + } + var firstViol *float64 + if len(viol) > 0 { + v := round(viol[0].T-tOff, 1) + firstViol = &v + } + marked := 0 + for k, v := range during.LoadQuality { + if k != "null" && k != "" && k != "measured" { + marked += v + } + } + m := metrics{"before": before, "during": during, "after": after, + "approx_while_all_read": len(viol), "approx_while_all_read_first": firstViol, + "load_w_missing_during": during.Samples - during.LoadWShown, "plug_in_approx_s": plugIn} + return m, []criterion{ + crit("Home use shown throughout the outage", during.LoadWShown == during.Samples), + crit("Home use marked estimated throughout the outage", marked == during.Samples), + crit("measured before the outage", before.LoadQuality["measured"] == before.Samples), + crit("measured after the outage", after.LoadQuality["measured"] == after.Samples), + crit("never estimated while every driver reads", len(viol) == 0), + } +} + +func judgeS7(d *runData) (metrics, []criterion) { + tCN, tStop, tUp, t1 := d.marks["charge_now"], d.marks["ftw_stop"], d.marks["ftw_up"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + pre := d.window(tStop-30, tStop) + preCar := meanOf(pre, car) + post := d.window(tUp, t1) + m := metrics{"pre_car_w": math.Round(preCar), "downtime_s": round(tUp-tStop, 1)} + var before *float64 + if len(pre) > 0 { + m["pre_load_w"] = status(pre[len(pre)-1]).LoadW + before = status(pre[len(pre)-1]).Energy.Today.LoadWh + } + m["today_load_wh_before"] = before + var afterFirst, afterLast *float64 + nullAfter := 0 + hq, lq := counter{}, counter{} + for _, r := range post { + // A sample before the rig's first status read after the restart + // has no reading; only a status without the day total counts. + if r.Status == nil { + continue + } + st := status(r) + v := st.Energy.Today.LoadWh + if v == nil { + nullAfter++ + } else if afterFirst == nil { + afterFirst = v + } + afterLast = v + hq[str(st.Energy.Today.HistoryQuality)]++ + lq[str(st.LoadQuality)]++ + } + m["today_load_wh_after_first"], m["today_load_wh_null_after"], m["today_load_wh_after_last"] = afterFirst, nullAfter, afterLast + m["history_quality_after"], m["load_quality_after"] = hq, lq + loadBack := d.firstT(tUp, func(r *sample) bool { return status(r).LoadW != nil }) + m["load_w_back_s"] = roundp(loadBack, 1) + m["car_back_s"] = roundp(d.firstT(tUp, func(r *sample) bool { return preCar > 0 && car(r) >= 0.95*preCar }), 1) + m["car_resumed_s"] = roundp(d.firstT(tUp, func(r *sample) bool { return car(r) >= 0.95*minW }), 1) + m["car_w_150s_after_up"] = math.Round(meanOf(d.window(tUp+90, tUp+150), car)) + rs := counter{} + for _, r := range d.window(tUp+10, t1) { + for _, x := range reasons(r) { + rs[x]++ + } + } + m["reasons_after_up"] = rs.top(4) + m["car_min_during_restart"] = math.Round(minOf(d.window(tStop, tUp+60), car)) + m["car_w_last60"] = math.Round(meanOf(d.window(t1-60, t1), car)) + if tk, ok := d.marks["today_known"]; ok { + m["today_known_after_s"] = math.Round(tk - tCN) + } + minAfterStop := minOf(d.window(tStop, t1), car) + m["car_min_after_stop"] = math.Round(minAfterStop) + var fullFrom *float64 + dip, dipMin := 0, preCar + for _, r := range post { + if car(r) >= 0.95*preCar { + if fullFrom == nil { + v := r.T - tUp + fullFrom = &v + } + } else { + fullFrom = nil + dip++ + dipMin = math.Min(dipMin, car(r)) + } + } + m["full_power_from_s"], m["dip_s"], m["dip_min_w"] = roundp(fullFrom, 1), dip, math.Round(dipMin) + cs := []criterion{ + crit("Home use back within 30 s", le(loadBack, 30)), + // Charge now continues without a stop and is back at full power + // within 420 s of Core being up, for at least the last 30 s. The + // dip is recorded. + crit("Charge now never stops", minAfterStop >= 0.95*minW), + crit("full power within 420 s", fullFrom != nil && *fullFrom <= 420 && t1-tUp-*fullFrom >= 30), + } + if d.meta["short"] != true { + cs = append([]criterion{crit("the day's Home use is kept", before != nil && nullAfter == 0 && + afterLast != nil && *afterLast >= *before*0.99)}, cs...) + } + return m, cs +} + +func minOf(ws []*sample, f func(*sample) float64) float64 { + if len(ws) == 0 { + return 0 + } + m := math.Inf(1) + for _, r := range ws { + m = math.Min(m, f(r)) + } + return m +} + +func judgeS8(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + t0, t1 := d.marks["charge_now"], d.marks["end"] + m := d.carSummary(t0, t1, maxW) + m["over_fuse_longest_s"], m["over_fuse_total_s"] = d.runsOver(t0, t1, fuse) + overLongest, overTotal := d.runsOver(t0, t1, fuse-1) + m["over_fuse_minus1_longest_s"], m["over_fuse_minus1_total_s"] = overLongest, overTotal + m["i2_over_fuse_minus1_s"], m["i2_peak_a"] = d.i2Over(t0, t1, fuse-1) + var prs [][2]float64 + for i, p := range connector1(profiles(d.frames, t0, t1)) { + if i >= 20 { + break + } + prs = append(prs, [2]float64{p.T, math.Round(p.offerW())}) + } + m["profiles"] = prs + allowed := d.metaF("charger_delay_s") + 2 + 2 + 3 + m["allowed_over_s"] = allowed + if plug, ok := d.marks["plug"]; ok { + // Before FTW's first profile reaches the charger it draws its maximum. + m["plugin_max_i"] = round(maxOf(d.window(plug, t0), maxAbsI), 2) + m["plugin_over_fuse_longest_s"], _ = d.runsOver(plug, t0, fuse) + } + return m, []criterion{ + crit("fuse holds", !m["fuse_blown"].(bool)), + crit("over fuse - 1 A no longer than the charger's delay + 7 s", overLongest <= allowed), + } +} + +func judgeS9(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tSw, t1 := d.marks["charge_now"], d.marks["switch"], d.marks["end"] + before, after := d.overage(tCN, tSw, fuse), d.overage(tSw, t1, fuse) + car := func(r *sample) float64 { return carW(r, 0) } + m := metrics{"before": before, "after": after, + "before_phase_a": d.phaseMean(tSw-30, tSw), "after_phase_a_last60": d.phaseMean(t1-60, t1), + "car_w_before": math.Round(meanOf(d.window(tSw-30, tSw), car)), + "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), car))} + var seq [][3]any + for _, p := range connector1(profiles(d.frames, tSw-5, t1)) { + a := round(p.Limit, 2) + if len(seq) == 0 || seq[len(seq)-1][1] != a || !samePhases(seq[len(seq)-1][2], p.Phases) { + seq = append(seq, [3]any{p.T, a, p.Phases}) + } + } + if len(seq) > 12 { + seq = seq[:12] + } + m["profiles_after_switch"] = seq + return m, []criterion{ + crit("fuse holds", !before["fuse_blown"].(bool) && !after["fuse_blown"].(bool)), + crit("over fuse - 1 A at most 3 s on 3p", before["over_limit_longest_s"].(float64) <= 3), + crit("over fuse - 1 A at most 3 s after the switch to 1p", after["over_limit_longest_s"].(float64) <= 3), + } +} + +func samePhases(a any, b *float64) bool { + pa, _ := a.(*float64) + if pa == nil || b == nil { + return pa == nil && b == nil + } + return *pa == *b +} + +func judgeS10(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tSt, t1 := d.marks["charge_now"], d.marks["step"], d.marks["end"] + before, after := d.overage(tCN, tSt, fuse), d.overage(tSt, t1, fuse) + m := metrics{"before": before, "after": after} + pre, last := d.window(tSt-30, tSt), d.window(t1-60, t1) + var preW, lastW [2]float64 + for i := 0; i < 2; i++ { + i := i + preW[i] = math.Round(meanOf(pre, func(r *sample) float64 { return carW(r, i) })) + lastW[i] = math.Round(meanOf(last, func(r *sample) float64 { return carW(r, i) })) + } + m["pre_step_w"], m["last60_w"] = preW, lastW + for i, name := range lpIDs { + var seq [][2]float64 + for _, p := range connector1(profiles(d.loadFrames(i), tSt-5, t1)) { + a := round(p.Limit, 1) + if len(seq) == 0 || seq[len(seq)-1][1] != a { + seq = append(seq, [2]float64{p.T, a}) + } + } + if len(seq) > 10 { + seq = seq[:10] + } + m[name+"_profiles_after_step"] = seq + } + return m, []criterion{ + crit("fuse holds", !before["fuse_blown"].(bool) && !after["fuse_blown"].(bool)), + crit("over fuse - 1 A at most 3 s after the step", after["over_limit_longest_s"].(float64) <= 3), + } +} + +func judgeS11(d *runData) (metrics, []criterion) { + fuse := d.metaF("fuse") + tCN, tP, tR, t1 := d.marks["charge_now"], d.marks["pause"], d.marks["restart"], d.marks["end"] + car := func(r *sample) float64 { return carW(r, 0) } + prs := connector1(profiles(d.frames, tCN-1, t1)) + var firstPos, paused *profile + for i := range prs { + p := &prs[i] + if firstPos == nil && p.T >= tR-tCN && p.Limit > 0 { + firstPos = p + } + if paused == nil && p.T >= tP-tCN && p.Limit == 0 { + paused = p + } + } + m := metrics{ + "car_w_before_pause": math.Round(meanOf(d.window(tP-20, tP), car)), + "car_started_s": roundp(d.firstT(tR, func(r *sample) bool { return car(r) >= 0.95*minW }), 1), + "car_w_last60": math.Round(meanOf(d.window(t1-60, t1), car)), + } + var firstPosS *float64 + if paused != nil { + m["pause_profile_at_s"] = round(paused.T-(tP-tCN), 1) + } + if firstPos != nil { + v := round(firstPos.T-(tR-tCN), 1) + firstPosS = &v + m["first_positive_offer_a"] = firstPos.Limit + } + m["first_positive_offer_s"] = firstPosS + rs := counter{} + for _, r := range d.window(tR+2, t1) { + for _, x := range reasons(r) { + rs[x]++ + } + } + m["reasons_after_restart"] = rs.top(4) + run := d.overage(tCN, t1, fuse) + for k, v := range run { + m["run_"+k] = v + } + return m, []criterion{ + crit("an offer within 60 s of the restart", le(firstPosS, 60)), + crit("over fuse - 1 A at most 3 s", run["over_limit_longest_s"].(float64) <= 3), + crit("fuse holds", !run["fuse_blown"].(bool)), + } +}