fix(live): Home use as the plain balance; display and energy load as two series

.changeset/live-home-use-quality.md

@@ -0,0 +1,7 @@
+---
+"ftw": patch
+---
+
+Show Home use again when one device is offline. Live Home use is the energy balance as is: the site meter minus every device that is read, as before 0.141. A device that is offline, or a setup problem, no longer blanks it; the dashboard marks it "estimated" with "≈". A negative value is shown, since it points at production FTW cannot see. After sunset at the site's map pin an offline inverter counts as 0 W. Home use is blank only when the site meter is missing or stale. `/api/status` adds `load_quality` and `load_reason` as hints.
+
+Home use now has two series. The live display value is logged as the `load_display_w` metric. The energy-based Home use from closed history intervals, with its quality flags, stays what load models train on. Home Assistant gets a second sensor, "Load Power (energy average)", next to "Load Power".

go/cmd/ftw/app_link.go

@@ -81,7 +81,7 @@ func (a *appSite) Snapshot() appproto.Snapshot {
 	live := a.tel.LiveMeasurementNow(siteMeterDriver, opts)
 	snap := appproto.Snapshot{Mode: mode, ControlRev: rev, FuseA: fuseA,
 		GridW: live.GridW, PVW: live.PVW, BatteryW: live.BatteryW, EVW: live.EVW,
-		LoadW: live.HouseholdW, LoadWKnown: live.Valid}
+		LoadW: live.LoadW, LoadWKnown: live.LoadKnown}
 
 	uptimeMs := time.Since(a.started).Milliseconds()
 	seen := map[string]bool{}

go/cmd/ftw/display_load.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"github.com/srcfl/ftw/go/internal/state"
+	"github.com/srcfl/ftw/go/internal/telemetry"
+)
+
+// Display load is stored as a Core metric series, apart from every driver.
+const (
+	displayLoadDriver = "ftw:site"
+	displayLoadMetric = "load_display_w"
+)
+
+// displayLoadSample is the live Home use clients show, for the metric store.
+// Nothing while the site meter is unknown: that is what clients show too.
+func displayLoadSample(tel *telemetry.Store, siteMeter string, nowMs int64, options telemetry.ForecastOptions, sun historySun) (state.Sample, bool) {
+	if tel == nil {
+		return state.Sample{}, false
+	}
+	options.PVDark = sun.darkness
+	live := tel.LiveMeasurementNow(siteMeter, options)
+	if !live.LoadKnown {
+		return state.Sample{}, false
+	}
+	return state.Sample{Driver: displayLoadDriver, Metric: displayLoadMetric, TsMs: nowMs, Value: live.LoadW, Unit: "W"}, true
+}

go/cmd/ftw/display_load_test.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/srcfl/ftw/go/internal/state"
+	"github.com/srcfl/ftw/go/internal/telemetry"
+)
+
+// Two series: the live display load is logged as a Core metric exactly as
+// clients show it (negative included). The history tick's Load stays the
+// placeholder the source-energy writer replaces, so training, which reads
+// HouseholdIntervals, never sees the display value.
+func TestDisplayLoadIsLoggedApartFromTrainingLoad(t *testing.T) {
+	st, err := state.Open(filepath.Join(t.TempDir(), "state.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+	if err := st.EnableHistoryAggregation(); err != nil {
+		t.Fatal(err)
+	}
+	tel := telemetry.NewStore()
+	for _, d := range []string{"meter", "battery"} {
+		tel.EnsureDriverHealth(d)
+	}
+	tel.Update("meter", telemetry.DerMeter, 1200, nil, nil)
+	tel.Update("battery", telemetry.DerBattery, 2000, nil, nil) // negative balance
+	tel.RecordDriverSuccess("meter")
+	tel.RecordDriverSuccess("battery")
+	ctrl := tickPersistControl{SiteMeterDriver: "meter"}
+	at := time.Now().UnixMilli()
+	if _, err := persistTelemetryTick(st, tel, ctrl, at, time.Minute, testEnergyIdentity(st)); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.FlushHistory(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	var got []state.Sample
+	for i := 0; i < 50 && len(got) == 0; i++ {
+		got, err = st.LoadSeries(displayLoadDriver, displayLoadMetric, at-1, at+1, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(got) == 0 {
+			time.Sleep(20 * time.Millisecond)
+		}
+	}
+	if len(got) != 1 || got[0].Value != -800 {
+		t.Fatalf("display load series = %+v, want one -800 W sample", got)
+	}
+	history, err := st.LoadHistory(at-1, at+1, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, h := range history {
+		if h.Missing&state.MissingLoad == 0 || h.LoadW != 0 {
+			t.Fatalf("display load leaked into the history Load column: %+v", h)
+		}
+	}
+	if intervals, _, err := st.HouseholdIntervals(context.Background(), 0, at, 15*60000); err != nil || len(intervals) != 0 {
+		t.Fatalf("training intervals from one display tick: %v %v", intervals, err)
+	}
+}
+
+func TestDisplayLoadNotLoggedWithoutSiteMeter(t *testing.T) {
+	tel := telemetry.NewStore()
+	if _, ok := displayLoadSample(tel, "meter", time.Now().UnixMilli(), telemetry.ForecastOptions{}, historySun{}); ok {
+		t.Fatal("display load logged without a site meter")
+	}
+}

go/cmd/ftw/history_dark_pv.go

@@ -36,6 +36,19 @@ func (s historySun) zenith(atMS int64) (float64, bool) {
 	return sunpos.At(time.UnixMilli(atMS), s.Lat, s.Lon).ZenithDeg, true
 }
 
+// darkness tells live Home use how dark it is at the site, by the same
+// sunset and civil-dusk limits as history.
+func (s historySun) darkness(at time.Time) telemetry.Darkness {
+	z, placed := s.zenith(at.UnixMilli())
+	switch {
+	case !placed || z <= sunsetZenithDeg:
+		return telemetry.DarknessUnknown
+	case z <= civilDuskZenithDeg:
+		return telemetry.DarknessTwilight
+	}
+	return telemetry.DarknessNight
+}
+
 // darkPV records an unknown PV source as 0 W while the sun is down at the
 // site: physics, not interpolation. Past civil dusk the 0 W is as good as a
 // reading and carries no marker, so a night with a sleeping inverter also

go/cmd/ftw/live_load_dark_test.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/srcfl/ftw/go/internal/telemetry"
+)
+
+func TestHistorySunDarknessForLiveLoad(t *testing.T) {
+	sun := historySunOf(59.33, 18.07, true) // Stockholm
+	cases := map[string]telemetry.Darkness{
+		"2026-06-21T12:00:00+02:00": telemetry.DarknessUnknown,
+		"2026-10-21T02:00:00+02:00": telemetry.DarknessNight,
+	}
+	for at, want := range cases {
+		ts, _ := time.Parse(time.RFC3339, at)
+		if got := sun.darkness(ts); got != want {
+			t.Fatalf("%s: darkness %d, want %d", at, got, want)
+		}
+	}
+	if (historySun{}).darkness(time.Now()) != telemetry.DarknessUnknown {
+		t.Fatal("an unplaced site inferred darkness")
+	}
+	// Through an October evening: daylight, then twilight, then night, in order.
+	start, _ := time.Parse(time.RFC3339, "2026-10-21T15:00:00+02:00")
+	seen := []telemetry.Darkness{}
+	for m := 0; m <= 300; m++ {
+		d := sun.darkness(start.Add(time.Duration(m) * time.Minute))
+		if len(seen) == 0 || seen[len(seen)-1] != d {
+			seen = append(seen, d)
+		}
+	}
+	if len(seen) != 3 || seen[0] != telemetry.DarknessUnknown || seen[1] != telemetry.DarknessTwilight || seen[2] != telemetry.DarknessNight {
+		t.Fatalf("evening sequence %v", seen)
+	}
+}

go/cmd/ftw/live_load_test.go

@@ -76,11 +76,12 @@ func TestLiveLoadAppAndLANShareSourcePowerAndValidity(t *testing.T) {
 				want = 1500
 			case "negative":
 				tel.Update("battery", telemetry.DerBattery, 2000, nil, nil)
-				valid = false
+				want = -1000 // shown as is: production Core cannot see
 			case "offline_ev":
 				tel.Update("charger", telemetry.DerEV, 500, nil, nil)
 				tel.DriverHealthMut("charger").SetOffline()
-				valid = false
+				// Left out, as before #1592: Home use is an estimate, not blank.
+				want = 1000
 			case "idle_ev", "paused_ev":
 				fields := map[string]any{"connected": scenario == "paused_ev", "charging": false,
 					"control_power_observed_at": time.Now().Add(-time.Hour).Format(time.RFC3339Nano),
@@ -136,11 +137,16 @@ func TestLiveLoadOptionsFollowConfigurationForAppLANAndCloud(t *testing.T) {
 		snap := site.Snapshot()
 		status := liveLoadStatus(t, tel, ctrl, options)
 		_, cloudKnown := (&cloudSite{site: site}).Live().Fields[strconv.Itoa(int(appproto.FidLoadW))]
-		known := !batteryDeclared
-		if snap.LoadWKnown != known || (status["load_w"] != nil) != known || cloudKnown != known {
+		// A declared battery that never reported is left out: Home use stays
+		// known on every client and is marked estimated on the LAN.
+		if !snap.LoadWKnown || status["load_w"] == nil || !cloudKnown {
 			t.Fatal("configured never-emitted flow differs between clients")
 		}
-		if known && (snap.LoadW != 1000 || status["load_w"] != float64(1000)) {
+		wantQuality := map[bool]string{true: "estimated", false: "measured"}[batteryDeclared]
+		if status["load_quality"] != wantQuality {
+			t.Fatalf("load_quality = %v, want %s", status["load_quality"], wantQuality)
+		}
+		if snap.LoadW != 1000 || status["load_w"] != float64(1000) {
 			t.Fatal("learning-only identity gate affected live load")
 		}
 		if batteryDeclared {

go/cmd/ftw/main.go

@@ -858,7 +858,10 @@ func main() {
 	}
 	forecastSettings := newForecastSiteConfig(st)
 	liveMeasurementOptions := func() telemetry.ForecastOptions {
-		return forecastSettings.Snapshot().HistoryOptions
+		site := forecastSettings.Snapshot()
+		opts := site.HistoryOptions
+		opts.PVDark = site.HistorySun.darkness
+		return opts
 	}
 	forecastSettings.identity = func(name string) (string, bool) {
 		if id, ok := runningDeviceID(reg, name); ok {
@@ -3169,6 +3172,8 @@ func main() {
 					"ev_w", evW,
 					"v2x_w", v2xW,
 					"load_valid", live.Valid,
+					"load_quality", live.LoadQuality,
+					"load_reason", live.LoadReason,
 					"self_tune_active", selfTuneActive,
 					"self_tune_driver", selfTuneName,
 					"self_tune_command_w", selfTuneCmd,
@@ -4031,6 +4036,12 @@ func persistTelemetryTick(st *state.Store, tel *telemetry.Store, ctrl tickPersis
 			Value: sm.Value, Unit: sm.Unit,
 		}
 	}
+	// Display load: the live balance exactly as clients show it. A record of
+	// what was on screen, never a training label; models read the household
+	// energy intervals the source writer closes (HouseholdIntervals).
+	if display, ok := displayLoadSample(tel, ctrl.SiteMeterDriver, nowMs, options, sun); ok {
+		stSamples = append(stSamples, display)
+	}
 	energyObservations := buildEnergyObservations(st, tel, ctrl, hp, identity)
 	if hp.Sources != nil {
 		hp.Sources.Topology += historyDevices(st, tel, identity, energyObservations, dark)

go/internal/api/api_history_quality.go

@@ -37,14 +37,26 @@ func (s *Server) qualifyStatusHistory(resp map[string]any, siteMeter string) {
 		missing |= state.BalanceMismatch
 	}
 	resp["grid_w"] = missing.Value(state.MissingGrid, r.GridW)
-	resp["pv_w"] = missing.Value(state.MissingPV, r.PVW)
+	// Live power is not history: a dark inverter is 0 W and Home use stays
+	// known while the site meter is read (load_quality says how).
+	if r.PVLiveKnown && opts.PVInvalidReason == "" {
+		resp["pv_w"] = r.PVW
+	} else {
+		resp["pv_w"] = nil
+	}
 	resp["bat_w"] = missing.Value(state.MissingBattery, r.BatteryW)
 	resp["ev_w"] = missing.Value(state.MissingEV, r.EVW)
 	if missing&state.MissingEV != 0 {
 		resp["ev_charging_w"] = nil
 	}
 	resp["v2x_w"] = missing.Value(state.MissingV2X, r.V2XW)
-	resp["load_w"] = missing.Value(state.MissingLoad, r.HouseholdW)
+	if r.LoadKnown {
+		resp["load_w"] = r.LoadW
+	} else {
+		resp["load_w"] = nil
+	}
+	resp["load_quality"] = r.LoadQuality
+	resp["load_reason"] = r.LoadReason
 	resp["history_quality"] = missing.Quality()
 	resp["history_reason"] = r.Reason
 	drivers, _ := resp["drivers"].(map[string]any)

go/internal/api/api_history_quality_test.go

@@ -77,13 +77,15 @@ func TestLiveHistoryQualitySeparatesOfflineFromInvalidBalance(t *testing.T) {
 	}})
 	resp := map[string]any{}
 	srv.qualifyStatusHistory(resp, "meter")
-	if resp["grid_w"] != float64(1200) || resp["bat_w"] != nil || resp["load_w"] != nil || resp["history_quality"] != "incomplete" {
+	// Live Home use leaves the offline battery out; history stays incomplete.
+	if resp["grid_w"] != float64(1200) || resp["bat_w"] != nil || resp["load_w"] != float64(1200) || resp["load_quality"] != "estimated" || resp["history_quality"] != "incomplete" {
 		t.Fatal(resp)
 	}
 	tel.Update("battery", telemetry.DerBattery, 2000, nil, nil)
 	tel.RecordDriverSuccess("battery")
 	srv.qualifyStatusHistory(resp, "meter")
-	if resp["bat_w"] != float64(2000) || resp["load_w"] != nil || resp["history_quality"] != "balance_mismatch" {
+	// Negative balance: shown as is, history still records the mismatch.
+	if resp["bat_w"] != float64(2000) || resp["load_w"] != float64(-800) || resp["load_reason"] != "invalid_balance" || resp["history_quality"] != "balance_mismatch" {
 		t.Fatal(resp)
 	}
 	tel.Update("battery", telemetry.DerBattery, 0, nil, nil)
@@ -130,7 +132,50 @@ func TestLiveHistoryQualityConfiguredPVWithoutMeasurementIsUnknown(t *testing.T)
 	}})
 	resp := map[string]any{}
 	srv.qualifyStatusHistory(resp, "meter")
-	if resp["pv_w"] != nil || resp["load_w"] != nil {
-		t.Fatal("unmeasured configured PV reported as zero", resp)
+	if resp["pv_w"] != nil || resp["load_w"] != float64(1200) || resp["load_quality"] != "estimated" || resp["history_quality"] != "incomplete" {
+		t.Fatal("unmeasured configured PV reported as zero, or Home use blanked", resp)
+	}
+}
+
+// Anders (Discord, v0.154.1): Growatt offline after sunset, Pixii meter and
+// battery read. /api/status showed pv_w and load_w null.
+func TestStatusAndersNightAndDay(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		dark     telemetry.Darkness
+		pv       any
+		quality  string
+		loadWant any
+	}{
+		{"night", telemetry.DarknessNight, float64(0), "measured", float64(980)},
+		{"day", telemetry.DarknessUnknown, nil, "estimated", float64(980)},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			tel := telemetry.NewStore()
+			for _, name := range []string{"pixii", "growatt"} {
+				tel.EnsureDriverHealth(name)
+			}
+			tel.Update("pixii", telemetry.DerMeter, 780, nil, nil)
+			tel.Update("pixii", telemetry.DerBattery, -200, nil, nil)
+			tel.RecordDriverSuccess("pixii")
+			tel.Update("growatt", telemetry.DerPV, -40, nil, nil)
+			tel.DriverHealthMut("growatt").SetOffline()
+			dark := tc.dark
+			srv := New(&Deps{Tel: tel, HistoryMeasurementSources: func() telemetry.ForecastOptions {
+				return telemetry.ForecastOptions{PVDark: func(time.Time) telemetry.Darkness { return dark }}
+			}})
+			resp := map[string]any{"drivers": map[string]any{"growatt": map[string]any{}}}
+			srv.qualifyStatusHistory(resp, "pixii")
+			if resp["grid_w"] != float64(780) || resp["pv_w"] != tc.pv || resp["load_w"] != tc.loadWant || resp["load_quality"] != tc.quality {
+				t.Fatal(resp)
+			}
+			// History quality stays separate from live Home use.
+			if resp["history_quality"] != "incomplete" || resp["history_reason"] != "offline:growatt:pv" {
+				t.Fatal("live display changed history quality", resp)
+			}
+			if tc.dark == telemetry.DarknessNight && resp["drivers"].(map[string]any)["growatt"].(map[string]any)["pv_w"] == nil {
+				t.Fatal("dark inverter not shown as 0 W")
+			}
+		})
 	}
 }

go/internal/api/api_home_estimate_test.go

@@ -65,14 +65,15 @@ func TestStatusKeepsPowerBalanceWhenSessionEnergyCannotFormAnInterval(t *testing
 	}
 	tel.DriverHealthMut("charger").SetOffline()
 	srv.qualifyStatusHistory(resp, "site")
-	if resp["load_w"] != nil || resp["load_estimate"].(telemetry.HomeEstimate).Watts != nil {
-		t.Fatal("offline charger became zero load")
+	// The offline charger is left out of live Home use, which says so.
+	if resp["load_w"] != float64(1234) || resp["load_quality"] != "estimated" || resp["load_estimate"].(telemetry.HomeEstimate).Watts != nil {
+		t.Fatal("offline charger blanked Home use or became an energy interval")
 	}
 	tel.RecordDriverSuccess("charger")
 	tel.Update("charger", telemetry.DerEV, 6000, nil, nil)
 	srv.qualifyStatusHistory(resp, "site")
-	if resp["load_w"] != nil || resp["history_reason"] != "invalid_balance" {
-		t.Fatal("negative residual became a zero household balance")
+	if resp["load_w"] != float64(1234-6000) || resp["history_reason"] != "invalid_balance" {
+		t.Fatal("negative residual was hidden or zeroed")
 	}
 }
 
@@ -96,12 +97,13 @@ func TestStatusHomePowerUsesPixiiACWithoutChangingRawHistoryInputs(t *testing.T)
 	}
 	tel.Update("battery", telemetry.DerBattery, -2500, nil, json.RawMessage(`{"control_power_available":false}`))
 	srv.qualifyStatusHistory(resp, "site")
-	if resp["load_w"] != nil || resp["bat_w"] != nil {
-		t.Fatal("missing AC measurement became DC power or zero")
+	// Unknown AC power leaves the battery out; the balance is shown as is.
+	if resp["load_w"] != float64(5000-6000) || resp["bat_w"] != nil || resp["load_quality"] != "estimated" {
+		t.Fatal("missing AC measurement became DC power or hid the balance")
 	}
 	tel.Update("battery", telemetry.DerBattery, -2500, nil, json.RawMessage(`{"control_power_w":-500,"control_power_available":true}`))
 	srv.qualifyStatusHistory(resp, "site")
-	if resp["load_w"] != nil || resp["history_reason"] != "invalid_balance" {
+	if resp["load_w"] != float64(5000-(-500)-6000) || resp["history_reason"] != "invalid_balance" {
 		t.Fatal("negative AC balance became zero or a DC estimate")
 	}
 }