From 0cfd82cdcb5696b3bf17a41cdaa62b0d789dbd6b Mon Sep 17 00:00:00 2001 From: alex289 Date: Wed, 28 Oct 2026 13:08:26 +0100 Subject: [PATCH] feat(health): report per-service health and add degraded status Agents now include the health of each compose service in their status reports. The hub derives a new degraded status when some services are healthy while others are unhealthy or stopped. Older agents keep working because the aggregate status is unchanged. --- .../internal/agent/docker/health_watcher.go | 34 ++--- .../agent/docker/health_watcher_test.go | 100 ++++++++------ backend/internal/agent/docker/status.go | 80 ++++++++++-- backend/internal/agent/docker/status_test.go | 59 ++++++++- backend/internal/hub/models/applications.go | 3 + backend/internal/hub/websocket/handler.go | 28 +++- .../hub/websocket/status_report_test.go | 34 +++++ backend/internal/proto/messages.pb.go | 123 ++++++++++++++---- backend/internal/proto/messages.proto | 10 ++ frontend/messages/de.json | 4 +- frontend/messages/en.json | 4 +- .../badges/application-status-badge.tsx | 5 + frontend/src/lib/applications.ts | 1 + frontend/src/lib/fleet.ts | 11 +- 14 files changed, 395 insertions(+), 101 deletions(-) diff --git a/backend/internal/agent/docker/health_watcher.go b/backend/internal/agent/docker/health_watcher.go index c3131a11..c8a98b70 100644 --- a/backend/internal/agent/docker/health_watcher.go +++ b/backend/internal/agent/docker/health_watcher.go @@ -47,12 +47,12 @@ type healthWatcher struct { debounce time.Duration maxDelay time.Duration snapshotTimeout time.Duration - checkHealth func(ctx context.Context, appID string) HealthState + checkHealth func(ctx context.Context, appID string) HealthReport sender MessageSender pending map[string]*pendingHealthEvaluation evaluating map[string]*healthEvaluation epochs map[string]uint64 - lastReported map[string]HealthState + lastReported map[string]HealthReport nextEpoch uint64 stopped bool } @@ -68,7 +68,7 @@ func newHealthWatcher(c *Client) *healthWatcher { pending: make(map[string]*pendingHealthEvaluation), evaluating: make(map[string]*healthEvaluation), epochs: make(map[string]uint64), - lastReported: make(map[string]HealthState), + lastReported: make(map[string]HealthReport), } } @@ -325,7 +325,7 @@ func (w *healthWatcher) runEvaluation(appID string, evaluation *healthEvaluation checkCtx, cancel := context.WithTimeout(evaluation.ctx, healthEvaluateTimeout) health := w.checkHealth(checkCtx, appID) cancel() - if health == HealthUnknown { + if health.State == HealthUnknown { return } @@ -335,7 +335,7 @@ func (w *healthWatcher) runEvaluation(appID string, evaluation *healthEvaluation return } last, seen := w.lastReported[appID] - if seen && last == health { + if seen && last.Equal(health) { w.mu.Unlock() return } @@ -349,7 +349,7 @@ func (w *healthWatcher) runEvaluation(appID string, evaluation *healthEvaluation if err := sender.SendMessage(&messages.ClientMessage{ Payload: &messages.ClientMessage_ApplicationStatusReport{ ApplicationStatusReport: &messages.ApplicationStatusReport{ - Statuses: []*messages.ApplicationStatus{{ApplicationId: appID, Health: health.Proto()}}, + Statuses: []*messages.ApplicationStatus{health.Proto(appID)}, }, }, }); err != nil { @@ -364,7 +364,7 @@ func (w *healthWatcher) markUnsent(appID string) { w.mu.Lock() defer w.mu.Unlock() if !w.stopped { - w.lastReported[appID] = HealthUnknown + w.lastReported[appID] = HealthReport{State: HealthUnknown} } } @@ -385,18 +385,18 @@ func (w *healthWatcher) reportApplicationStatus(ctx context.Context, sender Mess snapshotCtx, cancel := context.WithTimeout(ctx, w.snapshotTimeout) defer cancel() statuses := make([]*messages.ApplicationStatus, 0, len(appIDs)) - reported := make(map[string]HealthState, len(appIDs)) + reported := make(map[string]HealthReport, len(appIDs)) var unchecked []string for _, appID := range appIDs { - health := HealthUnknown + health := HealthReport{State: HealthUnknown} if snapshotCtx.Err() == nil { health = w.checkHealth(snapshotCtx, appID) } if snapshotCtx.Err() != nil { - health = HealthUnknown + health = HealthReport{State: HealthUnknown} unchecked = append(unchecked, appID) } - statuses = append(statuses, &messages.ApplicationStatus{ApplicationId: appID, Health: health.Proto()}) + statuses = append(statuses, health.Proto(appID)) reported[appID] = health } if ctx.Err() != nil { @@ -418,8 +418,8 @@ func (w *healthWatcher) reportApplicationStatus(ctx context.Context, sender Mess w.mu.Lock() if !w.stopped { for appID, health := range reported { - if health != HealthUnknown { - w.lastReported[appID] = HealthUnknown + if health.State != HealthUnknown { + w.lastReported[appID] = HealthReport{State: HealthUnknown} } } } @@ -430,7 +430,7 @@ func (w *healthWatcher) reportApplicationStatus(ctx context.Context, sender Mess w.mu.Lock() if !w.stopped { for appID, health := range reported { - if health == HealthUnknown { + if health.State == HealthUnknown { delete(w.lastReported, appID) continue } @@ -468,7 +468,7 @@ func (w *healthWatcher) reconcileMissingApplication(appID string) { sender := w.sender w.mu.Unlock() reported := false - reportedHealth := HealthUnknown + reportedHealth := HealthReport{State: HealthUnknown} if !stopped && sender != nil { checkCtx, cancel := context.WithTimeout(w.ctx, healthEvaluateTimeout) health := w.checkHealth(checkCtx, appID) @@ -478,7 +478,7 @@ func (w *healthWatcher) reconcileMissingApplication(appID string) { if err := sender.SendMessage(&messages.ClientMessage{ Payload: &messages.ClientMessage_ApplicationStatusReport{ ApplicationStatusReport: &messages.ApplicationStatusReport{ - Statuses: []*messages.ApplicationStatus{{ApplicationId: appID, Health: health.Proto()}}, + Statuses: []*messages.ApplicationStatus{health.Proto(appID)}, }, }, }); err != nil { @@ -492,7 +492,7 @@ func (w *healthWatcher) reconcileMissingApplication(appID string) { w.mu.Lock() if !w.stopped { switch { - case reported && reportedHealth != HealthUnknown: + case reported && reportedHealth.State != HealthUnknown: w.lastReported[appID] = reportedHealth case reported: delete(w.lastReported, appID) diff --git a/backend/internal/agent/docker/health_watcher_test.go b/backend/internal/agent/docker/health_watcher_test.go index cc4f0756..14567e12 100644 --- a/backend/internal/agent/docker/health_watcher_test.go +++ b/backend/internal/agent/docker/health_watcher_test.go @@ -13,7 +13,7 @@ import ( "github.com/rs/zerolog" ) -func newTestWatcher(t *testing.T, sender MessageSender, check func(ctx context.Context, appID string) HealthState) *healthWatcher { +func newTestWatcher(t *testing.T, sender MessageSender, check func(ctx context.Context, appID string) HealthReport) *healthWatcher { t.Helper() w := &healthWatcher{ ctx: t.Context(), @@ -26,14 +26,14 @@ func newTestWatcher(t *testing.T, sender MessageSender, check func(ctx context.C pending: make(map[string]*pendingHealthEvaluation), evaluating: make(map[string]*healthEvaluation), epochs: make(map[string]uint64), - lastReported: make(map[string]HealthState), + lastReported: make(map[string]HealthReport), } t.Cleanup(w.stop) return w } -func staticHealth(state HealthState) func(context.Context, string) HealthState { - return func(context.Context, string) HealthState { return state } +func staticHealth(state HealthState) func(context.Context, string) HealthReport { + return func(context.Context, string) HealthReport { return HealthReport{State: state} } } // awaitReports polls the sender until it has received at least n messages. @@ -110,7 +110,7 @@ func TestHealthWatcher_DeduplicatesUnchangedHealth(t *testing.T) { func TestHealthWatcher_ReportsChanges(t *testing.T) { sender := &stubMessageSender{} state := HealthHealthy - w := newTestWatcher(t, sender, func(context.Context, string) HealthState { return state }) + w := newTestWatcher(t, sender, func(context.Context, string) HealthReport { return HealthReport{State: state} }) w.schedule("app-1") awaitReports(t, sender, 1) @@ -125,6 +125,32 @@ func TestHealthWatcher_ReportsChanges(t *testing.T) { } } +func TestHealthWatcher_ReportsServiceChangesWithSameAggregate(t *testing.T) { + sender := &stubMessageSender{} + failing := "web" + w := newTestWatcher(t, sender, func(context.Context, string) HealthReport { + services := []ServiceHealth{{Name: "db", State: HealthHealthy}, {Name: "web", State: HealthHealthy}} + for i := range services { + if services[i].Name == failing { + services[i].State = HealthUnhealthy + } + } + return HealthReport{State: HealthUnhealthy, Services: services} + }) + + w.schedule("app-1") + awaitReports(t, sender, 1) + + failing = "db" + w.schedule("app-1") + + msgs := awaitReports(t, sender, 2) + services := msgs[1].GetApplicationStatusReport().Statuses[0].Services + if len(services) != 2 || services[0].Health != messages.HealthStatus_HEALTH_STATUS_UNHEALTHY || services[1].Health != messages.HealthStatus_HEALTH_STATUS_HEALTHY { + t.Errorf("expected db unhealthy and web healthy, got %v", services) + } +} + func TestHealthWatcher_DoesNotReportUnknown(t *testing.T) { sender := &stubMessageSender{} w := newTestWatcher(t, sender, staticHealth(HealthUnknown)) @@ -227,17 +253,17 @@ func TestHealthWatcher_SerializesEvaluationsPerApplication(t *testing.T) { secondStarted := make(chan struct{}) releaseFirst := make(chan struct{}) var calls atomic.Int32 - w := newTestWatcher(t, sender, func(context.Context, string) HealthState { + w := newTestWatcher(t, sender, func(context.Context, string) HealthReport { switch calls.Add(1) { case 1: close(firstStarted) <-releaseFirst - return HealthHealthy + return HealthReport{State: HealthHealthy} case 2: close(secondStarted) - return HealthUnhealthy + return HealthReport{State: HealthUnhealthy} default: - return HealthUnknown + return HealthReport{State: HealthUnknown} } }) w.debounce = time.Hour @@ -296,7 +322,7 @@ func TestHealthWatcher_SerializesEvaluationsPerApplication(t *testing.T) { w.mu.Lock() last := w.lastReported["app-1"] w.mu.Unlock() - if last != HealthUnhealthy { + if last.State != HealthUnhealthy { t.Errorf("last reported health = %v, want unhealthy", last) } } @@ -307,17 +333,17 @@ func TestHealthWatcher_SerializesSnapshotAndIncrementalReports(t *testing.T) { incrementalStarted := make(chan struct{}) releaseSnapshot := make(chan struct{}) var calls atomic.Int32 - w := newTestWatcher(t, sender, func(context.Context, string) HealthState { + w := newTestWatcher(t, sender, func(context.Context, string) HealthReport { switch calls.Add(1) { case 1: close(snapshotStarted) <-releaseSnapshot - return HealthHealthy + return HealthReport{State: HealthHealthy} case 2: close(incrementalStarted) - return HealthUnhealthy + return HealthReport{State: HealthUnhealthy} default: - return HealthUnknown + return HealthReport{State: HealthUnknown} } }) w.debounce = time.Hour @@ -389,12 +415,12 @@ func TestHealthWatcher_ForgetInvalidatesRunningEvaluation(t *testing.T) { started := make(chan struct{}) canceled := make(chan struct{}) releaseCheck := make(chan struct{}) - w := newTestWatcher(t, sender, func(ctx context.Context, _ string) HealthState { + w := newTestWatcher(t, sender, func(ctx context.Context, _ string) HealthReport { close(started) <-ctx.Done() close(canceled) <-releaseCheck - return HealthHealthy + return HealthReport{State: HealthHealthy} }) release := func() { select { @@ -450,7 +476,7 @@ func TestHealthWatcher_NilSenderStillTracksState(t *testing.T) { state, seen := w.lastReported["app-1"] w.mu.Unlock() if seen { - if state != HealthHealthy { + if state.State != HealthHealthy { t.Errorf("expected HealthHealthy tracked, got %v", state) } return @@ -527,7 +553,7 @@ func TestReconcileApplicationHealth_ReEvaluatesActiveApps(t *testing.T) { c.healthWatcher.sender = sender c.healthWatcher.debounce = time.Millisecond c.healthWatcher.checkHealth = staticHealth(HealthHealthy) - c.healthWatcher.lastReported["app-1"] = HealthUnhealthy + c.healthWatcher.lastReported["app-1"] = HealthReport{State: HealthUnhealthy} c.reconcileApplicationHealth(map[string]struct{}{"app-1": {}}) @@ -558,7 +584,7 @@ func TestReconcileApplicationHealth_ForgetsAppsWithoutContainers(t *testing.T) { sender := &stubMessageSender{} c.healthWatcher.sender = sender c.healthWatcher.debounce = time.Hour - c.healthWatcher.lastReported["removed-app"] = HealthHealthy + c.healthWatcher.lastReported["removed-app"] = HealthReport{State: HealthHealthy} c.healthWatcher.schedule("removed-app") c.reconcileApplicationHealth(map[string]struct{}{}) @@ -591,9 +617,9 @@ func TestReconcileApplicationHealth_RechecksStaleMissingSnapshot(t *testing.T) { c := newTestClient(t) sender := &stubMessageSender{} c.healthWatcher.sender = sender - c.healthWatcher.lastReported["recreated-app"] = HealthHealthy - c.healthWatcher.checkHealth = func(context.Context, string) HealthState { - return HealthHealthy + c.healthWatcher.lastReported["recreated-app"] = HealthReport{State: HealthHealthy} + c.healthWatcher.checkHealth = func(context.Context, string) HealthReport { + return HealthReport{State: HealthHealthy} } c.reconcileApplicationHealth(map[string]struct{}{}) @@ -603,7 +629,7 @@ func TestReconcileApplicationHealth_RechecksStaleMissingSnapshot(t *testing.T) { if status.Health != messages.HealthStatus_HEALTH_STATUS_HEALTHY { t.Errorf("health = %v, want healthy", status.Health) } - if got := c.healthWatcher.lastReported["recreated-app"]; got != HealthHealthy { + if got := c.healthWatcher.lastReported["recreated-app"]; got.State != HealthHealthy { t.Errorf("last reported health = %v, want healthy", got) } } @@ -612,13 +638,13 @@ func TestReconcileApplicationHealth_RetriesAfterRecheckCancellation(t *testing.T c := newTestClient(t) sender := &stubMessageSender{} c.healthWatcher.sender = sender - c.healthWatcher.lastReported["missing-app"] = HealthHealthy + c.healthWatcher.lastReported["missing-app"] = HealthReport{State: HealthHealthy} cancelled, cancel := context.WithCancel(context.Background()) cancel() c.healthWatcher.ctx = cancelled - c.healthWatcher.checkHealth = func(ctx context.Context, _ string) HealthState { + c.healthWatcher.checkHealth = func(ctx context.Context, _ string) HealthReport { <-ctx.Done() - return HealthUnknown + return HealthReport{State: HealthUnknown} } c.reconcileApplicationHealth(map[string]struct{}{}) @@ -634,7 +660,7 @@ func TestReconcileApplicationHealth_RetriesAfterRecheckCancellation(t *testing.T func TestReconcileApplicationHealth_RetriesAfterSendFailure(t *testing.T) { sender := &stubMessageSender{err: context.DeadlineExceeded} w := newTestWatcher(t, sender, staticHealth(HealthUnknown)) - w.lastReported["missing-app"] = HealthHealthy + w.lastReported["missing-app"] = HealthReport{State: HealthHealthy} w.reconcileMissingApplication("missing-app") @@ -672,10 +698,10 @@ func TestHealthWatcher_ResendsAfterFailedSend(t *testing.T) { func TestReportApplicationStatus_FailedSendAllowsResend(t *testing.T) { sender := &stubMessageSender{err: errors.New("connection closed")} w := newTestWatcher(t, sender, staticHealth(HealthHealthy)) - w.lastReported["app-1"] = HealthHealthy + w.lastReported["app-1"] = HealthReport{State: HealthHealthy} w.reportApplicationStatus(t.Context(), sender, []string{"app-1"}) - if got := w.lastReported["app-1"]; got == HealthHealthy { + if got := w.lastReported["app-1"]; got.State == HealthHealthy { t.Fatal("a snapshot the hub never received must not count as reported") } @@ -695,16 +721,16 @@ func TestReportApplicationStatus_TimeoutReportsUncheckedAppsAsUnknown(t *testing calls := make(map[string]int) // The first check of app-2 hangs until the snapshot budget is spent; every // later check, such as the follow-up evaluation, answers immediately. - w := newTestWatcher(t, sender, func(ctx context.Context, appID string) HealthState { + w := newTestWatcher(t, sender, func(ctx context.Context, appID string) HealthReport { mu.Lock() calls[appID]++ first := calls[appID] == 1 mu.Unlock() if appID == "app-2" && first { <-ctx.Done() - return HealthUnknown + return HealthReport{State: HealthUnknown} } - return HealthHealthy + return HealthReport{State: HealthHealthy} }) w.snapshotTimeout = 100 * time.Millisecond @@ -747,10 +773,10 @@ func TestReportApplicationStatus_TimeoutReportsUncheckedAppsAsUnknown(t *testing func TestReportApplicationStatus_CancelledContextSendsNothing(t *testing.T) { sender := &stubMessageSender{} ctx, cancel := context.WithCancel(t.Context()) - w := newTestWatcher(t, sender, func(context.Context, string) HealthState { + w := newTestWatcher(t, sender, func(context.Context, string) HealthReport { // The agent is shutting down while the snapshot is being collected. cancel() - return HealthUnknown + return HealthReport{State: HealthUnknown} }) w.reportApplicationStatus(ctx, sender, []string{"app-1", "app-2"}) @@ -775,7 +801,7 @@ func TestClientDeploy_ReportsHealthAfterFailure(t *testing.T) { c.healthWatcher.checkHealth = staticHealth(HealthHealthy) // The previous containers still run, so the agent believes the hub already // knows the app is healthy; the hub reset it to unknown on dispatch though. - c.healthWatcher.lastReported["app-1"] = HealthHealthy + c.healthWatcher.lastReported["app-1"] = HealthReport{State: HealthHealthy} err := c.Deploy(t.Context(), DeployRequest{ApplicationID: "app-1", ApplicationName: "app"}) if err == nil { @@ -811,7 +837,7 @@ func TestImagePoller_ExplicitPullWithoutUpdateReportsHealth(t *testing.T) { w.sender = sender w.debounce = time.Millisecond w.checkHealth = staticHealth(HealthHealthy) - w.lastReported["app-1"] = HealthHealthy + w.lastReported["app-1"] = HealthReport{State: HealthHealthy} p.runOnce("app-1", "app", "req-1", false) @@ -833,7 +859,7 @@ func TestImagePoller_PeriodicPullWithoutUpdateDoesNotReportHealth(t *testing.T) w.sender = sender w.debounce = time.Millisecond w.checkHealth = staticHealth(HealthHealthy) - w.lastReported["app-1"] = HealthHealthy + w.lastReported["app-1"] = HealthReport{State: HealthHealthy} p.runOnce("app-1", "app", "", false) time.Sleep(50 * time.Millisecond) diff --git a/backend/internal/agent/docker/status.go b/backend/internal/agent/docker/status.go index 63fa2a71..010369fc 100644 --- a/backend/internal/agent/docker/status.go +++ b/backend/internal/agent/docker/status.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "slices" + "strings" messages "github.com/OrcaCD/orca-cd/internal/proto" + "github.com/docker/compose/v5/pkg/api" "github.com/moby/moby/api/types/container" "github.com/moby/moby/client" ) @@ -35,14 +37,42 @@ func (s HealthState) Proto() messages.HealthStatus { } } -// ApplicationHealth reports the aggregate health of the containers belonging to -// an application, identified by the orca-cd.application-id label applied at -// deploy time. It returns HealthUnknown when the daemon is unreachable, no -// containers exist (nothing deployed yet), or a healthcheck is still starting. -// HealthStopped is returned when none of its containers run. -func (c *Client) ApplicationHealth(ctx context.Context, appID string) HealthState { +// ServiceHealth is the health of a single compose service of an application. +type ServiceHealth struct { + Name string + State HealthState +} + +// HealthReport is an application's aggregate health together with the health +// of each of its compose services, sorted by name. +type HealthReport struct { + State HealthState + Services []ServiceHealth +} + +// Equal reports whether both reports describe the same aggregate and service +// health. +func (r HealthReport) Equal(other HealthReport) bool { + return r.State == other.State && slices.Equal(r.Services, other.Services) +} + +// Proto converts the report to the status message sent to the hub. +func (r HealthReport) Proto(appID string) *messages.ApplicationStatus { + status := &messages.ApplicationStatus{ApplicationId: appID, Health: r.State.Proto()} + for _, service := range r.Services { + status.Services = append(status.Services, &messages.ServiceStatus{Name: service.Name, Health: service.State.Proto()}) + } + return status +} + +// ApplicationHealth reports the health of the containers belonging to an +// application, identified by the orca-cd.application-id label applied at +// deploy time. The aggregate state is HealthUnknown when the daemon is +// unreachable, no containers exist (nothing deployed yet), or a healthcheck is +// still starting. HealthStopped is returned when none of its containers run. +func (c *Client) ApplicationHealth(ctx context.Context, appID string) HealthReport { if !c.Ready() { - return HealthUnknown + return HealthReport{State: HealthUnknown} } result, err := c.cli.Client().ContainerList(ctx, client.ContainerListOptions{ @@ -51,16 +81,16 @@ func (c *Client) ApplicationHealth(ctx context.Context, appID string) HealthStat }) if err != nil { c.log.Error().Err(err).Str("application_id", appID).Msg("failed to list containers for health check") - return HealthUnknown + return HealthReport{State: HealthUnknown} } items, err := withInspectedHealth(ctx, c.cli.Client(), result.Items) if err != nil { c.log.Error().Err(err).Str("application_id", appID).Msg("failed to inspect containers for health check") - return HealthUnknown + return HealthReport{State: HealthUnknown} } - return aggregateHealth(items) + return HealthReport{State: aggregateHealth(items), Services: servicesHealth(items)} } // withInspectedHealth fills in the healthcheck summary of running containers @@ -125,6 +155,36 @@ func aggregateHealth(items []container.Summary) HealthState { return HealthHealthy } +// servicesHealth applies aggregateHealth to the containers of each compose +// service, so the hub can tell a partially failing application apart from one +// that is failing entirely. +func servicesHealth(items []container.Summary) []ServiceHealth { + byService := make(map[string][]container.Summary) + for _, item := range items { + name := serviceName(item) + byService[name] = append(byService[name], item) + } + + services := make([]ServiceHealth, 0, len(byService)) + for name, serviceItems := range byService { + services = append(services, ServiceHealth{Name: name, State: aggregateHealth(serviceItems)}) + } + slices.SortFunc(services, func(a, b ServiceHealth) int { return strings.Compare(a.Name, b.Name) }) + return services +} + +// serviceName returns the compose service a container belongs to, falling back +// to the container name for containers that lack the compose label. +func serviceName(item container.Summary) string { + if name := item.Labels[api.ServiceLabel]; name != "" { + return name + } + if len(item.Names) > 0 { + return strings.TrimPrefix(item.Names[0], "/") + } + return "" +} + // allStopped reports whether none of the containers run, as after // `docker compose stop` or `docker compose pause`. Restarting (crash loop), // dead and removing containers are not considered stopped. diff --git a/backend/internal/agent/docker/status_test.go b/backend/internal/agent/docker/status_test.go index 504c6aff..2714bdf2 100644 --- a/backend/internal/agent/docker/status_test.go +++ b/backend/internal/agent/docker/status_test.go @@ -7,6 +7,7 @@ import ( "testing" messages "github.com/OrcaCD/orca-cd/internal/proto" + "github.com/docker/compose/v5/pkg/api" "github.com/moby/moby/api/types/container" "github.com/moby/moby/client" "github.com/rs/zerolog" @@ -17,7 +18,7 @@ func TestApplicationHealth_NoContainersIsUnknown(t *testing.T) { // No application has ever been deployed with this id, so no containers carry // its label. - if got := c.ApplicationHealth(t.Context(), "no-such-application-id"); got != HealthUnknown { + if got := c.ApplicationHealth(t.Context(), "no-such-application-id"); got.State != HealthUnknown { t.Errorf("expected HealthUnknown for an app with no containers, got %v", got) } } @@ -31,7 +32,7 @@ func TestApplicationHealth_DaemonUnreachableIsUnknown(t *testing.T) { } t.Cleanup(c.Close) - if got := c.ApplicationHealth(t.Context(), "any"); got != HealthUnknown { + if got := c.ApplicationHealth(t.Context(), "any"); got.State != HealthUnknown { t.Errorf("expected HealthUnknown when daemon is unreachable, got %v", got) } } @@ -111,6 +112,60 @@ func TestAggregateHealth(t *testing.T) { } } +func TestServicesHealth(t *testing.T) { + service := func(name string, state container.ContainerState, health *container.HealthSummary) container.Summary { + return container.Summary{State: state, Health: health, Labels: map[string]string{api.ServiceLabel: name}} + } + items := []container.Summary{ + service("web", container.StateRunning, &container.HealthSummary{Status: container.Healthy}), + service("web", container.StateRunning, &container.HealthSummary{Status: container.Unhealthy}), + service("db", container.StateRunning, nil), + service("worker", container.StateExited, nil), + {State: container.StateRunning, Names: []string{"/standalone"}}, + } + + got := servicesHealth(items) + want := []ServiceHealth{ + {Name: "db", State: HealthHealthy}, + {Name: "standalone", State: HealthHealthy}, + {Name: "web", State: HealthUnhealthy}, + {Name: "worker", State: HealthStopped}, + } + if !slices.Equal(got, want) { + t.Errorf("servicesHealth() = %v, want %v", got, want) + } +} + +func TestHealthReport_Proto(t *testing.T) { + report := HealthReport{ + State: HealthUnhealthy, + Services: []ServiceHealth{{Name: "db", State: HealthHealthy}, {Name: "web", State: HealthUnhealthy}}, + } + + status := report.Proto("app-1") + if status.ApplicationId != "app-1" || status.Health != messages.HealthStatus_HEALTH_STATUS_UNHEALTHY { + t.Fatalf("unexpected status %v", status) + } + if len(status.Services) != 2 || + status.Services[0].Name != "db" || status.Services[0].Health != messages.HealthStatus_HEALTH_STATUS_HEALTHY || + status.Services[1].Name != "web" || status.Services[1].Health != messages.HealthStatus_HEALTH_STATUS_UNHEALTHY { + t.Errorf("unexpected services %v", status.Services) + } +} + +func TestHealthReport_Equal(t *testing.T) { + base := HealthReport{State: HealthUnhealthy, Services: []ServiceHealth{{Name: "web", State: HealthUnhealthy}}} + if !base.Equal(HealthReport{State: HealthUnhealthy, Services: []ServiceHealth{{Name: "web", State: HealthUnhealthy}}}) { + t.Error("identical reports must be equal") + } + if base.Equal(HealthReport{State: HealthUnhealthy, Services: []ServiceHealth{{Name: "db", State: HealthUnhealthy}}}) { + t.Error("a different failing service must not be equal") + } + if base.Equal(HealthReport{State: HealthUnhealthy}) { + t.Error("missing services must not be equal") + } +} + type fakeInspector struct { states map[string]*container.State err error diff --git a/backend/internal/hub/models/applications.go b/backend/internal/hub/models/applications.go index 9925384d..48b3be4a 100644 --- a/backend/internal/hub/models/applications.go +++ b/backend/internal/hub/models/applications.go @@ -30,6 +30,9 @@ const ( Unhealthy HealthStatus = "unhealthy" Stopped HealthStatus = "stopped" UnknownHealth HealthStatus = "unknown" + // Degraded means some services of the application are healthy while + // others are unhealthy or stopped. + Degraded HealthStatus = "degraded" ) type Application struct { diff --git a/backend/internal/hub/websocket/handler.go b/backend/internal/hub/websocket/handler.go index d4fd79f5..38828f54 100644 --- a/backend/internal/hub/websocket/handler.go +++ b/backend/internal/hub/websocket/handler.go @@ -304,7 +304,7 @@ func handleApplicationStatusReport(parent context.Context, client *Client, repor if ctx.Err() != nil { return } - health := protoHealthToModel(status.Health) + health := reportedApplicationHealth(status) event, err := applyReportedHealth(ctx, client.Id, status.ApplicationId, health) if err != nil { log.Error().Err(err). @@ -385,8 +385,8 @@ func sanitizeReportedVersion(v string) string { // the change triggers, if any. Transitions are detected with conditional // updates so concurrent reports cannot notify twice. // -// Only Healthy→Unhealthy/Stopped and Unhealthy/Stopped→Healthy notify; a -// stopped application counts as unhealthy for notifications. Unknown→Unhealthy +// Only Healthy→Unhealthy/Stopped/Degraded and back to Healthy notify; stopped +// and degraded applications count as unhealthy for notifications. Unknown→Unhealthy // is deliberately silent: health is reset to Unknown on every disconnect, so it // would re-announce unhealthy apps after each agent reconnect. Apps that are // currently deploying are skipped, as the deployment reports its own outcome. @@ -394,12 +394,12 @@ func applyReportedHealth(ctx context.Context, agentId, applicationId string, hea var event models.NotificationEvent transition := gorm.G[models.Application](db.DB).Where("id = ? AND agent_id = ?", applicationId, agentId) switch health { - case models.Unhealthy, models.Stopped: + case models.Unhealthy, models.Stopped, models.Degraded: event = models.NotificationEventHealthUnhealthy transition = transition.Where("health_status = ? AND sync_status <> ?", models.Healthy, models.Syncing) case models.Healthy: event = models.NotificationEventHealthRecovered - transition = transition.Where("health_status IN ?", []models.HealthStatus{models.Unhealthy, models.Stopped}) + transition = transition.Where("health_status IN ?", []models.HealthStatus{models.Unhealthy, models.Stopped, models.Degraded}) } if event != "" { @@ -431,10 +431,28 @@ func notifyHealthChange(ctx context.Context, applicationId string, event models. message = "Application " + app.Name.String() + " is unhealthy" case models.Stopped: message = "Application " + app.Name.String() + " is stopped" + case models.Degraded: + message = "Application " + app.Name.String() + " is degraded" } go notifications.SendForApplication(applicationId, event, message, log) } +// reportedApplicationHealth derives the application health from a status +// report. Agents report a partially failing application as unhealthy; when at +// least one of its services is still healthy, the hub records it as degraded. +func reportedApplicationHealth(status *messages.ApplicationStatus) models.HealthStatus { + health := protoHealthToModel(status.Health) + if health != models.Unhealthy { + return health + } + for _, service := range status.Services { + if service.Health == messages.HealthStatus_HEALTH_STATUS_HEALTHY { + return models.Degraded + } + } + return health +} + func protoHealthToModel(h messages.HealthStatus) models.HealthStatus { switch h { case messages.HealthStatus_HEALTH_STATUS_HEALTHY: diff --git a/backend/internal/hub/websocket/status_report_test.go b/backend/internal/hub/websocket/status_report_test.go index 478942cd..3826a49c 100644 --- a/backend/internal/hub/websocket/status_report_test.go +++ b/backend/internal/hub/websocket/status_report_test.go @@ -105,6 +105,10 @@ func TestApplyReportedHealth_Transitions(t *testing.T) { {"unhealthy to stopped", models.Unhealthy, models.Synced, models.Stopped, "", models.Stopped}, {"stopped to unhealthy", models.Stopped, models.Synced, models.Unhealthy, "", models.Unhealthy}, {"stopped to healthy", models.Stopped, models.Synced, models.Healthy, models.NotificationEventHealthRecovered, models.Healthy}, + {"healthy to degraded", models.Healthy, models.Synced, models.Degraded, models.NotificationEventHealthUnhealthy, models.Degraded}, + {"degraded to unhealthy", models.Degraded, models.Synced, models.Unhealthy, "", models.Unhealthy}, + {"unhealthy to degraded", models.Unhealthy, models.Synced, models.Degraded, "", models.Degraded}, + {"degraded to healthy", models.Degraded, models.Synced, models.Healthy, models.NotificationEventHealthRecovered, models.Healthy}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -169,3 +173,33 @@ func TestHandleApplicationStatusReport_NotifiesOnHealthTransition(t *testing.T) t.Errorf("expected unhealthy, got %q", got) } } + +func TestReportedApplicationHealth(t *testing.T) { + healthy := messages.HealthStatus_HEALTH_STATUS_HEALTHY + unhealthy := messages.HealthStatus_HEALTH_STATUS_UNHEALTHY + stopped := messages.HealthStatus_HEALTH_STATUS_STOPPED + service := func(name string, health messages.HealthStatus) *messages.ServiceStatus { + return &messages.ServiceStatus{Name: name, Health: health} + } + + tests := []struct { + name string + status *messages.ApplicationStatus + want models.HealthStatus + }{ + {"healthy", &messages.ApplicationStatus{Health: healthy, Services: []*messages.ServiceStatus{service("web", healthy)}}, models.Healthy}, + {"unhealthy without services from older agents", &messages.ApplicationStatus{Health: unhealthy}, models.Unhealthy}, + {"all services unhealthy", &messages.ApplicationStatus{Health: unhealthy, Services: []*messages.ServiceStatus{service("db", unhealthy), service("web", stopped)}}, models.Unhealthy}, + {"one healthy and one unhealthy service", &messages.ApplicationStatus{Health: unhealthy, Services: []*messages.ServiceStatus{service("db", healthy), service("web", unhealthy)}}, models.Degraded}, + {"one healthy and one stopped service", &messages.ApplicationStatus{Health: unhealthy, Services: []*messages.ServiceStatus{service("db", healthy), service("web", stopped)}}, models.Degraded}, + {"stopped", &messages.ApplicationStatus{Health: stopped, Services: []*messages.ServiceStatus{service("web", stopped)}}, models.Stopped}, + {"unknown", &messages.ApplicationStatus{Health: messages.HealthStatus_HEALTH_STATUS_UNSPECIFIED}, models.UnknownHealth}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := reportedApplicationHealth(tt.status); got != tt.want { + t.Errorf("reportedApplicationHealth() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/backend/internal/proto/messages.pb.go b/backend/internal/proto/messages.pb.go index 2e44fd2b..ba91b6b8 100644 --- a/backend/internal/proto/messages.pb.go +++ b/backend/internal/proto/messages.pb.go @@ -1209,17 +1209,74 @@ func (x *PullImagesResult) GetErrorMessage() string { return "" } +type ServiceStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Health HealthStatus `protobuf:"varint,2,opt,name=health,proto3,enum=messages.HealthStatus" json:"health,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ServiceStatus) Reset() { + *x = ServiceStatus{} + mi := &file_messages_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ServiceStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServiceStatus) ProtoMessage() {} + +func (x *ServiceStatus) ProtoReflect() protoreflect.Message { + mi := &file_messages_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServiceStatus.ProtoReflect.Descriptor instead. +func (*ServiceStatus) Descriptor() ([]byte, []int) { + return file_messages_proto_rawDescGZIP(), []int{15} +} + +func (x *ServiceStatus) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ServiceStatus) GetHealth() HealthStatus { + if x != nil { + return x.Health + } + return HealthStatus_HEALTH_STATUS_UNSPECIFIED +} + type ApplicationStatus struct { state protoimpl.MessageState `protogen:"open.v1"` ApplicationId string `protobuf:"bytes,1,opt,name=application_id,json=applicationId,proto3" json:"application_id,omitempty"` - Health HealthStatus `protobuf:"varint,2,opt,name=health,proto3,enum=messages.HealthStatus" json:"health,omitempty"` + // Aggregate health of all containers. A mix of healthy and failing services + // is reported as unhealthy; the hub derives the degraded state from services. + Health HealthStatus `protobuf:"varint,2,opt,name=health,proto3,enum=messages.HealthStatus" json:"health,omitempty"` + // Health per compose service, sorted by name. Empty for agents that predate + // per-service reporting. + Services []*ServiceStatus `protobuf:"bytes,3,rep,name=services,proto3" json:"services,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ApplicationStatus) Reset() { *x = ApplicationStatus{} - mi := &file_messages_proto_msgTypes[15] + mi := &file_messages_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1231,7 +1288,7 @@ func (x *ApplicationStatus) String() string { func (*ApplicationStatus) ProtoMessage() {} func (x *ApplicationStatus) ProtoReflect() protoreflect.Message { - mi := &file_messages_proto_msgTypes[15] + mi := &file_messages_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1244,7 +1301,7 @@ func (x *ApplicationStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ApplicationStatus.ProtoReflect.Descriptor instead. func (*ApplicationStatus) Descriptor() ([]byte, []int) { - return file_messages_proto_rawDescGZIP(), []int{15} + return file_messages_proto_rawDescGZIP(), []int{16} } func (x *ApplicationStatus) GetApplicationId() string { @@ -1261,6 +1318,13 @@ func (x *ApplicationStatus) GetHealth() HealthStatus { return HealthStatus_HEALTH_STATUS_UNSPECIFIED } +func (x *ApplicationStatus) GetServices() []*ServiceStatus { + if x != nil { + return x.Services + } + return nil +} + // ApplicationStatusReport carries the health of every application managed by an // agent in a single message, sent right after the agent connects. type ApplicationStatusReport struct { @@ -1272,7 +1336,7 @@ type ApplicationStatusReport struct { func (x *ApplicationStatusReport) Reset() { *x = ApplicationStatusReport{} - mi := &file_messages_proto_msgTypes[16] + mi := &file_messages_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1284,7 +1348,7 @@ func (x *ApplicationStatusReport) String() string { func (*ApplicationStatusReport) ProtoMessage() {} func (x *ApplicationStatusReport) ProtoReflect() protoreflect.Message { - mi := &file_messages_proto_msgTypes[16] + mi := &file_messages_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1297,7 +1361,7 @@ func (x *ApplicationStatusReport) ProtoReflect() protoreflect.Message { // Deprecated: Use ApplicationStatusReport.ProtoReflect.Descriptor instead. func (*ApplicationStatusReport) Descriptor() ([]byte, []int) { - return file_messages_proto_rawDescGZIP(), []int{16} + return file_messages_proto_rawDescGZIP(), []int{17} } func (x *ApplicationStatusReport) GetStatuses() []*ApplicationStatus { @@ -1318,7 +1382,7 @@ type AgentInfo struct { func (x *AgentInfo) Reset() { *x = AgentInfo{} - mi := &file_messages_proto_msgTypes[17] + mi := &file_messages_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1330,7 +1394,7 @@ func (x *AgentInfo) String() string { func (*AgentInfo) ProtoMessage() {} func (x *AgentInfo) ProtoReflect() protoreflect.Message { - mi := &file_messages_proto_msgTypes[17] + mi := &file_messages_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1343,7 +1407,7 @@ func (x *AgentInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use AgentInfo.ProtoReflect.Descriptor instead. func (*AgentInfo) Descriptor() ([]byte, []int) { - return file_messages_proto_rawDescGZIP(), []int{17} + return file_messages_proto_rawDescGZIP(), []int{18} } func (x *AgentInfo) GetVersion() string { @@ -1453,10 +1517,14 @@ const file_messages_proto_rawDesc = "" + "\x0eapplication_id\x18\x02 \x01(\tR\rapplicationId\x12\x18\n" + "\asuccess\x18\x03 \x01(\bR\asuccess\x12%\n" + "\x0eimages_updated\x18\x04 \x01(\bR\rimagesUpdated\x12#\n" + - "\rerror_message\x18\x05 \x01(\tR\ferrorMessage\"j\n" + + "\rerror_message\x18\x05 \x01(\tR\ferrorMessage\"S\n" + + "\rServiceStatus\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12.\n" + + "\x06health\x18\x02 \x01(\x0e2\x16.messages.HealthStatusR\x06health\"\x9f\x01\n" + "\x11ApplicationStatus\x12%\n" + "\x0eapplication_id\x18\x01 \x01(\tR\rapplicationId\x12.\n" + - "\x06health\x18\x02 \x01(\x0e2\x16.messages.HealthStatusR\x06health\"R\n" + + "\x06health\x18\x02 \x01(\x0e2\x16.messages.HealthStatusR\x06health\x123\n" + + "\bservices\x18\x03 \x03(\v2\x17.messages.ServiceStatusR\bservices\"R\n" + "\x17ApplicationStatusReport\x127\n" + "\bstatuses\x18\x01 \x03(\v2\x1b.messages.ApplicationStatusR\bstatuses\"{\n" + "\tAgentInfo\x12\x18\n" + @@ -1483,7 +1551,7 @@ func file_messages_proto_rawDescGZIP() []byte { } var file_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 19) var file_messages_proto_goTypes = []any{ (HealthStatus)(0), // 0: messages.HealthStatus (*ClientMessage)(nil), // 1: messages.ClientMessage @@ -1501,9 +1569,10 @@ var file_messages_proto_goTypes = []any{ (*ImagePollSettings)(nil), // 13: messages.ImagePollSettings (*PullImagesRequest)(nil), // 14: messages.PullImagesRequest (*PullImagesResult)(nil), // 15: messages.PullImagesResult - (*ApplicationStatus)(nil), // 16: messages.ApplicationStatus - (*ApplicationStatusReport)(nil), // 17: messages.ApplicationStatusReport - (*AgentInfo)(nil), // 18: messages.AgentInfo + (*ServiceStatus)(nil), // 16: messages.ServiceStatus + (*ApplicationStatus)(nil), // 17: messages.ApplicationStatus + (*ApplicationStatusReport)(nil), // 18: messages.ApplicationStatusReport + (*AgentInfo)(nil), // 19: messages.AgentInfo } var file_messages_proto_depIdxs = []int32{ 4, // 0: messages.ClientMessage.pong:type_name -> messages.PongResponse @@ -1512,8 +1581,8 @@ var file_messages_proto_depIdxs = []int32{ 11, // 3: messages.ClientMessage.deploy_result:type_name -> messages.DeployResult 15, // 4: messages.ClientMessage.pull_images_result:type_name -> messages.PullImagesResult 10, // 5: messages.ClientMessage.delete_result:type_name -> messages.DeleteResult - 17, // 6: messages.ClientMessage.application_status_report:type_name -> messages.ApplicationStatusReport - 18, // 7: messages.ClientMessage.agent_info:type_name -> messages.AgentInfo + 18, // 6: messages.ClientMessage.application_status_report:type_name -> messages.ApplicationStatusReport + 19, // 7: messages.ClientMessage.agent_info:type_name -> messages.AgentInfo 3, // 8: messages.ServerMessage.ping:type_name -> messages.PingRequest 5, // 9: messages.ServerMessage.key_exchange_init:type_name -> messages.KeyExchangeInit 7, // 10: messages.ServerMessage.encrypted_payload:type_name -> messages.EncryptedPayload @@ -1522,13 +1591,15 @@ var file_messages_proto_depIdxs = []int32{ 14, // 13: messages.ServerMessage.pull_images_request:type_name -> messages.PullImagesRequest 9, // 14: messages.ServerMessage.delete_request:type_name -> messages.DeleteRequest 13, // 15: messages.AgentSettings.image_poll_settings:type_name -> messages.ImagePollSettings - 0, // 16: messages.ApplicationStatus.health:type_name -> messages.HealthStatus - 16, // 17: messages.ApplicationStatusReport.statuses:type_name -> messages.ApplicationStatus - 18, // [18:18] is the sub-list for method output_type - 18, // [18:18] is the sub-list for method input_type - 18, // [18:18] is the sub-list for extension type_name - 18, // [18:18] is the sub-list for extension extendee - 0, // [0:18] is the sub-list for field type_name + 0, // 16: messages.ServiceStatus.health:type_name -> messages.HealthStatus + 0, // 17: messages.ApplicationStatus.health:type_name -> messages.HealthStatus + 16, // 18: messages.ApplicationStatus.services:type_name -> messages.ServiceStatus + 17, // 19: messages.ApplicationStatusReport.statuses:type_name -> messages.ApplicationStatus + 20, // [20:20] is the sub-list for method output_type + 20, // [20:20] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { file_messages_proto_init() } @@ -1561,7 +1632,7 @@ func file_messages_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_messages_proto_rawDesc), len(file_messages_proto_rawDesc)), NumEnums: 1, - NumMessages: 18, + NumMessages: 19, NumExtensions: 0, NumServices: 0, }, diff --git a/backend/internal/proto/messages.proto b/backend/internal/proto/messages.proto index 4a65d5b0..1f9ec888 100644 --- a/backend/internal/proto/messages.proto +++ b/backend/internal/proto/messages.proto @@ -116,9 +116,19 @@ enum HealthStatus { HEALTH_STATUS_STOPPED = 3; } +message ServiceStatus { + string name = 1; + HealthStatus health = 2; +} + message ApplicationStatus { string application_id = 1; + // Aggregate health of all containers. A mix of healthy and failing services + // is reported as unhealthy; the hub derives the degraded state from services. HealthStatus health = 2; + // Health per compose service, sorted by name. Empty for agents that predate + // per-service reporting. + repeated ServiceStatus services = 3; } // ApplicationStatusReport carries the health of every application managed by an diff --git a/frontend/messages/de.json b/frontend/messages/de.json index 5a7f4ed9..9e5eb28e 100644 --- a/frontend/messages/de.json +++ b/frontend/messages/de.json @@ -103,6 +103,7 @@ "statusHealthy": "Gesund", "statusUnhealthy": "Ungesund", "statusStopped": "Gestoppt", + "statusDegraded": "Beeinträchtigt", "userSettings": "Benutzereinstellungen", "signOut": "Abmelden", "navApplications": "Anwendungen", @@ -170,6 +171,7 @@ "applicationsAttentionAgentError": "Agent {agent} kann Docker nicht erreichen", "applicationsAttentionUnhealthy": "Container melden einen ungesunden Zustand", "applicationsAttentionStopped": "Alle Container sind gestoppt", + "applicationsAttentionDegraded": "Einige Dienste melden einen ungesunden Zustand", "applicationsAttentionOutOfSync": "Nicht auf dem neuesten Commit deployt", "applicationsAttentionRetry": "Erneut versuchen", "manifest": "Manifest", @@ -354,7 +356,7 @@ "notificationEventImageUpdateSucceeded": "Image-Update erfolgreich", "notificationEventImageUpdateFailed": "Image-Update fehlgeschlagen", "notificationEventSyncFailed": "Synchronisierung fehlgeschlagen", - "notificationEventHealthUnhealthy": "Wurde ungesund oder gestoppt", + "notificationEventHealthUnhealthy": "Wurde ungesund, beeinträchtigt oder gestoppt", "notificationEventHealthRecovered": "Wieder gesund", "notificationEventAgentOffline": "Offline (nach 1 Minute)", "notificationEventAgentOnline": "Wieder online", diff --git a/frontend/messages/en.json b/frontend/messages/en.json index ef0866de..b4ee8b62 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -103,6 +103,7 @@ "statusHealthy": "Healthy", "statusUnhealthy": "Unhealthy", "statusStopped": "Stopped", + "statusDegraded": "Degraded", "userSettings": "User Settings", "signOut": "Sign Out", "navApplications": "Applications", @@ -170,6 +171,7 @@ "applicationsAttentionAgentError": "Agent {agent} cannot reach Docker", "applicationsAttentionUnhealthy": "Containers report an unhealthy state", "applicationsAttentionStopped": "All containers are stopped", + "applicationsAttentionDegraded": "Some services report an unhealthy state", "applicationsAttentionOutOfSync": "Not deployed at the latest commit", "applicationsAttentionRetry": "Retry", "manifest": "Manifest", @@ -354,7 +356,7 @@ "notificationEventImageUpdateSucceeded": "Image update succeeded", "notificationEventImageUpdateFailed": "Image update failed", "notificationEventSyncFailed": "Sync failed", - "notificationEventHealthUnhealthy": "Became unhealthy or stopped", + "notificationEventHealthUnhealthy": "Became unhealthy, degraded or stopped", "notificationEventHealthRecovered": "Healthy again", "notificationEventAgentOffline": "Went offline (after 1 minute)", "notificationEventAgentOnline": "Back online", diff --git a/frontend/src/components/badges/application-status-badge.tsx b/frontend/src/components/badges/application-status-badge.tsx index f276707c..f81d1822 100644 --- a/frontend/src/components/badges/application-status-badge.tsx +++ b/frontend/src/components/badges/application-status-badge.tsx @@ -50,6 +50,11 @@ const healthConfig: Record< className: "bg-red-500/20 text-red-400 border-red-500/30", dotClass: "bg-red-400", }, + degraded: { + label: () => m.statusDegraded(), + className: "bg-yellow-500/20 text-yellow-300 border-yellow-500/30", + dotClass: "bg-yellow-300", + }, unknown: { label: () => m.unknown(), className: "bg-zinc-500/20 text-zinc-400 border-zinc-500/30", diff --git a/frontend/src/lib/applications.ts b/frontend/src/lib/applications.ts index 61b9b3df..d1d82b7b 100644 --- a/frontend/src/lib/applications.ts +++ b/frontend/src/lib/applications.ts @@ -76,6 +76,7 @@ export enum HealthStatus { Healthy = "healthy", Unhealthy = "unhealthy", Stopped = "stopped", + Degraded = "degraded", Unknown = "unknown", } diff --git a/frontend/src/lib/fleet.ts b/frontend/src/lib/fleet.ts index 4fbe41a3..fce42ea9 100644 --- a/frontend/src/lib/fleet.ts +++ b/frontend/src/lib/fleet.ts @@ -46,8 +46,13 @@ export const fleetCategoryConfig: Record< }; export function getFleetCategory(app: ApplicationListItem): FleetCategory { - // Stopped apps are not running anything, so they need the same attention. - if (app.healthStatus === HealthStatus.Unhealthy || app.healthStatus === HealthStatus.Stopped) { + // Stopped and degraded apps are (partially) not running, so they need the + // same attention. + if ( + app.healthStatus === HealthStatus.Unhealthy || + app.healthStatus === HealthStatus.Stopped || + app.healthStatus === HealthStatus.Degraded + ) { return "unhealthy"; } if (app.syncStatus === SyncStatus.OutOfSync) { @@ -113,6 +118,8 @@ export function getAttentionItems( reason = m.applicationsAttentionAgentError({ agent: app.agentName }); } else if (app.healthStatus === HealthStatus.Stopped) { reason = m.applicationsAttentionStopped(); + } else if (app.healthStatus === HealthStatus.Degraded) { + reason = m.applicationsAttentionDegraded(); } else if (category === "unhealthy") { reason = m.applicationsAttentionUnhealthy(); } else {