Skip to content

feat(health): report per-service health and add degraded status #424

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alex289 merged 1 commit into from
Oct 29, 2026
+395 −101
Merged

feat(health): report per-service health and add degraded status #424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions backend/internal/agent/docker/health_watcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ type healthWatcher struct {
debounce time.Duration
maxDelay time.Duration
snapshotTimeout time.Duration
checkHealth func(ctx context.Context, appID string) HealthState
checkHealth func(ctx context.Context, appID string) HealthReport
sender MessageSender
pending map[string]*pendingHealthEvaluation
evaluating map[string]*healthEvaluation
epochs map[string]uint64
lastReported map[string]HealthState
lastReported map[string]HealthReport
nextEpoch uint64
stopped bool
}
Expand All @@ -68,7 +68,7 @@ func newHealthWatcher(c *Client) *healthWatcher {
pending: make(map[string]*pendingHealthEvaluation),
evaluating: make(map[string]*healthEvaluation),
epochs: make(map[string]uint64),
lastReported: make(map[string]HealthState),
lastReported: make(map[string]HealthReport),
}
}

Expand Down Expand Up @@ -325,7 +325,7 @@ func (w *healthWatcher) runEvaluation(appID string, evaluation *healthEvaluation
checkCtx, cancel := context.WithTimeout(evaluation.ctx, healthEvaluateTimeout)
health := w.checkHealth(checkCtx, appID)
cancel()
if health == HealthUnknown {
if health.State == HealthUnknown {
return
}

Expand All @@ -335,7 +335,7 @@ func (w *healthWatcher) runEvaluation(appID string, evaluation *healthEvaluation
return
}
last, seen := w.lastReported[appID]
if seen && last == health {
if seen && last.Equal(health) {
w.mu.Unlock()
return
}
Expand All @@ -349,7 +349,7 @@ func (w *healthWatcher) runEvaluation(appID string, evaluation *healthEvaluation
if err := sender.SendMessage(&messages.ClientMessage{
Payload: &messages.ClientMessage_ApplicationStatusReport{
ApplicationStatusReport: &messages.ApplicationStatusReport{
Statuses: []*messages.ApplicationStatus{{ApplicationId: appID, Health: health.Proto()}},
Statuses: []*messages.ApplicationStatus{health.Proto(appID)},
},
},
}); err != nil {
Expand All @@ -364,7 +364,7 @@ func (w *healthWatcher) markUnsent(appID string) {
w.mu.Lock()
defer w.mu.Unlock()
if !w.stopped {
w.lastReported[appID] = HealthUnknown
w.lastReported[appID] = HealthReport{State: HealthUnknown}
}
}

Expand All @@ -385,18 +385,18 @@ func (w *healthWatcher) reportApplicationStatus(ctx context.Context, sender Mess
snapshotCtx, cancel := context.WithTimeout(ctx, w.snapshotTimeout)
defer cancel()
statuses := make([]*messages.ApplicationStatus, 0, len(appIDs))
reported := make(map[string]HealthState, len(appIDs))
reported := make(map[string]HealthReport, len(appIDs))
var unchecked []string
for _, appID := range appIDs {
health := HealthUnknown
health := HealthReport{State: HealthUnknown}
if snapshotCtx.Err() == nil {
health = w.checkHealth(snapshotCtx, appID)
}
if snapshotCtx.Err() != nil {
health = HealthUnknown
health = HealthReport{State: HealthUnknown}
unchecked = append(unchecked, appID)
}
statuses = append(statuses, &messages.ApplicationStatus{ApplicationId: appID, Health: health.Proto()})
statuses = append(statuses, health.Proto(appID))
reported[appID] = health
}
if ctx.Err() != nil {
Expand All @@ -418,8 +418,8 @@ func (w *healthWatcher) reportApplicationStatus(ctx context.Context, sender Mess
w.mu.Lock()
if !w.stopped {
for appID, health := range reported {
if health != HealthUnknown {
w.lastReported[appID] = HealthUnknown
if health.State != HealthUnknown {
w.lastReported[appID] = HealthReport{State: HealthUnknown}
}
}
}
Expand All @@ -430,7 +430,7 @@ func (w *healthWatcher) reportApplicationStatus(ctx context.Context, sender Mess
w.mu.Lock()
if !w.stopped {
for appID, health := range reported {
if health == HealthUnknown {
if health.State == HealthUnknown {
delete(w.lastReported, appID)
continue
}
Expand Down Expand Up @@ -468,7 +468,7 @@ func (w *healthWatcher) reconcileMissingApplication(appID string) {
sender := w.sender
w.mu.Unlock()
reported := false
reportedHealth := HealthUnknown
reportedHealth := HealthReport{State: HealthUnknown}
if !stopped && sender != nil {
checkCtx, cancel := context.WithTimeout(w.ctx, healthEvaluateTimeout)
health := w.checkHealth(checkCtx, appID)
Expand All @@ -478,7 +478,7 @@ func (w *healthWatcher) reconcileMissingApplication(appID string) {
if err := sender.SendMessage(&messages.ClientMessage{
Payload: &messages.ClientMessage_ApplicationStatusReport{
ApplicationStatusReport: &messages.ApplicationStatusReport{
Statuses: []*messages.ApplicationStatus{{ApplicationId: appID, Health: health.Proto()}},
Statuses: []*messages.ApplicationStatus{health.Proto(appID)},
},
},
}); err != nil {
Expand All @@ -492,7 +492,7 @@ func (w *healthWatcher) reconcileMissingApplication(appID string) {
w.mu.Lock()
if !w.stopped {
switch {
case reported && reportedHealth != HealthUnknown:
case reported && reportedHealth.State != HealthUnknown:
w.lastReported[appID] = reportedHealth
case reported:
delete(w.lastReported, appID)
Expand Down
Loading
Loading