fix: http action error attribution by splitting request deadlines

http_action/README.md

@@ -40,6 +40,9 @@ The HTTP Action Capability enables Chainlink Runtime Environment (CRE) workflows
 #### 2.1.4 Gateway Mode Client
 - **Purpose**: Gateway-proxied HTTP request execution
 - **Features**: Rate limiting, request deduplication via consistent hashing, exponential backoff retry
+- **Deadlines**: `timeoutMs` bounds delivery to a gateway. The response wait is `timeoutMs +
+  responseGraceMs`, started after the send, so the gateway (which applies `timeoutMs` to the endpoint
+  call itself) always reports back before we give up. Exhausting the grace means the gateway went silent.
 
 ---
 
@@ -96,6 +99,30 @@ type Response struct {
 - `maxAgeMs`: Must be non-negative and not exceed configured `maxCacheAgeMs`.
 - `store`: Can be true or false;
 
+### 3.4 Error Classification
+
+Failures are attributed to the party responsible for them. Only platform faults reach
+`http_action_execution_error_count`, which is what the "Execution Errors in More than F Nodes" alert
+reads. A slow or failing customer endpoint must never page us.
+
+| Condition | Error type | Capability error | Counters (`http_action_` prefix) |
+|---|---|---|---|
+| Input validation failed | `UserError` | user, `InvalidArgument` / `LimitExceeded` | `validation_failure_count` |
+| Gateway reports endpoint send/read failure | `UserError` | user, `InvalidArgument` | `external_endpoint_error_count` |
+| Gateway reports a blocked request | `UserError` | user, `InvalidArgument` | `validation_failure_count` |
+| Response exceeds the size limit | `UserError` | user, `LimitExceeded` | `external_endpoint_error_count` |
+| No gateway reachable | plain error | system, `Internal` | `capability_gateway_send_error_count`, `execution_error_count` |
+| Gateway returned an unclassified error | plain error | system, `Internal` | `execution_error_count` |
+| Gateway silent past the response deadline | `TimeoutError` | system, `DeadlineExceeded` | `execution_timeout_count`, `execution_error_count` |
+| Caller canceled before a response arrived | `CanceledError` | system, `Canceled` | `request_canceled_count` |
+
+Direct mode returns `InputValidationError` rather than `UserError` for the first row; both map to the
+same capability error.
+
+Cancellation is not the user's fault, but `caperrors.Origin` has only `System` and `User`, so it is
+reported as a system error carrying `Canceled`. Alerts over capability failures should exclude that
+code rather than treat it as a platform fault.
+
 ---
 
 ## 4. Configuration Specification
@@ -144,9 +171,13 @@ type GatewayConnectionConfig struct {
     InitialIntervalMs uint32  `json:"initialIntervalMs"` // Initial retry interval
     MaxElapsedTimeMs  uint32  `json:"maxElapsedTimeMs"`  // Maximum retry duration
     Multiplier        float64 `json:"multiplier"`        // Backoff multiplier
+    ResponseGraceMs   uint32  `json:"responseGraceMs"`   // Extra wait for a gateway response beyond timeoutMs
 }
 ```
 
+`responseGraceMs` defaults to 5000, mirroring the gateway's own budget for delivering a response back
+to the node. It is optional: omitting it keeps the default, so existing deployments need no change.
+
 ### 4.6 Configuration Examples
 
 #### 4.6.1 Gateway Mode Configuration
@@ -157,7 +188,8 @@ type GatewayConnectionConfig struct {
   "gatewayConnection": {
     "initialIntervalMs": 100,
     "maxElapsedTimeMs": 30000,
-    "multiplier": 2.0
+    "multiplier": 2.0,
+    "responseGraceMs": 5000
   }
 }
 ```

http_action/action/action.go

@@ -134,10 +134,24 @@
 		var userErr gateway.UserError
 		if errors.As(err, &userErr) {
 			return nil, caperrors.NewPublicUserError(
 				fmt.Errorf("request failed for workflowID %s (Owner: %s, Name: %s, ExecutionID: %s): %w",
 					metadata.WorkflowID, metadata.WorkflowOwner, metadata.WorkflowName, metadata.WorkflowExecutionID, err),
 				common.UserErrorCode(err))
 		}
+		var canceledErr gateway.CanceledError
+		if errors.As(err, &canceledErr) {
+			return nil, caperrors.NewPublicSystemError(
+				fmt.Errorf("request canceled for workflowID %s (Owner: %s, Name: %s, ExecutionID: %s): %w",
+					metadata.WorkflowID, metadata.WorkflowOwner, metadata.WorkflowName, metadata.WorkflowExecutionID, err),
+				caperrors.Canceled)
+		}
+		var gatewayTimeoutErr gateway.TimeoutError
+		if errors.As(err, &gatewayTimeoutErr) {
+			return nil, caperrors.NewPublicSystemError(
+				fmt.Errorf("request failed for workflowID %s (Owner: %s, Name: %s, ExecutionID: %s): %w",
+					metadata.WorkflowID, metadata.WorkflowOwner, metadata.WorkflowName, metadata.WorkflowExecutionID, err),
+				caperrors.DeadlineExceeded)
+		}
 		return nil, caperrors.NewPublicSystemError(
 			fmt.Errorf("request failed for workflowID %s (Owner: %s, Name: %s, ExecutionID: %s): %w",
 				metadata.WorkflowID, metadata.WorkflowOwner, metadata.WorkflowName, metadata.WorkflowExecutionID, err), caperrors.Internal)

http_action/action/action_test.go

@@ -389,4 +389,46 @@ func TestSendRequest_ErrorHandling(t *testing.T) {
 		assert.Equal(t, caperrors.Internal, capErr.Code())
 		assert.Equal(t, caperrors.VisibilityPublic, capErr.Visibility())
 	})
+
+	t.Run("client returns TimeoutError and service returns DeadlineExceeded system error", func(t *testing.T) {
+		setup := setupServiceTest(t)
+
+		input := &http.Request{
+			Url:           "https://example.com",
+			Method:        "GET",
+			Timeout:       durationpb.New(1000 * time.Millisecond),
+			CacheSettings: &http.CacheSettings{},
+		}
+
+		setup.mockClient.Err = gateway.NewTimeoutError(errors.New("gateway did not respond before deadline"))
+
+		_, err := setup.service.SendRequest(t.Context(), setup.metadata, input)
+		require.Error(t, err)
+
+		var capErr caperrors.Error
+		assert.True(t, errors.As(err, &capErr))
+		assert.Equal(t, caperrors.DeadlineExceeded, capErr.Code())
+		assert.Equal(t, caperrors.OriginSystem, capErr.Origin())
+	})
+
+	t.Run("client returns CanceledError and service returns Canceled system error", func(t *testing.T) {
+		setup := setupServiceTest(t)
+
+		input := &http.Request{
+			Url:           "https://example.com",
+			Method:        "GET",
+			Timeout:       durationpb.New(1000 * time.Millisecond),
+			CacheSettings: &http.CacheSettings{},
+		}
+
+		setup.mockClient.Err = gateway.NewCanceledError(context.Canceled)
+
+		_, err := setup.service.SendRequest(t.Context(), setup.metadata, input)
+		require.Error(t, err)
+
+		var capErr caperrors.Error
+		assert.True(t, errors.As(err, &capErr))
+		assert.Equal(t, caperrors.Canceled, capErr.Code())
+		assert.Equal(t, caperrors.OriginSystem, capErr.Origin())
+	})
 }

http_action/common/metrics.go

@@ -29,6 +29,7 @@ type Metrics struct {
 	successfulResponse              metric.Int64Counter
 	executionError                  metric.Int64Counter
 	executionTimeout                metric.Int64Counter
+	requestCanceled                 metric.Int64Counter
 	externalEndpointError           metric.Int64Counter
 	requestLatency                  metric.Int64Histogram
 	requestLatencyExcludingExternal metric.Int64Histogram
@@ -99,12 +100,20 @@ func (m *Metrics) init() error {
 
 	m.executionTimeout, err = meter.Int64Counter(
 		"http_action_execution_timeout_count",
-		metric.WithDescription("Number of HTTP action execution timeouts"),
+		metric.WithDescription("Number of HTTP action requests where the gateway did not respond before the response deadline"),
 	)
 	if err != nil {
 		return fmt.Errorf("failed to create execution timeout metric: %w", err)
 	}
 
+	m.requestCanceled, err = meter.Int64Counter(
+		"http_action_request_canceled_count",
+		metric.WithDescription("Number of HTTP action requests canceled by the caller before a gateway response arrived"),
+	)
+	if err != nil {
+		return fmt.Errorf("failed to create request canceled metric: %w", err)
+	}
+
 	m.externalEndpointError, err = meter.Int64Counter(
 		"http_action_external_endpoint_error_count",
 		metric.WithDescription("Number of HTTP action external endpoint errors"),
@@ -171,11 +180,17 @@ func (m *Metrics) IncrementExecutionError(ctx context.Context, proxyMode ProxyMo
 	m.executionError.Add(ctx, 1, metric.WithAttributes(attribute.String(AttrProxyMode, proxyMode.String())))
 }
 
+// Gateway silence is a platform fault, so it also counts as an execution error.
 func (m *Metrics) IncrementExecutionTimeout(ctx context.Context, proxyMode ProxyMode, lggr logger.Logger) {
 	m.executionTimeout.Add(ctx, 1, metric.WithAttributes(attribute.String(AttrProxyMode, proxyMode.String())))
 	m.executionError.Add(ctx, 1, metric.WithAttributes(attribute.String(AttrProxyMode, proxyMode.String())))
 }
 
+// Caller cancellation is nobody's fault, so it is deliberately not an execution error.
+func (m *Metrics) IncrementRequestCanceled(ctx context.Context, proxyMode ProxyMode, lggr logger.Logger) {
+	m.requestCanceled.Add(ctx, 1, metric.WithAttributes(attribute.String(AttrProxyMode, proxyMode.String())))
+}
+
 func (m *Metrics) IncrementExternalEndpointError(ctx context.Context, proxyMode ProxyMode, lggr logger.Logger) {
 	m.externalEndpointError.Add(ctx, 1, metric.WithAttributes(attribute.String(AttrProxyMode, proxyMode.String())))
 }

http_action/common/service_config.go

@@ -54,6 +54,8 @@ type GatewayConnectionConfig struct {
 	MaxElapsedTimeMs uint32 `json:"maxElapsedTimeMs"`
 	// Multiplier is the multiplier for the exponential backoff retry strategy.
 	Multiplier float64 `json:"multiplier"`
+	// ResponseGraceMs is the extra time the node waits for a gateway response beyond the request's own timeout.
+	ResponseGraceMs uint32 `json:"responseGraceMs"`
 }
 
 // HTTPClientConfig defines configuration options for the HTTP client used in "direct" mode.

http_action/gateway/gateway_outbound_proxy.go

@@ -28,6 +28,8 @@ const (
 	defaultGatewayConnectionInitialIntervalMs = 100 // 100 milliseconds
 	defaultGatewayConnectionMaxElapsedTimeMs  = 5_000
 	defaultGatewayConnectionMultiplier        = 2.0
+	// Mirrors the gateway's own budget for delivering a response back to us.
+	defaultGatewayResponseGraceMs = 5_000
 )
 
 var (
@@ -110,6 +112,9 @@ func applyDefaults(cfg common.GatewayConnectionConfig) common.GatewayConnectionC
 	if cfg.Multiplier == 0 {
 		cfg.Multiplier = defaultGatewayConnectionMultiplier
 	}
+	if cfg.ResponseGraceMs == 0 {
+		cfg.ResponseGraceMs = defaultGatewayResponseGraceMs
+	}
 	return cfg
 }
 
@@ -136,8 +141,9 @@ func (p *gatewayOutboundProxy) SendRequest(ctx context.Context, metadata capabil
 	}
 	input = validatedInput
 
-	ctx, cancel := context.WithTimeout(ctx, input.Timeout.AsDuration())
-	defer cancel()
+	// Bounds delivery to a gateway only; the response wait gets its own deadline after the send.
+	sendCtx, cancelSend := context.WithTimeout(ctx, input.Timeout.AsDuration())
+	defer cancelSend()
 
 	// Set only one of Headers or MultiHeaders on the outgoing request (MultiHeaders if input has it, else Headers).
 	gatewayHeaders, gatewayMultiHeaders := gatewayHeadersFromInput(input)
@@ -190,13 +196,13 @@ func (p *gatewayOutboundProxy) SendRequest(ctx context.Context, metadata capabil
 
 	p.metrics.IncrementRequestCount(ctx, lggr)
 
-	donID, err := p.validator.ResolveGatewayProxyDonID(ctx)
+	donID, err := p.validator.ResolveGatewayProxyDonID(sendCtx)
 	if err != nil {
 		p.metrics.IncrementExecutionError(ctx, common.ProxyModeGateway, lggr)
 		return nil, 0, fmt.Errorf("failed to resolve gateway proxy DON: %w", err)
 	}
 
-	selectedGateway, err := p.awaitConnection(ctx, lggr, donID, gatewayReq.Hash())
+	selectedGateway, err := p.awaitConnection(sendCtx, lggr, donID, gatewayReq.Hash())
 	if err != nil {
 		p.metrics.IncrementGatewaySendError(ctx, selectedGateway, donID, lggr)
 		return nil, 0, fmt.Errorf("failed to establish connection to gateway: %w", err)
@@ -205,11 +211,15 @@ func (p *gatewayOutboundProxy) SendRequest(ctx context.Context, metadata capabil
 	lggr.Debugw("sending request to gateway", "donID", donID, "selectedGateway", selectedGateway)
 
 	p.metrics.IncrementGatewaySendCount(ctx, selectedGateway, donID, lggr)
-	if err := p.gatewayConnector.SendToGateway(ctx, selectedGateway, &gatewayResp); err != nil {
+	if err := p.gatewayConnector.SendToGateway(sendCtx, selectedGateway, &gatewayResp); err != nil {
 		p.metrics.IncrementGatewaySendError(ctx, selectedGateway, donID, lggr)
 		return nil, 0, fmt.Errorf("failed to send request to gateway: %w", err)
 	}
 
+	responseTimeout := input.Timeout.AsDuration() + time.Duration(p.gatewayConnectionConfig.ResponseGraceMs)*time.Millisecond
+	waitCtx, cancelWait := context.WithTimeout(ctx, responseTimeout)
+	defer cancelWait()
+
 	select {
 	case resp := <-responseCh:
 		lggr.Debugw("received response from gateway")
@@ -241,18 +251,33 @@ func (p *gatewayOutboundProxy) SendRequest(ctx context.Context, metadata capabil
 		}
 
 		return response, resp.ExternalEndpointLatency, nil
-	case <-ctx.Done():
-		p.metrics.IncrementExecutionTimeout(ctx, common.ProxyModeGateway, lggr)
+	case <-waitCtx.Done():
 		elapsedMs := time.Since(startTime).Milliseconds()
-		timeoutMs := input.Timeout.AsDuration().Milliseconds()
-		cause := context.Cause(ctx)
-		lggr.Debugw(ErrMsgGatewayResponseWait,
+		timeoutMs := responseTimeout.Milliseconds()
+
+		// A live parent means we hit our own deadline; otherwise the caller cancelled us.
+		if parentErr := ctx.Err(); parentErr != nil {
+			p.metrics.IncrementRequestCanceled(ctx, common.ProxyModeGateway, lggr)
+			cause := context.Cause(ctx)
+			lggr.Debugw(ErrMsgGatewayResponseWait,
+				"elapsedMs", elapsedMs,
+				"timeoutMs", timeoutMs,
+				"cause", cause,
+			)
+			return nil, 0, NewCanceledError(
+				fmt.Errorf("%s (elapsedMs: %d, timeoutMs: %d): %w", ErrMsgGatewayResponseWait, elapsedMs, timeoutMs, cause),
+			)
+		}
+
+		p.metrics.IncrementExecutionTimeout(ctx, common.ProxyModeGateway, lggr)
+		lggr.Errorw(ErrMsgGatewayResponseTimeout,
 			"elapsedMs", elapsedMs,
 			"timeoutMs", timeoutMs,
-			"cause", cause,
+			"selectedGateway", selectedGateway,
+			"donID", donID,
 		)
-		return nil, 0, NewUserError(
-			fmt.Errorf("%s (elapsedMs: %d, timeoutMs: %d): %w", ErrMsgGatewayResponseWait, elapsedMs, timeoutMs, cause),
+		return nil, 0, NewTimeoutError(
+			fmt.Errorf("%s (elapsedMs: %d, timeoutMs: %d): %w", ErrMsgGatewayResponseTimeout, elapsedMs, timeoutMs, context.Cause(waitCtx)),
 		)
 	}
 }