From 933c554f8bffa2bf7d188ced712b69b20844ac2d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 2 Oct 2026 19:47:28 +0000 Subject: [PATCH] Explain cancellation cascades in workflow debug output --- docs/cli-reference.md | 15 ++ src/Commands/DebugCommand.php | 6 +- src/Support/CancellationCascadeRenderer.php | 133 ++++++++++++++ .../Commands/DebugCancellationCommandTest.php | 172 ++++++++++++++++++ 4 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 src/Support/CancellationCascadeRenderer.php create mode 100644 tests/Commands/DebugCancellationCommandTest.php diff --git a/docs/cli-reference.md b/docs/cli-reference.md index ea90d5d..1708204 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -324,6 +324,21 @@ setting, normalized auth-composition source names, TLS verification mode, server-advertised `auth_composition_contract` metadata, `/api/cluster/info`, and compatibility warnings derived from the protocol manifests and `client_compatibility` metadata. + +When Server supplies cancellation cascade diagnostics, `dw debug workflow` +shows the original root identity and cleanup deadline, each run's request, +delivery boundary and phase, child policy outcomes, activity stop receipts, +cleanup recovery grants and recorded cleanup outcomes. Independent roots keep +their own deadlines. A fence without a matching stop receipt keeps callback +state `unknown`. Recovery grants alone do not identify the cause of worker loss. + +`Evidence: complete` describes retained inspection evidence, not whether every +cleanup succeeded. Clipped or unavailable evidence is shown as partial with +Server findings. `--run-id` preserves historical selection. `--output=json` +retains the complete Server response, including fields this CLI does not +recognize. This candidate view requires a Server runtime that supplies +`durable-workflow.cancellation-cascade/v1` diagnostics. + Use `dw debug workflow ` when support needs a single stuck-run capture: execution state, pending workflow/activity tasks, task queue backlog and pollers, recent failures, and compatibility metadata. diff --git a/src/Commands/DebugCommand.php b/src/Commands/DebugCommand.php index e28b38c..f54a0be 100644 --- a/src/Commands/DebugCommand.php +++ b/src/Commands/DebugCommand.php @@ -4,6 +4,7 @@ namespace DurableWorkflow\Cli\Commands; +use DurableWorkflow\Cli\Support\CancellationCascadeRenderer; use DurableWorkflow\Cli\Support\InvalidOptionException; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; @@ -21,8 +22,8 @@ protected function configure(): void ->setDescription('Run one-shot diagnostics for a workflow') ->setHelp(<<<'HELP' Run a one-shot workflow diagnostic that combines execution state, pending -workflow/activity work, task queue state, recent failures, and compatibility -metadata from the server. +workflow/activity work, task queue state, recent failures, compatibility and +available cancellation cascade evidence from the server. Examples: @@ -131,6 +132,7 @@ private function renderHuman(InputInterface $input, OutputInterface $output, arr )); $output->writeln(''); + (new CancellationCascadeRenderer())->render($output, $result); $this->renderWorkflowTasks($output, $result['pending_workflow_tasks'] ?? []); $this->renderActivities($output, $result['pending_activities'] ?? []); $this->renderFailures($output, $result['recent_failures'] ?? []); diff --git a/src/Support/CancellationCascadeRenderer.php b/src/Support/CancellationCascadeRenderer.php new file mode 100644 index 0000000..66d9ab8 --- /dev/null +++ b/src/Support/CancellationCascadeRenderer.php @@ -0,0 +1,133 @@ + $diagnostics */ + public function render(OutputInterface $output, array $diagnostics): void + { + if (! array_key_exists('cancellation_cascade_supported', $diagnostics) + && ! array_key_exists('cancellation_cascade', $diagnostics)) { + return; + } + $output->writeln('Cancellation Cascade:'); + $view = $diagnostics['cancellation_cascade'] ?? null; + if (! is_array($view)) { + $output->writeln(($diagnostics['cancellation_cascade_supported'] ?? false) === true + ? ' No cooperative cancellation request for this run.' + : ' Unavailable with this Server runtime.'); + $output->writeln(''); + + return; + } + if (($view['schema'] ?? null) !== 'durable-workflow.cancellation-cascade/v1') { + $output->writeln(' Unrecognized cascade schema. Use --output=json to inspect the response.'); + $output->writeln(''); + + return; + } + + $root = $this->object($view['root'] ?? null); + $requester = $this->object($root['requester'] ?? null); + $this->line($output, ' Root request: %s', $root['root_request_id'] ?? null); + $this->line($output, ' Root workflow: %s run=%s', $root['root_workflow_instance_id'] ?? null, $root['root_workflow_run_id'] ?? null); + $this->line($output, ' Original request: %s deadline=%s', $root['requested_at'] ?? null, $root['cleanup_deadline_at'] ?? null); + $this->line($output, ' Reason: %s source=%s requester=%s', $root['reason'] ?? null, $root['source'] ?? null, $requester['label'] ?? $requester['id'] ?? null); + $this->line($output, ' Evidence: %s%s', + ($view['inspection_complete'] ?? false) === true ? 'complete' : 'partial', + ($view['truncated'] ?? false) === true ? ' (inspection limits reached)' : ''); + + foreach ($this->objects($view['runs'] ?? null) as $run) { + $request = $this->object($run['request'] ?? null); + $delivery = $this->object($run['delivery'] ?? null); + $cleanup = $this->object($run['cleanup'] ?? null); + $this->line($output, ' Run %s%s: %s workflow=%s phase=%s status=%s', + $run['run_id'] ?? null, ($run['run_id'] ?? null) === ($view['selected_run_id'] ?? null) ? ' [selected]' : '', + $run['workflow_type'] ?? null, $run['workflow_id'] ?? null, $run['lifecycle'] ?? null, $run['projected_status'] ?? null); + $this->line($output, ' Request: %s parent=%s budget=%s deadline=%s', + $request['request_id'] ?? null, $request['parent_request_id'] ?? null, $this->budget($run, $root), $request['cleanup_deadline_at'] ?? null); + $this->line($output, ' Delivery: event=%s sequence=%s span=%s kind=%s', + $delivery['history_event_id'] ?? null, $delivery['sequence'] ?? null, $delivery['sequence_span'] ?? null, $delivery['call_kind'] ?? null); + $this->line($output, ' Cleanup: %s finished=%s terminal=%s event=%s', + $cleanup['outcome'] ?? null, $cleanup['finished_at'] ?? null, $run['terminal_event_type'] ?? null, $run['terminal_history_event_id'] ?? null); + foreach ($this->objects($run['activity_stops'] ?? null) as $stop) { + $this->line($output, ' Activity %s (%s, %s): callback=%s fence=%s stop=%s source=%s', + $stop['activity_execution_id'] ?? null, $stop['activity_type'] ?? null, $stop['execution_mode'] ?? null, + $stop['callback_state'] ?? null, $stop['fence_history_event_id'] ?? null, $stop['stop_history_event_id'] ?? null, $stop['evidence_source'] ?? null); + $this->line($output, ' Attempt: %s acknowledged=%s after_deadline=%s', + $stop['activity_attempt_id'] ?? null, $stop['acknowledged_at'] ?? null, $stop['received_after_deadline'] ?? null); + } + foreach ($this->objects($run['cleanup_recovery'] ?? null) as $recovery) { + $attempt = $this->object($recovery['attempt'] ?? null); + $this->line($output, ' Cleanup recovery: activity=%s event=%s at=%s', + $recovery['activity_execution_id'] ?? null, $recovery['history_event_id'] ?? null, $recovery['recorded_at'] ?? null); + $this->line($output, ' %s attempt=%s owner=%s -> %s attempt=%s owner=%s callback=%s', + $attempt['original_workflow_task_id'] ?? null, $attempt['original_workflow_task_attempt'] ?? null, + $attempt['original_lease_owner'] ?? null, $attempt['workflow_task_id'] ?? null, + $attempt['workflow_task_attempt'] ?? null, $attempt['lease_owner'] ?? null, $attempt['callback_stop_state'] ?? null); + } + foreach ($this->objects($run['child_propagation'] ?? null) as $propagation) { + $this->line($output, ' Child %s: policy=%s outcome=%s rejection=%s event=%s terminal=%s', + $propagation['child_run_id'] ?? null, $propagation['policy'] ?? null, $propagation['request_outcome'] ?? null, + $propagation['rejection_reason'] ?? null, $propagation['history_event_id'] ?? null, $propagation['child_terminal_history_event_id'] ?? null); + } + } + foreach ($this->objects($view['edges'] ?? null) as $edge) { + $this->line($output, ' Relation: %s -> %s kind=%s state=%s', + $edge['parent_run_id'] ?? null, $edge['child_run_id'] ?? null, $edge['kind'] ?? null, $edge['reference_state'] ?? null); + } + foreach ($this->objects($view['findings'] ?? null) as $finding) { + $this->line($output, ' Finding [%s] run=%s: %s', $finding['code'] ?? null, $finding['run_id'] ?? null, $finding['message'] ?? null); + } + $output->writeln(''); + } + + /** @param array $run @param array $root */ + private function budget(array $run, array $root): string + { + $request = $this->object($run['request'] ?? null); + if ($request === [] || $root === []) { + return 'unverified'; + } + if (($run['same_root_budget'] ?? false) === true) { + return 'original root'; + } + + return ($request['root_request_id'] ?? null) !== ($root['root_request_id'] ?? null) + ? 'independent root' : 'conflict or unverified'; + } + + /** @return array */ + private function object(mixed $value): array + { + return is_array($value) ? $value : []; + } + + /** @return list> */ + private function objects(mixed $value): array + { + return is_array($value) ? array_values(array_filter($value, 'is_array')) : []; + } + + private function line(OutputInterface $output, string $format, mixed ...$values): void + { + $output->writeln(sprintf($format, ...array_map($this->text(...), $values))); + } + + private function text(mixed $value): string + { + if (! is_scalar($value)) { + return '-'; + } + $text = is_bool($value) ? ($value ? 'true' : 'false') : (string) $value; + + return OutputFormatter::escape(preg_replace('/[\x00-\x1F\x7F\x{80}-\x{9F}]/u', ' ', $text) ?? '-'); + } +} diff --git a/tests/Commands/DebugCancellationCommandTest.php b/tests/Commands/DebugCancellationCommandTest.php new file mode 100644 index 0000000..eabef9b --- /dev/null +++ b/tests/Commands/DebugCancellationCommandTest.php @@ -0,0 +1,172 @@ +display($this->payload()); + foreach ([ + 'Root request: root-request', 'Original request: 2026-10-02T00:00:00Z deadline=2026-10-02T00:00:30Z', + 'parent-run [selected]', 'phase=cleaning_up', 'phase=cancelled', 'budget=original root', + 'Request: child-request parent=root-request', 'Delivery: event=13 sequence=1 span=2 kind=parallel', + 'Cleanup: completed', 'Activity php-work (php.local.work, local): callback=reported_stopped', + 'Activity rust-work (rust.remote.work, remote): callback=reported_stopped', + 'Cleanup recovery: activity=php-cleanup event=17', + 'task-1 attempt=1 owner=old-worker -> task-2 attempt=2 owner=replacement callback=unknown', + 'Child child-run: policy=wait_cancellation_completed outcome=requested', + 'Relation: parent-run -> child-run kind=child_workflow state=resolved', 'Evidence: complete', + ] as $expected) { + self::assertStringContainsString($expected, $display); + } + self::assertStringNotContainsString('SIGKILL', $display, 'Grant recovery does not prove the cause of worker loss.'); + } + + public function test_json_preserves_the_entire_response_including_unrecognized_evidence(): void + { + $payload = $this->payload(); + $payload['cancellation_cascade']['future_evidence'] = ['retained' => true]; + $payload['cancellation_cascade']['schema'] = 'durable-workflow.cancellation-cascade/v2'; + $display = $this->display($payload, true); + self::assertSame($payload, json_decode(trim($display), true, 512, JSON_THROW_ON_ERROR)); + } + + public function test_fences_and_partial_history_do_not_imply_callback_exit_or_completed_cleanup(): void + { + $payload = $this->payload(); + $view = &$payload['cancellation_cascade']; + $view['inspection_complete'] = false; + $view['truncated'] = true; + $view['runs'][0]['activity_stops'][0]['callback_state'] = 'unknown'; + $view['runs'][0]['activity_stops'][0]['stop_history_event_id'] = null; + $view['runs'][0]['activity_stops'][0]['evidence_source'] = null; + $view['runs'][1]['cleanup'] = null; + $view['findings'] = [['code' => 'history_limit', 'run_id' => 'parent-run', 'message' => 'Open the run history for additional evidence.']]; + $display = $this->display($payload); + self::assertStringContainsString('Evidence: partial (inspection limits reached)', $display); + self::assertStringContainsString('callback=unknown fence=8 stop=- source=-', $display); + self::assertStringContainsString('Cleanup: - finished=- terminal=WorkflowCancelled', $display); + self::assertStringContainsString('Finding [history_limit] run=parent-run: Open the run history', $display); + } + + public function test_independent_child_roots_keep_their_own_deadlines(): void + { + $payload = $this->payload(); + $child = &$payload['cancellation_cascade']['runs'][1]; + $child['same_root_budget'] = false; + $child['request']['root_request_id'] = 'independent-request'; + $child['request']['cleanup_deadline_at'] = '2026-10-02T00:00:45Z'; + self::assertStringContainsString('budget=independent root deadline=2026-10-02T00:00:45Z', $this->display($payload)); + } + + public function test_missing_root_and_conflicting_budget_are_reported_as_unverified(): void + { + $payload = $this->payload(); + $payload['cancellation_cascade']['runs'][1]['same_root_budget'] = false; + self::assertStringContainsString('budget=conflict or unverified', $this->display($payload)); + $payload['cancellation_cascade']['root'] = null; + self::assertStringContainsString('Root request: -', $this->display($payload)); + self::assertStringContainsString('budget=unverified', $this->display($payload)); + } + + public function test_old_runtime_no_request_and_future_schema_are_distinguishable(): void + { + self::assertStringContainsString('Unavailable with this Server runtime.', $this->display(['cancellation_cascade_supported' => false])); + self::assertStringContainsString('No cooperative cancellation request for this run.', $this->display(['cancellation_cascade_supported' => true, 'cancellation_cascade' => null])); + self::assertStringContainsString('Unrecognized cascade schema.', $this->display(['cancellation_cascade' => ['schema' => 'future']])); + self::assertStringNotContainsString('Cancellation Cascade:', $this->display([])); + } + + public function test_caller_text_cannot_inject_console_markup_or_terminal_controls(): void + { + $payload = $this->payload(); + $payload['cancellation_cascade']['root']['reason'] = "literal reason\x1b]52;c;payload\x07\r\nforged line"; + $payload['cancellation_cascade']['runs'][0]['workflow_type'] = 'caller type'; + $display = $this->display($payload); + self::assertStringContainsString('literal reason', $display); + self::assertStringContainsString('caller type', $display); + self::assertStringNotContainsString("\x1b", $display); + self::assertStringNotContainsString("\x07", $display); + self::assertStringNotContainsString("\nforged line", $display); + } + + private function display(array $payload, bool $json = false): string + { + $client = new class($payload) extends ServerClient + { + public function __construct(private readonly array $payload) {} + + public function get(string $path, array $query = []): array + { + return $this->payload; + } + }; + $command = new DebugCommand(); + $command->setServerClient($client); + $tester = new CommandTester($command); + self::assertSame(Command::SUCCESS, $tester->execute([ + 'target' => 'workflow', 'workflow-id' => 'parent-workflow', '--run-id' => 'parent-run', + '--output' => $json ? 'json' : 'table', + ])); + + return $tester->getDisplay(); + } + + /** A diagnostic display fixture, not a runtime qualification result. */ + private function payload(): array + { + $root = [ + 'root_request_id' => 'root-request', 'request_id' => 'root-request', + 'root_workflow_instance_id' => 'parent-workflow', 'root_workflow_run_id' => 'parent-run', + 'parent_request_id' => null, 'reason' => 'Maintenance', 'source' => 'operator', 'requester' => ['id' => 'test-operator'], + 'requested_at' => '2026-10-02T00:00:00Z', 'cleanup_deadline_at' => '2026-10-02T00:00:30Z', + ]; + $stop = [ + 'activity_execution_id' => 'php-work', 'activity_attempt_id' => 'php-attempt', 'activity_type' => 'php.local.work', + 'execution_mode' => 'local', 'fence_history_event_id' => 8, 'callback_state' => 'reported_stopped', + 'stop_history_event_id' => 9, 'evidence_source' => 'joined_callback', + 'acknowledged_at' => '2026-10-02T00:00:01Z', 'received_after_deadline' => false, + ]; + $parent = [ + 'run_id' => 'parent-run', 'workflow_id' => 'parent-workflow', 'workflow_type' => 'php.parent', + 'projected_status' => 'waiting', 'lifecycle' => 'cleaning_up', 'same_root_budget' => true, 'request' => $root, + 'delivery' => ['history_event_id' => 13, 'sequence' => 1, 'sequence_span' => 2, 'call_kind' => 'parallel'], + 'cleanup' => null, 'terminal_event_type' => null, 'terminal_history_event_id' => null, + 'activity_stops' => [$stop], + 'cleanup_recovery' => [[ + 'history_event_id' => 17, 'activity_execution_id' => 'php-cleanup', 'recorded_at' => '2026-10-02T00:00:12Z', + 'attempt' => ['original_workflow_task_id' => 'task-1', 'original_workflow_task_attempt' => 1, + 'original_lease_owner' => 'old-worker', 'workflow_task_id' => 'task-2', 'workflow_task_attempt' => 2, + 'lease_owner' => 'replacement', 'callback_stop_state' => 'unknown'], + ]], + 'child_propagation' => [['child_run_id' => 'child-run', 'policy' => 'wait_cancellation_completed', + 'request_outcome' => 'requested', 'history_event_id' => 10]], + ]; + $child = array_replace($parent, [ + 'run_id' => 'child-run', 'workflow_id' => 'child-workflow', 'workflow_type' => 'python.child', + 'projected_status' => 'cancelled', 'lifecycle' => 'cancelled', + 'request' => array_replace($root, ['request_id' => 'child-request', 'parent_request_id' => 'root-request']), + 'cleanup' => ['outcome' => 'completed', 'finished_at' => '2026-10-02T00:00:03Z'], + 'terminal_event_type' => 'WorkflowCancelled', 'terminal_history_event_id' => 11, + 'activity_stops' => [array_replace($stop, ['activity_execution_id' => 'rust-work', 'activity_type' => 'rust.remote.work', 'execution_mode' => 'remote'])], + 'cleanup_recovery' => [], 'child_propagation' => [], + ]); + + return [ + 'workflow_id' => 'parent-workflow', 'run_id' => 'parent-run', 'cancellation_cascade_supported' => true, + 'cancellation_cascade' => ['schema' => 'durable-workflow.cancellation-cascade/v1', + 'selected_run_id' => 'parent-run', 'root' => $root, 'runs' => [$parent, $child], + 'edges' => [['parent_run_id' => 'parent-run', 'child_run_id' => 'child-run', 'kind' => 'child_workflow', 'reference_state' => 'resolved']], + 'inspection_complete' => true, 'truncated' => false, 'findings' => []], + ]; + } +}