From 4758cff5a6f9850339ee628f49bdf791595417a1 Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Fri, 23 Oct 2026 23:34:10 +0530 Subject: [PATCH 1/3] Plan workflows from descriptions --- .../src/lib/workflow/__tests__/engine.test.ts | 7 + .../lib/workflow/__tests__/generate.test.ts | 244 ++++++++++++++++++ apps/dashboard/src/lib/workflow/generate.ts | 235 +++++++++++++++++ apps/dashboard/src/lib/workflow/nodeKinds.ts | 27 +- apps/dashboard/src/lib/workflow/templates.ts | 7 +- 5 files changed, 512 insertions(+), 8 deletions(-) create mode 100644 apps/dashboard/src/lib/workflow/__tests__/generate.test.ts create mode 100644 apps/dashboard/src/lib/workflow/generate.ts diff --git a/apps/dashboard/src/lib/workflow/__tests__/engine.test.ts b/apps/dashboard/src/lib/workflow/__tests__/engine.test.ts index 9a67cbbf..2f37e3d4 100644 --- a/apps/dashboard/src/lib/workflow/__tests__/engine.test.ts +++ b/apps/dashboard/src/lib/workflow/__tests__/engine.test.ts @@ -79,6 +79,13 @@ describe("autofillModels", () => { ]); expect(filled.nodes.map((n) => n.config.model ?? null)).toEqual([null, "whisper-small", "llama-3-8b", "kokoro-tts", null]); }); + + it("leaves a model empty rather than binding a deployment for the other audio direction", () => { + const filled = autofillModels(TEMPLATES.bankingVoice.build(), [ + { id: "k", name: "kokoro-tts", modelName: "kokoro-tts", model_type: "audio_generation", endpointUrl: "", status: "READY" }, + ]); + expect(filled.nodes.map((n) => n.config.model ?? null)).toEqual([null, null, null, "kokoro-tts", null]); + }); }); describe("runWorkflow with the banking voice template", () => { diff --git a/apps/dashboard/src/lib/workflow/__tests__/generate.test.ts b/apps/dashboard/src/lib/workflow/__tests__/generate.test.ts new file mode 100644 index 00000000..ebee5434 --- /dev/null +++ b/apps/dashboard/src/lib/workflow/__tests__/generate.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, it, vi } from "vitest"; +import { extractJson, generateWorkflow, intentError, layout, planFromKeywords, plannerPrompt, validatePlan } from "../generate"; +import type { Workflow } from "../types"; + +const kinds = (w: Workflow) => w.nodes.map((n) => n.kind); +const pairs = (w: Workflow) => w.edges.map((e) => `${e.source}>${e.target}`); + +describe("plannerPrompt", () => { + it("lists every registry kind with its ports and plannable config", () => { + const prompt = plannerPrompt(); + expect(prompt).toContain("- llm: Chat completion with a system prompt. Input: text. Output: text. Config: systemPrompt (system prompt)."); + expect(prompt).toContain("- audioInput: Record from the mic or upload a file. Input: none. Output: audio. Config: none."); + expect(prompt).toContain("- textToSpeech: Synthesize speech with a TTS model. Input: text. Output: audio. Config: none."); + expect(prompt).not.toContain("model (model)"); + expect(prompt).not.toContain("voice (voice)"); + }); +}); + +describe("extractJson", () => { + it("parses a bare object", () => { + expect(extractJson('{"a":1}')).toEqual({ a: 1 }); + }); + + it("strips code fences and surrounding prose", () => { + expect(extractJson('Sure! Here it is:\n```json\n{"nodes":[{"id":"n1"}]}\n```\nEnjoy')).toEqual({ nodes: [{ id: "n1" }] }); + }); + + it("takes only the first object and ignores braces inside strings", () => { + expect(extractJson('{"text":"a } b {"} {"second":true}')).toEqual({ text: "a } b {" }); + }); + + it("returns null for no object or broken JSON", () => { + expect(extractJson("no json here")).toBeNull(); + expect(extractJson('{"a": }')).toBeNull(); + expect(extractJson('{"a": 1')).toBeNull(); + }); +}); + +describe("validatePlan", () => { + it("keeps a valid chain, drops model config, and lays it out in columns", () => { + const { workflow, errors } = validatePlan({ + nodes: [ + { id: "a", kind: "textInput", config: { text: "hi", bogus: "x" } }, + { id: "b", kind: "llm", config: { model: "gpt-9", systemPrompt: "Be brief" } }, + { id: "c", kind: "textOutput" }, + ], + edges: [ + { source: "a", target: "b" }, + { source: "b", target: "c" }, + ], + }); + expect(errors).toEqual([]); + expect(workflow).toEqual({ + nodes: [ + { id: "a", kind: "textInput", position: { x: 0, y: 0 }, config: { text: "hi" } }, + { id: "b", kind: "llm", position: { x: 280, y: 0 }, config: { systemPrompt: "Be brief" } }, + { id: "c", kind: "textOutput", position: { x: 560, y: 0 }, config: {} }, + ], + edges: [ + { id: "e-a-b", source: "a", target: "b" }, + { id: "e-b-c", source: "b", target: "c" }, + ], + }); + }); + + it("drops unknown kinds, duplicates and edges that fail connectionError", () => { + const { workflow, errors } = validatePlan({ + nodes: [ + { id: "a", kind: "audioInput" }, + { id: "b", kind: "videoInput" }, + { id: "c", kind: "textOutput" }, + { id: "c", kind: "llm" }, + { id: "d", kind: "speechToText" }, + ], + edges: [ + { source: "a", target: "c" }, + { source: "a", target: "b" }, + { source: "a", target: "d" }, + { source: "d", target: "c" }, + { source: "a", target: "d" }, + ], + }); + expect(kinds(workflow)).toEqual(["audioInput", "textOutput", "speechToText"]); + expect(pairs(workflow)).toEqual(["a>d", "d>c"]); + expect(errors).toEqual([ + 'Unknown node kind "videoInput"', + 'Duplicate node id "c"', + "a -> c: Cannot connect audio output to text input", + "a -> b: Unknown node", + "a -> d: Speech to text already has an input", + ]); + }); + + it("reports when nothing usable remains", () => { + expect(validatePlan(null).errors).toEqual(["Reply is not an object with a nodes array"]); + expect(validatePlan({ nodes: [{ id: "x", kind: "nope" }] }).errors).toEqual(['Unknown node kind "nope"', "No valid nodes"]); + expect(validatePlan({ nodes: [{ id: "a", kind: "textInput" }, { id: "b", kind: "textOutput" }], edges: [] }).errors).toEqual([ + "a is not on a path from an input to an output", + "b is not on a path from an input to an output", + "No valid nodes", + ]); + }); + + it("drops nodes off the input-to-output path and deployment-specific config", () => { + // Shape of a real small-model reply: a chain with no sink plus stray nodes. + const { workflow, errors } = validatePlan({ + nodes: [ + { id: "t", kind: "textInput", config: { text: "Hi" } }, + { id: "l", kind: "llm" }, + { id: "v", kind: "textToSpeech", config: { voice: "en-US-ElizabethVoice" } }, + { id: "s", kind: "speechToText" }, + { id: "o", kind: "textOutput" }, + { id: "p", kind: "audioOutput" }, + ], + edges: [ + { source: "t", target: "l" }, + { source: "l", target: "v" }, + { source: "v", target: "p" }, + ], + }); + expect(kinds(workflow)).toEqual(["textInput", "llm", "textToSpeech", "audioOutput"]); + expect(workflow.nodes[2].config).toEqual({}); + expect(errors).toEqual(["s is not on a path from an input to an output", "o is not on a path from an input to an output"]); + }); +}); + +describe("layout", () => { + it("places nodes by longest path depth and stacks shared columns", () => { + const wf: Workflow = { + nodes: ["t", "l", "o", "s", "a"].map((id) => ({ + id, + kind: "textInput", + position: { x: 9, y: 9 }, + config: {}, + })), + edges: [ + { id: "1", source: "l", target: "o" }, + { id: "2", source: "t", target: "l" }, + { id: "3", source: "t", target: "s" }, + { id: "4", source: "s", target: "a" }, + ], + }; + expect(Object.fromEntries(layout(wf).nodes.map((n) => [n.id, n.position]))).toEqual({ + t: { x: 0, y: 0 }, + l: { x: 280, y: 0 }, + o: { x: 560, y: 0 }, + s: { x: 280, y: 320 }, + a: { x: 560, y: 320 }, + }); + }); +}); + +describe("planFromKeywords", () => { + it("builds a voice pipeline for voice requests", () => { + const wf = planFromKeywords("A voice assistant that answers banking questions and talks back"); + expect(kinds(wf)).toEqual(["audioInput", "speechToText", "llm", "textToSpeech", "audioOutput"]); + expect(pairs(wf)).toEqual([ + "audioInput-1>speechToText-2", + "speechToText-2>llm-3", + "llm-3>textToSpeech-4", + "textToSpeech-4>audioOutput-5", + ]); + expect(wf.nodes.map((n) => n.position.x)).toEqual([0, 280, 560, 840, 1120]); + expect(wf.nodes[2].config.systemPrompt).toBe( + "You are an assistant for this use case: A voice assistant that answers banking questions and talks back\nReplies are spoken aloud, so answer in one to three short sentences with no markdown.", + ); + }); + + it("transcribes", () => { + expect(kinds(planFromKeywords("Transcribe my meeting recordings"))).toEqual(["audioInput", "speechToText", "textOutput"]); + }); + + it("reads text aloud", () => { + expect(kinds(planFromKeywords("Read aloud a paragraph I paste"))).toEqual(["textInput", "textToSpeech", "audioOutput"]); + expect(kinds(planFromKeywords("speak text for me"))).toEqual(["textInput", "textToSpeech", "audioOutput"]); + }); + + it("defaults to a text chat", () => { + const wf = planFromKeywords("Summarize support tickets"); + expect(kinds(wf)).toEqual(["textInput", "llm", "textOutput"]); + expect(wf.nodes[0].config).toEqual({ text: "Hello" }); + }); +}); + +describe("intentError", () => { + const plan = (k: string[]) => validatePlan({ nodes: k.map((kind, i) => ({ id: `n${i}`, kind })), edges: k.slice(1).map((_, i) => ({ source: `n${i}`, target: `n${i + 1}` })) }).workflow; + + it("requires the modality the description names", () => { + expect(intentError("a voice bot that talks back", plan(["textInput", "textOutput"]))).toBe("The request needs audioInput and audioOutput"); + expect(intentError("transcribe calls", plan(["audioInput", "speechToText", "textOutput"]))).toBeNull(); + expect(intentError("read my notes aloud", plan(["textInput", "llm", "textOutput"]))).toBe("The request needs audioOutput"); + }); + + it("accepts anything for descriptions without a keyword", () => { + expect(intentError("summarize tickets", plan(["audioInput", "speechToText", "textOutput"]))).toBeNull(); + }); +}); + +describe("generateWorkflow", () => { + const good = '{"nodes":[{"id":"i","kind":"textInput","config":{"text":"q"}},{"id":"o","kind":"textOutput"}],"edges":[{"source":"i","target":"o"}]}'; + + it("uses the model reply when valid", async () => { + const complete = vi.fn().mockResolvedValue("```json\n" + good + "\n```"); + const result = await generateWorkflow("echo text", "bank-llm", complete); + expect(result.source).toEqual({ kind: "model", model: "bank-llm" }); + expect(kinds(result.workflow)).toEqual(["textInput", "textOutput"]); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it("retries once with the validation errors", async () => { + const complete = vi.fn().mockResolvedValueOnce('{"nodes":[{"id":"x","kind":"video"}]}').mockResolvedValueOnce(good); + const result = await generateWorkflow("echo text", "bank-llm", complete); + expect(result.source).toEqual({ kind: "model", model: "bank-llm" }); + expect(complete).toHaveBeenCalledTimes(2); + expect(complete.mock.calls[1][1]).toBe( + 'echo text\n\nYour previous reply was invalid: Unknown node kind "video"; No valid nodes. Reply with only the JSON object.', + ); + }); + + it("rejects a valid plan that ignores the requested modality", async () => { + const complete = vi.fn().mockResolvedValue(good); + const result = await generateWorkflow("a voice assistant", "bank-llm", complete); + expect(result.source).toEqual({ kind: "fallback", reason: "The request needs audioInput and audioOutput" }); + expect(complete.mock.calls[1][1]).toBe( + "a voice assistant\n\nYour previous reply was invalid: The request needs audioInput and audioOutput. Reply with only the JSON object.", + ); + expect(kinds(result.workflow)).toEqual(["audioInput", "speechToText", "llm", "textToSpeech", "audioOutput"]); + }); + + it("falls back to the keyword planner after two bad replies", async () => { + const complete = vi.fn().mockResolvedValue("I cannot help with that"); + const result = await generateWorkflow("transcribe calls", "bank-llm", complete); + expect(result.source).toEqual({ kind: "fallback", reason: "Reply is not an object with a nodes array" }); + expect(kinds(result.workflow)).toEqual(["audioInput", "speechToText", "textOutput"]); + expect(complete).toHaveBeenCalledTimes(2); + }); + + it("falls back when the model call fails or no model exists", async () => { + const failed = await generateWorkflow("chat", "bank-llm", vi.fn().mockRejectedValue(new Error("API Error: 502"))); + expect(failed.source).toEqual({ kind: "fallback", reason: "API Error: 502" }); + const none = await generateWorkflow("chat", undefined, vi.fn()); + expect(none.source).toEqual({ kind: "fallback", reason: "No inference deployment" }); + }); +}); diff --git a/apps/dashboard/src/lib/workflow/generate.ts b/apps/dashboard/src/lib/workflow/generate.ts new file mode 100644 index 00000000..606a4933 --- /dev/null +++ b/apps/dashboard/src/lib/workflow/generate.ts @@ -0,0 +1,235 @@ +import { connectionError } from "./engine"; +import { NODE_KINDS } from "./nodeKinds"; +import type { ConfigField } from "./nodeKinds"; +import type { NodeKind, Workflow, WorkflowNode } from "./types"; + +const COLUMN_WIDTH = 280; +const ROW_HEIGHT = 320; + +// Only free text the user would write themselves; models and voice ids depend on what is deployed. +function plannable(field: ConfigField): boolean { + return field.type !== "model" && !field.deploymentSpecific; +} + +export function plannerPrompt(): string { + const kinds = (Object.keys(NODE_KINDS) as NodeKind[]).map((kind) => { + const spec = NODE_KINDS[kind]; + const config = spec.fields.filter(plannable).map((f) => `${f.key} (${f.label.toLowerCase()})`); + return `- ${kind}: ${spec.description}. Input: ${spec.input ?? "none"}. Output: ${spec.output ?? "none"}. Config: ${config.join(", ") || "none"}.`; + }); + return [ + "You design pipelines for a node-based workflow editor. The only node kinds are:", + ...kinds, + "Rules: an edge connects a node's output to another node's input of the same type. A node has at most one incoming edge. No cycles. Every node must sit on a path from an input node (no input) to an output node (no output). Use audioInput when the user speaks and audioOutput when the app talks back. Only use the config keys listed.", + 'Reply with only one JSON object, no prose and no code fences, shaped like {"nodes":[{"id":"n1","kind":"textInput","config":{"text":"Hi"}}],"edges":[{"source":"n1","target":"n2"}]}', + ].join("\n"); +} + +// Models wrap JSON in prose or code fences, so take the first balanced object instead of the whole reply. +export function extractJson(text: string): unknown { + const body = text.replace(/```(?:json)?/gi, ""); + const start = body.indexOf("{"); + if (start === -1) return null; + let depth = 0; + let inString = false; + for (let i = start; i < body.length; i++) { + const ch = body[i]; + if (inString) { + if (ch === "\\") i++; + else if (ch === '"') inString = false; + } else if (ch === '"') inString = true; + else if (ch === "{") depth++; + else if (ch === "}" && --depth === 0) { + try { + return JSON.parse(body.slice(start, i + 1)); + } catch { + return null; + } + } + } + return null; +} + +function isRecord(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v); +} + +// Keeps every node and edge the registry and connectionError accept, and reports the rest. +export function validatePlan(raw: unknown): { workflow: Workflow; errors: string[] } { + const errors: string[] = []; + const workflow: Workflow = { nodes: [], edges: [] }; + if (!isRecord(raw) || !Array.isArray(raw.nodes)) return { workflow, errors: ["Reply is not an object with a nodes array"] }; + + for (const n of raw.nodes) { + if (!isRecord(n) || typeof n.id !== "string" || !n.id) { + errors.push("Node without an id"); + continue; + } + if (typeof n.kind !== "string" || !(n.kind in NODE_KINDS)) { + errors.push(`Unknown node kind "${String(n.kind)}"`); + continue; + } + if (workflow.nodes.some((x) => x.id === n.id)) { + errors.push(`Duplicate node id "${n.id}"`); + continue; + } + const kind = n.kind as NodeKind; + const config: Record = {}; + for (const field of NODE_KINDS[kind].fields) { + const value = isRecord(n.config) ? n.config[field.key] : undefined; + if (plannable(field) && typeof value === "string") config[field.key] = value; + } + workflow.nodes.push({ id: n.id, kind, position: { x: 0, y: 0 }, config }); + } + + for (const e of Array.isArray(raw.edges) ? raw.edges : []) { + if (!isRecord(e) || typeof e.source !== "string" || typeof e.target !== "string") { + errors.push("Edge without source and target"); + continue; + } + const error = connectionError(workflow, e.source, e.target); + if (error) errors.push(`${e.source} -> ${e.target}: ${error}`); + else workflow.edges.push({ id: `e-${e.source}-${e.target}`, source: e.source, target: e.target }); + } + + const complete = onCompletePath(workflow); + for (const n of workflow.nodes) if (!complete.has(n.id)) errors.push(`${n.id} is not on a path from an input to an output`); + const kept: Workflow = { + nodes: workflow.nodes.filter((n) => complete.has(n.id)), + edges: workflow.edges.filter((e) => complete.has(e.source) && complete.has(e.target)), + }; + if (kept.nodes.length === 0) errors.push("No valid nodes"); + return { workflow: layout(kept), errors }; +} + +// Nodes reachable from a source kind that can also reach a sink kind; anything else would never run usefully. +function onCompletePath(workflow: Workflow): Set { + const walk = (start: (n: WorkflowNode) => boolean, next: (id: string) => string[]) => { + const seen = new Set(); + const stack = workflow.nodes.filter(start).map((n) => n.id); + while (stack.length) { + const id = stack.pop()!; + if (seen.has(id)) continue; + seen.add(id); + stack.push(...next(id)); + } + return seen; + }; + const fromSource = walk( + (n) => NODE_KINDS[n.kind].input === null, + (id) => workflow.edges.filter((e) => e.source === id).map((e) => e.target), + ); + const toSink = walk( + (n) => NODE_KINDS[n.kind].output === null, + (id) => workflow.edges.filter((e) => e.target === id).map((e) => e.source), + ); + return new Set([...fromSource].filter((id) => toSink.has(id))); +} + +// Column = longest path from a source, so every edge points rightwards. +export function layout(workflow: Workflow): Workflow { + const column = new Map(workflow.nodes.map((n) => [n.id, 0])); + for (let pass = 0; pass < workflow.nodes.length; pass++) { + for (const e of workflow.edges) { + const next = (column.get(e.source) ?? 0) + 1; + if (next > (column.get(e.target) ?? 0)) column.set(e.target, next); + } + } + const rows = new Map(); + const nodes = workflow.nodes.map((n) => { + const col = column.get(n.id)!; + const row = rows.get(col) ?? 0; + rows.set(col, row + 1); + return { ...n, position: { x: col * COLUMN_WIDTH, y: row * ROW_HEIGHT } }; + }); + return { ...workflow, nodes }; +} + +type Step = { kind: NodeKind; config?: Record }; + +function chain(steps: Step[]): Workflow { + const nodes: WorkflowNode[] = steps.map((s, i) => ({ + id: `${s.kind}-${i + 1}`, + kind: s.kind, + position: { x: 0, y: 0 }, + config: s.config ?? {}, + })); + const edges = nodes.slice(1).map((n, i) => ({ id: `e-${nodes[i].id}-${n.id}`, source: nodes[i].id, target: n.id })); + return layout({ nodes, edges }); +} + +const SPOKEN = "\nReplies are spoken aloud, so answer in one to three short sentences with no markdown."; + +// Checked in order; the first match wins. No match means a plain text chat. +const RULES: Array<{ test: RegExp; steps: (systemPrompt: string) => Step[] }> = [ + { + test: /transcri|dictat|caption/, + steps: () => [{ kind: "audioInput" }, { kind: "speechToText" }, { kind: "textOutput" }], + }, + { + test: /read\b.*\baloud|out loud|speak (the |this )?text|text to speech|narrat/, + steps: () => [{ kind: "textInput", config: { text: "Hello from your workflow." } }, { kind: "textToSpeech" }, { kind: "audioOutput" }], + }, + { + test: /voice|speak|call|talk|phone/, + steps: (systemPrompt) => [ + { kind: "audioInput" }, + { kind: "speechToText" }, + { kind: "llm", config: { systemPrompt: systemPrompt + SPOKEN } }, + { kind: "textToSpeech" }, + { kind: "audioOutput" }, + ], + }, +]; + +function matchRule(description: string) { + const text = description.toLowerCase(); + return RULES.find((r) => r.test.test(text)); +} + +// Deterministic last resort when no model reply is usable. +export function planFromKeywords(description: string): Workflow { + const systemPrompt = `You are an assistant for this use case: ${description.trim()}`; + const rule = matchRule(description); + if (rule) return chain(rule.steps(systemPrompt)); + return chain([{ kind: "textInput", config: { text: "Hello" } }, { kind: "llm", config: { systemPrompt } }, { kind: "textOutput" }]); +} + +// When the description clearly names a modality, the model's plan must start and end with the same kinds. +export function intentError(description: string, workflow: Workflow): string | null { + const rule = matchRule(description); + if (!rule) return null; + const steps = rule.steps(""); + const missing = [steps[0].kind, steps[steps.length - 1].kind].filter((k) => !workflow.nodes.some((n) => n.kind === k)); + return missing.length ? `The request needs ${missing.join(" and ")}` : null; +} + +export type PlanSource = { kind: "model"; model: string } | { kind: "fallback"; reason: string }; + +export async function generateWorkflow( + description: string, + model: string | undefined, + complete: (system: string, user: string) => Promise, +): Promise<{ workflow: Workflow; source: PlanSource }> { + if (!model) return { workflow: planFromKeywords(description), source: { kind: "fallback", reason: "No inference deployment" } }; + const system = plannerPrompt(); + let reason: string; + try { + const attempt = async (prompt: string) => { + const plan = validatePlan(extractJson(await complete(system, prompt))); + const found = plan.workflow.nodes.length > 0; + const intent = found ? intentError(description, plan.workflow) : null; + return { workflow: plan.workflow, errors: intent ? [...plan.errors, intent] : plan.errors, ok: found && !intent }; + }; + const first = await attempt(description); + if (first.ok) return { workflow: first.workflow, source: { kind: "model", model } }; + const second = await attempt( + `${description}\n\nYour previous reply was invalid: ${first.errors.join("; ")}. Reply with only the JSON object.`, + ); + if (second.ok) return { workflow: second.workflow, source: { kind: "model", model } }; + reason = second.errors.join("; "); + } catch (err) { + reason = err instanceof Error ? err.message : String(err); + } + return { workflow: planFromKeywords(description), source: { kind: "fallback", reason } }; +} diff --git a/apps/dashboard/src/lib/workflow/nodeKinds.ts b/apps/dashboard/src/lib/workflow/nodeKinds.ts index a040f374..3b2da0a0 100644 --- a/apps/dashboard/src/lib/workflow/nodeKinds.ts +++ b/apps/dashboard/src/lib/workflow/nodeKinds.ts @@ -1,3 +1,4 @@ +import type { Deployment } from "@/hooks/useReadyDeployments"; import { parseThinking } from "@/lib/parseThinking"; import type { NodeKind, Payload, PortType, RunContext, WorkflowNode } from "./types"; @@ -10,7 +11,14 @@ export type ConfigField = // STT and TTS share the audio_generation model_type, so the name is the only hint for auto-selection. prefer?: RegExp; } - | { key: string; label: string; type: "text" | "textarea"; placeholder?: string }; + | { + key: string; + label: string; + type: "text" | "textarea"; + placeholder?: string; + // Values that only make sense for one deployment (voice ids) are left to the user, not the planner. + deploymentSpecific?: boolean; + }; export interface NodeKindSpec { label: string; @@ -50,11 +58,21 @@ function requireModel(node: WorkflowNode): string { return model; } +const DEFAULT_VOICE = "af_heart"; + function audioFileName(blob: Blob): string { const subtype = blob.type.split("/")[1]?.split(";")[0]; return `audio.${subtype === "mpeg" ? "mp3" : subtype || "webm"}`; } +// Deployments a model field can use. STT and TTS share a model_type, so a field with a name hint only takes +// matching names: binding a TTS model to an STT node would fail at run time. +export function matchingDeployments(field: Extract, deployments: Deployment[]): Deployment[] { + return deployments.filter( + (d) => d.model_type === field.modelType && d.modelName !== "-" && (!field.prefer || field.prefer.test(d.modelName)), + ); +} + export const NODE_KINDS: Record = { audioInput: { label: "Audio input", @@ -145,14 +163,15 @@ export const NODE_KINDS: Record = { modelType: "audio_generation", prefer: /tts|speech|kokoro|voice|bark|piper|orpheus/i, }, - { key: "voice", label: "Voice", type: "text", placeholder: "default" }, + { key: "voice", label: "Voice", type: "text", placeholder: DEFAULT_VOICE, deploymentSpecific: true }, ], run: async (input, node, ctx) => { const text = expectText(input); - const voice = node.config.voice?.trim(); + // The speech API rejects a missing voice; af_heart is Kokoro's default and Piper ignores the name. + const voice = node.config.voice?.trim() || DEFAULT_VOICE; const res = await inference(ctx, "/v1/audio/speech", { headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model: requireModel(node), input: text, ...(voice ? { voice } : {}) }), + body: JSON.stringify({ model: requireModel(node), input: text, voice }), }); return { type: "audio", blob: await res.blob() }; }, diff --git a/apps/dashboard/src/lib/workflow/templates.ts b/apps/dashboard/src/lib/workflow/templates.ts index 66d60dec..cb3f77dc 100644 --- a/apps/dashboard/src/lib/workflow/templates.ts +++ b/apps/dashboard/src/lib/workflow/templates.ts @@ -1,5 +1,5 @@ import type { Deployment } from "@/hooks/useReadyDeployments"; -import { NODE_KINDS } from "./nodeKinds"; +import { NODE_KINDS, matchingDeployments } from "./nodeKinds"; import type { NodeKind, Workflow } from "./types"; export const BANKING_SYSTEM_PROMPT = `You are the voice assistant for Inferia Bank, a fictional bank used for a product demo. Your answers are spoken aloud, so reply in one to three short sentences with no markdown or lists. @@ -47,7 +47,7 @@ export const TEMPLATES: Record Workflow }> }, }; -// Fill empty model fields with the first ready deployment of the matching type. +// Fill empty model fields with the first matching ready deployment. export function autofillModels(workflow: Workflow, deployments: Deployment[]): Workflow { return { ...workflow, @@ -55,8 +55,7 @@ export function autofillModels(workflow: Workflow, deployments: Deployment[]): W const config = { ...node.config }; for (const field of NODE_KINDS[node.kind].fields) { if (field.type !== "model" || config[field.key]) continue; - const candidates = deployments.filter((d) => d.model_type === field.modelType && d.modelName !== "-"); - const match = candidates.find((d) => field.prefer?.test(d.modelName)) ?? candidates[0]; + const match = matchingDeployments(field, deployments)[0]; if (match) config[field.key] = match.modelName; } return { ...node, config }; From 5c9d9a1bd8ee02fa1c3f7b86eb7e1498dc507fae Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Fri, 23 Oct 2026 23:34:10 +0530 Subject: [PATCH 2/3] Derive node deployment readiness --- .../lib/workflow/__tests__/readiness.test.ts | 102 ++++++++++++++ apps/dashboard/src/lib/workflow/readiness.ts | 130 ++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts create mode 100644 apps/dashboard/src/lib/workflow/readiness.ts diff --git a/apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts b/apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts new file mode 100644 index 00000000..e8cb41d5 --- /dev/null +++ b/apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import type { Deployment } from "@/hooks/useReadyDeployments"; +import { MODEL_CATALOG, deployRequests, nodeReadiness, runBlocker } from "../readiness"; +import type { Workflow } from "../types"; + +function dep(modelName: string, model_type: string): Deployment { + return { id: modelName, name: modelName, modelName, model_type, endpointUrl: "", status: "READY" }; +} + +const stt = { kind: "speechToText" as const, config: {} }; +const whisperCatalog = MODEL_CATALOG.speechToText!; + +describe("nodeReadiness", () => { + it("treats nodes without a model field, or with a model set, as ready", () => { + expect(nodeReadiness({ kind: "textInput", config: {} }, [], undefined)).toEqual({ state: "ready" }); + expect(nodeReadiness({ kind: "llm", config: { model: "bank-llm" } }, [], undefined)).toEqual({ state: "ready" }); + }); + + it("asks to select a model when a matching deployment exists", () => { + expect(nodeReadiness(stt, [dep("whisper-stt", "audio_generation")])).toEqual({ state: "selectModel" }); + expect(nodeReadiness({ kind: "llm", config: {} }, [dep("bank-llm", "inference")])).toEqual({ state: "selectModel" }); + }); + + it("needs deployment when only non-matching deployments exist", () => { + expect(nodeReadiness(stt, [dep("kokoro-tts", "audio_generation"), dep("bank-llm", "inference")])).toEqual({ + state: "needsDeployment", + recommended: whisperCatalog, + }); + expect(nodeReadiness({ kind: "textToSpeech", config: {} }, [])).toEqual({ + state: "needsDeployment", + recommended: MODEL_CATALOG.textToSpeech, + }); + }); + + it("follows the pending deploy through deploying, waiting and deployed", () => { + expect(nodeReadiness(stt, [], { status: "deploying", deploymentName: "whisper-tiny-stt" })).toEqual({ + state: "deploying", + recommended: whisperCatalog, + }); + expect(nodeReadiness(stt, [], { status: "waiting", reason: "no active compute pool" })).toEqual({ + state: "waiting", + recommended: whisperCatalog, + reason: "no active compute pool", + }); + expect( + nodeReadiness(stt, [dep("whisper-tiny-stt", "audio_generation")], { status: "deploying", deploymentName: "whisper-tiny-stt" }), + ).toEqual({ state: "deployed", model: "whisper-tiny-stt" }); + }); +}); + +describe("runBlocker", () => { + const wf = (configs: Array>): Workflow => ({ + nodes: [ + { id: "a", kind: "audioInput", position: { x: 0, y: 0 }, config: {} }, + { id: "s", kind: "speechToText", position: { x: 0, y: 0 }, config: configs[0] }, + { id: "l", kind: "llm", position: { x: 0, y: 0 }, config: configs[1] }, + ], + edges: [], + }); + + it("is null when every node is ready", () => { + expect(runBlocker(wf([{ model: "w" }, { model: "l" }]), [], {})).toBeNull(); + }); + + it("names the first blocked node and its reason", () => { + expect(runBlocker(wf([{}, { model: "l" }]), [], {})).toBe("Speech to text: Needs deployment"); + expect(runBlocker(wf([{}, { model: "l" }]), [], { s: { status: "waiting", reason: "no active compute pool" } })).toBe( + "Speech to text: Waiting for compute: no active compute pool", + ); + expect(runBlocker(wf([{ model: "w" }, {}]), [], { l: { status: "deploying", deploymentName: "qwen2-5-1-5b" } })).toBe( + "LLM: Deploying qwen2-5-1-5b", + ); + expect(runBlocker(wf([{ model: "w" }, {}]), [dep("bank-llm", "inference")], {})).toBe("LLM: Select a model"); + }); +}); + +describe("deployRequests", () => { + it("builds the preflight and pool deploy payloads", () => { + const req = deployRequests(MODEL_CATALOG.llm!, "pool-1", { orgId: "org-1", ownerId: "user-1" }); + expect(req.preflight).toEqual({ + model_id: "qwen2.5:1.5b", + engine: "ollama", + gpu_per_replica: 1, + pool_id: "pool-1", + model_type: "inference", + }); + expect(req.deploy).toMatchObject({ + model_name: "qwen2-5-1-5b", + model_version: "latest", + replicas: 1, + gpu_per_replica: 1, + workload_type: "inference", + pool_id: "pool-1", + engine: "ollama", + model_type: "inference", + inference_model: "qwen2.5:1.5b", + owner_id: "user-1", + org_id: "org-1", + }); + expect(deployRequests(whisperCatalog, "p", { orgId: "o" }).deploy.gpu_per_replica).toBe(0); + }); +}); diff --git a/apps/dashboard/src/lib/workflow/readiness.ts b/apps/dashboard/src/lib/workflow/readiness.ts new file mode 100644 index 00000000..16f36893 --- /dev/null +++ b/apps/dashboard/src/lib/workflow/readiness.ts @@ -0,0 +1,130 @@ +import type { Deployment } from "@/hooks/useReadyDeployments"; +import { NODE_KINDS, matchingDeployments } from "./nodeKinds"; +import type { NodeKind, Workflow, WorkflowNode } from "./types"; + +export interface CatalogModel { + modelId: string; + engine: string; + modelType: "inference" | "audio_generation"; + // Becomes the deployment's model_name, which is also the name the inference API routes on. + deploymentName: string; + job: Record; +} + +const speaches = (modelId: string) => ({ + model_id: modelId, + engine: "speaches", + image: "ghcr.io/speaches-ai/speaches:latest-cpu", + env: { PRELOAD_MODELS: JSON.stringify([modelId]) }, + expose: [{ port: 8000, type: "http", health_checks: [{ path: "/health", type: "http", method: "GET", expected_status: 200 }] }], + gpu: false, +}); + +export const MODEL_CATALOG: Partial> = { + speechToText: { + modelId: "Systran/faster-whisper-tiny.en", + engine: "speaches", + modelType: "audio_generation", + deploymentName: "whisper-tiny-stt", + job: speaches("Systran/faster-whisper-tiny.en"), + }, + textToSpeech: { + modelId: "speaches-ai/piper-en_US-amy-medium", + engine: "speaches", + modelType: "audio_generation", + deploymentName: "piper-amy-tts", + job: speaches("speaches-ai/piper-en_US-amy-medium"), + }, + llm: { + modelId: "qwen2.5:1.5b", + engine: "ollama", + modelType: "inference", + deploymentName: "qwen2-5-1-5b", + job: { + model_id: "qwen2.5:1.5b", + engine: "ollama", + image: "ollama/ollama:latest", + cmd: ["serve"], + expose: [{ port: 11434, type: "http" }], + gpu: true, + }, + }, +}; + +// Deploy attempt the canvas started for a node; lives in UI state, never in the saved workflow. +export type PendingDeploy = { status: "deploying"; deploymentName: string } | { status: "waiting"; reason: string }; + +export type Readiness = + | { state: "ready" } + | { state: "selectModel" } + | { state: "needsDeployment"; recommended: CatalogModel } + | { state: "deploying"; recommended: CatalogModel } + | { state: "waiting"; recommended: CatalogModel; reason: string } + // The node's own deployment came up; the canvas selects it and the node becomes ready. + | { state: "deployed"; model: string }; + +export function nodeReadiness( + node: Pick, + deployments: Deployment[], + pending?: PendingDeploy, +): Readiness { + const field = NODE_KINDS[node.kind].fields.find((f) => f.type === "model"); + if (!field || field.type !== "model" || node.config[field.key]) return { state: "ready" }; + const recommended = MODEL_CATALOG[node.kind]; + if (pending?.status === "deploying" && deployments.some((d) => d.modelName === pending.deploymentName)) { + return { state: "deployed", model: pending.deploymentName }; + } + if (matchingDeployments(field, deployments).length > 0 || !recommended) return { state: "selectModel" }; + if (pending?.status === "deploying") return { state: "deploying", recommended }; + if (pending?.status === "waiting") return { state: "waiting", recommended, reason: pending.reason }; + return { state: "needsDeployment", recommended }; +} + +export function readinessMessage(r: Readiness): string | null { + switch (r.state) { + case "ready": + case "deployed": + return null; + case "selectModel": + return "Select a model"; + case "needsDeployment": + return "Needs deployment"; + case "deploying": + return `Deploying ${r.recommended.deploymentName}`; + case "waiting": + return `Waiting for compute: ${r.reason}`; + } +} + +export function runBlocker( + workflow: Workflow, + deployments: Deployment[], + pending: Record, +): string | null { + for (const node of workflow.nodes) { + const message = readinessMessage(nodeReadiness(node, deployments, pending[node.id])); + if (message) return `${NODE_KINDS[node.kind].label}: ${message}`; + } + return null; +} + +export function deployRequests(entry: CatalogModel, poolId: string, ids: { orgId: string; ownerId?: string }) { + return { + preflight: { model_id: entry.modelId, engine: entry.engine, gpu_per_replica: 1, pool_id: poolId, model_type: entry.modelType }, + deploy: { + model_name: entry.deploymentName, + model_version: "latest", + replicas: 1, + gpu_per_replica: entry.job.gpu ? 1 : 0, + workload_type: "inference", + pool_id: poolId, + engine: entry.engine, + model_type: entry.modelType, + configuration: entry.job, + job_definition: entry.job, + inference_model: entry.modelId, + owner_id: ids.ownerId, + org_id: ids.orgId, + }, + }; +} From 22a7643250455d991df3a7173a0144772f0751aa Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Fri, 23 Oct 2026 23:34:11 +0530 Subject: [PATCH 3/3] Add describe and deploy UI --- .../components/workflow/DescribeWorkflow.tsx | 98 +++++++++++++++++++ .../components/workflow/WorkflowNodeCard.tsx | 41 +++++++- .../src/components/workflow/canvasContext.ts | 3 + .../src/components/workflow/useNodeDeploys.ts | 55 +++++++++++ .../src/hooks/useReadyDeployments.ts | 4 +- apps/dashboard/src/pages/Workflows.tsx | 55 +++++++++-- 6 files changed, 245 insertions(+), 11 deletions(-) create mode 100644 apps/dashboard/src/components/workflow/DescribeWorkflow.tsx create mode 100644 apps/dashboard/src/components/workflow/useNodeDeploys.ts diff --git a/apps/dashboard/src/components/workflow/DescribeWorkflow.tsx b/apps/dashboard/src/components/workflow/DescribeWorkflow.tsx new file mode 100644 index 00000000..82763343 --- /dev/null +++ b/apps/dashboard/src/components/workflow/DescribeWorkflow.tsx @@ -0,0 +1,98 @@ +import { useState } from "react"; +import { Loader2, Sparkles } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import type { Deployment } from "@/hooks/useReadyDeployments"; +import { INFERENCE_URL } from "@/lib/api"; +import { getToken } from "@/lib/tokenStore"; +import { generateWorkflow, type PlanSource } from "@/lib/workflow/generate"; +import { NODE_KINDS } from "@/lib/workflow/nodeKinds"; +import type { Workflow } from "@/lib/workflow/types"; +import { cn } from "@/lib/utils"; + +const inputClass = + "w-full rounded-md border bg-background px-2 py-1.5 text-xs outline-none focus:ring-1 focus:ring-ember-500"; + +// The planner call goes through the LLM node itself so it shares the sandbox auth and reasoning stripping. +function complete(model: string) { + return async (system: string, user: string) => { + const out = await NODE_KINDS.llm.run( + { type: "text", text: user }, + { id: "planner", kind: "llm", position: { x: 0, y: 0 }, config: { model, systemPrompt: system } }, + { baseUrl: INFERENCE_URL, token: getToken(), fetch: window.fetch.bind(window), audioInputs: {} }, + ); + return out.type === "text" ? out.text : ""; + }; +} + +export function DescribeWorkflow({ + deployments, + onGenerated, +}: { + deployments: Deployment[]; + onGenerated: (workflow: Workflow) => void; +}) { + const models = deployments.filter((d) => d.model_type === "inference" && d.modelName !== "-").map((d) => d.modelName); + const [description, setDescription] = useState(""); + const [picked, setPicked] = useState(""); + const [busy, setBusy] = useState(false); + const [source, setSource] = useState(null); + const model = models.includes(picked) ? picked : models[0]; + + const submit = async () => { + if (!description.trim() || busy) return; + setBusy(true); + try { + const result = await generateWorkflow(description, model, model ? complete(model) : async () => ""); + setSource(result.source); + onGenerated(result.workflow); + } finally { + setBusy(false); + } + }; + + return ( +
+

Describe a workflow

+