@@ -98,6 +113,7 @@ function ReadinessBanner({ nodeId, readiness }: { nodeId: string; readiness: Rea
{readiness.state === "needsDeployment" && "Needs deployment"}
{readiness.state === "deploying" && "Deploying..."}
{readiness.state === "waiting" && "Waiting for compute"}
+ {readiness.state === "failed" && "Deploy failed"}
{readiness.state === "deploying" ? (
@@ -108,14 +124,14 @@ function ReadinessBanner({ nodeId, readiness }: { nodeId: string; readiness: Rea
className="nodrag flex items-center gap-1 rounded-md bg-ember-600 px-2 py-0.5 font-medium text-white hover:bg-ember-700"
>
- {readiness.state === "waiting" ? "Retry" : "Deploy"}
+ {retry ? "Retry" : "Deploy"}
)}
Recommended {recommended.modelId} on {recommended.engine}
- {readiness.state === "waiting" &&
{readiness.reason}
}
+ {retry &&
{readiness.reason}
}
);
}
diff --git a/apps/dashboard/src/components/workflow/useNodeDeploys.ts b/apps/dashboard/src/components/workflow/useNodeDeploys.ts
index 7448ac0a..9bd4d730 100644
--- a/apps/dashboard/src/components/workflow/useNodeDeploys.ts
+++ b/apps/dashboard/src/components/workflow/useNodeDeploys.ts
@@ -1,9 +1,12 @@
-import { useCallback, useState } from "react";
+import { useCallback, useEffect, useState } from "react";
import { useAuth } from "@/context/AuthContext";
import { computeApi } from "@/lib/api";
import { deployRequests, type CatalogModel, type PendingDeploy } from "@/lib/workflow/readiness";
import { listPools } from "@/services/poolService";
+// A deploy that ends in one of these never comes up, so the node stops waiting on it.
+const DEAD_STATES = new Set(["FAILED", "TERMINATED", "STOPPED"]);
+
interface PreflightResponse {
ready: boolean;
checks: Array<{ passed: boolean; message?: string | null; check: string }>;
@@ -43,12 +46,34 @@ export function useNodeDeploys() {
const failed = data.checks.filter((c) => !c.passed).map((c) => c.message || c.check);
return set(nodeId, { status: "waiting", reason: failed.join("; ") || "preflight failed" });
}
- await computeApi.post("/deployment/deploy", req.deploy);
+ const { data: created } = await computeApi.post<{ deployment_id?: string }>("/deployment/deploy", req.deploy);
+ set(nodeId, { status: "deploying", deploymentName: entry.deploymentName, deploymentId: created?.deployment_id });
} catch (err) {
set(nodeId, { status: "waiting", reason: errorMessage(err) });
}
};
+ // The ready-deployments list never shows a failed deploy, so poll each one we started.
+ useEffect(() => {
+ const watched = Object.entries(pending).flatMap(([nodeId, p]) =>
+ p.status === "deploying" && p.deploymentId ? [[nodeId, p.deploymentId] as const] : [],
+ );
+ if (watched.length === 0) return;
+ const timer = setInterval(() => {
+ for (const [nodeId, id] of watched) {
+ computeApi
+ .get<{ state?: string; error_message?: string | null }>(`/deployment/status/${id}`)
+ .then(({ data }) => {
+ if (data.state && DEAD_STATES.has(data.state)) {
+ set(nodeId, { status: "failed", reason: data.error_message || `deployment ${data.state.toLowerCase()}` });
+ }
+ })
+ .catch(() => {});
+ }
+ }, 5000);
+ return () => clearInterval(timer);
+ }, [pending, set]);
+
const clear = useCallback((nodeId: string) => set(nodeId, null), [set]);
const reset = useCallback(() => setPending({}), []);
return { pending, deploy, clear, reset };
diff --git a/apps/dashboard/src/lib/workflow/__tests__/multimodal.test.ts b/apps/dashboard/src/lib/workflow/__tests__/multimodal.test.ts
index bdefde37..3017b62a 100644
--- a/apps/dashboard/src/lib/workflow/__tests__/multimodal.test.ts
+++ b/apps/dashboard/src/lib/workflow/__tests__/multimodal.test.ts
@@ -130,7 +130,8 @@ describe("vision model choice", () => {
it("recommends the catalog models when nothing matches", () => {
expect(nodeReadiness({ kind: "vision", config: {} }, [dep("qwen2-5-1-5b", "inference")])).toEqual({ state: "needsDeployment", recommended: MODEL_CATALOG.vision });
expect(MODEL_CATALOG.vision).toMatchObject({ modelId: "moondream", engine: "ollama", modelType: "inference", deploymentName: "moondream-vision" });
- expect(MODEL_CATALOG.imageGeneration).toMatchObject({ modelId: "stabilityai/sd-turbo", engine: "generic", modelType: "image_generation", deploymentName: "sd-turbo-image" });
+ expect(MODEL_CATALOG.imageGeneration).toMatchObject({ modelId: "stabilityai/sd-turbo", engine: "inferia-diffusion", modelType: "image_generation", deploymentName: "sd-turbo-image" });
+ expect(MODEL_CATALOG.imageGeneration!.job).toMatchObject({ image: "docker.io/inferiaai/inferiadiffusion:latest", gpu: true });
});
});
diff --git a/apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts b/apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts
index e8cb41d5..d5eb3a8e 100644
--- a/apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts
+++ b/apps/dashboard/src/lib/workflow/__tests__/readiness.test.ts
@@ -8,7 +8,8 @@ function dep(modelName: string, model_type: string): Deployment {
}
const stt = { kind: "speechToText" as const, config: {} };
-const whisperCatalog = MODEL_CATALOG.speechToText!;
+const llm = { kind: "llm" as const, config: {} };
+const qwenCatalog = MODEL_CATALOG.llm!;
describe("nodeReadiness", () => {
it("treats nodes without a model field, or with a model set, as ready", () => {
@@ -22,29 +23,47 @@ describe("nodeReadiness", () => {
});
it("needs deployment when only non-matching deployments exist", () => {
- expect(nodeReadiness(stt, [dep("kokoro-tts", "audio_generation"), dep("bank-llm", "inference")])).toEqual({
+ expect(nodeReadiness(llm, [dep("whisper-stt", "audio_generation")])).toEqual({
state: "needsDeployment",
- recommended: whisperCatalog,
+ recommended: qwenCatalog,
});
- expect(nodeReadiness({ kind: "textToSpeech", config: {} }, [])).toEqual({
- state: "needsDeployment",
- recommended: MODEL_CATALOG.textToSpeech,
+ });
+
+ it("asks for an external deployment for speech, which no pool engine can run", () => {
+ expect(nodeReadiness(stt, [dep("kokoro-tts", "audio_generation"), dep("bank-llm", "inference")])).toEqual({
+ state: "needsExternal",
});
+ expect(nodeReadiness({ kind: "textToSpeech", config: {} }, [])).toEqual({ state: "needsExternal" });
});
- it("follows the pending deploy through deploying, waiting and deployed", () => {
- expect(nodeReadiness(stt, [], { status: "deploying", deploymentName: "whisper-tiny-stt" })).toEqual({
+ it("follows the pending deploy through deploying, waiting, failed and deployed", () => {
+ expect(nodeReadiness(llm, [], { status: "deploying", deploymentName: "qwen2-5-1-5b" })).toEqual({
state: "deploying",
- recommended: whisperCatalog,
+ recommended: qwenCatalog,
});
- expect(nodeReadiness(stt, [], { status: "waiting", reason: "no active compute pool" })).toEqual({
+ expect(nodeReadiness(llm, [], { status: "waiting", reason: "no active compute pool" })).toEqual({
state: "waiting",
- recommended: whisperCatalog,
+ recommended: qwenCatalog,
reason: "no active compute pool",
});
+ expect(nodeReadiness(llm, [], { status: "failed", reason: "image pull failed" })).toEqual({
+ state: "failed",
+ recommended: qwenCatalog,
+ reason: "image pull failed",
+ });
expect(
- nodeReadiness(stt, [dep("whisper-tiny-stt", "audio_generation")], { status: "deploying", deploymentName: "whisper-tiny-stt" }),
- ).toEqual({ state: "deployed", model: "whisper-tiny-stt" });
+ nodeReadiness(llm, [dep("qwen2-5-1-5b", "inference")], { status: "deploying", deploymentName: "qwen2-5-1-5b" }),
+ ).toEqual({ state: "deployed", model: "qwen2-5-1-5b" });
+ });
+});
+
+describe("MODEL_CATALOG", () => {
+ it("only recommends engines that worker pools can run", () => {
+ const poolEngines = ["vllm", "vllm-omni", "ollama", "infinity", "triton", "inferia-diffusion"];
+ for (const entry of Object.values(MODEL_CATALOG)) {
+ expect(poolEngines).toContain(entry!.engine);
+ expect(entry!.job.engine).toBe(entry!.engine);
+ }
});
});
@@ -63,9 +82,12 @@ describe("runBlocker", () => {
});
it("names the first blocked node and its reason", () => {
- expect(runBlocker(wf([{}, { model: "l" }]), [], {})).toBe("Speech to text: Needs deployment");
- expect(runBlocker(wf([{}, { model: "l" }]), [], { s: { status: "waiting", reason: "no active compute pool" } })).toBe(
- "Speech to text: Waiting for compute: no active compute pool",
+ expect(runBlocker(wf([{}, { model: "l" }]), [], {})).toBe("Speech to text: Needs an external deployment");
+ expect(runBlocker(wf([{ model: "w" }, {}]), [], { l: { status: "waiting", reason: "no active compute pool" } })).toBe(
+ "LLM: Waiting for compute: no active compute pool",
+ );
+ expect(runBlocker(wf([{ model: "w" }, {}]), [], { l: { status: "failed", reason: "image pull failed" } })).toBe(
+ "LLM: Deploy failed: image pull failed",
);
expect(runBlocker(wf([{ model: "w" }, {}]), [], { l: { status: "deploying", deploymentName: "qwen2-5-1-5b" } })).toBe(
"LLM: Deploying qwen2-5-1-5b",
@@ -97,6 +119,5 @@ describe("deployRequests", () => {
owner_id: "user-1",
org_id: "org-1",
});
- expect(deployRequests(whisperCatalog, "p", { orgId: "o" }).deploy.gpu_per_replica).toBe(0);
});
});
diff --git a/apps/dashboard/src/lib/workflow/readiness.ts b/apps/dashboard/src/lib/workflow/readiness.ts
index 8e335d9a..db603699 100644
--- a/apps/dashboard/src/lib/workflow/readiness.ts
+++ b/apps/dashboard/src/lib/workflow/readiness.ts
@@ -1,6 +1,7 @@
import type { Deployment } from "@/hooks/useReadyDeployments";
import { NODE_KINDS, matchingDeployments } from "./nodeKinds";
import type { NodeKind, Workflow, WorkflowNode } from "./types";
+import { buildDiffusionSpec } from "@/pages/newDeploymentSpec";
export interface CatalogModel {
modelId: string;
@@ -20,30 +21,9 @@ const ollama = (modelId: string) => ({
gpu: true,
});
-const speaches = (modelId: string) => ({
- model_id: modelId,
- engine: "speaches",
- image: "ghcr.io/speaches-ai/speaches:latest-cpu",
- env: { PRELOAD_MODELS: JSON.stringify([modelId]) },
- expose: [{ port: 8000, type: "http", health_checks: [{ path: "/health", type: "http", method: "GET", expected_status: 200 }] }],
- gpu: false,
-});
-
+// Only engines that worker pools have a recipe for. Speech has none yet, so
+// speech nodes ask for an external deployment instead of offering Deploy.
export const MODEL_CATALOG: Partial