From f64889fa764899fd1757ec9be9f8df8683fc77f0 Mon Sep 17 00:00:00 2001 From: Pento Date: Sun, 26 Apr 2026 01:36:41 +0200 Subject: [PATCH 1/8] Add ComfyUI custom workflow and UNet/Flux generation modes --- .../components/settings/tabs/images.svelte | 550 ++++++++++++++-- src/lib/services/ai/image/providers/comfy.ts | 597 +++++++++++++++--- .../comfyWorkflows/unet-txt2img-workflow.json | 63 ++ src/lib/services/ai/image/providers/types.ts | 17 + 4 files changed, 1083 insertions(+), 144 deletions(-) create mode 100644 src/lib/services/ai/image/providers/comfyWorkflows/unet-txt2img-workflow.json diff --git a/src/lib/components/settings/tabs/images.svelte b/src/lib/components/settings/tabs/images.svelte index 45a213ba..96aa0af0 100644 --- a/src/lib/components/settings/tabs/images.svelte +++ b/src/lib/components/settings/tabs/images.svelte @@ -18,6 +18,8 @@ ComfyMode, type ImageModelInfo, } from '$lib/services/ai/image' + import { validateApiWorkflow, detectWorkflowFields } from '$lib/services/ai/image/providers/comfy' + import type { ComfyCustomWorkflow } from '$lib/services/ai/image/providers/types' import ImageModelSelect from '$lib/components/settings/ImageModelSelect.svelte' import type { ImageProfile, ImageProviderType, APIProfile } from '$lib/types' import * as Tabs from '$lib/components/ui/tabs' @@ -65,8 +67,28 @@ { value: 'comfyui', label: 'ComfyUI' }, ] const profileModes = [ + { value: ComfyMode.CustomWorkflow, label: 'Custom Workflow' }, { value: ComfyMode.BasicTxt2Img, label: 'Basic Text to Image' }, { value: ComfyMode.LoraTxt2Img, label: 'LoRA Text to Image' }, + { value: ComfyMode.UnetTxt2Img, label: 'UNet / Flux Text to Image' }, + ] as const + + const clipTypeItems = [ + { value: 'lumina2', label: 'Lumina 2' }, + { value: 'flux', label: 'Flux' }, + { value: 'sd3', label: 'SD3' }, + { value: 'stable_audio', label: 'Stable Audio' }, + { value: 'mochi', label: 'Mochi' }, + { value: 'ltxv', label: 'LTX Video' }, + { value: 'cosmos', label: 'Cosmos' }, + { value: 'wan', label: 'Wan' }, + { value: 'hidream', label: 'HiDream' }, + ] as const + const weightDtypeItems = [ + { value: 'default', label: 'default' }, + { value: 'fp8_e4m3fn', label: 'fp8_e4m3fn' }, + { value: 'fp8_e4m3fn_fast', label: 'fp8_e4m3fn_fast' }, + { value: 'fp8_e5m2', label: 'fp8_e5m2' }, ] as const // Tab state @@ -143,6 +165,39 @@ const { trigger: triggerAutoSave, flush: flushAutoSave } = createDebouncedSave(autoSaveProfile) let loadingInEffect = false + // Custom workflow state + let profileCustomWorkflow = $state(null) + // Holds parsed workflow data while the user resolves an ambiguous picker — not yet saved to profileCustomWorkflow + let pendingWorkflowData = $state<{ + workflow: ComfyCustomWorkflow['workflow'] + seedPath: string + outputNodeId: string + negativePromptPath: string | null + } | null>(null) + let workflowAmbiguousNodes = $state< + Array<{ + nodeId: string + title: string + path: string + textPreview: string + kSamplerSlot: string | null + }> + >([]) + let workflowPickerSelection = $state('') + let workflowError = $state(null) + let workflowFileName = $state(null) + + // UNet mode specific state + let profileClipName = $state('') + let profileVaeName = $state('') + let profileClipType = $state('lumina2') + let profileWeightDtype = $state('default') + let availableClips = $state([]) + let availableVaes = $state([]) + let isLoadingUnetModels = $state(false) + const clipItems = $derived(availableClips.map((c) => ({ value: c, label: c }))) + const vaeItems = $derived(availableVaes.map((v) => ({ value: v, label: v }))) + onDestroy(() => flushAutoSave()) // Model info cache for active profiles @@ -349,6 +404,15 @@ strengthClip: profileLoraStrengthClip ?? 1, } } + if (profileCustomWorkflow) { + opts.customWorkflow = profileCustomWorkflow + } + if (profileMode === ComfyMode.UnetTxt2Img) { + if (profileClipName) opts.clipName = profileClipName + if (profileVaeName) opts.vaeName = profileVaeName + if (profileClipType) opts.clipType = profileClipType + if (profileWeightDtype) opts.weightDtype = profileWeightDtype + } return opts } @@ -385,6 +449,11 @@ profileLoraName, profileLoraStrengthModel, profileLoraStrengthClip, + profileCustomWorkflow, + profileClipName, + profileVaeName, + profileClipType, + profileWeightDtype, ] // Skip the first run after startEditProfile populates the form if (suppressAutoSave) { @@ -414,11 +483,23 @@ profileModels = [] profileSampler = 'dpmpp_2m_sde_gpu' profileScheduler = 'sgm_uniform' - profileMode = ComfyMode.BasicTxt2Img + profileMode = ComfyMode.CustomWorkflow profileCfg = 1 profileSteps = 6 profilePositivePrompt = '' profileNegativePrompt = '' + profileCustomWorkflow = null + pendingWorkflowData = null + workflowAmbiguousNodes = [] + workflowPickerSelection = '' + workflowError = null + workflowFileName = null + profileClipName = '' + profileVaeName = '' + profileClipType = 'lumina2' + profileWeightDtype = 'default' + availableClips = [] + availableVaes = [] showApiKey = false showCopyDropdown = false openProfileIds.clear() @@ -447,6 +528,18 @@ profileSteps = Number(opts.step) || 6 profilePositivePrompt = (opts.positivePrompt as string) || '' profileNegativePrompt = (opts.negativePrompt as string) || '' + profileCustomWorkflow = (opts.customWorkflow as ComfyCustomWorkflow) || null + pendingWorkflowData = null + workflowFileName = profileCustomWorkflow ? 'Loaded from profile' : null + workflowAmbiguousNodes = [] + workflowPickerSelection = '' + workflowError = null + profileClipName = (opts.clipName as string) || '' + profileVaeName = (opts.vaeName as string) || '' + profileClipType = (opts.clipType as string) || 'lumina2' + profileWeightDtype = (opts.weightDtype as string) || 'default' + availableClips = [] + availableVaes = [] if (opts.lora) { const lora = opts.lora as any profileLoraName = lora.name || '' @@ -506,6 +599,138 @@ openProfileIds.delete(profile.id) } } + + // --------------------------------------------------------------------------- + // Custom workflow handlers + // --------------------------------------------------------------------------- + + /** + * Called after a file is selected or JSON is pasted. + * Validates the JSON, runs detection, and either auto-confirms or shows the picker. + */ + function processWorkflowJson(json: unknown, filename: string) { + workflowError = null + workflowAmbiguousNodes = [] + workflowPickerSelection = '' + + const validationError = validateApiWorkflow(json) + if (validationError) { + workflowError = validationError + return + } + + const workflow = json as ComfyCustomWorkflow['workflow'] + const { positiveNodes, negativeNode, seedPath, outputNodeId, saveImageCount } = + detectWorkflowFields(workflow) + + if (positiveNodes.length === 0) { + workflowError = 'No CLIPTextEncode (positive) node found in workflow.' + return + } + if (!seedPath) { + workflowError = 'No KSampler or KSamplerAdvanced node found — cannot detect seed path.' + return + } + if (saveImageCount === 0) { + workflowError = 'No SaveImage node found in workflow.' + return + } + if (saveImageCount > 1) { + workflowError = `Workflow has ${saveImageCount} SaveImage nodes. Please keep exactly one SaveImage node.` + return + } + + workflowFileName = filename + + if (positiveNodes.length > 1) { + // Ambiguous — show picker. Store parsed data in pending state; do NOT write + // profileCustomWorkflow yet so the auto-save effect doesn't fire prematurely. + pendingWorkflowData = { + workflow, + seedPath, + outputNodeId: outputNodeId!, + negativePromptPath: negativeNode?.path ?? null, + } + workflowAmbiguousNodes = positiveNodes + workflowPickerSelection = positiveNodes[0].path + } else { + // Unambiguous — auto-confirm immediately + pendingWorkflowData = null + profileCustomWorkflow = { + workflow, + positivePromptPath: positiveNodes[0].path, + seedPath, + outputNodeId: outputNodeId!, + negativePromptPath: negativeNode?.path ?? null, + } + workflowAmbiguousNodes = [] + } + } + + function handleWorkflowFileInput(event: Event) { + const input = event.target as HTMLInputElement + const file = input.files?.[0] + if (!file) return + const reader = new FileReader() + reader.onload = () => { + try { + const json = JSON.parse(reader.result as string) + processWorkflowJson(json, file.name) + } catch { + workflowError = 'Invalid JSON file.' + } + } + reader.readAsText(file) + // Reset input so the same file can be re-selected after clearing + input.value = '' + } + + function confirmWorkflowPicker() { + if (!pendingWorkflowData || !workflowPickerSelection) return + profileCustomWorkflow = { ...pendingWorkflowData, positivePromptPath: workflowPickerSelection } + pendingWorkflowData = null + workflowAmbiguousNodes = [] + } + + function clearCustomWorkflow() { + profileCustomWorkflow = null + pendingWorkflowData = null + workflowAmbiguousNodes = [] + workflowPickerSelection = '' + workflowError = null + workflowFileName = null + } + + // --------------------------------------------------------------------------- + // UNet model loading + // --------------------------------------------------------------------------- + + async function loadUnetModels() { + if (profileProviderType !== 'comfyui') return + const baseUrl = profileBaseUrl?.trim() || 'http://localhost:8188' + isLoadingUnetModels = true + try { + const [clips, vaes] = await Promise.all([ + fetch(`${baseUrl}/models/text_encoders`) + .then((r) => (r.ok ? (r.json() as Promise) : [])) + .catch(() => [] as string[]), + fetch(`${baseUrl}/models/vae`) + .then((r) => (r.ok ? (r.json() as Promise) : [])) + .catch(() => [] as string[]), + ]) + availableClips = clips + availableVaes = vaes + } finally { + isLoadingUnetModels = false + } + } + + // Auto-load when switching to UNet mode + $effect(() => { + if (profileMode === ComfyMode.UnetTxt2Img && availableClips.length === 0) { + void loadUnetModels() + } + })
@@ -1082,7 +1307,7 @@
{/if} -
+ {#snippet modelSelectContent()} {/if} -
+ {/snippet} + + {#if profileProviderType !== 'comfyui'} +
+ {@render modelSelectContent()} +
+ {/if} {#if profileProviderType === 'comfyui'}
@@ -1125,48 +1356,194 @@ placeholder="Select mode" />
-
- - s.value === profileSampler)} - onSelect={(v) => { - profileSampler = (v as { value: string }).value - }} - itemLabel={(s: { label: string }) => s.label} - itemValue={(s: { value: string }) => s.value} - placeholder="Select sampler" - /> -
-
- - s.value === profileScheduler)} - onSelect={(v) => { - profileScheduler = (v as { value: string }).value - }} - itemLabel={(s: { label: string }) => s.label} - itemValue={(s: { value: string }) => s.value} - placeholder="Select scheduler" - /> -
-
- - -
-
- - -
+ {#if profileMode !== ComfyMode.CustomWorkflow} +
+ {@render modelSelectContent()} +
+ {/if} + {#if profileMode !== ComfyMode.CustomWorkflow} +
+ + s.value === profileSampler)} + onSelect={(v) => { + profileSampler = (v as { value: string }).value + }} + itemLabel={(s: { label: string }) => s.label} + itemValue={(s: { value: string }) => s.value} + placeholder="Select sampler" + /> +
+
+ + s.value === profileScheduler)} + onSelect={(v) => { + profileScheduler = (v as { value: string }).value + }} + itemLabel={(s: { label: string }) => s.label} + itemValue={(s: { value: string }) => s.value} + placeholder="Select scheduler" + /> +
+
+ + +
+
+ + +
+ {/if}