From fc9b5426ae2a76766e0cf30a5b1a92d24f9d6e6d Mon Sep 17 00:00:00 2001 From: huchenlei Date: Wed, 8 Jan 2025 13:49:52 -0500 Subject: [PATCH 1/7] rfc: Workflow JSON Widget Values Format --- rfcs/0002-widget-values-format.md | 279 ++++++++++++++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 rfcs/0002-widget-values-format.md diff --git a/rfcs/0002-widget-values-format.md b/rfcs/0002-widget-values-format.md new file mode 100644 index 0000000..2deddf8 --- /dev/null +++ b/rfcs/0002-widget-values-format.md @@ -0,0 +1,279 @@ +# RFC: Workflow JSON Widget Values Format + +- Start Date: 2025-01-08 +- Target Major Version: TBD + +## Summary + +## Basic example + +Current format node serialization format: + +```json +{ + "id": 3, + "type": "KSampler", + "pos": [863, 186], + "size": [315, 262], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { "name": "model", "type": "MODEL", "link": 1 }, + { "name": "positive", "type": "CONDITIONING", "link": 4 }, + { "name": "negative", "type": "CONDITIONING", "link": 6 }, + { "name": "latent_image", "type": "LATENT", "link": 2 } + ], + "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [7], "slot_index": 0 }], + "properties": {}, + "widgets_values": [156680208700286, true, 20, 8, "euler", "normal", 1] +} +``` + +Proposed format: + +```json +{ + "id": 3, + "type": "KSampler", + "pos": [863, 186], + "size": [315, 262], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { "name": "model", "type": "MODEL", "link": 1 }, + { "name": "positive", "type": "CONDITIONING", "link": 4 }, + { "name": "negative", "type": "CONDITIONING", "link": 6 }, + { "name": "latent_image", "type": "LATENT", "link": 2 }, + { "name": "seed", "type": "INT", "value": 156680208700286 }, + { "name": "denoise", "type": "FLOAT", "value": 1.0 }, + { "name": "steps", "type": "INT", "value": 20 }, + { "name": "cfg", "type": "FLOAT", "value": 8 }, + { "name": "sampler_name", "type": "COMBO", "value": "euler" }, + { "name": "scheduler", "type": "COMBO", "value": "normal" }, + { "name": "denoise", "type": "FLOAT", "value": 1.0 }, + ], + "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [7], "slot_index": 0 }], + "properties": {}, +} +``` + +## Motivation + +The proposed format change addresses several key limitations in the current widget values implementation: + +1. **Unified Input Handling**: By moving widget values into the inputs array, we create a single, consistent way to handle all node inputs. This simplifies the node processing logic and reduces the need for special-case handling of widget values versus connected inputs. + +2. **Self-Describing Nodes**: The new format makes nodes more self-documenting by including input names and types directly in the node definition. This allows: + + - Reconstruction of node displays without requiring access to the full node definition + - Better error checking and validation of values + - Improved debugging capabilities + - Easier serialization/deserialization of workflows + +3. **Flexible Parameter Management**: The array-based structure of the current format makes it difficult to: + + - Insert new parameters in the middle of the list + - Remove deprecated parameters + - Maintain backward compatibility The named input approach solves these issues by making parameter order irrelevant. + +4. **Type Safety**: Explicit type definitions for each input value helps prevent type-related errors and makes it easier to implement proper validation at both runtime and development time. + +## Detailed design + +### Schema Changes + +The workflow schema will be updated from version 1.0 to 1.1 to accommodate the new widget values format. Here's the proposed Zod schema changes: + +```typescript +// Version 1.1 Node Input Schema +const NodeInputV1_1 = z.object({ + name: z.string(), + type: z.string(), + link: z.number().optional(), + value: z.any().optional(), // For widget values +}); + +// Version 1.1 Node Schema +const NodeV1_1 = z.object({ + id: z.number(), + type: z.string(), + pos: z.tuple([z.number(), z.number()]), + size: z.tuple([z.number(), z.number()]), + flags: z.record(z.any()), + order: z.number(), + mode: z.number(), + inputs: z.array(NodeInputV1_1), + outputs: z.array(NodeOutput), + properties: z.record(z.any()), + // widgets_values field removed +}); +``` + +### Version Conversion + +To maintain backward compatibility, the system will include bidirectional converters between versions: + +```typescript +function convertTo1_1(nodeV1_0: NodeV1_0): NodeV1_1 { + const widgetDefinitions = getNodeWidgetDefinitions(nodeV1_0.type); + + // Convert widget values to input format + const widgetInputs = widgetDefinitions.map((def, index) => ({ + name: def.name, + type: def.type, + value: nodeV1_0.widgets_values[index] + })); + + return { + ...nodeV1_0, + inputs: [...nodeV1_0.inputs, ...widgetInputs], + widgets_values: undefined // Remove widget_values field + }; +} + +function convertTo1_0(nodeV1_1: NodeV1_1): NodeV1_0 { + const widgetDefinitions = getNodeWidgetDefinitions(nodeV1_1.type); + const regularInputs = nodeV1_1.inputs.filter(input => !widgetDefinitions.find(def => def.name === input.name)); + const widgetValues = widgetDefinitions.map(def => { + const input = nodeV1_1.inputs.find(i => i.name === def.name); + return input?.value; + }); + + return { + ...nodeV1_1, + inputs: regularInputs, + widgets_values: widgetValues + }; +} +``` + +### Workflow Export Options + +When exporting workflows, users will be presented with schema version choices: + +1. Latest Version (1.1) - Default +2. Legacy Version (1.0) - For compatibility with older systems (Beta Reroute) +3. Legacy Version (0.4) - For compatibility with older systems + +The export dialog will include version selection and automatically convert the workflow to the selected format. + +### Unknown Node Handling + +For nodes without available definitions, LiteGraph will use the input metadata to render a basic representation: + +1. Regular inputs will be rendered as connection points +2. Widget inputs will be rendered as appropriate UI controls based on their type: + + - INT/FLOAT -> Number input + - STRING -> Text input + - COMBO -> Dropdown (with available values if provided) + - BOOLEAN -> Checkbox + +```typescript +class LiteGraphNode { + // ... existing code ... + + renderUnknownNode() { + this.inputs.forEach(input => { + if (input.link !== undefined) { + // Render connection point + this.addInput(input.name, input.type); + } else if (input.value !== undefined) { + // Render widget based on type + this.addWidget( + this.getWidgetTypeFromInputType(input.type), + input.name, + input.value, + (v) => { input.value = v; } + ); + } + }); + } +} +``` + +This approach ensures that even unknown nodes remain editable and maintain their configuration, even if specialized behavior isn't available. + +## Drawbacks + +1. **Increased Storage Size**: The new format is more verbose due to including field names and types for each widget value, which will increase the size of saved workflow files. While modern storage and network capabilities make this a minor concern, it's worth noting for systems handling large numbers of workflows. + +2. **Migration Complexity**: Existing workflows will need to be migrated to the new format, requiring: + + - Development of reliable conversion utilities + - Testing of conversion edge cases + - Potential maintenance of multiple format versions during transition + - Additional documentation for handling legacy formats + +3. **Performance Considerations**: The new format requires: + + - More complex parsing logic compared to simple array access + - Additional memory usage due to storing metadata with each value + - Potentially slower lookup times when accessing widget values (object property access vs array index) + +4. **Backward Compatibility Challenges**: While the proposal includes conversion utilities, there may be: + + - Third-party tools that need updating + - Custom node implementations that assume the array-based format + - Existing scripts or automation that parse workflow files directly + +5. **Learning Curve**: Users and developers familiar with the current array-based format will need to adapt to the new structure, potentially leading to initial confusion and requiring updates to documentation and tutorials. + +Despite these drawbacks, the benefits of improved maintainability, type safety, and self-documentation likely outweigh these concerns in the long term. + +# Adoption strategy + +The transition to the new widget values format will be implemented through a phased approach to ensure smooth adoption: + +1. **Version Support** + + - ComfyUI will support both 1.0 and 1.1 formats simultaneously during the transition period + - The internal format will be 1.1, with automatic conversion happening at workflow load/save + - All new features will target the 1.1 format + +2. **Breaking Changes** + + - This is technically a breaking change, but will be implemented with backward compatibility + - Existing workflows using the 1.0 format will continue to work through automatic conversion + - Node developers will need to update their implementations to support the new format + +3. **Migration Path** + + - For ComfyUI Users: + + - Existing workflows will be automatically converted when loaded + - No manual intervention required + - Option to export in legacy format for compatibility with older systems + + - For Node Developers: + + - Deprecation notices for direct `widgets_values` access + - New helper functions for accessing widget values through the inputs array + - Migration guide and examples provided in documentation + - Grace period of 2-3 releases before removing `widgets_values` support + +4. **Ecosystem Impact** + - Code search shows only ~10 custom node repositories directly accessing `widget_values` + - ComfyUI team can directly contribute fixes to these repositories + - API clients and workflow manipulation tools will need modification + - Web UI extensions may require updates for the new format + - Compatibility layer will be provided during transition: + ```javascript + get widgets_values() { + console.warn("Deprecated: accessing widgets_values directly. Please migrate to input values."); + return this.inputs + .filter(input => input.value !== undefined) + .map(input => input.value); + } + ``` + +5. **Timeline** + - Beta release with dual format support + - 3-month transition period with both formats supported + - Full migration to 1.1 format in next major version + +# Unresolved questions + +TBD From f2d1b22d3e8086515917ef53fdd53403e33b2bec Mon Sep 17 00:00:00 2001 From: huchenlei Date: Wed, 8 Jan 2025 13:50:33 -0500 Subject: [PATCH 2/7] Add summary section --- rfcs/0002-widget-values-format.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rfcs/0002-widget-values-format.md b/rfcs/0002-widget-values-format.md index 2deddf8..ac225ce 100644 --- a/rfcs/0002-widget-values-format.md +++ b/rfcs/0002-widget-values-format.md @@ -5,6 +5,8 @@ ## Summary +This RFC proposes a new format for handling widget values in ComfyUI workflows by integrating them directly into the node inputs array instead of storing them in a separate `widgets_values` array. The new format improves type safety, maintainability, and self-documentation of workflows by making each widget value a named, typed input with explicit metadata. This change will require a version bump in the workflow schema from 1.0 to 1.1, but includes backward compatibility measures to ensure a smooth transition for existing workflows and custom nodes. + ## Basic example Current format node serialization format: From 7733ba699ce1e0b278c695f6eda7beb3c9fbfc31 Mon Sep 17 00:00:00 2001 From: huchenlei Date: Wed, 8 Jan 2025 13:55:08 -0500 Subject: [PATCH 3/7] nit --- rfcs/0002-widget-values-format.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rfcs/0002-widget-values-format.md b/rfcs/0002-widget-values-format.md index ac225ce..406a225 100644 --- a/rfcs/0002-widget-values-format.md +++ b/rfcs/0002-widget-values-format.md @@ -225,7 +225,7 @@ This approach ensures that even unknown nodes remain editable and maintain their Despite these drawbacks, the benefits of improved maintainability, type safety, and self-documentation likely outweigh these concerns in the long term. -# Adoption strategy +## Adoption strategy The transition to the new widget values format will be implemented through a phased approach to ensure smooth adoption: @@ -276,6 +276,6 @@ The transition to the new widget values format will be implemented through a pha - 3-month transition period with both formats supported - Full migration to 1.1 format in next major version -# Unresolved questions +## Unresolved questions TBD From da5abbc975a5e84fe89e281b124af4746e15df41 Mon Sep 17 00:00:00 2001 From: huchenlei Date: Wed, 8 Jan 2025 15:09:43 -0500 Subject: [PATCH 4/7] Add widget to input convertion to example --- rfcs/0002-widget-values-format.md | 51 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/rfcs/0002-widget-values-format.md b/rfcs/0002-widget-values-format.md index 406a225..a19c2f9 100644 --- a/rfcs/0002-widget-values-format.md +++ b/rfcs/0002-widget-values-format.md @@ -9,6 +9,8 @@ This RFC proposes a new format for handling widget values in ComfyUI workflows b ## Basic example +![image](https://github.com/user-attachments/assets/e36113a9-20d6-406a-9a83-209c86b91107) + Current format node serialization format: ```json @@ -24,7 +26,16 @@ Current format node serialization format: { "name": "model", "type": "MODEL", "link": 1 }, { "name": "positive", "type": "CONDITIONING", "link": 4 }, { "name": "negative", "type": "CONDITIONING", "link": 6 }, - { "name": "latent_image", "type": "LATENT", "link": 2 } + { "name": "latent_image", "type": "LATENT", "link": 2 }, + // Seed input converted from widget + { + "name": "seed", + "type": "INT", + "link": null, + "widget": { + "name": "seed" + } + } ], "outputs": [{ "name": "LATENT", "type": "LATENT", "links": [7], "slot_index": 0 }], "properties": {}, @@ -48,7 +59,8 @@ Proposed format: { "name": "positive", "type": "CONDITIONING", "link": 4 }, { "name": "negative", "type": "CONDITIONING", "link": 6 }, { "name": "latent_image", "type": "LATENT", "link": 2 }, - { "name": "seed", "type": "INT", "value": 156680208700286 }, + // Seed input converted from widget + { "name": "seed", "type": "INT", "value": 156680208700286, "link": null }, { "name": "denoise", "type": "FLOAT", "value": 1.0 }, { "name": "steps", "type": "INT", "value": 20 }, { "name": "cfg", "type": "FLOAT", "value": 8 }, @@ -257,24 +269,27 @@ The transition to the new widget values format will be implemented through a pha - Grace period of 2-3 releases before removing `widgets_values` support 4. **Ecosystem Impact** - - Code search shows only ~10 custom node repositories directly accessing `widget_values` - - ComfyUI team can directly contribute fixes to these repositories - - API clients and workflow manipulation tools will need modification - - Web UI extensions may require updates for the new format - - Compatibility layer will be provided during transition: - ```javascript - get widgets_values() { - console.warn("Deprecated: accessing widgets_values directly. Please migrate to input values."); - return this.inputs - .filter(input => input.value !== undefined) - .map(input => input.value); - } - ``` + + - Code search shows only ~10 custom node repositories directly accessing `widget_values` + - ComfyUI team can directly contribute fixes to these repositories + - API clients and workflow manipulation tools will need modification + - Web UI extensions may require updates for the new format + - Compatibility layer will be provided during transition: + + ```javascript + get widgets_values() { + console.warn("Deprecated: accessing widgets_values directly. Please migrate to input values."); + return this.inputs + .filter(input => input.value !== undefined) + .map(input => input.value); + } + ``` 5. **Timeline** - - Beta release with dual format support - - 3-month transition period with both formats supported - - Full migration to 1.1 format in next major version + + - Beta release with dual format support + - 3-month transition period with both formats supported + - Full migration to 1.1 format in next major version ## Unresolved questions From 93ff506a4f679197bf6a700f87d1673d0b37c5dc Mon Sep 17 00:00:00 2001 From: huchenlei Date: Wed, 8 Jan 2025 19:56:56 -0500 Subject: [PATCH 5/7] Add state examples --- rfcs/0002-widget-values-format.md | 32 +++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/rfcs/0002-widget-values-format.md b/rfcs/0002-widget-values-format.md index a19c2f9..94ff81f 100644 --- a/rfcs/0002-widget-values-format.md +++ b/rfcs/0002-widget-values-format.md @@ -27,7 +27,6 @@ Current format node serialization format: { "name": "positive", "type": "CONDITIONING", "link": 4 }, { "name": "negative", "type": "CONDITIONING", "link": 6 }, { "name": "latent_image", "type": "LATENT", "link": 2 }, - // Seed input converted from widget { "name": "seed", "type": "INT", @@ -59,7 +58,6 @@ Proposed format: { "name": "positive", "type": "CONDITIONING", "link": 4 }, { "name": "negative", "type": "CONDITIONING", "link": 6 }, { "name": "latent_image", "type": "LATENT", "link": 2 }, - // Seed input converted from widget { "name": "seed", "type": "INT", "value": 156680208700286, "link": null }, { "name": "denoise", "type": "FLOAT", "value": 1.0 }, { "name": "steps", "type": "INT", "value": 20 }, @@ -73,6 +71,28 @@ Proposed format: } ``` +In the new format, the `link` field determines the input's behavior and UI representation: + +- `link: undefined` - Rendered as a widget (e.g., text input, slider, dropdown) + + ```json + { "name": "steps", "type": "INT", "value": 20 } + ``` + +- `link: null` - Rendered as an unconnected input socket + + ```json + { "name": "seed", "type": "INT", "value": 156680208700286, "link": null } + ``` + +- `link: number` - Rendered as a connected input socket with the specified link ID + + ```json + { "name": "model", "type": "MODEL", "link": 1 } + ``` + +This distinction allows nodes to clearly indicate whether an input should be displayed as an interactive widget or as a connection point, while maintaining the ability to store default values for unconnected inputs. + ## Motivation The proposed format change addresses several key limitations in the current widget values implementation: @@ -278,10 +298,10 @@ The transition to the new widget values format will be implemented through a pha ```javascript get widgets_values() { - console.warn("Deprecated: accessing widgets_values directly. Please migrate to input values."); - return this.inputs - .filter(input => input.value !== undefined) - .map(input => input.value); + console.warn("Deprecated: accessing widgets_values directly. Please migrate to input values."); + return this.inputs + .filter(input => input.value !== undefined) + .map(input => input.value); } ``` From b466c50c9021e18c411ea0f8032c6fe8b795cc2c Mon Sep 17 00:00:00 2001 From: huchenlei Date: Wed, 8 Jan 2025 20:01:37 -0500 Subject: [PATCH 6/7] 1.1 => 2.0 as it is a breaking change --- rfcs/0002-widget-values-format.md | 44 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/rfcs/0002-widget-values-format.md b/rfcs/0002-widget-values-format.md index 94ff81f..3ad5334 100644 --- a/rfcs/0002-widget-values-format.md +++ b/rfcs/0002-widget-values-format.md @@ -5,7 +5,7 @@ ## Summary -This RFC proposes a new format for handling widget values in ComfyUI workflows by integrating them directly into the node inputs array instead of storing them in a separate `widgets_values` array. The new format improves type safety, maintainability, and self-documentation of workflows by making each widget value a named, typed input with explicit metadata. This change will require a version bump in the workflow schema from 1.0 to 1.1, but includes backward compatibility measures to ensure a smooth transition for existing workflows and custom nodes. +This RFC proposes a new format for handling widget values in ComfyUI workflows by integrating them directly into the node inputs array instead of storing them in a separate `widgets_values` array. The new format improves type safety, maintainability, and self-documentation of workflows by making each widget value a named, typed input with explicit metadata. This change will require a version bump in the workflow schema from 1.0 to 2.0, but includes backward compatibility measures to ensure a smooth transition for existing workflows and custom nodes. ## Basic example @@ -118,19 +118,19 @@ The proposed format change addresses several key limitations in the current widg ### Schema Changes -The workflow schema will be updated from version 1.0 to 1.1 to accommodate the new widget values format. Here's the proposed Zod schema changes: +The workflow schema will be updated from version 1.0 to 2.0 to accommodate the new widget values format. Here's the proposed Zod schema changes: ```typescript -// Version 1.1 Node Input Schema -const NodeInputV1_1 = z.object({ +// Version 2.0 Node Input Schema +const NodeInputV2 = z.object({ name: z.string(), type: z.string(), link: z.number().optional(), value: z.any().optional(), // For widget values }); -// Version 1.1 Node Schema -const NodeV1_1 = z.object({ +// Version 2.0 Node Schema +const NodeV2 = z.object({ id: z.number(), type: z.string(), pos: z.tuple([z.number(), z.number()]), @@ -138,7 +138,7 @@ const NodeV1_1 = z.object({ flags: z.record(z.any()), order: z.number(), mode: z.number(), - inputs: z.array(NodeInputV1_1), + inputs: z.array(NodeInputV2), outputs: z.array(NodeOutput), properties: z.record(z.any()), // widgets_values field removed @@ -150,33 +150,33 @@ const NodeV1_1 = z.object({ To maintain backward compatibility, the system will include bidirectional converters between versions: ```typescript -function convertTo1_1(nodeV1_0: NodeV1_0): NodeV1_1 { - const widgetDefinitions = getNodeWidgetDefinitions(nodeV1_0.type); +function convertTo2(nodeV1: NodeV1): NodeV2 { + const widgetDefinitions = getNodeWidgetDefinitions(nodeV1.type); // Convert widget values to input format const widgetInputs = widgetDefinitions.map((def, index) => ({ name: def.name, type: def.type, - value: nodeV1_0.widgets_values[index] + value: nodeV1.widgets_values[index] })); return { - ...nodeV1_0, - inputs: [...nodeV1_0.inputs, ...widgetInputs], + ...nodeV1, + inputs: [...nodeV1.inputs, ...widgetInputs], widgets_values: undefined // Remove widget_values field }; } -function convertTo1_0(nodeV1_1: NodeV1_1): NodeV1_0 { - const widgetDefinitions = getNodeWidgetDefinitions(nodeV1_1.type); - const regularInputs = nodeV1_1.inputs.filter(input => !widgetDefinitions.find(def => def.name === input.name)); +function convertTo1(nodeV2: NodeV2): NodeV1 { + const widgetDefinitions = getNodeWidgetDefinitions(nodeV2.type); + const regularInputs = nodeV2.inputs.filter(input => !widgetDefinitions.find(def => def.name === input.name)); const widgetValues = widgetDefinitions.map(def => { - const input = nodeV1_1.inputs.find(i => i.name === def.name); + const input = nodeV2.inputs.find(i => i.name === def.name); return input?.value; }); return { - ...nodeV1_1, + ...nodeV2, inputs: regularInputs, widgets_values: widgetValues }; @@ -187,7 +187,7 @@ function convertTo1_0(nodeV1_1: NodeV1_1): NodeV1_0 { When exporting workflows, users will be presented with schema version choices: -1. Latest Version (1.1) - Default +1. Latest Version (2.0) - Default 2. Legacy Version (1.0) - For compatibility with older systems (Beta Reroute) 3. Legacy Version (0.4) - For compatibility with older systems @@ -263,9 +263,9 @@ The transition to the new widget values format will be implemented through a pha 1. **Version Support** - - ComfyUI will support both 1.0 and 1.1 formats simultaneously during the transition period - - The internal format will be 1.1, with automatic conversion happening at workflow load/save - - All new features will target the 1.1 format + - ComfyUI will support both 1.0 and 2.0 formats simultaneously during the transition period + - The internal format will be 2.0, with automatic conversion happening at workflow load/save + - All new features will target the 2.0 format 2. **Breaking Changes** @@ -309,7 +309,7 @@ The transition to the new widget values format will be implemented through a pha - Beta release with dual format support - 3-month transition period with both formats supported - - Full migration to 1.1 format in next major version + - Full migration to 2.0 format in next major version ## Unresolved questions From cf62f9bed533a871bde019a7415ffa07da031315 Mon Sep 17 00:00:00 2001 From: Robin Huang Date: Sat, 7 Jun 2025 15:58:38 -0700 Subject: [PATCH 7/7] Change version of rfc --- ...{0002-widget-values-format.md => 0004-widget-values-format.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename rfcs/{0002-widget-values-format.md => 0004-widget-values-format.md} (100%) diff --git a/rfcs/0002-widget-values-format.md b/rfcs/0004-widget-values-format.md similarity index 100% rename from rfcs/0002-widget-values-format.md rename to rfcs/0004-widget-values-format.md