From b8bc2b4dd84b6f3a11c7bb358f6fcc50a0a9e899 Mon Sep 17 00:00:00 2001 From: Rico Furtado Date: Wed, 4 Feb 2026 23:38:56 -0500 Subject: [PATCH 1/3] Changes in ingestion flow to support docling chunking component. --- flows/ingestion_flow.json | 1063 ++++++++++++++----------------------- 1 file changed, 409 insertions(+), 654 deletions(-) diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json index 6f65c04e0..d6d8ec43f 100644 --- a/flows/ingestion_flow.json +++ b/flows/ingestion_flow.json @@ -117,93 +117,6 @@ "target": "AdvancedDynamicFormBuilder-81Exw", "targetHandle": "{œfieldNameœ:œdynamic_owner_nameœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "DoclingRemote", - "id": "DoclingRemote-Dp3PX", - "name": "dataframe", - "output_types": [ - "DataFrame" - ] - }, - "targetHandle": { - "fieldName": "data_inputs", - "id": "ExportDoclingDocument-zZdRg", - "inputTypes": [ - "Data", - "DataFrame" - ], - "type": "other" - } - }, - "id": "xy-edge__DoclingRemote-Dp3PX{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-ExportDoclingDocument-zZdRg{œfieldNameœ:œdata_inputsœ,œidœ:œExportDoclingDocument-zZdRgœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", - "selected": false, - "source": "DoclingRemote-Dp3PX", - "sourceHandle": "{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", - "target": "ExportDoclingDocument-zZdRg", - "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œExportDoclingDocument-zZdRgœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" - }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "ExportDoclingDocument", - "id": "ExportDoclingDocument-zZdRg", - "name": "dataframe", - "output_types": [ - "DataFrame" - ] - }, - "targetHandle": { - "fieldName": "df", - "id": "DataFrameOperations-1BWXB", - "inputTypes": [ - "DataFrame" - ], - "type": "other" - } - }, - "id": "xy-edge__ExportDoclingDocument-zZdRg{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-1BWXB{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", - "selected": false, - "source": "ExportDoclingDocument-zZdRg", - "sourceHandle": "{œdataTypeœ:œExportDoclingDocumentœ,œidœ:œExportDoclingDocument-zZdRgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", - "target": "DataFrameOperations-1BWXB", - "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" - }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "DataFrameOperations", - "id": "DataFrameOperations-N80fC", - "name": "output", - "output_types": [ - "DataFrame" - ] - }, - "targetHandle": { - "fieldName": "data_inputs", - "id": "SplitText-QIKhg", - "inputTypes": [ - "Data", - "DataFrame", - "Message" - ], - "type": "other" - } - }, - "id": "xy-edge__DataFrameOperations-N80fC{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-SplitText-QIKhg{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}", - "selected": false, - "source": "DataFrameOperations-N80fC", - "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", - "target": "SplitText-QIKhg", - "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œSplitText-QIKhgœ,œinputTypesœ:[œDataœ,œDataFrameœ,œMessageœ],œtypeœ:œotherœ}" - }, { "animated": false, "className": "", @@ -288,35 +201,6 @@ "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", "targetHandle": "{œfieldNameœ:œdocs_metadataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ],œtypeœ:œtableœ}" }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "SplitText", - "id": "SplitText-QIKhg", - "name": "dataframe", - "output_types": [ - "DataFrame" - ] - }, - "targetHandle": { - "fieldName": "ingest_data", - "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", - "inputTypes": [ - "Data", - "DataFrame" - ], - "type": "other" - } - }, - "id": "xy-edge__SplitText-QIKhg{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", - "selected": false, - "source": "SplitText-QIKhg", - "sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", - "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", - "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" - }, { "animated": false, "className": "", @@ -400,252 +284,95 @@ "sourceHandle": "{œdataTypeœ:œEmbeddingModelœ,œidœ:œEmbeddingModel-EAo9iœ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}", "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}" - } - ], - "nodes": [ + }, { + "animated": false, + "className": "", "data": { - "description": "Split text into chunks based on specified criteria.", - "display_name": "Split Text", - "id": "SplitText-QIKhg", - "node": { - "base_classes": [ + "sourceHandle": { + "dataType": "DoclingRemote", + "id": "DoclingRemote-Dp3PX", + "name": "dataframe", + "output_types": [ "DataFrame" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Split text into chunks based on specified criteria.", - "display_name": "Split Text", - "documentation": "https://docs.langflow.org/components-processing#split-text", - "edited": true, - "field_order": [ - "data_inputs", - "chunk_overlap", - "chunk_size", - "separator", - "text_key", - "keep_separator" - ], - "frozen": false, - "icon": "scissors-line-dashed", - "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": { - "code_hash": "f2867efda61f", - "dependencies": { - "dependencies": [ - { - "name": "langchain_text_splitters", - "version": "0.3.9" - }, - { - "name": "lfx", - "version": null - } - ], - "total_dependencies": 2 - }, - "module": "custom_components.split_text" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Chunks", - "group_outputs": false, - "hidden": null, - "method": "split_text", - "name": "dataframe", - "options": null, - "required_inputs": null, - "selected": "DataFrame", - "tool_mode": true, - "types": [ - "DataFrame" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "chunk_overlap": { - "_input_type": "IntInput", - "advanced": false, - "display_name": "Chunk Overlap", - "dynamic": false, - "info": "Number of characters to overlap between chunks.", - "list": false, - "list_add_label": "Add More", - "name": "chunk_overlap", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "int", - "value": 200 - }, - "chunk_size": { - "_input_type": "IntInput", - "advanced": false, - "display_name": "Chunk Size", - "dynamic": false, - "info": "The maximum length of each chunk. Text is first split by separator, then chunks are merged up to this size. Individual splits larger than this won't be further divided.", - "list": false, - "list_add_label": "Add More", - "name": "chunk_size", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "int", - "value": 1000 - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from langchain_text_splitters import CharacterTextSplitter\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.utils.util import unescape_string\n\n\nclass SplitTextComponent(Component):\n display_name: str = \"Split Text\"\n description: str = \"Split text into chunks based on specified criteria.\"\n documentation: str = \"https://docs.langflow.org/components-processing#split-text\"\n icon = \"scissors-line-dashed\"\n name = \"SplitText\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Input\",\n info=\"The data with texts to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\", \"Message\"],\n required=True,\n ),\n IntInput(\n name=\"chunk_overlap\",\n display_name=\"Chunk Overlap\",\n info=\"Number of characters to overlap between chunks.\",\n value=200,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n info=(\n \"The maximum length of each chunk. Text is first split by separator, \"\n \"then chunks are merged up to this size. \"\n \"Individual splits larger than this won't be further divided.\"\n ),\n value=1000,\n ),\n MessageTextInput(\n name=\"separator\",\n display_name=\"Separator\",\n info=(\n \"The character to split on. Use \\\\n for newline. \"\n \"Examples: \\\\n\\\\n for paragraphs, \\\\n for lines, . for sentences\"\n ),\n value=\"\\n\",\n ),\n MessageTextInput(\n name=\"text_key\",\n display_name=\"Text Key\",\n info=\"The key to use for the text column.\",\n value=\"text\",\n advanced=True,\n ),\n DropdownInput(\n name=\"keep_separator\",\n display_name=\"Keep Separator\",\n info=\"Whether to keep the separator in the output chunks and where to place it.\",\n options=[\"False\", \"True\", \"Start\", \"End\"],\n value=\"False\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Chunks\", name=\"dataframe\", method=\"split_text\"),\n ]\n\n def _docs_to_data(self, docs) -> list[Data]:\n return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n\n def _fix_separator(self, separator: str) -> str:\n \"\"\"Fix common separator issues and convert to proper format.\"\"\"\n if separator == \"/n\":\n return \"\\n\"\n if separator == \"/t\":\n return \"\\t\"\n return separator\n\n def split_text_base(self):\n separator = self._fix_separator(self.separator)\n separator = unescape_string(separator)\n\n if isinstance(self.data_inputs, DataFrame):\n if not len(self.data_inputs):\n msg = \"DataFrame is empty\"\n raise TypeError(msg)\n\n self.data_inputs.text_key = self.text_key\n try:\n documents = self.data_inputs.to_lc_documents()\n except Exception as e:\n msg = f\"Error converting DataFrame to documents: {e}\"\n raise TypeError(msg) from e\n elif isinstance(self.data_inputs, Message):\n self.data_inputs = [self.data_inputs.to_data()]\n return self.split_text_base()\n else:\n if not self.data_inputs:\n msg = \"No data inputs provided\"\n raise TypeError(msg)\n\n documents = []\n if isinstance(self.data_inputs, Data):\n self.data_inputs.text_key = self.text_key\n documents = [self.data_inputs.to_lc_document()]\n else:\n try:\n documents = [input_.to_lc_document() for input_ in self.data_inputs if isinstance(input_, Data)]\n if not documents:\n msg = f\"No valid Data inputs found in {type(self.data_inputs)}\"\n raise TypeError(msg)\n except AttributeError as e:\n msg = f\"Invalid input type in collection: {e}\"\n raise TypeError(msg) from e\n try:\n # Convert string 'False'/'True' to boolean\n keep_sep = self.keep_separator\n if isinstance(keep_sep, str):\n if keep_sep.lower() == \"false\":\n keep_sep = False\n elif keep_sep.lower() == \"true\":\n keep_sep = True\n # 'start' and 'end' are kept as strings\n\n splitter = CharacterTextSplitter(\n chunk_overlap=self.chunk_overlap,\n chunk_size=self.chunk_size,\n separator=separator,\n keep_separator=keep_sep,\n )\n return splitter.split_documents(documents)\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n def split_text(self) -> DataFrame:\n return DataFrame(self._docs_to_data(self.split_text_base()))\n" - }, - "data_inputs": { - "_input_type": "HandleInput", - "advanced": false, - "display_name": "Input", - "dynamic": false, - "info": "The data with texts to split in chunks.", - "input_types": [ - "Data", - "DataFrame", - "Message" - ], - "list": false, - "list_add_label": "Add More", - "name": "data_inputs", - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "trace_as_metadata": true, - "type": "other", - "value": "" - }, - "keep_separator": { - "_input_type": "DropdownInput", - "advanced": true, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Keep Separator", - "dynamic": false, - "external_options": {}, - "info": "Whether to keep the separator in the output chunks and where to place it.", - "name": "keep_separator", - "options": [ - "False", - "True", - "Start", - "End" - ], - "options_metadata": [], - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "toggle": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "str", - "value": "False" - }, - "separator": { - "_input_type": "MessageTextInput", - "advanced": false, - "display_name": "Separator", - "dynamic": false, - "info": "The character to split on. Use \\n for newline. Examples: \\n\\n for paragraphs, \\n for lines, . for sentences", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "separator", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "\n" - }, - "text_key": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "Text Key", - "dynamic": false, - "info": "The key to use for the text column.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "text_key", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "text" - } - }, - "tool_mode": false + ] }, - "selected_output": "chunks", - "type": "SplitText" - }, - "dragging": false, - "height": 475, - "id": "SplitText-QIKhg", - "measured": { - "height": 475, - "width": 320 - }, - "position": { - "x": 1711.934915237861, - "y": 1637.2034518030887 + "targetHandle": { + "fieldName": "data_inputs", + "id": "ChunkDoclingDocument-7Tsav", + "inputTypes": [ + "Data", + "DataFrame" + ], + "type": "other" + } }, - "positionAbsolute": { - "x": 1683.4543896546102, - "y": 1350.7871623588553 + "id": "xy-edge__DoclingRemote-Dp3PX{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-ChunkDoclingDocument-7Tsav{œfieldNameœ:œdata_inputsœ,œidœ:œChunkDoclingDocument-7Tsavœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DoclingRemote-Dp3PX", + "sourceHandle": "{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "ChunkDoclingDocument-7Tsav", + "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œChunkDoclingDocument-7Tsavœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-N80fC", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "ingest_data", + "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", + "inputTypes": [ + "Data", + "DataFrame" + ], + "type": "other" + } }, + "id": "xy-edge__DataFrameOperations-N80fC{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", "selected": false, - "type": "genericNode", - "width": 320 + "source": "DataFrameOperations-N80fC", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", + "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "ChunkDoclingDocument", + "id": "ChunkDoclingDocument-7Tsav", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-1BWXB", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__ChunkDoclingDocument-7Tsav{œdataTypeœ:œChunkDoclingDocumentœ,œidœ:œChunkDoclingDocument-7Tsavœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-1BWXB{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "ChunkDoclingDocument-7Tsav", + "sourceHandle": "{œdataTypeœ:œChunkDoclingDocumentœ,œidœ:œChunkDoclingDocument-7Tsavœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-1BWXB", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + } + ], + "nodes": [ { "data": { "id": "AdvancedDynamicFormBuilder-81Exw", @@ -667,7 +394,7 @@ ], "frozen": false, "icon": "braces", - "last_updated": "2025-12-12T20:12:18.129Z", + "last_updated": "2026-02-05T04:24:29.583Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": {}, @@ -717,7 +444,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" + "value": "15acea35-7c03-488d-8702-13327d5a4cea" }, "_type": "Component", "code": { @@ -1417,6 +1144,7 @@ "frozen": false, "icon": "Docling", "legacy": false, + "lf_version": "1.7.0.dev21", "metadata": { "code_hash": "5723576d00e5", "dependencies": { @@ -1724,7 +1452,9 @@ "bz2", "gz" ], - "file_path": [], + "file_path": [ + "e278d6a2-65ab-421e-b66c-f4f0258f40c9/ULTABEAUTY_2023Q4_EARNINGS.pdf" + ], "info": "Supported file extensions: adoc, asciidoc, asc, bmp, csv, dotx, dotm, docm, docx, htm, html, jpeg, jpg, json, md, pdf, png, potx, ppsx, pptm, potm, ppsm, pptx, tiff, txt, xls, xlsx, xhtml, xml, webp; optionally bundled in file extensions: zip, tar, tgz, bz2, gz", "list": true, "list_add_label": "Add More", @@ -1786,274 +1516,17 @@ "tool_mode": false }, "showNode": true, - "type": "DoclingRemote" - }, - "dragging": false, - "id": "DoclingRemote-Dp3PX", - "measured": { - "height": 312, - "width": 320 - }, - "position": { - "x": -175.49741984154275, - "y": 1505.0245107748437 - }, - "selected": false, - "type": "genericNode" - }, - { - "data": { - "id": "ExportDoclingDocument-zZdRg", - "node": { - "base_classes": [ - "Data", - "DataFrame" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Export DoclingDocument to markdown, html or other formats.", - "display_name": "Export DoclingDocument", - "documentation": "https://docling-project.github.io/docling/", - "edited": false, - "field_order": [ - "data_inputs", - "export_format", - "image_mode", - "md_image_placeholder", - "md_page_break_placeholder", - "doc_key" - ], - "frozen": false, - "icon": "Docling", - "last_updated": "2025-10-04T01:42:10.290Z", - "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": { - "code_hash": "4de16ddd37ac", - "dependencies": { - "dependencies": [ - { - "name": "docling_core", - "version": "2.48.4" - }, - { - "name": "lfx", - "version": "0.1.12.dev31" - } - ], - "total_dependencies": 2 - }, - "module": "lfx.components.docling.export_docling_document.ExportDoclingDocumentComponent" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Exported data", - "group_outputs": false, - "method": "export_document", - "name": "data", - "options": null, - "required_inputs": null, - "selected": "Data", - "tool_mode": true, - "types": [ - "Data" - ], - "value": "__UNDEFINED__" - }, - { - "allows_loop": false, - "cache": true, - "display_name": "DataFrame", - "group_outputs": false, - "method": "as_dataframe", - "name": "dataframe", - "options": null, - "required_inputs": null, - "selected": "DataFrame", - "tool_mode": true, - "types": [ - "DataFrame" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from typing import Any\n\nfrom docling_core.types.doc import ImageRefMode\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import DropdownInput, HandleInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ExportDoclingDocumentComponent(Component):\n display_name: str = \"Export DoclingDocument\"\n description: str = \"Export DoclingDocument to markdown, html or other formats.\"\n documentation = \"https://docling-project.github.io/docling/\"\n icon = \"Docling\"\n name = \"ExportDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Data or DataFrame\",\n info=\"The data with documents to export.\",\n input_types=[\"Data\", \"DataFrame\"],\n required=True,\n ),\n DropdownInput(\n name=\"export_format\",\n display_name=\"Export format\",\n options=[\"Markdown\", \"HTML\", \"Plaintext\", \"DocTags\"],\n info=\"Select the export format to convert the input.\",\n value=\"Markdown\",\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"image_mode\",\n display_name=\"Image export mode\",\n options=[\"placeholder\", \"embedded\"],\n info=(\n \"Specify how images are exported in the output. Placeholder will replace the images with a string, \"\n \"whereas Embedded will include them as base64 encoded images.\"\n ),\n value=\"placeholder\",\n ),\n StrInput(\n name=\"md_image_placeholder\",\n display_name=\"Image placeholder\",\n info=\"Specify the image placeholder for markdown exports.\",\n value=\"\",\n advanced=True,\n ),\n StrInput(\n name=\"md_page_break_placeholder\",\n display_name=\"Page break placeholder\",\n info=\"Add this placeholder betweek pages in the markdown output.\",\n value=\"\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Exported data\", name=\"data\", method=\"export_document\"),\n Output(display_name=\"DataFrame\", name=\"dataframe\", method=\"as_dataframe\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"export_format\" and field_value == \"Markdown\":\n build_config[\"md_image_placeholder\"][\"show\"] = True\n build_config[\"md_page_break_placeholder\"][\"show\"] = True\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value == \"HTML\":\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = True\n elif field_name == \"export_format\" and field_value in {\"Plaintext\", \"DocTags\"}:\n build_config[\"md_image_placeholder\"][\"show\"] = False\n build_config[\"md_page_break_placeholder\"][\"show\"] = False\n build_config[\"image_mode\"][\"show\"] = False\n\n return build_config\n\n def export_document(self) -> list[Data]:\n documents = extract_docling_documents(self.data_inputs, self.doc_key)\n\n results: list[Data] = []\n try:\n image_mode = ImageRefMode(self.image_mode)\n for doc in documents:\n content = \"\"\n if self.export_format == \"Markdown\":\n content = doc.export_to_markdown(\n image_mode=image_mode,\n image_placeholder=self.md_image_placeholder,\n page_break_placeholder=self.md_page_break_placeholder,\n )\n elif self.export_format == \"HTML\":\n content = doc.export_to_html(image_mode=image_mode)\n elif self.export_format == \"Plaintext\":\n content = doc.export_to_text()\n elif self.export_format == \"DocTags\":\n content = doc.export_to_doctags()\n\n results.append(Data(text=content))\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return results\n\n def as_dataframe(self) -> DataFrame:\n return DataFrame(self.export_document())\n" - }, - "data_inputs": { - "_input_type": "HandleInput", - "advanced": false, - "display_name": "Data or DataFrame", - "dynamic": false, - "info": "The data with documents to export.", - "input_types": [ - "Data", - "DataFrame" - ], - "list": false, - "list_add_label": "Add More", - "name": "data_inputs", - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "trace_as_metadata": true, - "type": "other", - "value": "" - }, - "doc_key": { - "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "Doc Key", - "dynamic": false, - "info": "The key to use for the DoclingDocument column.", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "doc_key", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "type": "str", - "value": "doc" - }, - "export_format": { - "_input_type": "DropdownInput", - "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Export format", - "dynamic": false, - "external_options": {}, - "info": "Select the export format to convert the input.", - "name": "export_format", - "options": [ - "Markdown", - "HTML", - "Plaintext", - "DocTags" - ], - "options_metadata": [], - "placeholder": "", - "real_time_refresh": true, - "required": false, - "show": true, - "title_case": false, - "toggle": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "str", - "value": "Markdown" - }, - "image_mode": { - "_input_type": "DropdownInput", - "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Image export mode", - "dynamic": false, - "external_options": {}, - "info": "Specify how images are exported in the output. Placeholder will replace the images with a string, whereas Embedded will include them as base64 encoded images.", - "name": "image_mode", - "options": [ - "placeholder", - "embedded" - ], - "options_metadata": [], - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "toggle": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "str", - "value": "placeholder" - }, - "md_image_placeholder": { - "_input_type": "StrInput", - "advanced": true, - "display_name": "Image placeholder", - "dynamic": false, - "info": "Specify the image placeholder for markdown exports.", - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "md_image_placeholder", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "str", - "value": "" - }, - "md_page_break_placeholder": { - "_input_type": "StrInput", - "advanced": true, - "display_name": "Page break placeholder", - "dynamic": false, - "info": "Add this placeholder betweek pages in the markdown output.", - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "md_page_break_placeholder", - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "type": "str", - "value": "" - } - }, - "tool_mode": false - }, - "selected_output": "dataframe", - "showNode": true, - "type": "ExportDoclingDocument" + "type": "DoclingRemote" }, "dragging": false, - "id": "ExportDoclingDocument-zZdRg", + "id": "DoclingRemote-Dp3PX", "measured": { - "height": 347, + "height": 316, "width": 320 }, "position": { - "x": 206.97755086947473, - "y": 1610.6498167995744 + "x": -32.29834135984382, + "y": 1602.5217556985535 }, "selected": false, "type": "genericNode" @@ -2090,7 +1563,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-12-12T20:12:18.208Z", + "last_updated": "2026-02-05T04:24:29.784Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2137,7 +1610,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" + "value": "15acea35-7c03-488d-8702-13327d5a4cea" }, "_type": "Component", "ascending": { @@ -2503,8 +1976,8 @@ "width": 320 }, "position": { - "x": 575.218683966709, - "y": 1607.3264908868193 + "x": 818.9617962759838, + "y": 1598.1861241752215 }, "selected": false, "type": "genericNode" @@ -2541,7 +2014,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-12-12T20:12:18.209Z", + "last_updated": "2026-02-05T04:24:29.784Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2588,7 +2061,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" + "value": "15acea35-7c03-488d-8702-13327d5a4cea" }, "_type": "Component", "ascending": { @@ -2954,8 +2427,8 @@ "width": 320 }, "position": { - "x": 1330.149865256777, - "y": 1619.9268393528012 + "x": 1636.3521500953036, + "y": 1600.122711477673 }, "selected": false, "type": "genericNode" @@ -2992,7 +2465,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-12-12T20:12:18.209Z", + "last_updated": "2026-02-05T04:24:29.785Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -3039,7 +2512,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" + "value": "15acea35-7c03-488d-8702-13327d5a4cea" }, "_type": "Component", "ascending": { @@ -3405,8 +2878,8 @@ "width": 320 }, "position": { - "x": 937.1310281139399, - "y": 1611.2186890450444 + "x": 1222.0057906254049, + "y": 1592.9379556218487 }, "selected": false, "type": "genericNode" @@ -4193,7 +3666,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-12-12T20:12:18.131Z", + "last_updated": "2026-02-05T04:24:29.584Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -4261,13 +3734,13 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" + "value": "15acea35-7c03-488d-8702-13327d5a4cea" }, "_type": "Component", "api_base": { "_input_type": "MessageTextInput", - "advanced": true, - "display_name": "API Base URL", + "advanced": false, + "display_name": "OpenAI API Base URL", "dynamic": false, "info": "Base URL for the API. Leave empty for default.", "input_types": [ @@ -4280,7 +3753,7 @@ "override_skip": false, "placeholder": "", "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, @@ -4292,7 +3765,7 @@ "api_key": { "_input_type": "SecretStrInput", "advanced": false, - "display_name": "IBM watsonx.ai API Key", + "display_name": "OpenAI API Key", "dynamic": false, "info": "Model Provider API key", "input_types": [], @@ -4307,7 +3780,7 @@ "title_case": false, "track_in_telemetry": false, "type": "str", - "value": "WATSONX_API_KEY" + "value": "OPENAI_API_KEY" }, "base_url_ibm_watsonx": { "_input_type": "DropdownInput", @@ -4332,7 +3805,7 @@ "placeholder": "", "real_time_refresh": true, "required": false, - "show": true, + "show": false, "title_case": false, "toggle": false, "tool_mode": false, @@ -4433,7 +3906,7 @@ "override_skip": false, "placeholder": "", "required": false, - "show": true, + "show": false, "title_case": false, "tool_mode": false, "trace_as_metadata": true, @@ -4474,7 +3947,7 @@ "load_from_db": false, "name": "model", "options": [ - "ibm/granite-embedding-278m-multilingual" + "text-embedding-3-small" ], "options_metadata": [], "override_skip": false, @@ -4489,7 +3962,7 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "ibm/granite-embedding-278m-multilingual" + "value": "text-embedding-3-small" }, "model_kwargs": { "_input_type": "DictInput", @@ -4553,7 +4026,7 @@ "override_skip": false, "placeholder": "", "required": false, - "show": true, + "show": false, "title_case": false, "tool_mode": false, "trace_as_input": true, @@ -4603,7 +4076,7 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "IBM watsonx.ai" + "value": "OpenAI" }, "request_timeout": { "_input_type": "FloatInput", @@ -4657,7 +4130,7 @@ "override_skip": false, "placeholder": "", "required": false, - "show": true, + "show": false, "title_case": false, "tool_mode": false, "trace_as_metadata": true, @@ -4674,7 +4147,7 @@ "dragging": false, "id": "EmbeddingModel-EAo9i", "measured": { - "height": 534, + "height": 451, "width": 320 }, "position": { @@ -4718,7 +4191,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-12-12T20:12:18.132Z", + "last_updated": "2026-02-05T04:24:29.585Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -4786,7 +4259,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" + "value": "15acea35-7c03-488d-8702-13327d5a4cea" }, "_type": "Component", "api_base": { @@ -5244,7 +4717,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-12-12T20:12:18.133Z", + "last_updated": "2026-02-05T04:24:29.585Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -5312,7 +4785,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" + "value": "15acea35-7c03-488d-8702-13327d5a4cea" }, "_type": "Component", "api_base": { @@ -5734,19 +5207,301 @@ }, "selected": false, "type": "genericNode" + }, + { + "data": { + "id": "ChunkDoclingDocument-7Tsav", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Use the DocumentDocument chunkers to split the document into chunks.", + "display_name": "Chunk DoclingDocument", + "documentation": "https://docling-project.github.io/docling/concepts/chunking/", + "edited": false, + "field_order": [ + "data_inputs", + "chunker", + "provider", + "hf_model_name", + "openai_model_name", + "max_tokens", + "doc_key" + ], + "frozen": false, + "icon": "Docling", + "last_updated": "2026-02-04T17:56:53.100Z", + "legacy": false, + "lf_version": "1.7.0.dev21", + "metadata": { + "code_hash": "397fa38f89d7", + "dependencies": { + "dependencies": [ + { + "name": "tiktoken", + "version": "0.12.0" + }, + { + "name": "docling_core", + "version": "2.49.0" + }, + { + "name": "lfx", + "version": "0.2.0.dev21" + } + ], + "total_dependencies": 3 + }, + "module": "lfx.components.docling.chunk_docling_document.ChunkDoclingDocumentComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "DataFrame", + "group_outputs": false, + "loop_types": null, + "method": "chunk_documents", + "name": "dataframe", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "15acea35-7c03-488d-8702-13327d5a4cea" + }, + "_type": "Component", + "chunker": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Chunker", + "dynamic": false, + "external_options": {}, + "info": "Which chunker to use.", + "name": "chunker", + "options": [ + "HybridChunker", + "HierarchicalChunker" + ], + "options_metadata": [], + "override_skip": false, + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "str", + "value": "HierarchicalChunker" + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "import json\n\nimport tiktoken\nfrom docling_core.transforms.chunker import BaseChunker, DocMeta\nfrom docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ChunkDoclingDocumentComponent(Component):\n display_name: str = \"Chunk DoclingDocument\"\n description: str = \"Use the DocumentDocument chunkers to split the document into chunks.\"\n documentation = \"https://docling-project.github.io/docling/concepts/chunking/\"\n icon = \"Docling\"\n name = \"ChunkDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Data or DataFrame\",\n info=\"The data with documents to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\"],\n required=True,\n ),\n DropdownInput(\n name=\"chunker\",\n display_name=\"Chunker\",\n options=[\"HybridChunker\", \"HierarchicalChunker\"],\n info=(\"Which chunker to use.\"),\n value=\"HybridChunker\",\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"provider\",\n display_name=\"Provider\",\n options=[\"Hugging Face\", \"OpenAI\"],\n info=(\"Which tokenizer provider.\"),\n value=\"Hugging Face\",\n show=True,\n real_time_refresh=True,\n advanced=True,\n dynamic=True,\n ),\n StrInput(\n name=\"hf_model_name\",\n display_name=\"HF model name\",\n info=(\n \"Model name of the tokenizer to use with the HybridChunker when Hugging Face is chosen as a tokenizer.\"\n ),\n value=\"sentence-transformers/all-MiniLM-L6-v2\",\n show=True,\n advanced=True,\n dynamic=True,\n ),\n StrInput(\n name=\"openai_model_name\",\n display_name=\"OpenAI model name\",\n info=(\"Model name of the tokenizer to use with the HybridChunker when OpenAI is chosen as a tokenizer.\"),\n value=\"gpt-4o\",\n show=False,\n advanced=True,\n dynamic=True,\n ),\n IntInput(\n name=\"max_tokens\",\n display_name=\"Maximum tokens\",\n info=(\"Maximum number of tokens for the HybridChunker.\"),\n show=True,\n required=False,\n advanced=True,\n dynamic=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"DataFrame\", name=\"dataframe\", method=\"chunk_documents\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n if field_name == \"chunker\":\n provider_type = build_config[\"provider\"][\"value\"]\n is_hf = provider_type == \"Hugging Face\"\n is_openai = provider_type == \"OpenAI\"\n if field_value == \"HybridChunker\":\n build_config[\"provider\"][\"show\"] = True\n build_config[\"hf_model_name\"][\"show\"] = is_hf\n build_config[\"openai_model_name\"][\"show\"] = is_openai\n build_config[\"max_tokens\"][\"show\"] = True\n else:\n build_config[\"provider\"][\"show\"] = False\n build_config[\"hf_model_name\"][\"show\"] = False\n build_config[\"openai_model_name\"][\"show\"] = False\n build_config[\"max_tokens\"][\"show\"] = False\n elif field_name == \"provider\" and build_config[\"chunker\"][\"value\"] == \"HybridChunker\":\n if field_value == \"Hugging Face\":\n build_config[\"hf_model_name\"][\"show\"] = True\n build_config[\"openai_model_name\"][\"show\"] = False\n elif field_value == \"OpenAI\":\n build_config[\"hf_model_name\"][\"show\"] = False\n build_config[\"openai_model_name\"][\"show\"] = True\n\n return build_config\n\n def _docs_to_data(self, docs) -> list[Data]:\n return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n\n def chunk_documents(self) -> DataFrame:\n documents = extract_docling_documents(self.data_inputs, self.doc_key)\n\n chunker: BaseChunker\n if self.chunker == \"HybridChunker\":\n try:\n from docling_core.transforms.chunker.hybrid_chunker import HybridChunker\n except ImportError as e:\n msg = (\n \"HybridChunker is not installed. Please install it with `uv pip install docling-core[chunking] \"\n \"or `uv pip install transformers`\"\n )\n raise ImportError(msg) from e\n max_tokens: int | None = self.max_tokens if self.max_tokens else None\n if self.provider == \"Hugging Face\":\n try:\n from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer\n except ImportError as e:\n msg = (\n \"HuggingFaceTokenizer is not installed.\"\n \" Please install it with `uv pip install docling-core[chunking]`\"\n )\n raise ImportError(msg) from e\n tokenizer = HuggingFaceTokenizer.from_pretrained(\n model_name=self.hf_model_name,\n max_tokens=max_tokens,\n )\n elif self.provider == \"OpenAI\":\n try:\n from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer\n except ImportError as e:\n msg = (\n \"OpenAITokenizer is not installed.\"\n \" Please install it with `uv pip install docling-core[chunking]`\"\n \" or `uv pip install transformers`\"\n )\n raise ImportError(msg) from e\n if max_tokens is None:\n max_tokens = 128 * 1024 # context window length required for OpenAI tokenizers\n tokenizer = OpenAITokenizer(\n tokenizer=tiktoken.encoding_for_model(self.openai_model_name), max_tokens=max_tokens\n )\n chunker = HybridChunker(\n tokenizer=tokenizer,\n )\n elif self.chunker == \"HierarchicalChunker\":\n chunker = HierarchicalChunker()\n\n results: list[Data] = []\n try:\n for doc in documents:\n for chunk in chunker.chunk(dl_doc=doc):\n enriched_text = chunker.contextualize(chunk=chunk)\n meta = DocMeta.model_validate(chunk.meta)\n\n results.append(\n Data(\n data={\n \"text\": enriched_text,\n \"document_id\": f\"{doc.origin.binary_hash}\",\n \"doc_items\": json.dumps([item.self_ref for item in meta.doc_items]),\n }\n )\n )\n\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return DataFrame(results)\n" + }, + "data_inputs": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Data or DataFrame", + "dynamic": false, + "info": "The data with documents to split in chunks.", + "input_types": [ + "Data", + "DataFrame" + ], + "list": false, + "list_add_label": "Add More", + "name": "data_inputs", + "override_skip": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "other", + "value": "" + }, + "doc_key": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Doc Key", + "dynamic": false, + "info": "The key to use for the DoclingDocument column.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "doc_key", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "doc" + }, + "hf_model_name": { + "_input_type": "StrInput", + "advanced": true, + "display_name": "HF model name", + "dynamic": true, + "info": "Model name of the tokenizer to use with the HybridChunker when Hugging Face is chosen as a tokenizer.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "hf_model_name", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "sentence-transformers/all-MiniLM-L6-v2" + }, + "is_refresh": false, + "max_tokens": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Maximum tokens", + "dynamic": true, + "info": "Maximum number of tokens for the HybridChunker.", + "list": false, + "list_add_label": "Add More", + "name": "max_tokens", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": "" + }, + "openai_model_name": { + "_input_type": "StrInput", + "advanced": true, + "display_name": "OpenAI model name", + "dynamic": true, + "info": "Model name of the tokenizer to use with the HybridChunker when OpenAI is chosen as a tokenizer.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "openai_model_name", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "gpt-4o" + }, + "provider": { + "_input_type": "DropdownInput", + "advanced": true, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Provider", + "dynamic": true, + "external_options": {}, + "info": "Which tokenizer provider.", + "name": "provider", + "options": [ + "Hugging Face", + "OpenAI" + ], + "options_metadata": [], + "override_skip": false, + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": false, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "str", + "value": "Hugging Face" + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "ChunkDoclingDocument" + }, + "dragging": false, + "id": "ChunkDoclingDocument-7Tsav", + "measured": { + "height": 347, + "width": 320 + }, + "position": { + "x": 383.4363765185066, + "y": 1602.9945490955463 + }, + "selected": false, + "type": "genericNode" } ], "viewport": { - "x": 249.3666737262397, - "y": -156.8776378758762, - "zoom": 0.38977017930844676 + "x": 257.7690459293176, + "y": -154.96170253117464, + "zoom": 0.38900766436126777 } }, "description": "Load your data for chat context with Retrieval Augmented Generation.", "endpoint_name": null, "id": "5488df7c-b93f-4f87-a446-b67028bc0813", "is_component": false, - "locked": true, "last_tested_version": "1.7.0.dev21", "name": "OpenSearch Ingestion Flow", "tags": [ From 593103f66b8474ce0fba0004d0150a315723933e Mon Sep 17 00:00:00 2001 From: Rico Furtado Date: Thu, 5 Feb 2026 10:27:37 -0500 Subject: [PATCH 2/3] Changed default chunk node for hybrid, removed default file. --- flows/ingestion_flow.json | 46 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json index d6d8ec43f..fe31bca9c 100644 --- a/flows/ingestion_flow.json +++ b/flows/ingestion_flow.json @@ -394,7 +394,7 @@ ], "frozen": false, "icon": "braces", - "last_updated": "2026-02-05T04:24:29.583Z", + "last_updated": "2026-02-05T15:22:55.809Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": {}, @@ -1452,9 +1452,7 @@ "bz2", "gz" ], - "file_path": [ - "e278d6a2-65ab-421e-b66c-f4f0258f40c9/ULTABEAUTY_2023Q4_EARNINGS.pdf" - ], + "file_path": [], "info": "Supported file extensions: adoc, asciidoc, asc, bmp, csv, dotx, dotm, docm, docx, htm, html, jpeg, jpg, json, md, pdf, png, potx, ppsx, pptm, potm, ppsm, pptx, tiff, txt, xls, xlsx, xhtml, xml, webp; optionally bundled in file extensions: zip, tar, tgz, bz2, gz", "list": true, "list_add_label": "Add More", @@ -1521,12 +1519,12 @@ "dragging": false, "id": "DoclingRemote-Dp3PX", "measured": { - "height": 316, + "height": 312, "width": 320 }, "position": { - "x": -32.29834135984382, - "y": 1602.5217556985535 + "x": -0.2871317656367012, + "y": 1594.9000391285042 }, "selected": false, "type": "genericNode" @@ -1563,7 +1561,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2026-02-05T04:24:29.784Z", + "last_updated": "2026-02-05T15:22:55.931Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -1976,8 +1974,8 @@ "width": 320 }, "position": { - "x": 818.9617962759838, - "y": 1598.1861241752215 + "x": 826.5835128460332, + "y": 1595.137437547202 }, "selected": false, "type": "genericNode" @@ -2014,7 +2012,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2026-02-05T04:24:29.784Z", + "last_updated": "2026-02-05T15:22:55.932Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2427,7 +2425,7 @@ "width": 320 }, "position": { - "x": 1636.3521500953036, + "x": 1697.325882655698, "y": 1600.122711477673 }, "selected": false, @@ -2465,7 +2463,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2026-02-05T04:24:29.785Z", + "last_updated": "2026-02-05T15:22:55.932Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2878,8 +2876,8 @@ "width": 320 }, "position": { - "x": 1222.0057906254049, - "y": 1592.9379556218487 + "x": 1252.4926569056022, + "y": 1589.889268993829 }, "selected": false, "type": "genericNode" @@ -3666,7 +3664,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2026-02-05T04:24:29.584Z", + "last_updated": "2026-02-05T15:22:55.812Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -4191,7 +4189,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2026-02-05T04:24:29.585Z", + "last_updated": "2026-02-05T15:22:55.813Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -4717,7 +4715,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2026-02-05T04:24:29.585Z", + "last_updated": "2026-02-05T15:22:55.814Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -5233,7 +5231,7 @@ ], "frozen": false, "icon": "Docling", - "last_updated": "2026-02-04T17:56:53.100Z", + "last_updated": "2026-02-05T15:22:20.975Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -5485,17 +5483,17 @@ "width": 320 }, "position": { - "x": 383.4363765185066, - "y": 1602.9945490955463 + "x": 410.87455617068423, + "y": 1595.372832525497 }, "selected": false, "type": "genericNode" } ], "viewport": { - "x": 257.7690459293176, - "y": -154.96170253117464, - "zoom": 0.38900766436126777 + "x": 492.2079532383307, + "y": -215.27371168994932, + "zoom": 0.5404962259634922 } }, "description": "Load your data for chat context with Retrieval Augmented Generation.", From 40e8d3c7f01525bd84867c8b198ba78195e673e1 Mon Sep 17 00:00:00 2001 From: Rico Furtado Date: Thu, 12 Mar 2026 11:52:57 -0400 Subject: [PATCH 3/3] Update ingestion_flow.json --- flows/ingestion_flow.json | 4800 +++++++++++++++++++++++-------------- 1 file changed, 3057 insertions(+), 1743 deletions(-) diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json index fe31bca9c..71fd64a4f 100644 --- a/flows/ingestion_flow.json +++ b/flows/ingestion_flow.json @@ -1,122 +1,6 @@ { "data": { "edges": [ - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "SecretInput", - "id": "SecretInput-F34VJ", - "name": "text", - "output_types": [ - "Message" - ] - }, - "targetHandle": { - "fieldName": "dynamic_connector_type", - "id": "AdvancedDynamicFormBuilder-81Exw", - "inputTypes": [ - "Text", - "Message" - ], - "type": "str" - } - }, - "id": "xy-edge__SecretInput-F34VJ{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-F34VJœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_connector_typeœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", - "selected": false, - "source": "SecretInput-F34VJ", - "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-F34VJœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", - "target": "AdvancedDynamicFormBuilder-81Exw", - "targetHandle": "{œfieldNameœ:œdynamic_connector_typeœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" - }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "SecretInput", - "id": "SecretInput-b2cab", - "name": "text", - "output_types": [ - "Message" - ] - }, - "targetHandle": { - "fieldName": "dynamic_owner", - "id": "AdvancedDynamicFormBuilder-81Exw", - "inputTypes": [ - "Text", - "Message" - ], - "type": "str" - } - }, - "id": "xy-edge__SecretInput-b2cab{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-b2cabœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_ownerœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", - "selected": false, - "source": "SecretInput-b2cab", - "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-b2cabœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", - "target": "AdvancedDynamicFormBuilder-81Exw", - "targetHandle": "{œfieldNameœ:œdynamic_ownerœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" - }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "SecretInput", - "id": "SecretInput-ZVfuS", - "name": "text", - "output_types": [ - "Message" - ] - }, - "targetHandle": { - "fieldName": "dynamic_owner_email", - "id": "AdvancedDynamicFormBuilder-81Exw", - "inputTypes": [ - "Text", - "Message" - ], - "type": "str" - } - }, - "id": "xy-edge__SecretInput-ZVfuS{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-ZVfuSœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_owner_emailœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", - "selected": false, - "source": "SecretInput-ZVfuS", - "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-ZVfuSœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", - "target": "AdvancedDynamicFormBuilder-81Exw", - "targetHandle": "{œfieldNameœ:œdynamic_owner_emailœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" - }, - { - "animated": false, - "className": "", - "data": { - "sourceHandle": { - "dataType": "SecretInput", - "id": "SecretInput-Iqtxd", - "name": "text", - "output_types": [ - "Message" - ] - }, - "targetHandle": { - "fieldName": "dynamic_owner_name", - "id": "AdvancedDynamicFormBuilder-81Exw", - "inputTypes": [ - "Text", - "Message" - ], - "type": "str" - } - }, - "id": "xy-edge__SecretInput-Iqtxd{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-Iqtxdœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_owner_nameœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", - "selected": false, - "source": "SecretInput-Iqtxd", - "sourceHandle": "{œdataTypeœ:œSecretInputœ,œidœ:œSecretInput-Iqtxdœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", - "target": "AdvancedDynamicFormBuilder-81Exw", - "targetHandle": "{œfieldNameœ:œdynamic_owner_nameœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" - }, { "animated": false, "className": "", @@ -290,162 +174,394 @@ "className": "", "data": { "sourceHandle": { - "dataType": "DoclingRemote", - "id": "DoclingRemote-Dp3PX", - "name": "dataframe", + "dataType": "TextInput", + "id": "TextInput-CTKlr", + "name": "text", "output_types": [ - "DataFrame" + "Message" ] }, "targetHandle": { - "fieldName": "data_inputs", - "id": "ChunkDoclingDocument-7Tsav", + "fieldName": "dynamic_allowed_groups", + "id": "AdvancedDynamicFormBuilder-81Exw", "inputTypes": [ - "Data", - "DataFrame" + "Text", + "Message" ], - "type": "other" + "type": "str" } }, - "id": "xy-edge__DoclingRemote-Dp3PX{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-ChunkDoclingDocument-7Tsav{œfieldNameœ:œdata_inputsœ,œidœ:œChunkDoclingDocument-7Tsavœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "id": "xy-edge__TextInput-CTKlr{œdataTypeœ:œTextInputœ,œidœ:œTextInput-CTKlrœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_allowed_groupsœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", "selected": false, - "source": "DoclingRemote-Dp3PX", - "sourceHandle": "{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", - "target": "ChunkDoclingDocument-7Tsav", - "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œChunkDoclingDocument-7Tsavœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + "source": "TextInput-CTKlr", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-CTKlrœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_allowed_groupsœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" }, { "animated": false, "className": "", "data": { "sourceHandle": { - "dataType": "DataFrameOperations", - "id": "DataFrameOperations-N80fC", - "name": "output", + "dataType": "TextInput", + "id": "TextInput-hlgVv", + "name": "text", "output_types": [ - "DataFrame" + "Message" ] }, "targetHandle": { - "fieldName": "ingest_data", - "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", + "fieldName": "dynamic_allowed_users", + "id": "AdvancedDynamicFormBuilder-81Exw", "inputTypes": [ - "Data", - "DataFrame" + "Text", + "Message" ], - "type": "other" + "type": "str" } }, - "id": "xy-edge__DataFrameOperations-N80fC{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "id": "xy-edge__TextInput-hlgVv{œdataTypeœ:œTextInputœ,œidœ:œTextInput-hlgVvœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_allowed_usersœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", "selected": false, - "source": "DataFrameOperations-N80fC", - "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", - "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", - "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + "source": "TextInput-hlgVv", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-hlgVvœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_allowed_usersœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" }, { "animated": false, "className": "", "data": { "sourceHandle": { - "dataType": "ChunkDoclingDocument", - "id": "ChunkDoclingDocument-7Tsav", - "name": "dataframe", + "dataType": "TextInput", + "id": "TextInput-OGCeZ", + "name": "text", "output_types": [ - "DataFrame" + "Message" ] }, "targetHandle": { - "fieldName": "df", - "id": "DataFrameOperations-1BWXB", + "fieldName": "dynamic_connector_type", + "id": "AdvancedDynamicFormBuilder-81Exw", "inputTypes": [ - "DataFrame" + "Text", + "Message" ], - "type": "other" + "type": "str" } }, - "id": "xy-edge__ChunkDoclingDocument-7Tsav{œdataTypeœ:œChunkDoclingDocumentœ,œidœ:œChunkDoclingDocument-7Tsavœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-1BWXB{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "id": "xy-edge__TextInput-OGCeZ{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OGCeZœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_connector_typeœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", "selected": false, - "source": "ChunkDoclingDocument-7Tsav", - "sourceHandle": "{œdataTypeœ:œChunkDoclingDocumentœ,œidœ:œChunkDoclingDocument-7Tsavœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", - "target": "DataFrameOperations-1BWXB", - "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" - } - ], - "nodes": [ + "source": "TextInput-OGCeZ", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OGCeZœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_connector_typeœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" + }, { + "animated": false, + "className": "", "data": { - "id": "AdvancedDynamicFormBuilder-81Exw", - "node": { - "base_classes": [ - "Data", + "sourceHandle": { + "dataType": "TextInput", + "id": "TextInput-PI6at", + "name": "text", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "dynamic_document_id", + "id": "AdvancedDynamicFormBuilder-81Exw", + "inputTypes": [ + "Text", "Message" ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Creates dynamic input fields that can receive data from other components or manual input.", - "display_name": "Create Data", - "documentation": "", - "edited": true, - "field_order": [ - "form_fields", - "include_metadata" - ], - "frozen": false, - "icon": "braces", - "last_updated": "2026-02-05T15:22:55.809Z", - "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": {}, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Data", - "group_outputs": false, - "hidden": null, - "loop_types": null, - "method": "process_form", - "name": "form_data", - "options": null, - "required_inputs": null, - "selected": "Data", - "tool_mode": true, - "types": [ - "Data" - ], - "value": "__UNDEFINED__" - }, - { - "allows_loop": false, - "cache": true, - "display_name": "Message", - "group_outputs": false, - "hidden": null, - "loop_types": null, - "method": "get_message", - "name": "message", - "options": null, - "required_inputs": null, - "selected": "Message", - "tool_mode": true, - "types": [ - "Message" - ], - "value": "__UNDEFINED__" - } + "type": "str" + } + }, + "id": "xy-edge__TextInput-PI6at{œdataTypeœ:œTextInputœ,œidœ:œTextInput-PI6atœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_document_idœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "TextInput-PI6at", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-PI6atœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_document_idœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "TextInput", + "id": "TextInput-gRPNR", + "name": "text", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "dynamic_owner", + "id": "AdvancedDynamicFormBuilder-81Exw", + "inputTypes": [ + "Text", + "Message" ], - "pinned": false, - "template": { - "_frontend_node_flow_id": { - "value": "5488df7c-b93f-4f87-a446-b67028bc0813" - }, - "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" - }, + "type": "str" + } + }, + "id": "xy-edge__TextInput-gRPNR{œdataTypeœ:œTextInputœ,œidœ:œTextInput-gRPNRœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_ownerœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "TextInput-gRPNR", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-gRPNRœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_ownerœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "TextInput", + "id": "TextInput-lTHSx", + "name": "text", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "dynamic_owner_email", + "id": "AdvancedDynamicFormBuilder-81Exw", + "inputTypes": [ + "Text", + "Message" + ], + "type": "str" + } + }, + "id": "xy-edge__TextInput-lTHSx{œdataTypeœ:œTextInputœ,œidœ:œTextInput-lTHSxœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_owner_emailœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "TextInput-lTHSx", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-lTHSxœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_owner_emailœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "TextInput", + "id": "TextInput-UZQ8v", + "name": "text", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "dynamic_source_url", + "id": "AdvancedDynamicFormBuilder-81Exw", + "inputTypes": [ + "Text", + "Message" + ], + "type": "str" + } + }, + "id": "xy-edge__TextInput-UZQ8v{œdataTypeœ:œTextInputœ,œidœ:œTextInput-UZQ8vœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_source_urlœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "TextInput-UZQ8v", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-UZQ8vœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_source_urlœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "TextInput", + "id": "TextInput-68n9L", + "name": "text", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "dynamic_owner_name", + "id": "AdvancedDynamicFormBuilder-81Exw", + "inputTypes": [ + "Text", + "Message" + ], + "type": "str" + } + }, + "id": "xy-edge__TextInput-68n9L{œdataTypeœ:œTextInputœ,œidœ:œTextInput-68n9Lœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-AdvancedDynamicFormBuilder-81Exw{œfieldNameœ:œdynamic_owner_nameœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "TextInput-68n9L", + "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-68n9Lœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}", + "target": "AdvancedDynamicFormBuilder-81Exw", + "targetHandle": "{œfieldNameœ:œdynamic_owner_nameœ,œidœ:œAdvancedDynamicFormBuilder-81Exwœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-N80fC", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "ingest_data", + "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", + "inputTypes": [ + "Data", + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-N80fC{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DataFrameOperations-N80fC", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-N80fCœ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", + "targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "ChunkDoclingDocument", + "id": "ChunkDoclingDocument-DdOYd", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-1BWXB", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__ChunkDoclingDocument-DdOYd{œdataTypeœ:œChunkDoclingDocumentœ,œidœ:œChunkDoclingDocument-DdOYdœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-1BWXB{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "ChunkDoclingDocument-DdOYd", + "sourceHandle": "{œdataTypeœ:œChunkDoclingDocumentœ,œidœ:œChunkDoclingDocument-DdOYdœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-1BWXB", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-1BWXBœ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DoclingRemote", + "id": "DoclingRemote-Dp3PX", + "name": "dataframe", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "data_inputs", + "id": "ChunkDoclingDocument-DdOYd", + "inputTypes": [ + "Data", + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__DoclingRemote-Dp3PX{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}-ChunkDoclingDocument-DdOYd{œfieldNameœ:œdata_inputsœ,œidœ:œChunkDoclingDocument-DdOYdœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DoclingRemote-Dp3PX", + "sourceHandle": "{œdataTypeœ:œDoclingRemoteœ,œidœ:œDoclingRemote-Dp3PXœ,œnameœ:œdataframeœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "ChunkDoclingDocument-DdOYd", + "targetHandle": "{œfieldNameœ:œdata_inputsœ,œidœ:œChunkDoclingDocument-DdOYdœ,œinputTypesœ:[œDataœ,œDataFrameœ],œtypeœ:œotherœ}" + } + ], + "nodes": [ + { + "data": { + "id": "AdvancedDynamicFormBuilder-81Exw", + "node": { + "base_classes": [ + "Data", + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Creates dynamic input fields that can receive data from other components or manual input.", + "display_name": "Create Data", + "documentation": "", + "edited": true, + "field_order": [ + "form_fields", + "include_metadata" + ], + "frozen": false, + "icon": "braces", + "last_updated": "2026-03-12T15:33:27.314Z", + "legacy": false, + "lf_version": "1.7.0.dev21", + "metadata": {}, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Data", + "group_outputs": false, + "hidden": null, + "loop_types": null, + "method": "process_form", + "name": "form_data", + "options": null, + "required_inputs": null, + "selected": "Data", + "tool_mode": true, + "types": [ + "Data" + ], + "value": "__UNDEFINED__" + }, + { + "allows_loop": false, + "cache": true, + "display_name": "Message", + "group_outputs": false, + "hidden": null, + "loop_types": null, + "method": "get_message", + "name": "message", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "573ff6c4-003c-4e1d-aa3f-8a83e1f3e020" + }, "_type": "Component", "code": { "advanced": true, @@ -465,12 +581,12 @@ "type": "code", "value": "from typing import Any\r\n\r\nfrom langflow.custom import Component\r\nfrom langflow.io import (\r\n BoolInput,\r\n FloatInput,\r\n HandleInput,\r\n IntInput,\r\n MultilineInput,\r\n Output,\r\n StrInput,\r\n TableInput,\r\n)\r\nfrom langflow.schema.data import Data\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass CrateData(Component):\r\n \"\"\"Dynamic Form Component\r\n\r\n This component creates dynamic inputs that can receive data from other components\r\n or be filled manually. It demonstrates advanced dynamic input functionality with\r\n component connectivity.\r\n\r\n ## Features\r\n - **Dynamic Input Generation**: Create inputs based on table configuration\r\n - **Component Connectivity**: Inputs can receive data from other components\r\n - **Multiple Input Types**: Support for text, number, boolean, and handle inputs\r\n - **Flexible Data Sources**: Manual input OR component connections\r\n - **Real-time Updates**: Form fields update immediately when table changes\r\n - **Multiple Output Formats**: Data and formatted Message outputs\r\n - **JSON Output**: Collects all dynamic inputs into a structured JSON response\r\n\r\n ## Use Cases\r\n - Dynamic API parameter collection from multiple sources\r\n - Variable data aggregation from different components\r\n - Flexible pipeline configuration\r\n - Multi-source data processing\r\n\r\n ## Field Types Available\r\n - **text**: Single-line text input (can connect to Text/String outputs)\r\n - **multiline**: Multi-line text input (can connect to Text outputs)\r\n - **number**: Integer input (can connect to Number outputs)\r\n - **float**: Decimal number input (can connect to Number outputs)\r\n - **boolean**: True/false checkbox (can connect to Boolean outputs)\r\n - **handle**: Generic data input (can connect to any component output)\r\n - **data**: Structured data input (can connect to Data outputs)\r\n\r\n ## Input Types for Connections\r\n - **Text**: Text/String data from components\r\n - **Data**: Structured data objects\r\n - **Message**: Message objects with text content\r\n - **Number**: Numeric values\r\n - **Boolean**: True/false values\r\n - **Any**: Accepts any type of connection\r\n - **Combinations**: Text,Message | Data,Text | Text,Data,Message | etc.\r\n \"\"\"\r\n\r\n display_name = \"Create Data\"\r\n description = \"Creates dynamic input fields that can receive data from other components or manual input.\"\r\n icon = \"braces\"\r\n name = \"AdvancedDynamicFormBuilder\"\r\n\r\n def __init__(self, **kwargs):\r\n super().__init__(**kwargs)\r\n self._dynamic_inputs = {}\r\n\r\n inputs = [\r\n TableInput(\r\n name=\"form_fields\",\r\n display_name=\"Input Configuration\",\r\n info=\"Define the dynamic form fields. Each row creates a new input field that can connect to other components.\",\r\n table_schema=[\r\n {\r\n \"name\": \"field_name\",\r\n \"display_name\": \"Field Name\",\r\n \"type\": \"str\",\r\n \"description\": \"Name for the field (used as both internal name and display label)\",\r\n },\r\n {\r\n \"name\": \"field_type\",\r\n \"display_name\": \"Field Type\",\r\n \"type\": \"str\",\r\n \"description\": \"Type of input field to create\",\r\n \"options\": [\"Text\", \"Data\", \"Number\", \"Handle\", \"Boolean\"],\r\n \"value\": \"Text\",\r\n },\r\n ],\r\n value=[{\"field_name\": \"field_name\", \"field_type\": \"Text\"}],\r\n real_time_refresh=True,\r\n ),\r\n BoolInput(\r\n name=\"include_metadata\",\r\n display_name=\"Include Metadata\",\r\n info=\"Include form configuration metadata in the output.\",\r\n value=False,\r\n advanced=True,\r\n ),\r\n ]\r\n\r\n outputs = [\r\n Output(display_name=\"Data\", name=\"form_data\", method=\"process_form\"),\r\n Output(display_name=\"Message\", name=\"message\", method=\"get_message\"),\r\n ]\r\n\r\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str = None) -> dict:\r\n \"\"\"Update build configuration to add dynamic inputs that can connect to other components.\"\"\"\r\n if field_name == \"form_fields\":\r\n # Store current values before clearing dynamic inputs\r\n current_values = {}\r\n keys_to_remove = [key for key in build_config if key.startswith(\"dynamic_\")]\r\n for key in keys_to_remove:\r\n # Preserve the current value before deletion\r\n if hasattr(self, key):\r\n current_values[key] = getattr(self, key)\r\n del build_config[key]\r\n\r\n # Add dynamic inputs based on table configuration\r\n # Safety check to ensure field_value is not None and is iterable\r\n if field_value is None:\r\n field_value = []\r\n\r\n for i, field_config in enumerate(field_value):\r\n # Safety check to ensure field_config is not None\r\n if field_config is None:\r\n continue\r\n\r\n field_name = field_config.get(\"field_name\", f\"field_{i}\")\r\n display_name = field_name # Use field_name as display_name\r\n field_type_option = field_config.get(\"field_type\", \"Text\")\r\n default_value = \"\" # All fields have empty default value\r\n required = False # All fields are optional by default\r\n help_text = \"\" # All fields have empty help text\r\n\r\n # Map field type options to actual field types and input types\r\n field_type_mapping = {\r\n \"Text\": {\"field_type\": \"multiline\", \"input_types\": [\"Text\", \"Message\"]},\r\n \"Data\": {\"field_type\": \"data\", \"input_types\": [\"Data\"]},\r\n \"Number\": {\"field_type\": \"number\", \"input_types\": [\"Text\", \"Message\"]},\r\n \"Handle\": {\"field_type\": \"handle\", \"input_types\": [\"Text\", \"Data\", \"Message\"]},\r\n \"Boolean\": {\"field_type\": \"boolean\", \"input_types\": None},\r\n }\r\n\r\n field_config_mapped = field_type_mapping.get(\r\n field_type_option, {\"field_type\": \"text\", \"input_types\": []}\r\n )\r\n field_type = field_config_mapped[\"field_type\"]\r\n input_types_list = field_config_mapped[\"input_types\"]\r\n\r\n # Create the appropriate input type based on field_type\r\n dynamic_input_name = f\"dynamic_{field_name}\"\r\n\r\n if field_type == \"text\":\r\n # Use preserved value if available, otherwise use default\r\n current_value = current_values.get(dynamic_input_name, default_value)\r\n if current_value is None:\r\n current_value = default_value\r\n \r\n if input_types_list:\r\n build_config[dynamic_input_name] = StrInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=f\"{help_text} (Can connect to: {', '.join(input_types_list)})\",\r\n value=current_value,\r\n required=required,\r\n input_types=input_types_list,\r\n )\r\n else:\r\n build_config[dynamic_input_name] = StrInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=help_text,\r\n value=current_value,\r\n required=required,\r\n )\r\n\r\n elif field_type == \"multiline\":\r\n # Use preserved value if available, otherwise use default\r\n current_value = current_values.get(dynamic_input_name, default_value)\r\n if current_value is None:\r\n current_value = default_value\r\n \r\n if input_types_list:\r\n build_config[dynamic_input_name] = MultilineInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=f\"{help_text} (Can connect to: {', '.join(input_types_list)})\",\r\n value=current_value,\r\n required=required,\r\n input_types=input_types_list,\r\n )\r\n else:\r\n build_config[dynamic_input_name] = MultilineInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=help_text,\r\n value=current_value,\r\n required=required,\r\n )\r\n\r\n elif field_type == \"number\":\r\n # Use preserved value if available, otherwise use default\r\n current_value = current_values.get(dynamic_input_name, default_value)\r\n if current_value is None:\r\n current_value = default_value\r\n \r\n try:\r\n if current_value:\r\n current_int = int(current_value)\r\n else:\r\n current_int = 0\r\n except (ValueError, TypeError):\r\n try:\r\n current_int = int(default_value) if default_value else 0\r\n except ValueError:\r\n current_int = 0\r\n\r\n if input_types_list:\r\n build_config[dynamic_input_name] = IntInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=f\"{help_text} (Can connect to: {', '.join(input_types_list)})\",\r\n value=current_int,\r\n required=required,\r\n input_types=input_types_list,\r\n )\r\n else:\r\n build_config[dynamic_input_name] = IntInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=help_text,\r\n value=current_int,\r\n required=required,\r\n )\r\n\r\n elif field_type == \"float\":\r\n # Use preserved value if available, otherwise use default\r\n current_value = current_values.get(dynamic_input_name, default_value)\r\n if current_value is None:\r\n current_value = default_value\r\n \r\n try:\r\n if current_value:\r\n current_float = float(current_value)\r\n else:\r\n current_float = 0.0\r\n except (ValueError, TypeError):\r\n try:\r\n current_float = float(default_value) if default_value else 0.0\r\n except ValueError:\r\n current_float = 0.0\r\n\r\n if input_types_list:\r\n build_config[dynamic_input_name] = FloatInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=f\"{help_text} (Can connect to: {', '.join(input_types_list)})\",\r\n value=current_float,\r\n required=required,\r\n input_types=input_types_list,\r\n )\r\n else:\r\n build_config[dynamic_input_name] = FloatInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=help_text,\r\n value=current_float,\r\n required=required,\r\n )\r\n\r\n elif field_type == \"boolean\":\r\n # Use preserved value if available, otherwise use default\r\n current_value = current_values.get(dynamic_input_name, default_value)\r\n if current_value is None:\r\n current_value = default_value\r\n \r\n # Convert current value to boolean\r\n if isinstance(current_value, bool):\r\n current_bool = current_value\r\n else:\r\n current_bool = str(current_value).lower() in [\"true\", \"1\", \"yes\"] if current_value else False\r\n\r\n # Boolean fields don't use input_types parameter to avoid errors\r\n build_config[dynamic_input_name] = BoolInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=help_text,\r\n value=current_bool,\r\n input_types=[],\r\n required=required,\r\n )\r\n\r\n elif field_type == \"handle\":\r\n # HandleInput for generic data connections\r\n build_config[dynamic_input_name] = HandleInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=f\"{help_text} (Accepts: {', '.join(input_types_list) if input_types_list else 'Any'})\",\r\n input_types=input_types_list if input_types_list else [\"Data\", \"Text\", \"Message\"],\r\n required=required,\r\n )\r\n\r\n elif field_type == \"data\":\r\n # Specialized for Data type connections\r\n build_config[dynamic_input_name] = HandleInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=f\"{help_text} (Data input)\",\r\n input_types=[\"Data\"] if not input_types_list else input_types_list,\r\n required=required,\r\n )\r\n\r\n else:\r\n # Default to text input for unknown types\r\n # Use preserved value if available, otherwise use default\r\n current_value = current_values.get(dynamic_input_name, default_value)\r\n if current_value is None:\r\n current_value = default_value\r\n \r\n build_config[dynamic_input_name] = StrInput(\r\n name=dynamic_input_name,\r\n display_name=display_name,\r\n info=f\"{help_text} (Unknown type '{field_type}', defaulting to text)\",\r\n value=current_value,\r\n required=required,\r\n )\r\n\r\n return build_config\r\n\r\n def get_dynamic_values(self) -> dict[str, Any]:\r\n \"\"\"Extract simple values from all dynamic inputs, handling both manual and connected inputs.\"\"\"\r\n dynamic_values = {}\r\n connection_info = {}\r\n form_fields = getattr(self, \"form_fields\", [])\r\n\r\n for field_config in form_fields:\r\n # Safety check to ensure field_config is not None\r\n if field_config is None:\r\n continue\r\n\r\n field_name = field_config.get(\"field_name\", \"\")\r\n if field_name:\r\n dynamic_input_name = f\"dynamic_{field_name}\"\r\n value = getattr(self, dynamic_input_name, None)\r\n\r\n # Extract simple values from connections or manual input\r\n if value is not None:\r\n try:\r\n extracted_value = self._extract_simple_value(value)\r\n dynamic_values[field_name] = extracted_value\r\n\r\n # Determine connection type for status\r\n if hasattr(value, \"text\") and hasattr(value, \"timestamp\"):\r\n connection_info[field_name] = \"Connected (Message)\"\r\n elif hasattr(value, \"data\"):\r\n connection_info[field_name] = \"Connected (Data)\"\r\n elif isinstance(value, (str, int, float, bool, list, dict)):\r\n connection_info[field_name] = \"Manual input\"\r\n else:\r\n connection_info[field_name] = \"Connected (Object)\"\r\n\r\n except Exception:\r\n # Fallback to string representation if all else fails\r\n dynamic_values[field_name] = str(value)\r\n connection_info[field_name] = \"Error\"\r\n else:\r\n # Use empty default value if nothing connected\r\n dynamic_values[field_name] = \"\"\r\n connection_info[field_name] = \"Empty default\"\r\n\r\n # Store connection info for status output\r\n self._connection_info = connection_info\r\n return dynamic_values\r\n\r\n def _extract_simple_value(self, value: Any) -> Any:\r\n \"\"\"Extract the simplest, most useful value from any input type.\"\"\"\r\n # Handle None\r\n if value is None:\r\n return None\r\n\r\n # Handle simple types directly\r\n if isinstance(value, (str, int, float, bool)):\r\n return value\r\n\r\n # Handle lists and tuples - keep simple\r\n if isinstance(value, (list, tuple)):\r\n return [self._extract_simple_value(item) for item in value]\r\n\r\n # Handle dictionaries - keep simple\r\n if isinstance(value, dict):\r\n return {str(k): self._extract_simple_value(v) for k, v in value.items()}\r\n\r\n # Handle Message objects - extract only the text\r\n if hasattr(value, \"text\"):\r\n return str(value.text) if value.text is not None else \"\"\r\n\r\n # Handle Data objects - extract the data content\r\n if hasattr(value, \"data\") and value.data is not None:\r\n return self._extract_simple_value(value.data)\r\n\r\n # For any other object, convert to string\r\n return str(value)\r\n\r\n def process_form(self) -> Data:\r\n \"\"\"Process all dynamic form inputs and return clean data with just field values.\"\"\"\r\n # Get all dynamic values (just the key:value pairs)\r\n dynamic_values = self.get_dynamic_values()\r\n\r\n # Update status with connection info\r\n connected_fields = len([v for v in getattr(self, \"_connection_info\", {}).values() if \"Connected\" in v])\r\n total_fields = len(dynamic_values)\r\n\r\n self.status = f\"Form processed successfully. {connected_fields}/{total_fields} fields connected to components.\"\r\n\r\n # Return clean Data object with just the field values\r\n return Data(data=dynamic_values)\r\n\r\n def get_message(self) -> Message:\r\n \"\"\"Return form data as a formatted text message.\"\"\"\r\n # Get all dynamic values\r\n dynamic_values = self.get_dynamic_values()\r\n\r\n if not dynamic_values:\r\n return Message(text=\"No form data available\")\r\n\r\n # Format as text message\r\n message_lines = [\"📋 Form Data:\"]\r\n message_lines.append(\"=\" * 40)\r\n\r\n for field_name, value in dynamic_values.items():\r\n # Use field_name as display_name\r\n display_name = field_name\r\n\r\n message_lines.append(f\"• {display_name}: {value}\")\r\n\r\n message_lines.append(\"=\" * 40)\r\n message_lines.append(f\"Total fields: {len(dynamic_values)}\")\r\n\r\n message_text = \"\\n\".join(message_lines)\r\n self.status = f\"Message formatted with {len(dynamic_values)} fields\"\r\n\r\n return Message(text=message_text)" }, - "dynamic_connector_type": { + "dynamic_allowed_groups": { "_input_type": "MultilineInput", "advanced": false, "ai_enabled": false, "copy_field": false, - "display_name": "connector_type", + "display_name": "allowed_groups", "dynamic": false, "helper_text": null, "info": " (Can connect to: Text, Message)", @@ -482,8 +598,9 @@ "list_add_label": "Add More", "load_from_db": false, "multiline": true, - "name": "dynamic_connector_type", + "name": "dynamic_allowed_groups", "override_skip": false, + "password": false, "placeholder": "", "real_time_refresh": null, "refresh_button": null, @@ -498,12 +615,12 @@ "type": "str", "value": "" }, - "dynamic_owner": { + "dynamic_allowed_users": { "_input_type": "MultilineInput", "advanced": false, "ai_enabled": false, "copy_field": false, - "display_name": "owner", + "display_name": "allowed_users", "dynamic": false, "helper_text": null, "info": " (Can connect to: Text, Message)", @@ -515,8 +632,9 @@ "list_add_label": "Add More", "load_from_db": false, "multiline": true, - "name": "dynamic_owner", + "name": "dynamic_allowed_users", "override_skip": false, + "password": false, "placeholder": "", "real_time_refresh": null, "refresh_button": null, @@ -531,12 +649,12 @@ "type": "str", "value": "" }, - "dynamic_owner_email": { + "dynamic_connector_type": { "_input_type": "MultilineInput", "advanced": false, "ai_enabled": false, "copy_field": false, - "display_name": "owner_email", + "display_name": "connector_type", "dynamic": false, "helper_text": null, "info": " (Can connect to: Text, Message)", @@ -548,8 +666,9 @@ "list_add_label": "Add More", "load_from_db": false, "multiline": true, - "name": "dynamic_owner_email", + "name": "dynamic_connector_type", "override_skip": false, + "password": false, "placeholder": "", "real_time_refresh": null, "refresh_button": null, @@ -564,12 +683,12 @@ "type": "str", "value": "" }, - "dynamic_owner_name": { + "dynamic_document_id": { "_input_type": "MultilineInput", "advanced": false, "ai_enabled": false, "copy_field": false, - "display_name": "owner_name", + "display_name": "document_id", "dynamic": false, "helper_text": null, "info": " (Can connect to: Text, Message)", @@ -581,8 +700,9 @@ "list_add_label": "Add More", "load_from_db": false, "multiline": true, - "name": "dynamic_owner_name", + "name": "dynamic_document_id", "override_skip": false, + "password": false, "placeholder": "", "real_time_refresh": null, "refresh_button": null, @@ -597,12 +717,148 @@ "type": "str", "value": "" }, - "form_fields": { - "_input_type": "TableInput", + "dynamic_owner": { + "_input_type": "MultilineInput", "advanced": false, - "display_name": "Input Configuration", + "ai_enabled": false, + "copy_field": false, + "display_name": "owner", "dynamic": false, - "info": "Define the dynamic form fields. Each row creates a new input field that can connect to other components.", + "helper_text": null, + "info": " (Can connect to: Text, Message)", + "input_types": [ + "Text", + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "dynamic_owner", + "override_skip": false, + "password": false, + "placeholder": "", + "real_time_refresh": null, + "refresh_button": null, + "refresh_button_text": null, + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "dynamic_owner_email": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "owner_email", + "dynamic": false, + "helper_text": null, + "info": " (Can connect to: Text, Message)", + "input_types": [ + "Text", + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "dynamic_owner_email", + "override_skip": false, + "password": false, + "placeholder": "", + "real_time_refresh": null, + "refresh_button": null, + "refresh_button_text": null, + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "dynamic_owner_name": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "owner_name", + "dynamic": false, + "helper_text": null, + "info": " (Can connect to: Text, Message)", + "input_types": [ + "Text", + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "dynamic_owner_name", + "override_skip": false, + "password": false, + "placeholder": "", + "real_time_refresh": null, + "refresh_button": null, + "refresh_button_text": null, + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "dynamic_source_url": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "source_url", + "dynamic": false, + "helper_text": null, + "info": " (Can connect to: Text, Message)", + "input_types": [ + "Text", + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "dynamic_source_url", + "override_skip": false, + "password": false, + "placeholder": "", + "real_time_refresh": null, + "refresh_button": null, + "refresh_button_text": null, + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "form_fields": { + "_input_type": "TableInput", + "advanced": false, + "display_name": "Input Configuration", + "dynamic": false, + "info": "Define the dynamic form fields. Each row creates a new input field that can connect to other components.", "is_list": true, "list_add_label": "Add More", "load_from_db": false, @@ -671,6 +927,22 @@ { "field_name": "connector_type", "field_type": "Text" + }, + { + "field_name": "source_url", + "field_type": "Text" + }, + { + "field_name": "document_id", + "field_type": "Text" + }, + { + "field_name": "allowed_users", + "field_type": "Text" + }, + { + "field_name": "allowed_groups", + "field_type": "Text" } ] }, @@ -703,62 +975,152 @@ "dragging": false, "id": "AdvancedDynamicFormBuilder-81Exw", "measured": { - "height": 552, + "height": 881, "width": 320 }, "position": { - "x": 1778.514096901592, - "y": 673.5870187063417 + "x": 1331.3793686192853, + "y": 868.8499669098999 }, "selected": false, "type": "genericNode" }, { "data": { - "id": "SecretInput-F34VJ", + "description": "Uses Docling to process input documents connecting to your instance of Docling Serve.", + "display_name": "Docling Serve", + "id": "DoclingRemote-Dp3PX", "node": { "base_classes": [ - "Message" + "DataFrame" ], "beta": false, "conditional_paths": [], "custom_fields": {}, - "description": "Allows the selection of a secret to be generated as output..", - "display_name": "Secret Input", - "documentation": "https://docs.langflow.org/components-io#text-input", - "edited": true, + "description": "Uses Docling to process input documents connecting to your instance of Docling Serve.", + "display_name": "Docling Serve", + "documentation": "https://docling-project.github.io/docling/", + "edited": false, "field_order": [ - "input_value" + "path", + "file_path", + "separator", + "silent_errors", + "delete_server_file_after_processing", + "ignore_unsupported_extensions", + "ignore_unspecified_files", + "api_url", + "max_concurrency", + "max_poll_timeout", + "api_headers", + "docling_serve_opts" ], "frozen": false, - "icon": "type", + "icon": "Docling", + "last_updated": "2026-03-12T15:10:50.735Z", "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": {}, + "lf_version": "1.8.0", + "metadata": { + "code_hash": "409d771a961e", + "dependencies": { + "dependencies": [ + { + "name": "httpx", + "version": "0.28.1" + }, + { + "name": "docling_core", + "version": "2.60.1" + }, + { + "name": "pydantic", + "version": "2.11.10" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 4 + }, + "module": "custom_components.docling_serve" + }, "minimized": false, "output_types": [], "outputs": [ { "allows_loop": false, "cache": true, - "display_name": "Output Text", + "display_name": "Files", "group_outputs": false, "hidden": null, - "method": "text_response", - "name": "text", + "loop_types": null, + "method": "load_files", + "name": "dataframe", "options": null, "required_inputs": null, - "selected": "Message", + "selected": "DataFrame", "tool_mode": true, "types": [ - "Message" + "DataFrame" ], "value": "__UNDEFINED__" } ], "pinned": false, "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "573ff6c4-003c-4e1d-aa3f-8a83e1f3e020" + }, "_type": "Component", + "api_headers": { + "_input_type": "NestedDictInput", + "advanced": true, + "display_name": "HTTP headers", + "dynamic": false, + "info": "Optional dictionary of additional headers required for connecting to Docling Serve.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "name": "api_headers", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "NestedDict", + "value": {} + }, + "api_url": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Server address", + "dynamic": false, + "info": "URL of the Docling Serve instance.", + "list": false, + "list_add_label": "Add More", + "load_from_db": true, + "name": "api_url", + "override_skip": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "DOCLING_SERVE_URL" + }, "code": { "advanced": true, "dynamic": true, @@ -775,494 +1137,17 @@ "show": true, "title_case": false, "type": "code", - "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n" - }, - "input_value": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Secret", - "dynamic": false, - "info": "Secret to be passed as input.", - "input_types": [], - "load_from_db": true, - "name": "input_value", - "password": true, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "type": "str", - "value": "CONNECTOR_TYPE" - } - }, - "tool_mode": false - }, - "showNode": true, - "type": "SecretInput" - }, - "dragging": false, - "id": "SecretInput-F34VJ", - "measured": { - "height": 220, - "width": 320 - }, - "position": { - "x": 1343.2266447254406, - "y": 503.74766485111434 - }, - "selected": false, - "type": "genericNode" - }, - { - "data": { - "id": "SecretInput-b2cab", - "node": { - "base_classes": [ - "Message" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Allows the selection of a secret to be generated as output..", - "display_name": "Secret Input", - "documentation": "https://docs.langflow.org/components-io#text-input", - "edited": true, - "field_order": [ - "input_value" - ], - "frozen": false, - "icon": "type", - "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": {}, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Output Text", - "group_outputs": false, - "hidden": null, - "method": "text_response", - "name": "text", - "options": null, - "required_inputs": null, - "selected": "Message", - "tool_mode": true, - "types": [ - "Message" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n" - }, - "input_value": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Secret", - "dynamic": false, - "info": "Secret to be passed as input.", - "input_types": [], - "load_from_db": true, - "name": "input_value", - "password": true, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "type": "str", - "value": "OWNER" - } - }, - "tool_mode": false - }, - "showNode": true, - "type": "SecretInput" - }, - "dragging": false, - "id": "SecretInput-b2cab", - "measured": { - "height": 220, - "width": 320 - }, - "position": { - "x": 1341.4694747797632, - "y": 753.9209691638071 - }, - "selected": false, - "type": "genericNode" - }, - { - "data": { - "id": "SecretInput-ZVfuS", - "node": { - "base_classes": [ - "Message" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Allows the selection of a secret to be generated as output..", - "display_name": "Secret Input", - "documentation": "https://docs.langflow.org/components-io#text-input", - "edited": true, - "field_order": [ - "input_value" - ], - "frozen": false, - "icon": "type", - "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": {}, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Output Text", - "group_outputs": false, - "hidden": null, - "method": "text_response", - "name": "text", - "options": null, - "required_inputs": null, - "selected": "Message", - "tool_mode": true, - "types": [ - "Message" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n" - }, - "input_value": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Secret", - "dynamic": false, - "info": "Secret to be passed as input.", - "input_types": [], - "load_from_db": true, - "name": "input_value", - "password": true, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "type": "str", - "value": "OWNER_EMAIL" - } - }, - "tool_mode": false - }, - "showNode": true, - "type": "SecretInput" - }, - "dragging": false, - "id": "SecretInput-ZVfuS", - "measured": { - "height": 220, - "width": 320 - }, - "position": { - "x": 1336.2669044250051, - "y": 1000.9543699805594 - }, - "selected": false, - "type": "genericNode" - }, - { - "data": { - "id": "SecretInput-Iqtxd", - "node": { - "base_classes": [ - "Message" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Allows the selection of a secret to be generated as output..", - "display_name": "Secret Input", - "documentation": "https://docs.langflow.org/components-io#text-input", - "edited": true, - "field_order": [ - "input_value" - ], - "frozen": false, - "icon": "type", - "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": {}, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Output Text", - "group_outputs": false, - "hidden": null, - "method": "text_response", - "name": "text", - "options": null, - "required_inputs": null, - "selected": "Message", - "tool_mode": true, - "types": [ - "Message" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "from langflow.base.io.text import TextComponent\r\nfrom langflow.io import MultilineInput, Output, SecretStrInput\r\nfrom langflow.schema.message import Message\r\n\r\n\r\nclass SecretInputComponent(TextComponent):\r\n display_name = \"Secret Input\"\r\n description = \"Allows the selection of a secret to be generated as output..\"\r\n documentation: str = \"https://docs.langflow.org/components-io#text-input\"\r\n icon = \"type\"\r\n name = \"SecretInput\"\r\n\r\n inputs = [\r\n SecretStrInput(\r\n name=\"input_value\",\r\n display_name=\"Secret\",\r\n info=\"Secret to be passed as input.\",\r\n ),\r\n ]\r\n outputs = [\r\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\r\n ]\r\n\r\n def text_response(self) -> Message:\r\n return Message(\r\n text=self.input_value,\r\n )\r\n" - }, - "input_value": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "Secret", - "dynamic": false, - "info": "Secret to be passed as input.", - "input_types": [], - "load_from_db": true, - "name": "input_value", - "password": true, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "type": "str", - "value": "OWNER_NAME" - } - }, - "tool_mode": false - }, - "showNode": true, - "type": "SecretInput" - }, - "dragging": false, - "id": "SecretInput-Iqtxd", - "measured": { - "height": 220, - "width": 320 - }, - "position": { - "x": 1342.0034534756019, - "y": 1239.4741232342342 - }, - "selected": false, - "type": "genericNode" - }, - { - "data": { - "id": "DoclingRemote-Dp3PX", - "node": { - "base_classes": [ - "DataFrame" - ], - "beta": false, - "conditional_paths": [], - "custom_fields": {}, - "description": "Uses Docling to process input documents connecting to your instance of Docling Serve.", - "display_name": "Docling Serve", - "documentation": "https://docling-project.github.io/docling/", - "edited": true, - "field_order": [ - "path", - "file_path", - "separator", - "silent_errors", - "delete_server_file_after_processing", - "ignore_unsupported_extensions", - "ignore_unspecified_files", - "api_url", - "max_concurrency", - "max_poll_timeout", - "api_headers", - "docling_serve_opts" - ], - "frozen": false, - "icon": "Docling", - "legacy": false, - "lf_version": "1.7.0.dev21", - "metadata": { - "code_hash": "5723576d00e5", - "dependencies": { - "dependencies": [ - { - "name": "httpx", - "version": "0.28.1" - }, - { - "name": "docling_core", - "version": "2.49.0" - }, - { - "name": "pydantic", - "version": "2.11.10" - }, - { - "name": "lfx", - "version": "0.2.0.dev21" - } - ], - "total_dependencies": 4 - }, - "module": "custom_components.docling_serve" - }, - "minimized": false, - "output_types": [], - "outputs": [ - { - "allows_loop": false, - "cache": true, - "display_name": "Files", - "group_outputs": false, - "hidden": null, - "loop_types": null, - "method": "load_files", - "name": "dataframe", - "options": null, - "required_inputs": null, - "selected": "DataFrame", - "tool_mode": true, - "types": [ - "DataFrame" - ], - "value": "__UNDEFINED__" - } - ], - "pinned": false, - "template": { - "_type": "Component", - "api_headers": { - "_input_type": "NestedDictInput", - "advanced": true, - "display_name": "HTTP headers", - "dynamic": false, - "info": "Optional dictionary of additional headers required for connecting to Docling Serve.", - "list": false, - "list_add_label": "Add More", - "name": "api_headers", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "NestedDict", - "value": {} - }, - "api_url": { - "_input_type": "StrInput", - "advanced": false, - "display_name": "Server address", - "dynamic": false, - "info": "URL of the Docling Serve instance.", - "list": false, - "list_add_label": "Add More", - "load_from_db": true, - "name": "api_url", - "override_skip": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "DOCLING_SERVE_URL" - }, - "code": { - "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", - "list": false, - "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, - "placeholder": "", - "required": true, - "show": true, - "title_case": false, - "type": "code", - "value": "import base64\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n ),\n NestedDictInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional dictionary of additional headers required for connecting to Docling Serve.\"),\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n # Transform localhost URLs to container-accessible hosts when running in a container\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n # Check if processing exceeds the maximum poll timeout\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n # Call for a new status update\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task['task_id']}\")\n\n # Check if the status call gets into 5xx errors and retry\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n # Update task status\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task['task_id']}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n return Data(data={\"doc\": doc, \"file_path\": str(file_path)})\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self.api_headers) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n" + "value": "import base64\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n input_types=[\"Message\"],\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n input_types=[\"Message\"],\n ),\n NestedDictInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional dictionary of additional headers required for connecting to Docling Serve.\"),\n input_types=[\"Message\"],\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n input_types=[\"Message\"],\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n # Transform localhost URLs to container-accessible hosts when running in a container\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n # Check if processing exceeds the maximum poll timeout\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n # Call for a new status update\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task['task_id']}\")\n\n # Check if the status call gets into 5xx errors and retry\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n # Update task status\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task['task_id']}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n return Data(data={\"doc\": doc, \"file_path\": str(file_path)})\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self.api_headers) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n" }, "delete_server_file_after_processing": { "_input_type": "BoolInput", "advanced": true, "display_name": "Delete Server File After Processing", "dynamic": false, - "info": "If true, the Server File Path will be deleted after processing.", + "info": "If true, the File Path will be deleted after processing, if it is hosted on the Server.", "list": false, "list_add_label": "Add More", + "load_from_db": false, "name": "delete_server_file_after_processing", "override_skip": false, "placeholder": "", @@ -1281,6 +1166,9 @@ "display_name": "Docling options", "dynamic": false, "info": "Optional dictionary of additional options. See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", "load_from_db": false, @@ -1300,7 +1188,7 @@ "do_picture_classification": false, "do_picture_description": false, "do_table_structure": true, - "ocr_engine": "easyocr", + "ocr_engine": "ocrmac", "picture_description_local": { "prompt": "Describe this image in a few sentences.", "repo_id": "HuggingFaceTB/SmolVLM-256M-Instruct" @@ -1310,9 +1198,9 @@ "file_path": { "_input_type": "HandleInput", "advanced": true, - "display_name": "Server File Path", + "display_name": "File Path", "dynamic": false, - "info": "Data object with a 'file_path' property pointing to server file or a Message object with a path to the file. Supercedes 'Path' but supports same file types.", + "info": "Data object (or list of Data objects) with a 'file_path' property pointing to a file or a Message object with a path to the file or list of paths to the file. The file can be hosted on the server or on Langflow's file system. Supercedes 'Path' but supports same file types.", "input_types": [ "Data", "Message" @@ -1370,12 +1258,16 @@ "type": "bool", "value": true }, + "is_refresh": false, "max_concurrency": { "_input_type": "IntInput", "advanced": true, "display_name": "Concurrency", "dynamic": false, "info": "Maximum number of concurrent requests for the server.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", "name": "max_concurrency", @@ -1396,6 +1288,9 @@ "display_name": "Maximum poll time", "dynamic": false, "info": "Maximum waiting time for the document conversion to complete.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", "name": "max_poll_timeout", @@ -1513,18 +1408,18 @@ }, "tool_mode": false }, - "showNode": true, + "showNode": false, "type": "DoclingRemote" }, "dragging": false, "id": "DoclingRemote-Dp3PX", "measured": { - "height": 312, - "width": 320 + "height": 52, + "width": 192 }, "position": { - "x": -0.2871317656367012, - "y": 1594.9000391285042 + "x": 96.24733042785462, + "y": 1776.8766711328894 }, "selected": false, "type": "genericNode" @@ -1561,7 +1456,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2026-02-05T15:22:55.931Z", + "last_updated": "2026-02-06T17:43:52.579Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -1608,7 +1503,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" + "value": "c557e756-bc37-48fd-972d-366ef037c8cf" }, "_type": "Component", "ascending": { @@ -1964,18 +1859,18 @@ }, "tool_mode": false }, - "showNode": true, + "showNode": false, "type": "DataFrameOperations" }, "dragging": false, "id": "DataFrameOperations-1BWXB", "measured": { - "height": 399, - "width": 320 + "height": 52, + "width": 192 }, "position": { - "x": 826.5835128460332, - "y": 1595.137437547202 + "x": 705.5150898330688, + "y": 1778.4580906262447 }, "selected": false, "type": "genericNode" @@ -2012,7 +1907,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2026-02-05T15:22:55.932Z", + "last_updated": "2026-02-06T17:43:52.579Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2059,7 +1954,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" + "value": "c557e756-bc37-48fd-972d-366ef037c8cf" }, "_type": "Component", "ascending": { @@ -2415,18 +2310,18 @@ }, "tool_mode": false }, - "showNode": true, + "showNode": false, "type": "DataFrameOperations" }, "dragging": false, "id": "DataFrameOperations-N80fC", "measured": { - "height": 399, - "width": 320 + "height": 52, + "width": 192 }, "position": { - "x": 1697.325882655698, - "y": 1600.122711477673 + "x": 1318.1587449554054, + "y": 1780.9746899582965 }, "selected": false, "type": "genericNode" @@ -2463,7 +2358,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2026-02-05T15:22:55.932Z", + "last_updated": "2026-02-06T17:43:52.580Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2510,7 +2405,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" + "value": "c557e756-bc37-48fd-972d-366ef037c8cf" }, "_type": "Component", "ascending": { @@ -2866,18 +2761,18 @@ }, "tool_mode": false }, - "showNode": true, + "showNode": false, "type": "DataFrameOperations" }, "dragging": false, "id": "DataFrameOperations-9vMrp", "measured": { - "height": 399, - "width": 320 + "height": 52, + "width": 192 }, "position": { - "x": 1252.4926569056022, - "y": 1589.889268993829 + "x": 1015.5573550682316, + "y": 1779.5281179964784 }, "selected": false, "type": "genericNode" @@ -2901,8 +2796,8 @@ "width": 1000 }, "position": { - "x": -184.83853691310878, - "y": 944.7352701051674 + "x": -21.449808371464087, + "y": 1292.5429595750052 }, "resizing": true, "selected": false, @@ -2911,20 +2806,21 @@ }, { "data": { + "description": "Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. To search use the tools search_documents and raw_search. Search documents takes a query for vector search, for example\n {search_query: \"components in openrag\"}", + "display_name": "OpenSearch (Multi-Model Multi-Embedding)", "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", "node": { "base_classes": [ "Data", - "DataFrame", "VectorStore" ], "beta": false, "conditional_paths": [], "custom_fields": {}, - "description": "Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search.", + "description": "Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. To search use the tools search_documents and raw_search. Search documents takes a query for vector search, for example\n {search_query: \"components in openrag\"}", "display_name": "OpenSearch (Multi-Model Multi-Embedding)", "documentation": "", - "edited": true, + "edited": false, "field_order": [ "docs_metadata", "opensearch_url", @@ -2949,14 +2845,16 @@ "jwt_header", "bearer_prefix", "use_ssl", - "verify_certs" + "verify_certs", + "request_timeout", + "max_retries" ], "frozen": false, "icon": "OpenSearch", - "last_updated": "2025-12-03T21:41:24.832Z", + "last_updated": "2026-02-27T18:36:17.049Z", "legacy": false, "metadata": { - "code_hash": "00bd730431a2", + "code_hash": "6a3df45b55c5", "dependencies": { "dependencies": [ { @@ -2965,10 +2863,14 @@ }, { "name": "lfx", - "version": "0.2.0.dev21" + "version": null + }, + { + "name": "tenacity", + "version": "8.5.0" } ], - "total_dependencies": 2 + "total_dependencies": 3 }, "module": "custom_components.opensearch_multimodel_multiembedding" }, @@ -2996,18 +2898,18 @@ { "allows_loop": false, "cache": true, - "display_name": "DataFrame", + "display_name": "Raw Search", "group_outputs": false, "hidden": null, "loop_types": null, - "method": "as_dataframe", - "name": "dataframe", + "method": "raw_search", + "name": "raw_search", "options": null, "required_inputs": null, - "selected": "DataFrame", + "selected": "Data", "tool_mode": true, "types": [ - "DataFrame" + "Data" ], "value": "__UNDEFINED__" }, @@ -3036,28 +2938,200 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" + }, + "_type": "Component", + "auth_mode": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Authentication Mode", + "dynamic": false, + "external_options": {}, + "info": "Authentication method: 'basic' for username/password authentication, or 'jwt' for JSON Web Token (Bearer) authentication.", + "load_from_db": false, + "name": "auth_mode", + "options": [ + "basic", + "jwt" + ], + "options_metadata": [], + "override_skip": false, + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "str", + "value": "jwt" + }, + "bearer_prefix": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Prefix 'Bearer '", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "bearer_prefix", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport time\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"basic\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=True,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = raw_query\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Generate embeddings with rate-limit-aware retry logic using tenacity\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is retryable but not a rate limit error.\"\"\"\n # Retry on most exceptions except for specific non-retryable ones\n # Add other non-retryable exceptions here if needed\n return not is_rate_limit_error(exception)\n\n # Create retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Create retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n # Restrict concurrency for IBM/Watsonx models to avoid rate limits\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM: {is_ibm}\")\n\n # For IBM models, use sequential processing with rate limiting\n # For other models, use parallel processing\n vectors: list[list[float]] = [None] * len(texts)\n\n if is_ibm:\n # Sequential processing with inter-request delay for IBM models\n inter_request_delay = 0.6 # ~1.67 req/s, safely under 2 req/s limit\n logger.info(f\"Using sequential processing for IBM model with {inter_request_delay}s delay between requests\")\n\n for idx, chunk in enumerate(texts):\n if idx > 0:\n # Add delay between requests (but not before the first one)\n time.sleep(inter_request_delay)\n vectors[idx] = embed_chunk_with_retry(chunk, idx)\n else:\n # Parallel processing for non-IBM models\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = None\n if getattr(self, \"filter_expression\", \"\") and self.filter_expression.strip():\n try:\n filter_obj = json.loads(self.filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = (filter_obj or {}).get(\"limit\", self.number_of_results)\n score_threshold = (filter_obj or {}).get(\"score_threshold\", 0)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if isinstance(score_threshold, (int, float)) and score_threshold > 0:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Data format.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List of Data objects containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n return [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n" + }, + "docs_metadata": { + "_input_type": "TableInput", + "advanced": false, + "display_name": "Document Metadata", + "dynamic": false, + "info": "Additional metadata key-value pairs to be added to all ingested documents. Useful for tagging documents with source information, categories, or other custom attributes.", + "input_types": [ + "Data" + ], + "is_list": true, + "list_add_label": "Add More", + "name": "docs_metadata", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "table_icon": "Table", + "table_schema": [ + { + "description": "Key name", + "display_name": "Key", + "formatter": "text", + "name": "key", + "type": "str" + }, + { + "description": "Value of the metadata", + "display_name": "Value", + "formatter": "text", + "name": "value", + "type": "str" + } + ], + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "trigger_icon": "Table", + "trigger_text": "Open table", + "type": "table", + "value": [] + }, + "ef_construction": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "EF Construction", + "dynamic": false, + "info": "Size of the dynamic candidate list during index construction. Higher values improve recall but increase indexing time and memory usage.", + "list": false, + "list_add_label": "Add More", + "name": "ef_construction", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": 512 + }, + "embedding": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Embedding", + "dynamic": false, + "info": "", + "input_types": [ + "Embeddings" + ], + "list": true, + "list_add_label": "Add More", + "name": "embedding", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "other", + "value": "" + }, + "embedding_model_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Embedding Model Name", + "dynamic": false, + "info": "Name of the embedding model to use for ingestion. This selects which embedding from the list will be used to embed documents. Matches on deployment, model, model_id, or model_name. For duplicate deployments, use combined format: 'deployment:model' (e.g., 'text-embedding-ada-002:text-embedding-3-large'). Leave empty to use the first embedding. Error message will show all available identifiers.", + "list": false, + "list_add_label": "Add More", + "load_from_db": true, + "name": "embedding_model_name", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "SELECTED_EMBEDDING_MODEL" }, - "_type": "Component", - "auth_mode": { + "engine": { "_input_type": "DropdownInput", - "advanced": false, + "advanced": true, "combobox": false, "dialog_inputs": {}, - "display_name": "Authentication Mode", + "display_name": "Vector Engine", "dynamic": false, "external_options": {}, - "info": "Authentication method: 'basic' for username/password authentication, or 'jwt' for JSON Web Token (Bearer) authentication.", - "load_from_db": false, - "name": "auth_mode", + "info": "Vector search engine for similarity calculations. 'nmslib' works with standard OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.", + "name": "engine", "options": [ - "basic", - "jwt" + "nmslib", + "faiss", + "lucene", + "jvector" ], "options_metadata": [], "override_skip": false, "placeholder": "", - "real_time_refresh": true, "required": false, "show": true, "title_case": false, @@ -3066,17 +3140,131 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "jwt" + "value": "jvector" }, - "bearer_prefix": { - "_input_type": "BoolInput", + "filter_expression": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "Search Filters (JSON)", + "dynamic": false, + "info": "Optional JSON configuration for search filtering, result limits, and score thresholds.\n\nFormat 1 - Explicit filters:\n{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, {\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\n\nFormat 2 - Context-style mapping:\n{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\n\nUse __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "filter_expression", + "override_skip": false, + "password": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "index_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Index Name", + "dynamic": false, + "info": "The OpenSearch index name where documents will be stored and searched. Will be created automatically if it doesn't exist.", + "list": false, + "list_add_label": "Add More", + "load_from_db": true, + "name": "index_name", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "OPENSEARCH_INDEX_NAME" + }, + "ingest_data": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Ingest Data", + "dynamic": false, + "info": "", + "input_types": [ + "Data", + "DataFrame" + ], + "list": true, + "list_add_label": "Add More", + "name": "ingest_data", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "other", + "value": "" + }, + "is_refresh": false, + "jwt_header": { + "_input_type": "StrInput", "advanced": true, - "display_name": "Prefix 'Bearer '", + "display_name": "JWT Header Name", "dynamic": false, "info": "", "list": false, "list_add_label": "Add More", - "name": "bearer_prefix", + "load_from_db": false, + "name": "jwt_header", + "override_skip": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "Authorization" + }, + "jwt_token": { + "_input_type": "SecretStrInput", + "advanced": false, + "display_name": "JWT Token", + "dynamic": false, + "info": "Valid JSON Web Token for authentication. Will be sent in the Authorization header (with optional 'Bearer ' prefix).", + "input_types": [], + "load_from_db": true, + "name": "jwt_token", + "override_skip": false, + "password": true, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "track_in_telemetry": false, + "type": "str", + "value": "JWT" + }, + "m": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "M Parameter", + "dynamic": false, + "info": "Number of bidirectional connections for each vector in the HNSW graph. Higher values improve search quality but increase memory usage and indexing time.", + "list": false, + "list_add_label": "Add More", + "name": "m", "override_skip": false, "placeholder": "", "required": false, @@ -3085,78 +3273,120 @@ "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "bool", - "value": true + "type": "int", + "value": 16 }, - "code": { + "max_retries": { + "_input_type": "StrInput", "advanced": true, - "dynamic": true, - "fileTypes": [], - "file_path": "", - "info": "", + "display_name": "Max Retries", + "dynamic": false, + "info": "Number of retries for failed connections before raising an error.", "list": false, + "list_add_label": "Add More", "load_from_db": false, - "multiline": true, - "name": "code", - "password": false, + "name": "max_retries", + "override_skip": false, "placeholder": "", - "required": true, + "required": false, "show": true, "title_case": false, - "type": "code", - "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport time\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.inputs.inputs import DictInput\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search.\"\n \"To search use the tools search_documents and raw_search. Search documents takes a query for vector search, for example\\n\"\n \" {search_query: \\\"components in openrag\\\"}\"\n \"\\n\"\n \"you can also override the filter_expression to limit the hybrid query in search_documents by also passing filter_expression\\n\"\n \"for example:\\n\"\n \" {search_query: \\\"components in openrag\\\", filter_expression: {\\\"data_sources\\\":[\\\"my_doc.md\\\"],\\\"document_types\\\":[\\\"*\\\"],\\\"owners\\\":[\\\"*\\\"],\\\"connector_types\\\":[\\\"*\\\"]},\\\"limit\\\":10,\\\"scoreThreshold\\\":0}\"\n \"\\n\"\n \"raw_search takes actual opensearch queries for example:\"\n \" {\"\n \" \\\"size\\\": 100,\"\n \" \\\"query\\\": {\"\n \" \\\"term\\\": {\\\"filename\\\": \\\"my_doc.md\\\"}\"\n \" }\"\n \" \\\"_source\\\": [\\\"filename\\\", \\\"text\\\", \\\"page\\\"]\"\n \" }\"\n \"\\n\"\n \"or:\"\n \"\\n\"\n \" {\"\n \" \\\"size\\\": 0,\"\n \" \\\"aggs\\\": {\"\n \" \\\"distinct_filenames\\\": {\"\n \" \\\"cardinality\\\": {\\\"field\\\": \\\"filename\\\"}\"\n \" }\"\n \" },\"\n \" }\"\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"jvector\", \"nmslib\", \"faiss\", \"lucene\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'jvector' is recommended for most use cases. \"\n \"Note: Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"basic\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=True,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # DictInput(name=\"query\", display_name=\"Query\", input_types=[\"Data\"], is_list=False, tool_mode=True),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"DataFrame\", name=\"dataframe\", method=\"as_dataframe\"),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n query = self.search_query\n if isinstance(query, str):\n query = json.loads(query)\n client = self.build_client()\n logger.info(f\"query: {query}\")\n resp = client.search(\n index=self.index_name,\n body=query,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length (>3)\n return (\n isinstance(val, list) and len(val) > 100 and all(isinstance(x, (float, int)) for x in val)\n )\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except Exception as e:\n logger.warning(f\"Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.aerror(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"Starting ingestion mode...\")\n\n logger.warning(f\"Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Generate embeddings (threaded for concurrency) with retries\n def embed_chunk(chunk_text: str) -> list[float]:\n return selected_embedding.embed_documents([chunk_text])[0]\n\n vectors: list[list[float]] | None = None\n last_exception: Exception | None = None\n delay = 1.0\n attempts = 0\n max_attempts = 3\n\n while attempts < max_attempts:\n attempts += 1\n try:\n # Restrict concurrency for IBM/Watsonx models to avoid rate limits\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM: {is_ibm}\")\n max_workers = 1 if is_ibm else min(max(len(texts), 1), 8)\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk, chunk): idx for idx, chunk in enumerate(texts)}\n vectors = [None] * len(texts)\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n break\n except Exception as exc:\n last_exception = exc\n if attempts >= max_attempts:\n logger.error(\n f\"Embedding generation failed for model {embedding_model} after retries\",\n error=str(exc),\n )\n raise\n logger.warning(\n \"Threaded embedding generation failed for model %s (attempt %s/%s), retrying in %.1fs\",\n embedding_model,\n attempts,\n max_attempts,\n delay,\n )\n time.sleep(delay)\n delay = min(delay * 2, 8.0)\n\n if vectors is None:\n raise RuntimeError(\n f\"Embedding generation failed for {embedding_model}: {last_exception}\"\n if last_exception\n else f\"Embedding generation failed for {embedding_model}\"\n )\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error != \"resource_already_exists_exception\":\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = None\n if getattr(self, \"filter_expression\", \"\") and self.filter_expression.strip():\n try:\n filter_obj = json.loads(self.filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = (filter_obj or {}).get(\"limit\", self.number_of_results)\n score_threshold = (filter_obj or {}).get(\"score_threshold\", 0)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": \"filename\", \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if isinstance(score_threshold, (int, float)) and score_threshold > 0:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Data format.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List of Data objects containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n return [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n" + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "3" }, - "docs_metadata": { - "_input_type": "TableInput", + "num_candidates": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Candidate Pool Size", + "dynamic": false, + "info": "Number of approximate neighbors to consider for each KNN query. Some OpenSearch deployments do not support this parameter; set to 0 to disable.", + "list": false, + "list_add_label": "Add More", + "name": "num_candidates", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": 1000 + }, + "number_of_results": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Default Result Limit", + "dynamic": false, + "info": "Default maximum number of search results to return when no limit is specified in the filter expression.", + "list": false, + "list_add_label": "Add More", + "name": "number_of_results", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": 10 + }, + "opensearch_url": { + "_input_type": "StrInput", "advanced": false, - "display_name": "Document Metadata", + "display_name": "OpenSearch URL", "dynamic": false, - "info": "Additional metadata key-value pairs to be added to all ingested documents. Useful for tagging documents with source information, categories, or other custom attributes.", - "input_types": [ - "Data" - ], - "is_list": true, + "info": "The connection URL for your OpenSearch cluster (e.g., http://localhost:9200 for local development or your cloud endpoint).", + "list": false, "list_add_label": "Add More", - "name": "docs_metadata", + "load_from_db": true, + "name": "opensearch_url", "override_skip": false, "placeholder": "", "required": false, "show": true, - "table_icon": "Table", - "table_schema": [ - { - "description": "Key name", - "display_name": "Key", - "formatter": "text", - "name": "key", - "type": "str" - }, - { - "description": "Value of the metadata", - "display_name": "Value", - "formatter": "text", - "name": "value", - "type": "str" - } - ], "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "OPENSEARCH_URL" + }, + "password": { + "_input_type": "SecretStrInput", + "advanced": false, + "display_name": "OpenSearch Password", + "dynamic": false, + "info": "", + "input_types": [], + "load_from_db": false, + "name": "password", + "override_skip": false, + "password": true, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, "track_in_telemetry": false, - "trigger_icon": "Table", - "trigger_text": "Open table", - "type": "table", - "value": [] + "type": "str", + "value": "MyStrongOpenSearchPassword123!" }, - "ef_construction": { - "_input_type": "IntInput", + "request_timeout": { + "_input_type": "StrInput", "advanced": true, - "display_name": "EF Construction", + "display_name": "Request Timeout (seconds)", "dynamic": false, - "info": "Size of the dynamic candidate list during index construction. Higher values improve recall but increase indexing time and memory usage.", + "info": "Time in seconds to wait for a response from OpenSearch. Increase for large bulk ingestion or complex hybrid queries.", "list": false, "list_add_label": "Add More", - "name": "ef_construction", + "load_from_db": false, + "name": "request_timeout", "override_skip": false, "placeholder": "", "required": false, @@ -3164,42 +3394,44 @@ "title_case": false, "tool_mode": false, "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 512 + "track_in_telemetry": false, + "type": "str", + "value": "60" }, - "embedding": { - "_input_type": "HandleInput", + "search_query": { + "_input_type": "QueryInput", "advanced": false, - "display_name": "Embedding", + "display_name": "Search Query", "dynamic": false, - "info": "", + "info": "Enter a query to run a similarity search.", "input_types": [ - "Embeddings" + "Message" ], - "list": true, + "list": false, "list_add_label": "Add More", - "name": "embedding", + "load_from_db": false, + "name": "search_query", "override_skip": false, - "placeholder": "", + "placeholder": "Enter a query...", "required": false, "show": true, "title_case": false, + "tool_mode": true, + "trace_as_input": true, "trace_as_metadata": true, "track_in_telemetry": false, - "type": "other", + "type": "query", "value": "" }, - "embedding_model_name": { - "_input_type": "StrInput", - "advanced": false, - "display_name": "Embedding Model Name", + "should_cache_vector_store": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Cache Vector Store", "dynamic": false, - "info": "Name of the embedding model to use for ingestion. This selects which embedding from the list will be used to embed documents. Matches on deployment, model, model_id, or model_name. For duplicate deployments, use combined format: 'deployment:model' (e.g., 'text-embedding-ada-002:text-embedding-3-large'). Leave empty to use the first embedding. Error message will show all available identifiers.", + "info": "If True, the vector store will be cached for the current build of the component. This is useful for components that have multiple output methods and want to share the same vector store.", "list": false, "list_add_label": "Add More", - "load_from_db": true, - "name": "embedding_model_name", + "name": "should_cache_vector_store", "override_skip": false, "placeholder": "", "required": false, @@ -3207,25 +3439,26 @@ "title_case": false, "tool_mode": false, "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "SELECTED_EMBEDDING_MODEL" + "track_in_telemetry": true, + "type": "bool", + "value": true }, - "engine": { + "space_type": { "_input_type": "DropdownInput", "advanced": true, "combobox": false, "dialog_inputs": {}, - "display_name": "Vector Engine", + "display_name": "Distance Metric", "dynamic": false, "external_options": {}, - "info": "Vector search engine for similarity calculations. 'jvector' is recommended for most use cases. Note: Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.", - "name": "engine", + "info": "Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, 'cosinesimil' for cosine similarity, 'innerproduct' for dot product.", + "name": "space_type", "options": [ - "jvector", - "nmslib", - "faiss", - "lucene" + "l2", + "l1", + "cosinesimil", + "linf", + "innerproduct" ], "options_metadata": [], "override_skip": false, @@ -3238,46 +3471,59 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "jvector" + "value": "l2" }, - "filter_expression": { - "_input_type": "MultilineInput", + "use_ssl": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Use SSL/TLS", + "dynamic": false, + "info": "Enable SSL/TLS encryption for secure connections to OpenSearch.", + "list": false, + "list_add_label": "Add More", + "name": "use_ssl", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, + "username": { + "_input_type": "StrInput", "advanced": false, - "ai_enabled": false, - "copy_field": false, - "display_name": "Search Filters (JSON)", + "display_name": "Username", "dynamic": false, - "info": "Optional JSON configuration for search filtering, result limits, and score thresholds.\n\nFormat 1 - Explicit filters:\n{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, {\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\n\nFormat 2 - Context-style mapping:\n{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\n\nUse __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.", - "input_types": [ - "Message" - ], + "info": "", "list": false, "list_add_label": "Add More", "load_from_db": false, - "multiline": true, - "name": "filter_expression", + "name": "username", "override_skip": false, "placeholder": "", "required": false, - "show": true, + "show": false, "title_case": false, "tool_mode": false, - "trace_as_input": true, "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "admin" }, - "index_name": { + "vector_field": { "_input_type": "StrInput", - "advanced": false, - "display_name": "Index Name", + "advanced": true, + "display_name": "Legacy Vector Field Name", "dynamic": false, - "info": "The OpenSearch index name where documents will be stored and searched. Will be created automatically if it doesn't exist.", + "info": "Legacy field name for backward compatibility. New documents use dynamic fields (chunk_embedding_{model_name}) based on the embedding_model_name.", "list": false, "list_add_label": "Add More", "load_from_db": false, - "name": "index_name", + "name": "vector_field", "override_skip": false, "placeholder": "", "required": false, @@ -3287,101 +3533,217 @@ "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "documents" + "value": "chunk_embedding" }, - "ingest_data": { - "_input_type": "HandleInput", - "advanced": false, - "display_name": "Ingest Data", + "verify_certs": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Verify SSL Certificates", "dynamic": false, - "info": "", - "input_types": [ - "Data", - "DataFrame" - ], - "list": true, + "info": "Verify SSL certificates when connecting. Disable for self-signed certificates in development environments.", + "list": false, "list_add_label": "Add More", - "name": "ingest_data", + "name": "verify_certs", "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, + "tool_mode": false, "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "other", - "value": "" + "track_in_telemetry": true, + "type": "bool", + "value": false + } + }, + "tool_mode": false + }, + "selected_output": "search_results", + "showNode": true, + "type": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding" + }, + "dragging": false, + "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", + "measured": { + "height": 967, + "width": 320 + }, + "position": { + "x": 1756.949537574781, + "y": 1525.6023642029047 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "description": "Generate embeddings using a specified provider.", + "display_name": "Embedding Model", + "id": "EmbeddingModel-EAo9i", + "node": { + "base_classes": [ + "Embeddings" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Generate embeddings using a specified provider.", + "display_name": "Embedding Model", + "documentation": "https://docs.langflow.org/components-embedding-models", + "edited": false, + "field_order": [ + "model", + "api_key", + "api_base", + "base_url_ibm_watsonx", + "project_id", + "dimensions", + "chunk_size", + "request_timeout", + "max_retries", + "show_progress_bar", + "model_kwargs", + "truncate_input_tokens", + "input_text" + ], + "frozen": false, + "icon": "binary", + "last_updated": "2026-02-27T18:40:06.707Z", + "legacy": false, + "metadata": { + "code_hash": "c5ce0982da48", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + }, + { + "name": "langchain_core", + "version": "0.3.83" + } + ], + "total_dependencies": 2 + }, + "module": "custom_components.embedding_model" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Embedding Model", + "group_outputs": false, + "hidden": null, + "loop_types": null, + "method": "build_embeddings", + "name": "embeddings", + "options": null, + "required_inputs": null, + "selected": "Embeddings", + "tool_mode": true, + "types": [ + "Embeddings" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "573ff6c4-003c-4e1d-aa3f-8a83e1f3e020" }, - "is_refresh": false, - "jwt_header": { - "_input_type": "StrInput", + "_type": "Component", + "api_base": { + "_input_type": "MessageTextInput", "advanced": true, - "display_name": "JWT Header Name", + "display_name": "API Base URL", "dynamic": false, - "info": "", + "info": "Base URL for the API. Leave empty for default.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", "load_from_db": false, - "name": "jwt_header", + "name": "api_base", "override_skip": false, "placeholder": "", - "required": true, + "required": false, "show": true, "title_case": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "Authorization" + "value": "" }, - "jwt_token": { + "api_key": { "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "JWT Token", + "advanced": true, + "display_name": "API Key", "dynamic": false, - "info": "Valid JSON Web Token for authentication. Will be sent in the Authorization header (with optional 'Bearer ' prefix).", + "info": "Model Provider API key", "input_types": [], "load_from_db": false, - "name": "jwt_token", + "name": "api_key", "override_skip": false, "password": true, "placeholder": "", - "required": true, + "real_time_refresh": true, + "required": false, "show": true, "title_case": false, "track_in_telemetry": false, "type": "str", - "value": "jwt" + "value": "" }, - "m": { - "_input_type": "IntInput", - "advanced": true, - "display_name": "M Parameter", + "base_url_ibm_watsonx": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "watsonx API Endpoint", "dynamic": false, - "info": "Number of bidirectional connections for each vector in the HNSW graph. Higher values improve search quality but increase memory usage and indexing time.", - "list": false, - "list_add_label": "Add More", - "name": "m", + "external_options": {}, + "info": "The base URL of the API (IBM watsonx.ai only)", + "name": "base_url_ibm_watsonx", + "options": [ + "https://us-south.ml.cloud.ibm.com", + "https://eu-de.ml.cloud.ibm.com", + "https://eu-gb.ml.cloud.ibm.com", + "https://au-syd.ml.cloud.ibm.com", + "https://jp-tok.ml.cloud.ibm.com", + "https://ca-tor.ml.cloud.ibm.com" + ], + "options_metadata": [], "override_skip": false, "placeholder": "", + "real_time_refresh": true, "required": false, - "show": true, + "show": false, "title_case": false, + "toggle": false, "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "int", - "value": 16 + "type": "str", + "value": "https://us-south.ml.cloud.ibm.com" }, - "num_candidates": { + "chunk_size": { "_input_type": "IntInput", "advanced": true, - "display_name": "Candidate Pool Size", + "display_name": "Chunk Size", "dynamic": false, - "info": "Number of approximate neighbors to consider for each KNN query. Some OpenSearch deployments do not support this parameter; set to 0 to disable.", + "info": "", "list": false, "list_add_label": "Add More", - "name": "num_candidates", + "name": "chunk_size", "override_skip": false, "placeholder": "", "required": false, @@ -3393,36 +3755,34 @@ "type": "int", "value": 1000 }, - "number_of_results": { - "_input_type": "IntInput", + "code": { "advanced": true, - "display_name": "Default Result Limit", - "dynamic": false, - "info": "Default maximum number of search results to return when no limit is specified in the filter expression.", + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", "list": false, - "list_add_label": "Add More", - "name": "number_of_results", - "override_skip": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, "placeholder": "", - "required": false, + "required": true, "show": true, "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 10 + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.embeddings.embeddings_class import EmbeddingsWithModels\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.unified_models import (\n get_api_key_for_provider,\n get_embedding_class,\n get_embedding_model_options,\n get_unified_models_detailed,\n update_model_options_in_build_config,\n)\nfrom lfx.base.models.watsonx_constants import IBM_WATSONX_URLS\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import (\n BoolInput,\n DictInput,\n DropdownInput,\n FloatInput,\n IntInput,\n MessageTextInput,\n ModelInput,\n SecretStrInput,\n)\nfrom lfx.log.logger import logger\n\n\nclass EmbeddingModelComponent(LCEmbeddingsModel):\n display_name = \"Embedding Model\"\n description = \"Generate embeddings using a specified provider.\"\n documentation: str = \"https://docs.langflow.org/components-embedding-models\"\n icon = \"binary\"\n name = \"EmbeddingModel\"\n category = \"models\"\n\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None):\n \"\"\"Dynamically update build config with user-filtered model options.\"\"\"\n # Update model options\n build_config = update_model_options_in_build_config(\n component=self,\n build_config=build_config,\n cache_key_prefix=\"embedding_model_options\",\n get_options_func=get_embedding_model_options,\n field_name=field_name,\n field_value=field_value,\n )\n\n # Show/hide provider-specific fields based on selected model\n if field_name == \"model\" and isinstance(field_value, list) and len(field_value) > 0:\n selected_model = field_value[0]\n provider = selected_model.get(\"provider\", \"\")\n\n # Show/hide watsonx fields\n is_watsonx = provider == \"IBM WatsonX\"\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = is_watsonx\n build_config[\"project_id\"][\"show\"] = is_watsonx\n build_config[\"truncate_input_tokens\"][\"show\"] = is_watsonx\n build_config[\"input_text\"][\"show\"] = is_watsonx\n if is_watsonx:\n build_config[\"base_url_ibm_watsonx\"][\"required\"] = True\n build_config[\"project_id\"][\"required\"] = True\n\n return build_config\n\n inputs = [\n ModelInput(\n name=\"model\",\n display_name=\"Embedding Model\",\n info=\"Select your model provider\",\n real_time_refresh=True,\n required=True,\n model_type=\"embedding\",\n input_types=[\"Embeddings\"], # Override default to accept Embeddings instead of LanguageModel\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"API Key\",\n info=\"Model Provider API key\",\n real_time_refresh=True,\n advanced=True,\n ),\n MessageTextInput(\n name=\"api_base\",\n display_name=\"API Base URL\",\n info=\"Base URL for the API. Leave empty for default.\",\n advanced=True,\n ),\n # Watson-specific inputs\n DropdownInput(\n name=\"base_url_ibm_watsonx\",\n display_name=\"watsonx API Endpoint\",\n info=\"The base URL of the API (IBM watsonx.ai only)\",\n options=IBM_WATSONX_URLS,\n value=IBM_WATSONX_URLS[0],\n show=False,\n real_time_refresh=True,\n ),\n MessageTextInput(\n name=\"project_id\",\n display_name=\"Project ID\",\n info=\"IBM watsonx.ai Project ID (required for IBM watsonx.ai)\",\n show=False,\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n advanced=True,\n value=1000,\n ),\n FloatInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout\",\n advanced=True,\n ),\n IntInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n advanced=True,\n value=3,\n ),\n BoolInput(\n name=\"show_progress_bar\",\n display_name=\"Show Progress Bar\",\n advanced=True,\n ),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n IntInput(\n name=\"truncate_input_tokens\",\n display_name=\"Truncate Input Tokens\",\n advanced=True,\n value=200,\n show=False,\n ),\n BoolInput(\n name=\"input_text\",\n display_name=\"Include the original text in the output\",\n value=True,\n advanced=True,\n show=False,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n \"\"\"Build and return an embeddings instance based on the selected model.\n\n Returns an EmbeddingsWithModels wrapper that contains:\n - The primary embedding instance (for the selected model)\n - available_models dict mapping all available model names to their instances\n \"\"\"\n # If an Embeddings object is directly connected, return it\n try:\n from langchain_core.embeddings import Embeddings as BaseEmbeddings\n\n if isinstance(self.model, BaseEmbeddings):\n return self.model\n except ImportError:\n pass\n\n # Safely extract model configuration\n if not self.model or not isinstance(self.model, list):\n msg = \"Model must be a non-empty list\"\n raise ValueError(msg)\n\n model = self.model[0]\n model_name = model.get(\"name\")\n provider = model.get(\"provider\")\n metadata = model.get(\"metadata\", {})\n\n # Get API key from user input or global variables\n api_key = get_api_key_for_provider(self.user_id, provider, self.api_key)\n\n # Validate required fields (Ollama doesn't require API key)\n if not api_key and provider != \"Ollama\":\n msg = (\n f\"{provider} API key is required. \"\n f\"Please provide it in the component or configure it globally as \"\n f\"{provider.upper().replace(' ', '_')}_API_KEY.\"\n )\n raise ValueError(msg)\n\n if not model_name:\n msg = \"Model name is required\"\n raise ValueError(msg)\n\n # Get embedding class\n embedding_class_name = metadata.get(\"embedding_class\")\n if not embedding_class_name:\n msg = f\"No embedding class defined in metadata for {model_name}\"\n raise ValueError(msg)\n\n embedding_class = get_embedding_class(embedding_class_name)\n\n # Build kwargs using parameter mapping for primary instance\n kwargs = self._build_kwargs(model, metadata)\n primary_instance = embedding_class(**kwargs)\n\n # Get all available embedding models for this provider\n available_models_dict = self._build_available_models(\n provider=provider,\n embedding_class=embedding_class,\n metadata=metadata,\n api_key=api_key,\n )\n\n # Wrap with EmbeddingsWithModels to provide available_models metadata\n return EmbeddingsWithModels(\n embeddings=primary_instance,\n available_models=available_models_dict,\n )\n\n def _build_available_models(\n self,\n provider: str,\n embedding_class: type,\n metadata: dict[str, Any],\n api_key: str | None,\n ) -> dict[str, Embeddings]:\n \"\"\"Build a dictionary of all available embedding model instances for the provider.\n\n Args:\n provider: The provider name (e.g., \"OpenAI\", \"Ollama\")\n embedding_class: The embedding class to instantiate\n metadata: Metadata containing param_mapping\n api_key: The API key for the provider\n\n Returns:\n Dict mapping model names to their embedding instances\n \"\"\"\n available_models_dict: dict[str, Embeddings] = {}\n\n # Get all embedding models for this provider from unified models\n all_embedding_models = get_unified_models_detailed(\n providers=[provider],\n model_type=\"embeddings\",\n include_deprecated=False,\n include_unsupported=False,\n )\n\n if not all_embedding_models:\n return available_models_dict\n\n # Extract models from the provider data\n for provider_data in all_embedding_models:\n if provider_data.get(\"provider\") != provider:\n continue\n\n for model_data in provider_data.get(\"models\", []):\n model_name = model_data.get(\"model_name\")\n if not model_name:\n continue\n\n # Create a model dict compatible with _build_kwargs\n model_dict = {\n \"name\": model_name,\n \"provider\": provider,\n \"metadata\": metadata, # Reuse the same metadata/param_mapping\n }\n\n try:\n # Build kwargs for this model\n model_kwargs = self._build_kwargs_for_model(model_dict, metadata, api_key)\n # Create the embedding instance\n available_models_dict[model_name] = embedding_class(**model_kwargs)\n except Exception: # noqa: BLE001\n # Skip models that fail to instantiate\n # This handles cases where specific models have incompatible parameters\n logger.debug(\"Failed to instantiate embedding model %s: skipping\", model_name, exc_info=True)\n continue\n\n return available_models_dict\n\n def _build_kwargs_for_model(\n self,\n model: dict[str, Any],\n metadata: dict[str, Any],\n api_key: str | None,\n ) -> dict[str, Any]:\n \"\"\"Build kwargs dictionary for a specific model using parameter mapping.\n\n This is similar to _build_kwargs but uses the provided api_key directly\n instead of looking it up again.\n\n Args:\n model: Model dict with name and provider\n metadata: Metadata containing param_mapping\n api_key: The API key to use\n\n Returns:\n kwargs dict for embedding class instantiation\n \"\"\"\n param_mapping = metadata.get(\"param_mapping\", {})\n if not param_mapping:\n msg = \"Parameter mapping not found in metadata\"\n raise ValueError(msg)\n\n kwargs = {}\n provider = model.get(\"provider\")\n\n # Required parameters - handle both \"model\" and \"model_id\" (for watsonx)\n if \"model\" in param_mapping:\n kwargs[param_mapping[\"model\"]] = model.get(\"name\")\n elif \"model_id\" in param_mapping:\n kwargs[param_mapping[\"model_id\"]] = model.get(\"name\")\n\n # Add API key if mapped\n if \"api_key\" in param_mapping and api_key:\n kwargs[param_mapping[\"api_key\"]] = api_key\n\n # Optional parameters with their values\n optional_params = {\n \"api_base\": self.api_base if self.api_base else None,\n \"dimensions\": int(self.dimensions) if self.dimensions else None,\n \"chunk_size\": int(self.chunk_size) if self.chunk_size else None,\n \"request_timeout\": float(self.request_timeout) if self.request_timeout else None,\n \"max_retries\": int(self.max_retries) if self.max_retries else None,\n \"show_progress_bar\": self.show_progress_bar if hasattr(self, \"show_progress_bar\") else None,\n \"model_kwargs\": self.model_kwargs if self.model_kwargs else None,\n }\n\n # Watson-specific parameters\n if provider in {\"IBM WatsonX\", \"IBM watsonx.ai\"}:\n # Map base_url_ibm_watsonx to \"url\" parameter for watsonx\n if \"url\" in param_mapping:\n url_value = (\n self.base_url_ibm_watsonx\n if hasattr(self, \"base_url_ibm_watsonx\") and self.base_url_ibm_watsonx\n else \"https://us-south.ml.cloud.ibm.com\"\n )\n kwargs[param_mapping[\"url\"]] = url_value\n # Map project_id for watsonx\n if hasattr(self, \"project_id\") and self.project_id and \"project_id\" in param_mapping:\n kwargs[param_mapping[\"project_id\"]] = self.project_id\n\n # Ollama-specific parameters\n if provider == \"Ollama\" and \"base_url\" in param_mapping:\n # Map api_base to \"base_url\" parameter for Ollama\n base_url_value = self.api_base if hasattr(self, \"api_base\") and self.api_base else \"http://localhost:11434\"\n kwargs[param_mapping[\"base_url\"]] = base_url_value\n\n # Add optional parameters if they have values and are mapped\n for param_name, param_value in optional_params.items():\n if param_value is not None and param_name in param_mapping:\n # Special handling for request_timeout with Google provider\n if param_name == \"request_timeout\":\n if provider == \"Google Generative AI\" and isinstance(param_value, (int, float)):\n kwargs[param_mapping[param_name]] = {\"timeout\": param_value}\n else:\n kwargs[param_mapping[param_name]] = param_value\n else:\n kwargs[param_mapping[param_name]] = param_value\n\n return kwargs\n\n def _build_kwargs(self, model: dict[str, Any], metadata: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Build kwargs dictionary using parameter mapping.\"\"\"\n param_mapping = metadata.get(\"param_mapping\", {})\n if not param_mapping:\n msg = \"Parameter mapping not found in metadata\"\n raise ValueError(msg)\n\n kwargs = {}\n\n # Required parameters - handle both \"model\" and \"model_id\" (for watsonx)\n if \"model\" in param_mapping:\n kwargs[param_mapping[\"model\"]] = model.get(\"name\")\n elif \"model_id\" in param_mapping:\n kwargs[param_mapping[\"model_id\"]] = model.get(\"name\")\n if \"api_key\" in param_mapping:\n kwargs[param_mapping[\"api_key\"]] = get_api_key_for_provider(\n self.user_id,\n model.get(\"provider\"),\n self.api_key,\n )\n\n # Optional parameters with their values\n provider = model.get(\"provider\")\n optional_params = {\n \"api_base\": self.api_base if self.api_base else None,\n \"dimensions\": int(self.dimensions) if self.dimensions else None,\n \"chunk_size\": int(self.chunk_size) if self.chunk_size else None,\n \"request_timeout\": float(self.request_timeout) if self.request_timeout else None,\n \"max_retries\": int(self.max_retries) if self.max_retries else None,\n \"show_progress_bar\": self.show_progress_bar if hasattr(self, \"show_progress_bar\") else None,\n \"model_kwargs\": self.model_kwargs if self.model_kwargs else None,\n }\n\n # Watson-specific parameters\n if provider in {\"IBM WatsonX\", \"IBM watsonx.ai\"}:\n # Map base_url_ibm_watsonx to \"url\" parameter for watsonx\n if \"url\" in param_mapping:\n url_value = (\n self.base_url_ibm_watsonx\n if hasattr(self, \"base_url_ibm_watsonx\") and self.base_url_ibm_watsonx\n else \"https://us-south.ml.cloud.ibm.com\"\n )\n kwargs[param_mapping[\"url\"]] = url_value\n # Map project_id for watsonx\n if hasattr(self, \"project_id\") and self.project_id and \"project_id\" in param_mapping:\n kwargs[param_mapping[\"project_id\"]] = self.project_id\n\n # Ollama-specific parameters\n if provider == \"Ollama\" and \"base_url\" in param_mapping:\n # Map api_base to \"base_url\" parameter for Ollama\n base_url_value = self.api_base if hasattr(self, \"api_base\") and self.api_base else \"http://localhost:11434\"\n kwargs[param_mapping[\"base_url\"]] = base_url_value\n\n # Add optional parameters if they have values and are mapped\n for param_name, param_value in optional_params.items():\n if param_value is not None and param_name in param_mapping:\n # Special handling for request_timeout with Google provider\n if param_name == \"request_timeout\":\n if provider == \"Google Generative AI\" and isinstance(param_value, (int, float)):\n kwargs[param_mapping[param_name]] = {\"timeout\": param_value}\n else:\n kwargs[param_mapping[param_name]] = param_value\n else:\n kwargs[param_mapping[param_name]] = param_value\n\n return kwargs\n" }, - "opensearch_url": { - "_input_type": "StrInput", - "advanced": false, - "display_name": "OpenSearch URL", + "dimensions": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Dimensions", "dynamic": false, - "info": "The connection URL for your OpenSearch cluster (e.g., http://localhost:9200 for local development or your cloud endpoint).", + "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", "list": false, "list_add_label": "Add More", - "load_from_db": true, - "name": "opensearch_url", + "load_from_db": false, + "name": "dimensions", "override_skip": false, "placeholder": "", "required": false, @@ -3430,63 +3790,40 @@ "title_case": false, "tool_mode": false, "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "OPENSEARCH_URL" + "track_in_telemetry": true, + "type": "int", + "value": "" }, - "password": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "OpenSearch Password", + "input_text": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Include the original text in the output", "dynamic": false, "info": "", - "input_types": [], - "load_from_db": false, - "name": "password", - "override_skip": false, - "password": true, - "placeholder": "", - "required": false, - "show": false, - "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "MyStrongOpenSearchPassword123!" - }, - "search_query": { - "_input_type": "QueryInput", - "advanced": false, - "display_name": "Search Query", - "dynamic": false, - "info": "Enter a query to run a similarity search.", - "input_types": [ - "Message" - ], "list": false, "list_add_label": "Add More", - "load_from_db": false, - "name": "search_query", + "name": "input_text", "override_skip": false, - "placeholder": "Enter a query...", + "placeholder": "", "required": false, - "show": true, + "show": false, "title_case": false, - "tool_mode": true, - "trace_as_input": true, + "tool_mode": false, "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "query", - "value": "" + "track_in_telemetry": true, + "type": "bool", + "value": true }, - "should_cache_vector_store": { - "_input_type": "BoolInput", + "is_refresh": true, + "max_retries": { + "_input_type": "IntInput", "advanced": true, - "display_name": "Cache Vector Store", + "display_name": "Max Retries", "dynamic": false, - "info": "If True, the vector store will be cached for the current build of the component. This is useful for components that have multiple output methods and want to share the same vector store.", + "info": "", "list": false, "list_add_label": "Add More", - "name": "should_cache_vector_store", + "name": "max_retries", "override_skip": false, "placeholder": "", "required": false, @@ -3495,90 +3832,241 @@ "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "bool", - "value": true + "type": "int", + "value": 3 }, - "space_type": { - "_input_type": "DropdownInput", - "advanced": true, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Distance Metric", + "model": { + "_input_type": "ModelInput", + "advanced": false, + "display_name": "Embedding Model", "dynamic": false, - "external_options": {}, - "info": "Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, 'cosinesimil' for cosine similarity, 'innerproduct' for dot product.", - "name": "space_type", + "external_options": { + "fields": { + "data": { + "node": { + "display_name": "Connect other models", + "icon": "CornerDownLeft", + "name": "connect_other_models" + } + } + } + }, + "info": "Select your model provider", + "input_types": [ + "Embeddings" + ], + "list": false, + "list_add_label": "Add More", + "model_type": "embedding", + "name": "model", "options": [ - "l2", - "l1", - "cosinesimil", - "linf", - "innerproduct" + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-small", + "provider": "OpenAI" + }, + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-large", + "provider": "OpenAI" + }, + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-ada-002", + "provider": "OpenAI" + }, + { + "category": "Ollama", + "icon": "Ollama", + "metadata": { + "embedding_class": "OllamaEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "base_url": "base_url", + "model": "model", + "model_kwargs": "model_kwargs", + "num_ctx": "num_ctx", + "request_timeout": "request_timeout" + } + }, + "name": "nomic-embed-text:latest", + "provider": "Ollama" + }, + { + "category": "Ollama", + "icon": "Ollama", + "metadata": { + "embedding_class": "OllamaEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "base_url": "base_url", + "model": "model", + "model_kwargs": "model_kwargs", + "num_ctx": "num_ctx", + "request_timeout": "request_timeout" + } + }, + "name": "qwen3-embedding:latest", + "provider": "Ollama" + }, + { + "category": "Google Generative AI", + "icon": "GoogleGenerativeAI", + "metadata": { + "is_disabled_provider": true, + "variable_name": null + }, + "name": "__enable_provider_Google Generative AI__", + "provider": "Google Generative AI" + }, + { + "category": "IBM WatsonX", + "icon": "WatsonxAI", + "metadata": { + "is_disabled_provider": true, + "variable_name": null + }, + "name": "__enable_provider_IBM WatsonX__", + "provider": "IBM WatsonX" + } ], - "options_metadata": [], "override_skip": false, - "placeholder": "", - "required": false, + "placeholder": "Setup Provider", + "real_time_refresh": true, + "refresh_button": true, + "required": true, "show": true, "title_case": false, - "toggle": false, "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "str", - "value": "l2" + "trace_as_input": true, + "track_in_telemetry": false, + "type": "model", + "value": [ + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-small", + "provider": "OpenAI" + } + ] }, - "use_ssl": { - "_input_type": "BoolInput", + "model_kwargs": { + "_input_type": "DictInput", "advanced": true, - "display_name": "Use SSL/TLS", + "display_name": "Model Kwargs", "dynamic": false, - "info": "Enable SSL/TLS encryption for secure connections to OpenSearch.", + "info": "Additional keyword arguments to pass to the model.", "list": false, "list_add_label": "Add More", - "name": "use_ssl", + "name": "model_kwargs", "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": true + "trace_as_input": true, + "track_in_telemetry": false, + "type": "dict", + "value": {} }, - "username": { - "_input_type": "StrInput", + "project_id": { + "_input_type": "MessageTextInput", "advanced": false, - "display_name": "Username", + "display_name": "Project ID", "dynamic": false, - "info": "", + "info": "IBM watsonx.ai Project ID (required for IBM watsonx.ai)", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", "load_from_db": false, - "name": "username", + "name": "project_id", "override_skip": false, "placeholder": "", "required": false, "show": false, "title_case": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "admin" + "value": "" }, - "vector_field": { - "_input_type": "StrInput", + "request_timeout": { + "_input_type": "FloatInput", "advanced": true, - "display_name": "Legacy Vector Field Name", + "display_name": "Request Timeout", "dynamic": false, - "info": "Legacy field name for backward compatibility. New documents use dynamic fields (chunk_embedding_{model_name}) based on the embedding_model_name.", + "info": "", "list": false, "list_add_label": "Add More", - "load_from_db": false, - "name": "vector_field", + "name": "request_timeout", "override_skip": false, "placeholder": "", "required": false, @@ -3586,19 +4074,19 @@ "title_case": false, "tool_mode": false, "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "chunk_embedding" + "track_in_telemetry": true, + "type": "float", + "value": "" }, - "verify_certs": { + "show_progress_bar": { "_input_type": "BoolInput", "advanced": true, - "display_name": "Verify SSL Certificates", + "display_name": "Show Progress Bar", "dynamic": false, - "info": "Verify SSL certificates when connecting. Disable for self-signed certificates in development environments.", + "info": "", "list": false, "list_add_label": "Add More", - "name": "verify_certs", + "name": "show_progress_bar", "override_skip": false, "placeholder": "", "required": false, @@ -3609,30 +4097,51 @@ "track_in_telemetry": true, "type": "bool", "value": false + }, + "truncate_input_tokens": { + "_input_type": "IntInput", + "advanced": true, + "display_name": "Truncate Input Tokens", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "truncate_input_tokens", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": 200 } }, "tool_mode": false }, - "selected_output": "search_results", "showNode": true, - "type": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding" + "type": "EmbeddingModel" }, "dragging": false, - "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4", + "id": "EmbeddingModel-EAo9i", "measured": { - "height": 904, + "height": 207, "width": 320 }, "position": { - "x": 2261.865622928042, - "y": 1349.2821108833643 + "x": 1311.1440056916672, + "y": 1890.167434226358 }, "selected": false, "type": "genericNode" }, { "data": { - "id": "EmbeddingModel-EAo9i", + "description": "Generate embeddings using a specified provider.", + "display_name": "Embedding Model", + "id": "EmbeddingModel-E0hvR", "node": { "base_classes": [ "Embeddings" @@ -3643,14 +4152,12 @@ "description": "Generate embeddings using a specified provider.", "display_name": "Embedding Model", "documentation": "https://docs.langflow.org/components-embedding-models", - "edited": true, + "edited": false, "field_order": [ - "provider", - "api_base", - "ollama_base_url", - "base_url_ibm_watsonx", "model", "api_key", + "api_base", + "base_url_ibm_watsonx", "project_id", "dimensions", "chunk_size", @@ -3659,48 +4166,26 @@ "show_progress_bar", "model_kwargs", "truncate_input_tokens", - "input_text", - "fail_safe_mode" + "input_text" ], "frozen": false, "icon": "binary", - "last_updated": "2026-02-05T15:22:55.812Z", + "last_updated": "2026-02-27T18:40:19.256Z", "legacy": false, - "lf_version": "1.7.0.dev21", "metadata": { - "code_hash": "0e2d6fe67a26", + "code_hash": "c5ce0982da48", "dependencies": { "dependencies": [ - { - "name": "requests", - "version": "2.32.5" - }, - { - "name": "ibm_watsonx_ai", - "version": "1.4.2" - }, - { - "name": "langchain_openai", - "version": "0.3.23" - }, { "name": "lfx", - "version": "0.2.0.dev21" - }, - { - "name": "langchain_ollama", - "version": "0.3.10" - }, - { - "name": "langchain_community", - "version": "0.3.21" + "version": null }, { - "name": "langchain_ibm", - "version": "0.3.19" + "name": "langchain_core", + "version": "0.3.83" } ], - "total_dependencies": 7 + "total_dependencies": 2 }, "module": "custom_components.embedding_model" }, @@ -3732,13 +4217,13 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" + "value": "573ff6c4-003c-4e1d-aa3f-8a83e1f3e020" }, "_type": "Component", "api_base": { "_input_type": "MessageTextInput", - "advanced": false, - "display_name": "OpenAI API Base URL", + "advanced": true, + "display_name": "API Base URL", "dynamic": false, "info": "Base URL for the API. Leave empty for default.", "input_types": [ @@ -3762,12 +4247,12 @@ }, "api_key": { "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "OpenAI API Key", + "advanced": true, + "display_name": "API Key", "dynamic": false, "info": "Model Provider API key", "input_types": [], - "load_from_db": true, + "load_from_db": false, "name": "api_key", "override_skip": false, "password": true, @@ -3778,7 +4263,7 @@ "title_case": false, "track_in_telemetry": false, "type": "str", - "value": "OPENAI_API_KEY" + "value": "" }, "base_url_ibm_watsonx": { "_input_type": "DropdownInput", @@ -3848,7 +4333,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from typing import Any\n\nimport requests\nfrom ibm_watsonx_ai.metanames import EmbedTextParamsMetaNames\nfrom langchain_openai import OpenAIEmbeddings\n\nfrom lfx.base.embeddings.embeddings_class import EmbeddingsWithModels\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.model_utils import get_ollama_models, is_valid_ollama_url\nfrom lfx.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom lfx.base.models.watsonx_constants import (\n IBM_WATSONX_URLS,\n WATSONX_EMBEDDING_MODEL_NAMES,\n)\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import (\n BoolInput,\n DictInput,\n DropdownInput,\n FloatInput,\n IntInput,\n MessageTextInput,\n SecretStrInput,\n)\nfrom lfx.log.logger import logger\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.util import transform_localhost_url\n\n# Ollama API constants\nHTTP_STATUS_OK = 200\nJSON_MODELS_KEY = \"models\"\nJSON_NAME_KEY = \"name\"\nJSON_CAPABILITIES_KEY = \"capabilities\"\nDESIRED_CAPABILITY = \"embedding\"\nDEFAULT_OLLAMA_URL = \"http://localhost:11434\"\n\n\nclass EmbeddingModelComponent(LCEmbeddingsModel):\n display_name = \"Embedding Model\"\n description = \"Generate embeddings using a specified provider.\"\n documentation: str = \"https://docs.langflow.org/components-embedding-models\"\n icon = \"binary\"\n name = \"EmbeddingModel\"\n category = \"models\"\n\n inputs = [\n DropdownInput(\n name=\"provider\",\n display_name=\"Model Provider\",\n options=[\"OpenAI\", \"Ollama\", \"IBM watsonx.ai\"],\n value=\"OpenAI\",\n info=\"Select the embedding model provider\",\n real_time_refresh=True,\n options_metadata=[{\"icon\": \"OpenAI\"}, {\"icon\": \"Ollama\"}, {\"icon\": \"WatsonxAI\"}],\n ),\n MessageTextInput(\n name=\"api_base\",\n display_name=\"API Base URL\",\n info=\"Base URL for the API. Leave empty for default.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"ollama_base_url\",\n display_name=\"Ollama API URL\",\n info=f\"Endpoint of the Ollama API (Ollama only). Defaults to {DEFAULT_OLLAMA_URL}\",\n value=DEFAULT_OLLAMA_URL,\n show=False,\n real_time_refresh=True,\n load_from_db=True,\n ),\n DropdownInput(\n name=\"base_url_ibm_watsonx\",\n display_name=\"watsonx API Endpoint\",\n info=\"The base URL of the API (IBM watsonx.ai only)\",\n options=IBM_WATSONX_URLS,\n value=IBM_WATSONX_URLS[0],\n show=False,\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"model\",\n display_name=\"Model Name\",\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=OPENAI_EMBEDDING_MODEL_NAMES[0],\n info=\"Select the embedding model to use\",\n real_time_refresh=True,\n refresh_button=True,\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"OpenAI API Key\",\n info=\"Model Provider API key\",\n required=True,\n show=True,\n real_time_refresh=True,\n ),\n # Watson-specific inputs\n MessageTextInput(\n name=\"project_id\",\n display_name=\"Project ID\",\n info=\"IBM watsonx.ai Project ID (required for IBM watsonx.ai)\",\n show=False,\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", advanced=True, value=3),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n IntInput(\n name=\"truncate_input_tokens\",\n display_name=\"Truncate Input Tokens\",\n advanced=True,\n value=200,\n show=False,\n ),\n BoolInput(\n name=\"input_text\",\n display_name=\"Include the original text in the output\",\n value=True,\n advanced=True,\n show=False,\n ),\n BoolInput(\n name=\"fail_safe_mode\",\n display_name=\"Fail-Safe Mode\",\n value=False,\n advanced=True,\n info=\"When enabled, errors will be logged instead of raising exceptions. \"\n \"The component will return None on error.\",\n real_time_refresh=True,\n ),\n ]\n\n @staticmethod\n def fetch_ibm_models(base_url: str) -> list[str]:\n \"\"\"Fetch available models from the watsonx.ai API.\"\"\"\n try:\n endpoint = f\"{base_url}/ml/v1/foundation_model_specs\"\n params = {\n \"version\": \"2024-09-16\",\n \"filters\": \"function_embedding,!lifecycle_withdrawn:and\",\n }\n response = requests.get(endpoint, params=params, timeout=10)\n response.raise_for_status()\n data = response.json()\n models = [model[\"model_id\"] for model in data.get(\"resources\", [])]\n return sorted(models)\n except Exception: # noqa: BLE001\n logger.exception(\"Error fetching models\")\n return WATSONX_EMBEDDING_MODEL_NAMES\n async def fetch_ollama_models(self) -> list[str]:\n try:\n return await get_ollama_models(\n base_url_value=self.ollama_base_url,\n desired_capability=DESIRED_CAPABILITY,\n json_models_key=JSON_MODELS_KEY,\n json_name_key=JSON_NAME_KEY,\n json_capabilities_key=JSON_CAPABILITIES_KEY,\n )\n except Exception: # noqa: BLE001\n\n logger.exception(\"Error fetching models\")\n return []\n async def build_embeddings(self) -> Embeddings:\n provider = self.provider\n model = self.model\n api_key = self.api_key\n api_base = self.api_base\n base_url_ibm_watsonx = self.base_url_ibm_watsonx\n ollama_base_url = self.ollama_base_url\n dimensions = self.dimensions\n chunk_size = self.chunk_size\n request_timeout = self.request_timeout\n max_retries = self.max_retries\n show_progress_bar = self.show_progress_bar\n model_kwargs = self.model_kwargs or {}\n\n if provider == \"OpenAI\":\n if not api_key:\n msg = \"OpenAI API key is required when using OpenAI provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n try:\n # Create the primary embedding instance\n embeddings_instance = OpenAIEmbeddings(\n model=model,\n dimensions=dimensions or None,\n base_url=api_base or None,\n api_key=api_key,\n chunk_size=chunk_size,\n max_retries=max_retries,\n timeout=request_timeout or None,\n show_progress_bar=show_progress_bar,\n model_kwargs=model_kwargs,\n )\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in OPENAI_EMBEDDING_MODEL_NAMES:\n available_models_dict[model_name] = OpenAIEmbeddings(\n model=model_name,\n dimensions=dimensions or None, # Use same dimensions config for all\n base_url=api_base or None,\n api_key=api_key,\n chunk_size=chunk_size,\n max_retries=max_retries,\n timeout=request_timeout or None,\n show_progress_bar=show_progress_bar,\n model_kwargs=model_kwargs,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to initialize OpenAI embeddings: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n if provider == \"Ollama\":\n try:\n from langchain_ollama import OllamaEmbeddings\n except ImportError:\n try:\n from langchain_community.embeddings import OllamaEmbeddings\n except ImportError:\n msg = \"Please install langchain-ollama: pip install langchain-ollama\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ImportError(msg) from None\n\n try:\n transformed_base_url = transform_localhost_url(ollama_base_url)\n\n # Check if URL contains /v1 suffix (OpenAI-compatible mode)\n if transformed_base_url and transformed_base_url.rstrip(\"/\").endswith(\"/v1\"):\n # Strip /v1 suffix and log warning\n transformed_base_url = transformed_base_url.rstrip(\"/\").removesuffix(\"/v1\")\n logger.warning(\n \"Detected '/v1' suffix in base URL. The Ollama component uses the native Ollama API, \"\n \"not the OpenAI-compatible API. The '/v1' suffix has been automatically removed. \"\n \"If you want to use the OpenAI-compatible API, please use the OpenAI component instead. \"\n \"Learn more at https://docs.ollama.com/openai#openai-compatibility\"\n )\n\n final_base_url = transformed_base_url or \"http://localhost:11434\"\n\n # Create the primary embedding instance\n embeddings_instance = OllamaEmbeddings(\n model=model,\n base_url=final_base_url,\n **model_kwargs,\n )\n\n # Fetch available Ollama models\n available_model_names = await self.fetch_ollama_models()\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in available_model_names:\n available_models_dict[model_name] = OllamaEmbeddings(\n model=model_name,\n base_url=final_base_url,\n **model_kwargs,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to initialize Ollama embeddings: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n if provider == \"IBM watsonx.ai\":\n try:\n from langchain_ibm import WatsonxEmbeddings\n except ImportError:\n msg = \"Please install langchain-ibm: pip install langchain-ibm\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ImportError(msg) from None\n\n if not api_key:\n msg = \"IBM watsonx.ai API key is required when using IBM watsonx.ai provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n project_id = self.project_id\n\n if not project_id:\n msg = \"Project ID is required for IBM watsonx.ai provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n try:\n from ibm_watsonx_ai import APIClient, Credentials\n\n final_url = base_url_ibm_watsonx or \"https://us-south.ml.cloud.ibm.com\"\n\n credentials = Credentials(\n api_key=self.api_key,\n url=final_url,\n )\n\n api_client = APIClient(credentials)\n\n params = {\n EmbedTextParamsMetaNames.TRUNCATE_INPUT_TOKENS: self.truncate_input_tokens,\n EmbedTextParamsMetaNames.RETURN_OPTIONS: {\"input_text\": self.input_text},\n }\n\n # Create the primary embedding instance\n embeddings_instance = WatsonxEmbeddings(\n model_id=model,\n params=params,\n watsonx_client=api_client,\n project_id=project_id,\n )\n\n # Fetch available IBM watsonx.ai models\n available_model_names = self.fetch_ibm_models(final_url)\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in available_model_names:\n available_models_dict[model_name] = WatsonxEmbeddings(\n model_id=model_name,\n params=params,\n watsonx_client=api_client,\n project_id=project_id,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to authenticate with IBM watsonx.ai: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n msg = f\"Unknown provider: {provider}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n async def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n # Handle fail_safe_mode changes first - set all required fields to False if enabled\n if field_name == \"fail_safe_mode\":\n if field_value: # If fail_safe_mode is enabled\n build_config[\"api_key\"][\"required\"] = False\n elif hasattr(self, \"provider\"):\n # If fail_safe_mode is disabled, restore required flags based on provider\n if self.provider in [\"OpenAI\", \"IBM watsonx.ai\"]:\n build_config[\"api_key\"][\"required\"] = True\n else: # Ollama\n build_config[\"api_key\"][\"required\"] = False\n\n if field_name == \"provider\":\n if field_value == \"OpenAI\":\n build_config[\"model\"][\"options\"] = OPENAI_EMBEDDING_MODEL_NAMES\n build_config[\"model\"][\"value\"] = OPENAI_EMBEDDING_MODEL_NAMES[0]\n build_config[\"api_key\"][\"display_name\"] = \"OpenAI API Key\"\n # Only set required=True if fail_safe_mode is not enabled\n build_config[\"api_key\"][\"required\"] = not (hasattr(self, \"fail_safe_mode\") and self.fail_safe_mode)\n build_config[\"api_key\"][\"show\"] = True\n build_config[\"api_base\"][\"display_name\"] = \"OpenAI API Base URL\"\n build_config[\"api_base\"][\"advanced\"] = True\n build_config[\"api_base\"][\"show\"] = True\n build_config[\"ollama_base_url\"][\"show\"] = False\n build_config[\"project_id\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = False\n build_config[\"truncate_input_tokens\"][\"show\"] = False\n build_config[\"input_text\"][\"show\"] = False\n elif field_value == \"Ollama\":\n build_config[\"ollama_base_url\"][\"show\"] = True\n\n if await is_valid_ollama_url(url=self.ollama_base_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n build_config[\"model\"][\"value\"] = models[0] if models else \"\"\n except ValueError:\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n else:\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n build_config[\"truncate_input_tokens\"][\"show\"] = False\n build_config[\"input_text\"][\"show\"] = False\n build_config[\"api_key\"][\"display_name\"] = \"API Key (Optional)\"\n build_config[\"api_key\"][\"required\"] = False\n build_config[\"api_key\"][\"show\"] = False\n build_config[\"api_base\"][\"show\"] = False\n build_config[\"project_id\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = False\n\n elif field_value == \"IBM watsonx.ai\":\n build_config[\"model\"][\"options\"] = self.fetch_ibm_models(base_url=self.base_url_ibm_watsonx)\n build_config[\"model\"][\"value\"] = self.fetch_ibm_models(base_url=self.base_url_ibm_watsonx)[0]\n build_config[\"api_key\"][\"display_name\"] = \"IBM watsonx.ai API Key\"\n # Only set required=True if fail_safe_mode is not enabled\n build_config[\"api_key\"][\"required\"] = not (hasattr(self, \"fail_safe_mode\") and self.fail_safe_mode)\n build_config[\"api_key\"][\"show\"] = True\n build_config[\"api_base\"][\"show\"] = False\n build_config[\"ollama_base_url\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = True\n build_config[\"project_id\"][\"show\"] = True\n build_config[\"truncate_input_tokens\"][\"show\"] = True\n build_config[\"input_text\"][\"show\"] = True\n elif field_name == \"base_url_ibm_watsonx\":\n build_config[\"model\"][\"options\"] = self.fetch_ibm_models(base_url=field_value)\n build_config[\"model\"][\"value\"] = self.fetch_ibm_models(base_url=field_value)[0]\n elif field_name == \"ollama_base_url\":\n # # Refresh Ollama models when base URL changes\n # if hasattr(self, \"provider\") and self.provider == \"Ollama\":\n # Use field_value if provided, otherwise fall back to instance attribute\n ollama_url = self.ollama_base_url\n if await is_valid_ollama_url(url=ollama_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n build_config[\"model\"][\"value\"] = models[0] if models else \"\"\n except ValueError:\n await logger.awarning(\"Failed to fetch Ollama embedding models.\")\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n\n elif field_name == \"model\" and self.provider == \"Ollama\":\n ollama_url = self.ollama_base_url\n if await is_valid_ollama_url(url=ollama_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n except ValueError:\n await logger.awarning(\"Failed to refresh Ollama embedding models.\")\n build_config[\"model\"][\"options\"] = []\n\n return build_config\n" + "value": "from typing import Any\n\nfrom lfx.base.embeddings.embeddings_class import EmbeddingsWithModels\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.unified_models import (\n get_api_key_for_provider,\n get_embedding_class,\n get_embedding_model_options,\n get_unified_models_detailed,\n update_model_options_in_build_config,\n)\nfrom lfx.base.models.watsonx_constants import IBM_WATSONX_URLS\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import (\n BoolInput,\n DictInput,\n DropdownInput,\n FloatInput,\n IntInput,\n MessageTextInput,\n ModelInput,\n SecretStrInput,\n)\nfrom lfx.log.logger import logger\n\n\nclass EmbeddingModelComponent(LCEmbeddingsModel):\n display_name = \"Embedding Model\"\n description = \"Generate embeddings using a specified provider.\"\n documentation: str = \"https://docs.langflow.org/components-embedding-models\"\n icon = \"binary\"\n name = \"EmbeddingModel\"\n category = \"models\"\n\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None):\n \"\"\"Dynamically update build config with user-filtered model options.\"\"\"\n # Update model options\n build_config = update_model_options_in_build_config(\n component=self,\n build_config=build_config,\n cache_key_prefix=\"embedding_model_options\",\n get_options_func=get_embedding_model_options,\n field_name=field_name,\n field_value=field_value,\n )\n\n # Show/hide provider-specific fields based on selected model\n if field_name == \"model\" and isinstance(field_value, list) and len(field_value) > 0:\n selected_model = field_value[0]\n provider = selected_model.get(\"provider\", \"\")\n\n # Show/hide watsonx fields\n is_watsonx = provider == \"IBM WatsonX\"\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = is_watsonx\n build_config[\"project_id\"][\"show\"] = is_watsonx\n build_config[\"truncate_input_tokens\"][\"show\"] = is_watsonx\n build_config[\"input_text\"][\"show\"] = is_watsonx\n if is_watsonx:\n build_config[\"base_url_ibm_watsonx\"][\"required\"] = True\n build_config[\"project_id\"][\"required\"] = True\n\n return build_config\n\n inputs = [\n ModelInput(\n name=\"model\",\n display_name=\"Embedding Model\",\n info=\"Select your model provider\",\n real_time_refresh=True,\n required=True,\n model_type=\"embedding\",\n input_types=[\"Embeddings\"], # Override default to accept Embeddings instead of LanguageModel\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"API Key\",\n info=\"Model Provider API key\",\n real_time_refresh=True,\n advanced=True,\n ),\n MessageTextInput(\n name=\"api_base\",\n display_name=\"API Base URL\",\n info=\"Base URL for the API. Leave empty for default.\",\n advanced=True,\n ),\n # Watson-specific inputs\n DropdownInput(\n name=\"base_url_ibm_watsonx\",\n display_name=\"watsonx API Endpoint\",\n info=\"The base URL of the API (IBM watsonx.ai only)\",\n options=IBM_WATSONX_URLS,\n value=IBM_WATSONX_URLS[0],\n show=False,\n real_time_refresh=True,\n ),\n MessageTextInput(\n name=\"project_id\",\n display_name=\"Project ID\",\n info=\"IBM watsonx.ai Project ID (required for IBM watsonx.ai)\",\n show=False,\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n advanced=True,\n value=1000,\n ),\n FloatInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout\",\n advanced=True,\n ),\n IntInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n advanced=True,\n value=3,\n ),\n BoolInput(\n name=\"show_progress_bar\",\n display_name=\"Show Progress Bar\",\n advanced=True,\n ),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n IntInput(\n name=\"truncate_input_tokens\",\n display_name=\"Truncate Input Tokens\",\n advanced=True,\n value=200,\n show=False,\n ),\n BoolInput(\n name=\"input_text\",\n display_name=\"Include the original text in the output\",\n value=True,\n advanced=True,\n show=False,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n \"\"\"Build and return an embeddings instance based on the selected model.\n\n Returns an EmbeddingsWithModels wrapper that contains:\n - The primary embedding instance (for the selected model)\n - available_models dict mapping all available model names to their instances\n \"\"\"\n # If an Embeddings object is directly connected, return it\n try:\n from langchain_core.embeddings import Embeddings as BaseEmbeddings\n\n if isinstance(self.model, BaseEmbeddings):\n return self.model\n except ImportError:\n pass\n\n # Safely extract model configuration\n if not self.model or not isinstance(self.model, list):\n msg = \"Model must be a non-empty list\"\n raise ValueError(msg)\n\n model = self.model[0]\n model_name = model.get(\"name\")\n provider = model.get(\"provider\")\n metadata = model.get(\"metadata\", {})\n\n # Get API key from user input or global variables\n api_key = get_api_key_for_provider(self.user_id, provider, self.api_key)\n\n # Validate required fields (Ollama doesn't require API key)\n if not api_key and provider != \"Ollama\":\n msg = (\n f\"{provider} API key is required. \"\n f\"Please provide it in the component or configure it globally as \"\n f\"{provider.upper().replace(' ', '_')}_API_KEY.\"\n )\n raise ValueError(msg)\n\n if not model_name:\n msg = \"Model name is required\"\n raise ValueError(msg)\n\n # Get embedding class\n embedding_class_name = metadata.get(\"embedding_class\")\n if not embedding_class_name:\n msg = f\"No embedding class defined in metadata for {model_name}\"\n raise ValueError(msg)\n\n embedding_class = get_embedding_class(embedding_class_name)\n\n # Build kwargs using parameter mapping for primary instance\n kwargs = self._build_kwargs(model, metadata)\n primary_instance = embedding_class(**kwargs)\n\n # Get all available embedding models for this provider\n available_models_dict = self._build_available_models(\n provider=provider,\n embedding_class=embedding_class,\n metadata=metadata,\n api_key=api_key,\n )\n\n # Wrap with EmbeddingsWithModels to provide available_models metadata\n return EmbeddingsWithModels(\n embeddings=primary_instance,\n available_models=available_models_dict,\n )\n\n def _build_available_models(\n self,\n provider: str,\n embedding_class: type,\n metadata: dict[str, Any],\n api_key: str | None,\n ) -> dict[str, Embeddings]:\n \"\"\"Build a dictionary of all available embedding model instances for the provider.\n\n Args:\n provider: The provider name (e.g., \"OpenAI\", \"Ollama\")\n embedding_class: The embedding class to instantiate\n metadata: Metadata containing param_mapping\n api_key: The API key for the provider\n\n Returns:\n Dict mapping model names to their embedding instances\n \"\"\"\n available_models_dict: dict[str, Embeddings] = {}\n\n # Get all embedding models for this provider from unified models\n all_embedding_models = get_unified_models_detailed(\n providers=[provider],\n model_type=\"embeddings\",\n include_deprecated=False,\n include_unsupported=False,\n )\n\n if not all_embedding_models:\n return available_models_dict\n\n # Extract models from the provider data\n for provider_data in all_embedding_models:\n if provider_data.get(\"provider\") != provider:\n continue\n\n for model_data in provider_data.get(\"models\", []):\n model_name = model_data.get(\"model_name\")\n if not model_name:\n continue\n\n # Create a model dict compatible with _build_kwargs\n model_dict = {\n \"name\": model_name,\n \"provider\": provider,\n \"metadata\": metadata, # Reuse the same metadata/param_mapping\n }\n\n try:\n # Build kwargs for this model\n model_kwargs = self._build_kwargs_for_model(model_dict, metadata, api_key)\n # Create the embedding instance\n available_models_dict[model_name] = embedding_class(**model_kwargs)\n except Exception: # noqa: BLE001\n # Skip models that fail to instantiate\n # This handles cases where specific models have incompatible parameters\n logger.debug(\"Failed to instantiate embedding model %s: skipping\", model_name, exc_info=True)\n continue\n\n return available_models_dict\n\n def _build_kwargs_for_model(\n self,\n model: dict[str, Any],\n metadata: dict[str, Any],\n api_key: str | None,\n ) -> dict[str, Any]:\n \"\"\"Build kwargs dictionary for a specific model using parameter mapping.\n\n This is similar to _build_kwargs but uses the provided api_key directly\n instead of looking it up again.\n\n Args:\n model: Model dict with name and provider\n metadata: Metadata containing param_mapping\n api_key: The API key to use\n\n Returns:\n kwargs dict for embedding class instantiation\n \"\"\"\n param_mapping = metadata.get(\"param_mapping\", {})\n if not param_mapping:\n msg = \"Parameter mapping not found in metadata\"\n raise ValueError(msg)\n\n kwargs = {}\n provider = model.get(\"provider\")\n\n # Required parameters - handle both \"model\" and \"model_id\" (for watsonx)\n if \"model\" in param_mapping:\n kwargs[param_mapping[\"model\"]] = model.get(\"name\")\n elif \"model_id\" in param_mapping:\n kwargs[param_mapping[\"model_id\"]] = model.get(\"name\")\n\n # Add API key if mapped\n if \"api_key\" in param_mapping and api_key:\n kwargs[param_mapping[\"api_key\"]] = api_key\n\n # Optional parameters with their values\n optional_params = {\n \"api_base\": self.api_base if self.api_base else None,\n \"dimensions\": int(self.dimensions) if self.dimensions else None,\n \"chunk_size\": int(self.chunk_size) if self.chunk_size else None,\n \"request_timeout\": float(self.request_timeout) if self.request_timeout else None,\n \"max_retries\": int(self.max_retries) if self.max_retries else None,\n \"show_progress_bar\": self.show_progress_bar if hasattr(self, \"show_progress_bar\") else None,\n \"model_kwargs\": self.model_kwargs if self.model_kwargs else None,\n }\n\n # Watson-specific parameters\n if provider in {\"IBM WatsonX\", \"IBM watsonx.ai\"}:\n # Map base_url_ibm_watsonx to \"url\" parameter for watsonx\n if \"url\" in param_mapping:\n url_value = (\n self.base_url_ibm_watsonx\n if hasattr(self, \"base_url_ibm_watsonx\") and self.base_url_ibm_watsonx\n else \"https://us-south.ml.cloud.ibm.com\"\n )\n kwargs[param_mapping[\"url\"]] = url_value\n # Map project_id for watsonx\n if hasattr(self, \"project_id\") and self.project_id and \"project_id\" in param_mapping:\n kwargs[param_mapping[\"project_id\"]] = self.project_id\n\n # Ollama-specific parameters\n if provider == \"Ollama\" and \"base_url\" in param_mapping:\n # Map api_base to \"base_url\" parameter for Ollama\n base_url_value = self.api_base if hasattr(self, \"api_base\") and self.api_base else \"http://localhost:11434\"\n kwargs[param_mapping[\"base_url\"]] = base_url_value\n\n # Add optional parameters if they have values and are mapped\n for param_name, param_value in optional_params.items():\n if param_value is not None and param_name in param_mapping:\n # Special handling for request_timeout with Google provider\n if param_name == \"request_timeout\":\n if provider == \"Google Generative AI\" and isinstance(param_value, (int, float)):\n kwargs[param_mapping[param_name]] = {\"timeout\": param_value}\n else:\n kwargs[param_mapping[param_name]] = param_value\n else:\n kwargs[param_mapping[param_name]] = param_value\n\n return kwargs\n\n def _build_kwargs(self, model: dict[str, Any], metadata: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Build kwargs dictionary using parameter mapping.\"\"\"\n param_mapping = metadata.get(\"param_mapping\", {})\n if not param_mapping:\n msg = \"Parameter mapping not found in metadata\"\n raise ValueError(msg)\n\n kwargs = {}\n\n # Required parameters - handle both \"model\" and \"model_id\" (for watsonx)\n if \"model\" in param_mapping:\n kwargs[param_mapping[\"model\"]] = model.get(\"name\")\n elif \"model_id\" in param_mapping:\n kwargs[param_mapping[\"model_id\"]] = model.get(\"name\")\n if \"api_key\" in param_mapping:\n kwargs[param_mapping[\"api_key\"]] = get_api_key_for_provider(\n self.user_id,\n model.get(\"provider\"),\n self.api_key,\n )\n\n # Optional parameters with their values\n provider = model.get(\"provider\")\n optional_params = {\n \"api_base\": self.api_base if self.api_base else None,\n \"dimensions\": int(self.dimensions) if self.dimensions else None,\n \"chunk_size\": int(self.chunk_size) if self.chunk_size else None,\n \"request_timeout\": float(self.request_timeout) if self.request_timeout else None,\n \"max_retries\": int(self.max_retries) if self.max_retries else None,\n \"show_progress_bar\": self.show_progress_bar if hasattr(self, \"show_progress_bar\") else None,\n \"model_kwargs\": self.model_kwargs if self.model_kwargs else None,\n }\n\n # Watson-specific parameters\n if provider in {\"IBM WatsonX\", \"IBM watsonx.ai\"}:\n # Map base_url_ibm_watsonx to \"url\" parameter for watsonx\n if \"url\" in param_mapping:\n url_value = (\n self.base_url_ibm_watsonx\n if hasattr(self, \"base_url_ibm_watsonx\") and self.base_url_ibm_watsonx\n else \"https://us-south.ml.cloud.ibm.com\"\n )\n kwargs[param_mapping[\"url\"]] = url_value\n # Map project_id for watsonx\n if hasattr(self, \"project_id\") and self.project_id and \"project_id\" in param_mapping:\n kwargs[param_mapping[\"project_id\"]] = self.project_id\n\n # Ollama-specific parameters\n if provider == \"Ollama\" and \"base_url\" in param_mapping:\n # Map api_base to \"base_url\" parameter for Ollama\n base_url_value = self.api_base if hasattr(self, \"api_base\") and self.api_base else \"http://localhost:11434\"\n kwargs[param_mapping[\"base_url\"]] = base_url_value\n\n # Add optional parameters if they have values and are mapped\n for param_name, param_value in optional_params.items():\n if param_value is not None and param_name in param_mapping:\n # Special handling for request_timeout with Google provider\n if param_name == \"request_timeout\":\n if provider == \"Google Generative AI\" and isinstance(param_value, (int, float)):\n kwargs[param_mapping[param_name]] = {\"timeout\": param_value}\n else:\n kwargs[param_mapping[param_name]] = param_value\n else:\n kwargs[param_mapping[param_name]] = param_value\n\n return kwargs\n" }, "dimensions": { "_input_type": "IntInput", @@ -3858,6 +4343,7 @@ "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", "list": false, "list_add_label": "Add More", + "load_from_db": false, "name": "dimensions", "override_skip": false, "placeholder": "", @@ -3870,28 +4356,6 @@ "type": "int", "value": "" }, - "fail_safe_mode": { - "_input_type": "BoolInput", - "advanced": true, - "display_name": "Fail-Safe Mode", - "dynamic": false, - "info": "When enabled, errors will be logged instead of raising exceptions. The component will return None on error.", - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "fail_safe_mode", - "override_skip": false, - "placeholder": "", - "real_time_refresh": true, - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": true - }, "input_text": { "_input_type": "BoolInput", "advanced": true, @@ -3912,7 +4376,7 @@ "type": "bool", "value": true }, - "is_refresh": false, + "is_refresh": true, "max_retries": { "_input_type": "IntInput", "advanced": true, @@ -3934,33 +4398,181 @@ "value": 3 }, "model": { - "_input_type": "DropdownInput", + "_input_type": "ModelInput", "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Model Name", + "display_name": "Embedding Model", "dynamic": false, - "external_options": {}, - "info": "Select the embedding model to use", - "load_from_db": false, + "external_options": { + "fields": { + "data": { + "node": { + "display_name": "Connect other models", + "icon": "CornerDownLeft", + "name": "connect_other_models" + } + } + } + }, + "info": "Select your model provider", + "input_types": [ + "Embeddings" + ], + "list": false, + "list_add_label": "Add More", + "model_type": "embedding", "name": "model", "options": [ - "text-embedding-3-small" + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-small", + "provider": "OpenAI" + }, + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-large", + "provider": "OpenAI" + }, + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-ada-002", + "provider": "OpenAI" + }, + { + "category": "Ollama", + "icon": "Ollama", + "metadata": { + "embedding_class": "OllamaEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "base_url": "base_url", + "model": "model", + "model_kwargs": "model_kwargs", + "num_ctx": "num_ctx", + "request_timeout": "request_timeout" + } + }, + "name": "nomic-embed-text:latest", + "provider": "Ollama" + }, + { + "category": "Ollama", + "icon": "Ollama", + "metadata": { + "embedding_class": "OllamaEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "base_url": "base_url", + "model": "model", + "model_kwargs": "model_kwargs", + "num_ctx": "num_ctx", + "request_timeout": "request_timeout" + } + }, + "name": "qwen3-embedding:latest", + "provider": "Ollama" + }, + { + "category": "Google Generative AI", + "icon": "GoogleGenerativeAI", + "metadata": { + "is_disabled_provider": true, + "variable_name": null + }, + "name": "__enable_provider_Google Generative AI__", + "provider": "Google Generative AI" + }, + { + "category": "IBM WatsonX", + "icon": "WatsonxAI", + "metadata": { + "is_disabled_provider": true, + "variable_name": null + }, + "name": "__enable_provider_IBM WatsonX__", + "provider": "IBM WatsonX" + } ], - "options_metadata": [], "override_skip": false, - "placeholder": "", + "placeholder": "Setup Provider", "real_time_refresh": true, "refresh_button": true, - "required": false, + "required": true, "show": true, "title_case": false, - "toggle": false, "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "str", - "value": "text-embedding-3-small" + "trace_as_input": true, + "track_in_telemetry": false, + "type": "model", + "value": [ + { + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-large", + "provider": "OpenAI" + } + ] }, "model_kwargs": { "_input_type": "DictInput", @@ -3982,32 +4594,6 @@ "type": "dict", "value": {} }, - "ollama_base_url": { - "_input_type": "MessageTextInput", - "advanced": false, - "display_name": "Ollama API URL", - "dynamic": false, - "info": "Endpoint of the Ollama API (Ollama only). Defaults to http://localhost:11434", - "input_types": [ - "Message" - ], - "list": false, - "list_add_label": "Add More", - "load_from_db": false, - "name": "ollama_base_url", - "override_skip": false, - "placeholder": "", - "real_time_refresh": true, - "required": false, - "show": false, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, "project_id": { "_input_type": "MessageTextInput", "advanced": false, @@ -4019,7 +4605,7 @@ ], "list": false, "list_add_label": "Add More", - "load_from_db": true, + "load_from_db": false, "name": "project_id", "override_skip": false, "placeholder": "", @@ -4031,50 +4617,7 @@ "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "WATSONX_PROJECT_ID" - }, - "provider": { - "_input_type": "DropdownInput", - "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Model Provider", - "dynamic": false, - "external_options": {}, - "info": "Select the embedding model provider", - "load_from_db": false, - "name": "provider", - "options": [ - "OpenAI", - "Ollama", - "IBM watsonx.ai" - ], - "options_metadata": [ - { - "icon": "OpenAI" - }, - { - "icon": "Ollama" - }, - { - "icon": "WatsonxAI" - } - ], - "override_skip": false, - "placeholder": "", - "real_time_refresh": true, - "required": false, - "selected_metadata": { - "icon": "WatsonxAI" - }, - "show": true, - "title_case": false, - "toggle": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "str", - "value": "OpenAI" + "value": "" }, "request_timeout": { "_input_type": "FloatInput", @@ -4143,21 +4686,23 @@ "type": "EmbeddingModel" }, "dragging": false, - "id": "EmbeddingModel-EAo9i", + "id": "EmbeddingModel-E0hvR", "measured": { - "height": 451, + "height": 207, "width": 320 }, "position": { - "x": 838.9563350647003, - "y": 2191.523005861695 + "x": 1308.6253288695966, + "y": 2125.2916788143903 }, "selected": false, "type": "genericNode" }, { "data": { - "id": "EmbeddingModel-E0hvR", + "description": "Generate embeddings using a specified provider.", + "display_name": "Embedding Model", + "id": "EmbeddingModel-3LsIP", "node": { "base_classes": [ "Embeddings" @@ -4168,14 +4713,12 @@ "description": "Generate embeddings using a specified provider.", "display_name": "Embedding Model", "documentation": "https://docs.langflow.org/components-embedding-models", - "edited": true, + "edited": false, "field_order": [ - "provider", - "api_base", - "ollama_base_url", - "base_url_ibm_watsonx", "model", "api_key", + "api_base", + "base_url_ibm_watsonx", "project_id", "dimensions", "chunk_size", @@ -4184,48 +4727,26 @@ "show_progress_bar", "model_kwargs", "truncate_input_tokens", - "input_text", - "fail_safe_mode" + "input_text" ], "frozen": false, "icon": "binary", - "last_updated": "2026-02-05T15:22:55.813Z", + "last_updated": "2026-02-27T18:40:44.054Z", "legacy": false, - "lf_version": "1.7.0.dev21", "metadata": { - "code_hash": "0e2d6fe67a26", + "code_hash": "c5ce0982da48", "dependencies": { "dependencies": [ - { - "name": "requests", - "version": "2.32.5" - }, - { - "name": "ibm_watsonx_ai", - "version": "1.4.2" - }, - { - "name": "langchain_openai", - "version": "0.3.23" - }, { "name": "lfx", - "version": "0.2.0.dev21" - }, - { - "name": "langchain_ollama", - "version": "0.3.10" - }, - { - "name": "langchain_community", - "version": "0.3.21" + "version": null }, { - "name": "langchain_ibm", - "version": "0.3.19" + "name": "langchain_core", + "version": "0.3.83" } ], - "total_dependencies": 7 + "total_dependencies": 2 }, "module": "custom_components.embedding_model" }, @@ -4257,7 +4778,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" + "value": "573ff6c4-003c-4e1d-aa3f-8a83e1f3e020" }, "_type": "Component", "api_base": { @@ -4276,7 +4797,7 @@ "override_skip": false, "placeholder": "", "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, @@ -4287,8 +4808,8 @@ }, "api_key": { "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "API Key (Optional)", + "advanced": true, + "display_name": "API Key", "dynamic": false, "info": "Model Provider API key", "input_types": [], @@ -4299,7 +4820,7 @@ "placeholder": "", "real_time_refresh": true, "required": false, - "show": false, + "show": true, "title_case": false, "track_in_telemetry": false, "type": "str", @@ -4373,7 +4894,7 @@ "show": true, "title_case": false, "type": "code", - "value": "from typing import Any\n\nimport requests\nfrom ibm_watsonx_ai.metanames import EmbedTextParamsMetaNames\nfrom langchain_openai import OpenAIEmbeddings\n\nfrom lfx.base.embeddings.embeddings_class import EmbeddingsWithModels\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.model_utils import get_ollama_models, is_valid_ollama_url\nfrom lfx.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom lfx.base.models.watsonx_constants import (\n IBM_WATSONX_URLS,\n WATSONX_EMBEDDING_MODEL_NAMES,\n)\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import (\n BoolInput,\n DictInput,\n DropdownInput,\n FloatInput,\n IntInput,\n MessageTextInput,\n SecretStrInput,\n)\nfrom lfx.log.logger import logger\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.util import transform_localhost_url\n\n# Ollama API constants\nHTTP_STATUS_OK = 200\nJSON_MODELS_KEY = \"models\"\nJSON_NAME_KEY = \"name\"\nJSON_CAPABILITIES_KEY = \"capabilities\"\nDESIRED_CAPABILITY = \"embedding\"\nDEFAULT_OLLAMA_URL = \"http://localhost:11434\"\n\n\nclass EmbeddingModelComponent(LCEmbeddingsModel):\n display_name = \"Embedding Model\"\n description = \"Generate embeddings using a specified provider.\"\n documentation: str = \"https://docs.langflow.org/components-embedding-models\"\n icon = \"binary\"\n name = \"EmbeddingModel\"\n category = \"models\"\n\n inputs = [\n DropdownInput(\n name=\"provider\",\n display_name=\"Model Provider\",\n options=[\"OpenAI\", \"Ollama\", \"IBM watsonx.ai\"],\n value=\"OpenAI\",\n info=\"Select the embedding model provider\",\n real_time_refresh=True,\n options_metadata=[{\"icon\": \"OpenAI\"}, {\"icon\": \"Ollama\"}, {\"icon\": \"WatsonxAI\"}],\n ),\n MessageTextInput(\n name=\"api_base\",\n display_name=\"API Base URL\",\n info=\"Base URL for the API. Leave empty for default.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"ollama_base_url\",\n display_name=\"Ollama API URL\",\n info=f\"Endpoint of the Ollama API (Ollama only). Defaults to {DEFAULT_OLLAMA_URL}\",\n value=DEFAULT_OLLAMA_URL,\n show=False,\n real_time_refresh=True,\n load_from_db=True,\n ),\n DropdownInput(\n name=\"base_url_ibm_watsonx\",\n display_name=\"watsonx API Endpoint\",\n info=\"The base URL of the API (IBM watsonx.ai only)\",\n options=IBM_WATSONX_URLS,\n value=IBM_WATSONX_URLS[0],\n show=False,\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"model\",\n display_name=\"Model Name\",\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=OPENAI_EMBEDDING_MODEL_NAMES[0],\n info=\"Select the embedding model to use\",\n real_time_refresh=True,\n refresh_button=True,\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"OpenAI API Key\",\n info=\"Model Provider API key\",\n required=True,\n show=True,\n real_time_refresh=True,\n ),\n # Watson-specific inputs\n MessageTextInput(\n name=\"project_id\",\n display_name=\"Project ID\",\n info=\"IBM watsonx.ai Project ID (required for IBM watsonx.ai)\",\n show=False,\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", advanced=True, value=3),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n IntInput(\n name=\"truncate_input_tokens\",\n display_name=\"Truncate Input Tokens\",\n advanced=True,\n value=200,\n show=False,\n ),\n BoolInput(\n name=\"input_text\",\n display_name=\"Include the original text in the output\",\n value=True,\n advanced=True,\n show=False,\n ),\n BoolInput(\n name=\"fail_safe_mode\",\n display_name=\"Fail-Safe Mode\",\n value=False,\n advanced=True,\n info=\"When enabled, errors will be logged instead of raising exceptions. \"\n \"The component will return None on error.\",\n real_time_refresh=True,\n ),\n ]\n\n @staticmethod\n def fetch_ibm_models(base_url: str) -> list[str]:\n \"\"\"Fetch available models from the watsonx.ai API.\"\"\"\n try:\n endpoint = f\"{base_url}/ml/v1/foundation_model_specs\"\n params = {\n \"version\": \"2024-09-16\",\n \"filters\": \"function_embedding,!lifecycle_withdrawn:and\",\n }\n response = requests.get(endpoint, params=params, timeout=10)\n response.raise_for_status()\n data = response.json()\n models = [model[\"model_id\"] for model in data.get(\"resources\", [])]\n return sorted(models)\n except Exception: # noqa: BLE001\n logger.exception(\"Error fetching models\")\n return WATSONX_EMBEDDING_MODEL_NAMES\n async def fetch_ollama_models(self) -> list[str]:\n try:\n return await get_ollama_models(\n base_url_value=self.ollama_base_url,\n desired_capability=DESIRED_CAPABILITY,\n json_models_key=JSON_MODELS_KEY,\n json_name_key=JSON_NAME_KEY,\n json_capabilities_key=JSON_CAPABILITIES_KEY,\n )\n except Exception: # noqa: BLE001\n\n logger.exception(\"Error fetching models\")\n return []\n async def build_embeddings(self) -> Embeddings:\n provider = self.provider\n model = self.model\n api_key = self.api_key\n api_base = self.api_base\n base_url_ibm_watsonx = self.base_url_ibm_watsonx\n ollama_base_url = self.ollama_base_url\n dimensions = self.dimensions\n chunk_size = self.chunk_size\n request_timeout = self.request_timeout\n max_retries = self.max_retries\n show_progress_bar = self.show_progress_bar\n model_kwargs = self.model_kwargs or {}\n\n if provider == \"OpenAI\":\n if not api_key:\n msg = \"OpenAI API key is required when using OpenAI provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n try:\n # Create the primary embedding instance\n embeddings_instance = OpenAIEmbeddings(\n model=model,\n dimensions=dimensions or None,\n base_url=api_base or None,\n api_key=api_key,\n chunk_size=chunk_size,\n max_retries=max_retries,\n timeout=request_timeout or None,\n show_progress_bar=show_progress_bar,\n model_kwargs=model_kwargs,\n )\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in OPENAI_EMBEDDING_MODEL_NAMES:\n available_models_dict[model_name] = OpenAIEmbeddings(\n model=model_name,\n dimensions=dimensions or None, # Use same dimensions config for all\n base_url=api_base or None,\n api_key=api_key,\n chunk_size=chunk_size,\n max_retries=max_retries,\n timeout=request_timeout or None,\n show_progress_bar=show_progress_bar,\n model_kwargs=model_kwargs,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to initialize OpenAI embeddings: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n if provider == \"Ollama\":\n try:\n from langchain_ollama import OllamaEmbeddings\n except ImportError:\n try:\n from langchain_community.embeddings import OllamaEmbeddings\n except ImportError:\n msg = \"Please install langchain-ollama: pip install langchain-ollama\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ImportError(msg) from None\n\n try:\n transformed_base_url = transform_localhost_url(ollama_base_url)\n\n # Check if URL contains /v1 suffix (OpenAI-compatible mode)\n if transformed_base_url and transformed_base_url.rstrip(\"/\").endswith(\"/v1\"):\n # Strip /v1 suffix and log warning\n transformed_base_url = transformed_base_url.rstrip(\"/\").removesuffix(\"/v1\")\n logger.warning(\n \"Detected '/v1' suffix in base URL. The Ollama component uses the native Ollama API, \"\n \"not the OpenAI-compatible API. The '/v1' suffix has been automatically removed. \"\n \"If you want to use the OpenAI-compatible API, please use the OpenAI component instead. \"\n \"Learn more at https://docs.ollama.com/openai#openai-compatibility\"\n )\n\n final_base_url = transformed_base_url or \"http://localhost:11434\"\n\n # Create the primary embedding instance\n embeddings_instance = OllamaEmbeddings(\n model=model,\n base_url=final_base_url,\n **model_kwargs,\n )\n\n # Fetch available Ollama models\n available_model_names = await self.fetch_ollama_models()\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in available_model_names:\n available_models_dict[model_name] = OllamaEmbeddings(\n model=model_name,\n base_url=final_base_url,\n **model_kwargs,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to initialize Ollama embeddings: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n if provider == \"IBM watsonx.ai\":\n try:\n from langchain_ibm import WatsonxEmbeddings\n except ImportError:\n msg = \"Please install langchain-ibm: pip install langchain-ibm\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ImportError(msg) from None\n\n if not api_key:\n msg = \"IBM watsonx.ai API key is required when using IBM watsonx.ai provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n project_id = self.project_id\n\n if not project_id:\n msg = \"Project ID is required for IBM watsonx.ai provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n try:\n from ibm_watsonx_ai import APIClient, Credentials\n\n final_url = base_url_ibm_watsonx or \"https://us-south.ml.cloud.ibm.com\"\n\n credentials = Credentials(\n api_key=self.api_key,\n url=final_url,\n )\n\n api_client = APIClient(credentials)\n\n params = {\n EmbedTextParamsMetaNames.TRUNCATE_INPUT_TOKENS: self.truncate_input_tokens,\n EmbedTextParamsMetaNames.RETURN_OPTIONS: {\"input_text\": self.input_text},\n }\n\n # Create the primary embedding instance\n embeddings_instance = WatsonxEmbeddings(\n model_id=model,\n params=params,\n watsonx_client=api_client,\n project_id=project_id,\n )\n\n # Fetch available IBM watsonx.ai models\n available_model_names = self.fetch_ibm_models(final_url)\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in available_model_names:\n available_models_dict[model_name] = WatsonxEmbeddings(\n model_id=model_name,\n params=params,\n watsonx_client=api_client,\n project_id=project_id,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to authenticate with IBM watsonx.ai: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n msg = f\"Unknown provider: {provider}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n async def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n # Handle fail_safe_mode changes first - set all required fields to False if enabled\n if field_name == \"fail_safe_mode\":\n if field_value: # If fail_safe_mode is enabled\n build_config[\"api_key\"][\"required\"] = False\n elif hasattr(self, \"provider\"):\n # If fail_safe_mode is disabled, restore required flags based on provider\n if self.provider in [\"OpenAI\", \"IBM watsonx.ai\"]:\n build_config[\"api_key\"][\"required\"] = True\n else: # Ollama\n build_config[\"api_key\"][\"required\"] = False\n\n if field_name == \"provider\":\n if field_value == \"OpenAI\":\n build_config[\"model\"][\"options\"] = OPENAI_EMBEDDING_MODEL_NAMES\n build_config[\"model\"][\"value\"] = OPENAI_EMBEDDING_MODEL_NAMES[0]\n build_config[\"api_key\"][\"display_name\"] = \"OpenAI API Key\"\n # Only set required=True if fail_safe_mode is not enabled\n build_config[\"api_key\"][\"required\"] = not (hasattr(self, \"fail_safe_mode\") and self.fail_safe_mode)\n build_config[\"api_key\"][\"show\"] = True\n build_config[\"api_base\"][\"display_name\"] = \"OpenAI API Base URL\"\n build_config[\"api_base\"][\"advanced\"] = True\n build_config[\"api_base\"][\"show\"] = True\n build_config[\"ollama_base_url\"][\"show\"] = False\n build_config[\"project_id\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = False\n build_config[\"truncate_input_tokens\"][\"show\"] = False\n build_config[\"input_text\"][\"show\"] = False\n elif field_value == \"Ollama\":\n build_config[\"ollama_base_url\"][\"show\"] = True\n\n if await is_valid_ollama_url(url=self.ollama_base_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n build_config[\"model\"][\"value\"] = models[0] if models else \"\"\n except ValueError:\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n else:\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n build_config[\"truncate_input_tokens\"][\"show\"] = False\n build_config[\"input_text\"][\"show\"] = False\n build_config[\"api_key\"][\"display_name\"] = \"API Key (Optional)\"\n build_config[\"api_key\"][\"required\"] = False\n build_config[\"api_key\"][\"show\"] = False\n build_config[\"api_base\"][\"show\"] = False\n build_config[\"project_id\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = False\n\n elif field_value == \"IBM watsonx.ai\":\n build_config[\"model\"][\"options\"] = self.fetch_ibm_models(base_url=self.base_url_ibm_watsonx)\n build_config[\"model\"][\"value\"] = self.fetch_ibm_models(base_url=self.base_url_ibm_watsonx)[0]\n build_config[\"api_key\"][\"display_name\"] = \"IBM watsonx.ai API Key\"\n # Only set required=True if fail_safe_mode is not enabled\n build_config[\"api_key\"][\"required\"] = not (hasattr(self, \"fail_safe_mode\") and self.fail_safe_mode)\n build_config[\"api_key\"][\"show\"] = True\n build_config[\"api_base\"][\"show\"] = False\n build_config[\"ollama_base_url\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = True\n build_config[\"project_id\"][\"show\"] = True\n build_config[\"truncate_input_tokens\"][\"show\"] = True\n build_config[\"input_text\"][\"show\"] = True\n elif field_name == \"base_url_ibm_watsonx\":\n build_config[\"model\"][\"options\"] = self.fetch_ibm_models(base_url=field_value)\n build_config[\"model\"][\"value\"] = self.fetch_ibm_models(base_url=field_value)[0]\n elif field_name == \"ollama_base_url\":\n # # Refresh Ollama models when base URL changes\n # if hasattr(self, \"provider\") and self.provider == \"Ollama\":\n # Use field_value if provided, otherwise fall back to instance attribute\n ollama_url = self.ollama_base_url\n if await is_valid_ollama_url(url=ollama_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n build_config[\"model\"][\"value\"] = models[0] if models else \"\"\n except ValueError:\n await logger.awarning(\"Failed to fetch Ollama embedding models.\")\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n\n elif field_name == \"model\" and self.provider == \"Ollama\":\n ollama_url = self.ollama_base_url\n if await is_valid_ollama_url(url=ollama_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n except ValueError:\n await logger.awarning(\"Failed to refresh Ollama embedding models.\")\n build_config[\"model\"][\"options\"] = []\n\n return build_config\n" + "value": "from typing import Any\n\nfrom lfx.base.embeddings.embeddings_class import EmbeddingsWithModels\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.unified_models import (\n get_api_key_for_provider,\n get_embedding_class,\n get_embedding_model_options,\n get_unified_models_detailed,\n update_model_options_in_build_config,\n)\nfrom lfx.base.models.watsonx_constants import IBM_WATSONX_URLS\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import (\n BoolInput,\n DictInput,\n DropdownInput,\n FloatInput,\n IntInput,\n MessageTextInput,\n ModelInput,\n SecretStrInput,\n)\nfrom lfx.log.logger import logger\n\n\nclass EmbeddingModelComponent(LCEmbeddingsModel):\n display_name = \"Embedding Model\"\n description = \"Generate embeddings using a specified provider.\"\n documentation: str = \"https://docs.langflow.org/components-embedding-models\"\n icon = \"binary\"\n name = \"EmbeddingModel\"\n category = \"models\"\n\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None):\n \"\"\"Dynamically update build config with user-filtered model options.\"\"\"\n # Update model options\n build_config = update_model_options_in_build_config(\n component=self,\n build_config=build_config,\n cache_key_prefix=\"embedding_model_options\",\n get_options_func=get_embedding_model_options,\n field_name=field_name,\n field_value=field_value,\n )\n\n # Show/hide provider-specific fields based on selected model\n if field_name == \"model\" and isinstance(field_value, list) and len(field_value) > 0:\n selected_model = field_value[0]\n provider = selected_model.get(\"provider\", \"\")\n\n # Show/hide watsonx fields\n is_watsonx = provider == \"IBM WatsonX\"\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = is_watsonx\n build_config[\"project_id\"][\"show\"] = is_watsonx\n build_config[\"truncate_input_tokens\"][\"show\"] = is_watsonx\n build_config[\"input_text\"][\"show\"] = is_watsonx\n if is_watsonx:\n build_config[\"base_url_ibm_watsonx\"][\"required\"] = True\n build_config[\"project_id\"][\"required\"] = True\n\n return build_config\n\n inputs = [\n ModelInput(\n name=\"model\",\n display_name=\"Embedding Model\",\n info=\"Select your model provider\",\n real_time_refresh=True,\n required=True,\n model_type=\"embedding\",\n input_types=[\"Embeddings\"], # Override default to accept Embeddings instead of LanguageModel\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"API Key\",\n info=\"Model Provider API key\",\n real_time_refresh=True,\n advanced=True,\n ),\n MessageTextInput(\n name=\"api_base\",\n display_name=\"API Base URL\",\n info=\"Base URL for the API. Leave empty for default.\",\n advanced=True,\n ),\n # Watson-specific inputs\n DropdownInput(\n name=\"base_url_ibm_watsonx\",\n display_name=\"watsonx API Endpoint\",\n info=\"The base URL of the API (IBM watsonx.ai only)\",\n options=IBM_WATSONX_URLS,\n value=IBM_WATSONX_URLS[0],\n show=False,\n real_time_refresh=True,\n ),\n MessageTextInput(\n name=\"project_id\",\n display_name=\"Project ID\",\n info=\"IBM watsonx.ai Project ID (required for IBM watsonx.ai)\",\n show=False,\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n IntInput(\n name=\"chunk_size\",\n display_name=\"Chunk Size\",\n advanced=True,\n value=1000,\n ),\n FloatInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout\",\n advanced=True,\n ),\n IntInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n advanced=True,\n value=3,\n ),\n BoolInput(\n name=\"show_progress_bar\",\n display_name=\"Show Progress Bar\",\n advanced=True,\n ),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n IntInput(\n name=\"truncate_input_tokens\",\n display_name=\"Truncate Input Tokens\",\n advanced=True,\n value=200,\n show=False,\n ),\n BoolInput(\n name=\"input_text\",\n display_name=\"Include the original text in the output\",\n value=True,\n advanced=True,\n show=False,\n ),\n ]\n\n def build_embeddings(self) -> Embeddings:\n \"\"\"Build and return an embeddings instance based on the selected model.\n\n Returns an EmbeddingsWithModels wrapper that contains:\n - The primary embedding instance (for the selected model)\n - available_models dict mapping all available model names to their instances\n \"\"\"\n # If an Embeddings object is directly connected, return it\n try:\n from langchain_core.embeddings import Embeddings as BaseEmbeddings\n\n if isinstance(self.model, BaseEmbeddings):\n return self.model\n except ImportError:\n pass\n\n # Safely extract model configuration\n if not self.model or not isinstance(self.model, list):\n msg = \"Model must be a non-empty list\"\n raise ValueError(msg)\n\n model = self.model[0]\n model_name = model.get(\"name\")\n provider = model.get(\"provider\")\n metadata = model.get(\"metadata\", {})\n\n # Get API key from user input or global variables\n api_key = get_api_key_for_provider(self.user_id, provider, self.api_key)\n\n # Validate required fields (Ollama doesn't require API key)\n if not api_key and provider != \"Ollama\":\n msg = (\n f\"{provider} API key is required. \"\n f\"Please provide it in the component or configure it globally as \"\n f\"{provider.upper().replace(' ', '_')}_API_KEY.\"\n )\n raise ValueError(msg)\n\n if not model_name:\n msg = \"Model name is required\"\n raise ValueError(msg)\n\n # Get embedding class\n embedding_class_name = metadata.get(\"embedding_class\")\n if not embedding_class_name:\n msg = f\"No embedding class defined in metadata for {model_name}\"\n raise ValueError(msg)\n\n embedding_class = get_embedding_class(embedding_class_name)\n\n # Build kwargs using parameter mapping for primary instance\n kwargs = self._build_kwargs(model, metadata)\n primary_instance = embedding_class(**kwargs)\n\n # Get all available embedding models for this provider\n available_models_dict = self._build_available_models(\n provider=provider,\n embedding_class=embedding_class,\n metadata=metadata,\n api_key=api_key,\n )\n\n # Wrap with EmbeddingsWithModels to provide available_models metadata\n return EmbeddingsWithModels(\n embeddings=primary_instance,\n available_models=available_models_dict,\n )\n\n def _build_available_models(\n self,\n provider: str,\n embedding_class: type,\n metadata: dict[str, Any],\n api_key: str | None,\n ) -> dict[str, Embeddings]:\n \"\"\"Build a dictionary of all available embedding model instances for the provider.\n\n Args:\n provider: The provider name (e.g., \"OpenAI\", \"Ollama\")\n embedding_class: The embedding class to instantiate\n metadata: Metadata containing param_mapping\n api_key: The API key for the provider\n\n Returns:\n Dict mapping model names to their embedding instances\n \"\"\"\n available_models_dict: dict[str, Embeddings] = {}\n\n # Get all embedding models for this provider from unified models\n all_embedding_models = get_unified_models_detailed(\n providers=[provider],\n model_type=\"embeddings\",\n include_deprecated=False,\n include_unsupported=False,\n )\n\n if not all_embedding_models:\n return available_models_dict\n\n # Extract models from the provider data\n for provider_data in all_embedding_models:\n if provider_data.get(\"provider\") != provider:\n continue\n\n for model_data in provider_data.get(\"models\", []):\n model_name = model_data.get(\"model_name\")\n if not model_name:\n continue\n\n # Create a model dict compatible with _build_kwargs\n model_dict = {\n \"name\": model_name,\n \"provider\": provider,\n \"metadata\": metadata, # Reuse the same metadata/param_mapping\n }\n\n try:\n # Build kwargs for this model\n model_kwargs = self._build_kwargs_for_model(model_dict, metadata, api_key)\n # Create the embedding instance\n available_models_dict[model_name] = embedding_class(**model_kwargs)\n except Exception: # noqa: BLE001\n # Skip models that fail to instantiate\n # This handles cases where specific models have incompatible parameters\n logger.debug(\"Failed to instantiate embedding model %s: skipping\", model_name, exc_info=True)\n continue\n\n return available_models_dict\n\n def _build_kwargs_for_model(\n self,\n model: dict[str, Any],\n metadata: dict[str, Any],\n api_key: str | None,\n ) -> dict[str, Any]:\n \"\"\"Build kwargs dictionary for a specific model using parameter mapping.\n\n This is similar to _build_kwargs but uses the provided api_key directly\n instead of looking it up again.\n\n Args:\n model: Model dict with name and provider\n metadata: Metadata containing param_mapping\n api_key: The API key to use\n\n Returns:\n kwargs dict for embedding class instantiation\n \"\"\"\n param_mapping = metadata.get(\"param_mapping\", {})\n if not param_mapping:\n msg = \"Parameter mapping not found in metadata\"\n raise ValueError(msg)\n\n kwargs = {}\n provider = model.get(\"provider\")\n\n # Required parameters - handle both \"model\" and \"model_id\" (for watsonx)\n if \"model\" in param_mapping:\n kwargs[param_mapping[\"model\"]] = model.get(\"name\")\n elif \"model_id\" in param_mapping:\n kwargs[param_mapping[\"model_id\"]] = model.get(\"name\")\n\n # Add API key if mapped\n if \"api_key\" in param_mapping and api_key:\n kwargs[param_mapping[\"api_key\"]] = api_key\n\n # Optional parameters with their values\n optional_params = {\n \"api_base\": self.api_base if self.api_base else None,\n \"dimensions\": int(self.dimensions) if self.dimensions else None,\n \"chunk_size\": int(self.chunk_size) if self.chunk_size else None,\n \"request_timeout\": float(self.request_timeout) if self.request_timeout else None,\n \"max_retries\": int(self.max_retries) if self.max_retries else None,\n \"show_progress_bar\": self.show_progress_bar if hasattr(self, \"show_progress_bar\") else None,\n \"model_kwargs\": self.model_kwargs if self.model_kwargs else None,\n }\n\n # Watson-specific parameters\n if provider in {\"IBM WatsonX\", \"IBM watsonx.ai\"}:\n # Map base_url_ibm_watsonx to \"url\" parameter for watsonx\n if \"url\" in param_mapping:\n url_value = (\n self.base_url_ibm_watsonx\n if hasattr(self, \"base_url_ibm_watsonx\") and self.base_url_ibm_watsonx\n else \"https://us-south.ml.cloud.ibm.com\"\n )\n kwargs[param_mapping[\"url\"]] = url_value\n # Map project_id for watsonx\n if hasattr(self, \"project_id\") and self.project_id and \"project_id\" in param_mapping:\n kwargs[param_mapping[\"project_id\"]] = self.project_id\n\n # Ollama-specific parameters\n if provider == \"Ollama\" and \"base_url\" in param_mapping:\n # Map api_base to \"base_url\" parameter for Ollama\n base_url_value = self.api_base if hasattr(self, \"api_base\") and self.api_base else \"http://localhost:11434\"\n kwargs[param_mapping[\"base_url\"]] = base_url_value\n\n # Add optional parameters if they have values and are mapped\n for param_name, param_value in optional_params.items():\n if param_value is not None and param_name in param_mapping:\n # Special handling for request_timeout with Google provider\n if param_name == \"request_timeout\":\n if provider == \"Google Generative AI\" and isinstance(param_value, (int, float)):\n kwargs[param_mapping[param_name]] = {\"timeout\": param_value}\n else:\n kwargs[param_mapping[param_name]] = param_value\n else:\n kwargs[param_mapping[param_name]] = param_value\n\n return kwargs\n\n def _build_kwargs(self, model: dict[str, Any], metadata: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Build kwargs dictionary using parameter mapping.\"\"\"\n param_mapping = metadata.get(\"param_mapping\", {})\n if not param_mapping:\n msg = \"Parameter mapping not found in metadata\"\n raise ValueError(msg)\n\n kwargs = {}\n\n # Required parameters - handle both \"model\" and \"model_id\" (for watsonx)\n if \"model\" in param_mapping:\n kwargs[param_mapping[\"model\"]] = model.get(\"name\")\n elif \"model_id\" in param_mapping:\n kwargs[param_mapping[\"model_id\"]] = model.get(\"name\")\n if \"api_key\" in param_mapping:\n kwargs[param_mapping[\"api_key\"]] = get_api_key_for_provider(\n self.user_id,\n model.get(\"provider\"),\n self.api_key,\n )\n\n # Optional parameters with their values\n provider = model.get(\"provider\")\n optional_params = {\n \"api_base\": self.api_base if self.api_base else None,\n \"dimensions\": int(self.dimensions) if self.dimensions else None,\n \"chunk_size\": int(self.chunk_size) if self.chunk_size else None,\n \"request_timeout\": float(self.request_timeout) if self.request_timeout else None,\n \"max_retries\": int(self.max_retries) if self.max_retries else None,\n \"show_progress_bar\": self.show_progress_bar if hasattr(self, \"show_progress_bar\") else None,\n \"model_kwargs\": self.model_kwargs if self.model_kwargs else None,\n }\n\n # Watson-specific parameters\n if provider in {\"IBM WatsonX\", \"IBM watsonx.ai\"}:\n # Map base_url_ibm_watsonx to \"url\" parameter for watsonx\n if \"url\" in param_mapping:\n url_value = (\n self.base_url_ibm_watsonx\n if hasattr(self, \"base_url_ibm_watsonx\") and self.base_url_ibm_watsonx\n else \"https://us-south.ml.cloud.ibm.com\"\n )\n kwargs[param_mapping[\"url\"]] = url_value\n # Map project_id for watsonx\n if hasattr(self, \"project_id\") and self.project_id and \"project_id\" in param_mapping:\n kwargs[param_mapping[\"project_id\"]] = self.project_id\n\n # Ollama-specific parameters\n if provider == \"Ollama\" and \"base_url\" in param_mapping:\n # Map api_base to \"base_url\" parameter for Ollama\n base_url_value = self.api_base if hasattr(self, \"api_base\") and self.api_base else \"http://localhost:11434\"\n kwargs[param_mapping[\"base_url\"]] = base_url_value\n\n # Add optional parameters if they have values and are mapped\n for param_name, param_value in optional_params.items():\n if param_value is not None and param_name in param_mapping:\n # Special handling for request_timeout with Google provider\n if param_name == \"request_timeout\":\n if provider == \"Google Generative AI\" and isinstance(param_value, (int, float)):\n kwargs[param_mapping[param_name]] = {\"timeout\": param_value}\n else:\n kwargs[param_mapping[param_name]] = param_value\n else:\n kwargs[param_mapping[param_name]] = param_value\n\n return kwargs\n" }, "dimensions": { "_input_type": "IntInput", @@ -4383,6 +4904,7 @@ "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", "list": false, "list_add_label": "Add More", + "load_from_db": false, "name": "dimensions", "override_skip": false, "placeholder": "", @@ -4395,21 +4917,19 @@ "type": "int", "value": "" }, - "fail_safe_mode": { + "input_text": { "_input_type": "BoolInput", "advanced": true, - "display_name": "Fail-Safe Mode", + "display_name": "Include the original text in the output", "dynamic": false, - "info": "When enabled, errors will be logged instead of raising exceptions. The component will return None on error.", + "info": "", "list": false, "list_add_label": "Add More", - "load_from_db": false, - "name": "fail_safe_mode", + "name": "input_text", "override_skip": false, "placeholder": "", - "real_time_refresh": true, "required": false, - "show": true, + "show": false, "title_case": false, "tool_mode": false, "trace_as_metadata": true, @@ -4417,343 +4937,821 @@ "type": "bool", "value": true }, - "input_text": { - "_input_type": "BoolInput", + "is_refresh": true, + "max_retries": { + "_input_type": "IntInput", "advanced": true, - "display_name": "Include the original text in the output", + "display_name": "Max Retries", "dynamic": false, "info": "", "list": false, "list_add_label": "Add More", - "name": "input_text", + "name": "max_retries", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": 3 + }, + "model": { + "_input_type": "ModelInput", + "advanced": false, + "display_name": "Embedding Model", + "dynamic": false, + "external_options": { + "fields": { + "data": { + "node": { + "display_name": "Connect other models", + "icon": "CornerDownLeft", + "name": "connect_other_models" + } + } + } + }, + "info": "Select your model provider", + "input_types": [ + "Embeddings" + ], + "list": false, + "list_add_label": "Add More", + "model_type": "embedding", + "name": "model", + "options": [ + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-small", + "provider": "OpenAI" + }, + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-large", + "provider": "OpenAI" + }, + { + "category": "OpenAI", + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-ada-002", + "provider": "OpenAI" + }, + { + "category": "Ollama", + "icon": "Ollama", + "metadata": { + "embedding_class": "OllamaEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "base_url": "base_url", + "model": "model", + "model_kwargs": "model_kwargs", + "num_ctx": "num_ctx", + "request_timeout": "request_timeout" + } + }, + "name": "nomic-embed-text:latest", + "provider": "Ollama" + }, + { + "category": "Ollama", + "icon": "Ollama", + "metadata": { + "embedding_class": "OllamaEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "base_url": "base_url", + "model": "model", + "model_kwargs": "model_kwargs", + "num_ctx": "num_ctx", + "request_timeout": "request_timeout" + } + }, + "name": "qwen3-embedding:latest", + "provider": "Ollama" + }, + { + "category": "Google Generative AI", + "icon": "GoogleGenerativeAI", + "metadata": { + "is_disabled_provider": true, + "variable_name": null + }, + "name": "__enable_provider_Google Generative AI__", + "provider": "Google Generative AI" + }, + { + "category": "IBM WatsonX", + "icon": "WatsonxAI", + "metadata": { + "is_disabled_provider": true, + "variable_name": null + }, + "name": "__enable_provider_IBM WatsonX__", + "provider": "IBM WatsonX" + } + ], + "override_skip": false, + "placeholder": "Setup Provider", + "real_time_refresh": true, + "refresh_button": true, + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "track_in_telemetry": false, + "type": "model", + "value": [ + { + "icon": "OpenAI", + "metadata": { + "embedding_class": "OpenAIEmbeddings", + "model_type": "embeddings", + "param_mapping": { + "api_base": "base_url", + "api_key": "api_key", + "chunk_size": "chunk_size", + "dimensions": "dimensions", + "max_retries": "max_retries", + "model": "model", + "model_kwargs": "model_kwargs", + "request_timeout": "timeout", + "show_progress_bar": "show_progress_bar" + } + }, + "name": "text-embedding-3-large", + "provider": "OpenAI" + } + ] + }, + "model_kwargs": { + "_input_type": "DictInput", + "advanced": true, + "display_name": "Model Kwargs", + "dynamic": false, + "info": "Additional keyword arguments to pass to the model.", + "list": false, + "list_add_label": "Add More", + "name": "model_kwargs", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "track_in_telemetry": false, + "type": "dict", + "value": {} + }, + "project_id": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Project ID", + "dynamic": false, + "info": "IBM watsonx.ai Project ID (required for IBM watsonx.ai)", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "project_id", "override_skip": false, "placeholder": "", "required": false, "show": false, "title_case": false, "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "request_timeout": { + "_input_type": "FloatInput", + "advanced": true, + "display_name": "Request Timeout", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "request_timeout", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "float", + "value": "" + }, + "show_progress_bar": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Show Progress Bar", + "dynamic": false, + "info": "", + "list": false, + "list_add_label": "Add More", + "name": "show_progress_bar", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, "type": "bool", - "value": true + "value": false }, - "is_refresh": false, - "max_retries": { + "truncate_input_tokens": { "_input_type": "IntInput", "advanced": true, - "display_name": "Max Retries", + "display_name": "Truncate Input Tokens", "dynamic": false, "info": "", "list": false, "list_add_label": "Add More", - "name": "max_retries", + "name": "truncate_input_tokens", "override_skip": false, "placeholder": "", - "required": false, + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": 200 + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "EmbeddingModel" + }, + "dragging": false, + "id": "EmbeddingModel-3LsIP", + "measured": { + "height": 207, + "width": 320 + }, + "position": { + "x": 1307.8810223474497, + "y": 2360.3146379131413 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "TextInput-CTKlr", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, + "field_order": [ + "input_value", + "use_global_variable" + ], + "frozen": false, + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", + "legacy": false, + "metadata": { + "code_hash": "518f16485886", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 + }, + "module": "lfx.components.input_output.text.TextInputComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Text", + "group_outputs": false, + "loop_types": null, + "method": "text_response", + "name": "text", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" + }, + "_type": "Component", + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, "show": true, "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 3 + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "model": { - "_input_type": "DropdownInput", + "input_value": { + "_input_type": "MultilineInput", "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Model Name", + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "external_options": {}, - "info": "Select the embedding model to use", - "load_from_db": false, - "name": "model", - "options": [ - "all-minilm:latest", - "nomic-embed-text:latest" + "info": "Text to be passed as input.", + "input_types": [ + "Message" ], - "options_metadata": [], + "list": false, + "list_add_label": "Add More", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", - "real_time_refresh": true, - "refresh_button": true, "required": false, "show": true, "title_case": false, - "toggle": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, - "track_in_telemetry": true, + "track_in_telemetry": false, "type": "str", - "value": "all-minilm:latest" + "value": "ALLOWED_GROUPS" }, - "model_kwargs": { - "_input_type": "DictInput", + "is_refresh": false, + "use_global_variable": { + "_input_type": "BoolInput", "advanced": true, - "display_name": "Model Kwargs", + "display_name": "Use Global Variable", "dynamic": false, - "info": "Additional keyword arguments to pass to the model.", + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", "list": false, "list_add_label": "Add More", - "name": "model_kwargs", + "name": "use_global_variable", "override_skip": false, "placeholder": "", + "real_time_refresh": true, "required": false, "show": true, "title_case": false, "tool_mode": false, - "trace_as_input": true, - "track_in_telemetry": false, - "type": "dict", - "value": {} + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "showNode": false, + "type": "TextInput" + }, + "dragging": false, + "id": "TextInput-CTKlr", + "measured": { + "height": 52, + "width": 192 + }, + "position": { + "x": 1028.9084907232098, + "y": 1062.849997200131 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "TextInput-hlgVv", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, + "field_order": [ + "input_value", + "use_global_variable" + ], + "frozen": false, + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", + "legacy": false, + "metadata": { + "code_hash": "518f16485886", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 }, - "ollama_base_url": { - "_input_type": "MessageTextInput", - "advanced": false, - "display_name": "Ollama API URL", - "dynamic": false, - "info": "Endpoint of the Ollama API (Ollama only). Defaults to http://localhost:11434", - "input_types": [ + "module": "lfx.components.input_output.text.TextInputComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Text", + "group_outputs": false, + "loop_types": null, + "method": "text_response", + "name": "text", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ "Message" ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" + }, + "_type": "Component", + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", "list": false, - "list_add_label": "Add More", - "load_from_db": true, - "name": "ollama_base_url", - "override_skip": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, "placeholder": "", - "real_time_refresh": true, - "required": false, + "required": true, "show": true, "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "OLLAMA_BASE_URL" + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "project_id": { - "_input_type": "MessageTextInput", + "input_value": { + "_input_type": "MultilineInput", "advanced": false, - "display_name": "Project ID", + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "info": "IBM watsonx.ai Project ID (required for IBM watsonx.ai)", + "info": "Text to be passed as input.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", - "load_from_db": false, - "name": "project_id", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "ALLOWED_USERS" }, - "provider": { - "_input_type": "DropdownInput", - "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Model Provider", + "is_refresh": false, + "use_global_variable": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Use Global Variable", "dynamic": false, - "external_options": {}, - "info": "Select the embedding model provider", - "load_from_db": false, - "name": "provider", - "options": [ - "OpenAI", - "Ollama", - "IBM watsonx.ai" - ], - "options_metadata": [ - { - "icon": "OpenAI" - }, - { - "icon": "Ollama" - }, - { - "icon": "WatsonxAI" - } - ], + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", + "list": false, + "list_add_label": "Add More", + "name": "use_global_variable", "override_skip": false, "placeholder": "", "real_time_refresh": true, "required": false, - "selected_metadata": { - "icon": "Ollama" - }, "show": true, "title_case": false, - "toggle": false, "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "str", - "value": "Ollama" + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "showNode": false, + "type": "TextInput" + }, + "dragging": false, + "id": "TextInput-hlgVv", + "measured": { + "height": 52, + "width": 192 + }, + "position": { + "x": 1032.6873315933058, + "y": 1145.9844963422515 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "TextInput-OGCeZ", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, + "field_order": [ + "input_value", + "use_global_variable" + ], + "frozen": false, + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", + "legacy": false, + "metadata": { + "code_hash": "518f16485886", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 }, - "request_timeout": { - "_input_type": "FloatInput", - "advanced": true, - "display_name": "Request Timeout", - "dynamic": false, + "module": "lfx.components.input_output.text.TextInputComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Text", + "group_outputs": false, + "loop_types": null, + "method": "text_response", + "name": "text", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" + }, + "_type": "Component", + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", "info": "", "list": false, - "list_add_label": "Add More", - "name": "request_timeout", - "override_skip": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, "placeholder": "", - "required": false, + "required": true, "show": true, "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "float", - "value": "" + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "show_progress_bar": { - "_input_type": "BoolInput", - "advanced": true, - "display_name": "Show Progress Bar", + "input_value": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "info": "", + "info": "Text to be passed as input.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", - "name": "show_progress_bar", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false + "track_in_telemetry": false, + "type": "str", + "value": "CONNECTOR_TYPE" }, - "truncate_input_tokens": { - "_input_type": "IntInput", + "is_refresh": false, + "use_global_variable": { + "_input_type": "BoolInput", "advanced": true, - "display_name": "Truncate Input Tokens", + "display_name": "Use Global Variable", "dynamic": false, - "info": "", + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", "list": false, "list_add_label": "Add More", - "name": "truncate_input_tokens", + "name": "use_global_variable", "override_skip": false, "placeholder": "", + "real_time_refresh": true, "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "int", - "value": 200 + "type": "bool", + "value": true } }, "tool_mode": false }, - "showNode": true, - "type": "EmbeddingModel" + "showNode": false, + "type": "TextInput" }, "dragging": false, - "id": "EmbeddingModel-E0hvR", + "id": "TextInput-OGCeZ", "measured": { - "height": 369, - "width": 320 + "height": 52, + "width": 192 }, "position": { - "x": 1223.4804629271505, - "y": 2198.8989246514284 + "x": 1034.5767520283541, + "y": 1227.2295750493238 }, "selected": false, "type": "genericNode" }, { "data": { - "id": "EmbeddingModel-3LsIP", + "id": "TextInput-PI6at", "node": { "base_classes": [ - "Embeddings" + "Message" ], "beta": false, "conditional_paths": [], "custom_fields": {}, - "description": "Generate embeddings using a specified provider.", - "display_name": "Embedding Model", - "documentation": "https://docs.langflow.org/components-embedding-models", - "edited": true, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, "field_order": [ - "provider", - "api_base", - "ollama_base_url", - "base_url_ibm_watsonx", - "model", - "api_key", - "project_id", - "dimensions", - "chunk_size", - "request_timeout", - "max_retries", - "show_progress_bar", - "model_kwargs", - "truncate_input_tokens", - "input_text", - "fail_safe_mode" + "input_value", + "use_global_variable" ], "frozen": false, - "icon": "binary", - "last_updated": "2026-02-05T15:22:55.814Z", + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", "legacy": false, - "lf_version": "1.7.0.dev21", "metadata": { - "code_hash": "0e2d6fe67a26", + "code_hash": "518f16485886", "dependencies": { "dependencies": [ - { - "name": "requests", - "version": "2.32.5" - }, - { - "name": "ibm_watsonx_ai", - "version": "1.4.2" - }, - { - "name": "langchain_openai", - "version": "0.3.23" - }, { "name": "lfx", - "version": "0.2.0.dev21" - }, - { - "name": "langchain_ollama", - "version": "0.3.10" - }, - { - "name": "langchain_community", - "version": "0.3.21" - }, - { - "name": "langchain_ibm", - "version": "0.3.19" + "version": null } ], - "total_dependencies": 7 + "total_dependencies": 1 }, - "module": "custom_components.embedding_model" + "module": "lfx.components.input_output.text.TextInputComponent" }, "minimized": false, "output_types": [], @@ -4761,18 +5759,17 @@ { "allows_loop": false, "cache": true, - "display_name": "Embedding Model", + "display_name": "Output Text", "group_outputs": false, - "hidden": null, "loop_types": null, - "method": "build_embeddings", - "name": "embeddings", + "method": "text_response", + "name": "text", "options": null, "required_inputs": null, - "selected": "Embeddings", + "selected": "Message", "tool_mode": true, "types": [ - "Embeddings" + "Message" ], "value": "__UNDEFINED__" } @@ -4783,106 +5780,162 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" }, "_type": "Component", - "api_base": { - "_input_type": "MessageTextInput", + "code": { "advanced": true, - "display_name": "OpenAI API Base URL", - "dynamic": false, - "info": "Base URL for the API. Leave empty for default.", - "input_types": [ - "Message" - ], + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", "list": false, - "list_add_label": "Add More", "load_from_db": false, - "name": "api_base", - "override_skip": false, - "placeholder": "", - "required": false, - "show": true, - "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" - }, - "api_key": { - "_input_type": "SecretStrInput", - "advanced": false, - "display_name": "OpenAI API Key", - "dynamic": false, - "info": "Model Provider API key", - "input_types": [], - "load_from_db": true, - "name": "api_key", - "override_skip": false, - "password": true, + "multiline": true, + "name": "code", + "password": false, "placeholder": "", - "real_time_refresh": true, "required": true, "show": true, "title_case": false, - "track_in_telemetry": false, - "type": "str", - "value": "OPENAI_API_KEY" + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "base_url_ibm_watsonx": { - "_input_type": "DropdownInput", + "input_value": { + "_input_type": "MultilineInput", "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "watsonx API Endpoint", + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "external_options": {}, - "info": "The base URL of the API (IBM watsonx.ai only)", - "name": "base_url_ibm_watsonx", - "options": [ - "https://us-south.ml.cloud.ibm.com", - "https://eu-de.ml.cloud.ibm.com", - "https://eu-gb.ml.cloud.ibm.com", - "https://au-syd.ml.cloud.ibm.com", - "https://jp-tok.ml.cloud.ibm.com", - "https://ca-tor.ml.cloud.ibm.com" + "info": "Text to be passed as input.", + "input_types": [ + "Message" ], - "options_metadata": [], + "list": false, + "list_add_label": "Add More", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", - "real_time_refresh": true, "required": false, - "show": false, + "show": true, "title_case": false, - "toggle": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, - "track_in_telemetry": true, + "track_in_telemetry": false, "type": "str", - "value": "https://us-south.ml.cloud.ibm.com" + "value": "DOCUMENT_ID" }, - "chunk_size": { - "_input_type": "IntInput", + "is_refresh": false, + "use_global_variable": { + "_input_type": "BoolInput", "advanced": true, - "display_name": "Chunk Size", + "display_name": "Use Global Variable", "dynamic": false, - "info": "", + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", "list": false, "list_add_label": "Add More", - "name": "chunk_size", + "name": "use_global_variable", "override_skip": false, "placeholder": "", + "real_time_refresh": true, "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "int", - "value": 1000 + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "showNode": false, + "type": "TextInput" + }, + "dragging": false, + "id": "TextInput-PI6at", + "measured": { + "height": 52, + "width": 192 + }, + "position": { + "x": 1034.5767520283543, + "y": 1310.364074191445 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "TextInput-gRPNR", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, + "field_order": [ + "input_value", + "use_global_variable" + ], + "frozen": false, + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", + "legacy": false, + "metadata": { + "code_hash": "518f16485886", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 + }, + "module": "lfx.components.input_output.text.TextInputComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Text", + "group_outputs": false, + "loop_types": null, + "method": "text_response", + "name": "text", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" }, + "_type": "Component", "code": { "advanced": true, "dynamic": true, @@ -4899,38 +5952,47 @@ "show": true, "title_case": false, "type": "code", - "value": "from typing import Any\n\nimport requests\nfrom ibm_watsonx_ai.metanames import EmbedTextParamsMetaNames\nfrom langchain_openai import OpenAIEmbeddings\n\nfrom lfx.base.embeddings.embeddings_class import EmbeddingsWithModels\nfrom lfx.base.embeddings.model import LCEmbeddingsModel\nfrom lfx.base.models.model_utils import get_ollama_models, is_valid_ollama_url\nfrom lfx.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES\nfrom lfx.base.models.watsonx_constants import (\n IBM_WATSONX_URLS,\n WATSONX_EMBEDDING_MODEL_NAMES,\n)\nfrom lfx.field_typing import Embeddings\nfrom lfx.io import (\n BoolInput,\n DictInput,\n DropdownInput,\n FloatInput,\n IntInput,\n MessageTextInput,\n SecretStrInput,\n)\nfrom lfx.log.logger import logger\nfrom lfx.schema.dotdict import dotdict\nfrom lfx.utils.util import transform_localhost_url\n\n# Ollama API constants\nHTTP_STATUS_OK = 200\nJSON_MODELS_KEY = \"models\"\nJSON_NAME_KEY = \"name\"\nJSON_CAPABILITIES_KEY = \"capabilities\"\nDESIRED_CAPABILITY = \"embedding\"\nDEFAULT_OLLAMA_URL = \"http://localhost:11434\"\n\n\nclass EmbeddingModelComponent(LCEmbeddingsModel):\n display_name = \"Embedding Model\"\n description = \"Generate embeddings using a specified provider.\"\n documentation: str = \"https://docs.langflow.org/components-embedding-models\"\n icon = \"binary\"\n name = \"EmbeddingModel\"\n category = \"models\"\n\n inputs = [\n DropdownInput(\n name=\"provider\",\n display_name=\"Model Provider\",\n options=[\"OpenAI\", \"Ollama\", \"IBM watsonx.ai\"],\n value=\"OpenAI\",\n info=\"Select the embedding model provider\",\n real_time_refresh=True,\n options_metadata=[{\"icon\": \"OpenAI\"}, {\"icon\": \"Ollama\"}, {\"icon\": \"WatsonxAI\"}],\n ),\n MessageTextInput(\n name=\"api_base\",\n display_name=\"API Base URL\",\n info=\"Base URL for the API. Leave empty for default.\",\n advanced=True,\n ),\n MessageTextInput(\n name=\"ollama_base_url\",\n display_name=\"Ollama API URL\",\n info=f\"Endpoint of the Ollama API (Ollama only). Defaults to {DEFAULT_OLLAMA_URL}\",\n value=DEFAULT_OLLAMA_URL,\n show=False,\n real_time_refresh=True,\n load_from_db=True,\n ),\n DropdownInput(\n name=\"base_url_ibm_watsonx\",\n display_name=\"watsonx API Endpoint\",\n info=\"The base URL of the API (IBM watsonx.ai only)\",\n options=IBM_WATSONX_URLS,\n value=IBM_WATSONX_URLS[0],\n show=False,\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"model\",\n display_name=\"Model Name\",\n options=OPENAI_EMBEDDING_MODEL_NAMES,\n value=OPENAI_EMBEDDING_MODEL_NAMES[0],\n info=\"Select the embedding model to use\",\n real_time_refresh=True,\n refresh_button=True,\n ),\n SecretStrInput(\n name=\"api_key\",\n display_name=\"OpenAI API Key\",\n info=\"Model Provider API key\",\n required=True,\n show=True,\n real_time_refresh=True,\n ),\n # Watson-specific inputs\n MessageTextInput(\n name=\"project_id\",\n display_name=\"Project ID\",\n info=\"IBM watsonx.ai Project ID (required for IBM watsonx.ai)\",\n show=False,\n ),\n IntInput(\n name=\"dimensions\",\n display_name=\"Dimensions\",\n info=\"The number of dimensions the resulting output embeddings should have. \"\n \"Only supported by certain models.\",\n advanced=True,\n ),\n IntInput(name=\"chunk_size\", display_name=\"Chunk Size\", advanced=True, value=1000),\n FloatInput(name=\"request_timeout\", display_name=\"Request Timeout\", advanced=True),\n IntInput(name=\"max_retries\", display_name=\"Max Retries\", advanced=True, value=3),\n BoolInput(name=\"show_progress_bar\", display_name=\"Show Progress Bar\", advanced=True),\n DictInput(\n name=\"model_kwargs\",\n display_name=\"Model Kwargs\",\n advanced=True,\n info=\"Additional keyword arguments to pass to the model.\",\n ),\n IntInput(\n name=\"truncate_input_tokens\",\n display_name=\"Truncate Input Tokens\",\n advanced=True,\n value=200,\n show=False,\n ),\n BoolInput(\n name=\"input_text\",\n display_name=\"Include the original text in the output\",\n value=True,\n advanced=True,\n show=False,\n ),\n BoolInput(\n name=\"fail_safe_mode\",\n display_name=\"Fail-Safe Mode\",\n value=False,\n advanced=True,\n info=\"When enabled, errors will be logged instead of raising exceptions. \"\n \"The component will return None on error.\",\n real_time_refresh=True,\n ),\n ]\n\n @staticmethod\n def fetch_ibm_models(base_url: str) -> list[str]:\n \"\"\"Fetch available models from the watsonx.ai API.\"\"\"\n try:\n endpoint = f\"{base_url}/ml/v1/foundation_model_specs\"\n params = {\n \"version\": \"2024-09-16\",\n \"filters\": \"function_embedding,!lifecycle_withdrawn:and\",\n }\n response = requests.get(endpoint, params=params, timeout=10)\n response.raise_for_status()\n data = response.json()\n models = [model[\"model_id\"] for model in data.get(\"resources\", [])]\n return sorted(models)\n except Exception: # noqa: BLE001\n logger.exception(\"Error fetching models\")\n return WATSONX_EMBEDDING_MODEL_NAMES\n async def fetch_ollama_models(self) -> list[str]:\n try:\n return await get_ollama_models(\n base_url_value=self.ollama_base_url,\n desired_capability=DESIRED_CAPABILITY,\n json_models_key=JSON_MODELS_KEY,\n json_name_key=JSON_NAME_KEY,\n json_capabilities_key=JSON_CAPABILITIES_KEY,\n )\n except Exception: # noqa: BLE001\n\n logger.exception(\"Error fetching models\")\n return []\n async def build_embeddings(self) -> Embeddings:\n provider = self.provider\n model = self.model\n api_key = self.api_key\n api_base = self.api_base\n base_url_ibm_watsonx = self.base_url_ibm_watsonx\n ollama_base_url = self.ollama_base_url\n dimensions = self.dimensions\n chunk_size = self.chunk_size\n request_timeout = self.request_timeout\n max_retries = self.max_retries\n show_progress_bar = self.show_progress_bar\n model_kwargs = self.model_kwargs or {}\n\n if provider == \"OpenAI\":\n if not api_key:\n msg = \"OpenAI API key is required when using OpenAI provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n try:\n # Create the primary embedding instance\n embeddings_instance = OpenAIEmbeddings(\n model=model,\n dimensions=dimensions or None,\n base_url=api_base or None,\n api_key=api_key,\n chunk_size=chunk_size,\n max_retries=max_retries,\n timeout=request_timeout or None,\n show_progress_bar=show_progress_bar,\n model_kwargs=model_kwargs,\n )\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in OPENAI_EMBEDDING_MODEL_NAMES:\n available_models_dict[model_name] = OpenAIEmbeddings(\n model=model_name,\n dimensions=dimensions or None, # Use same dimensions config for all\n base_url=api_base or None,\n api_key=api_key,\n chunk_size=chunk_size,\n max_retries=max_retries,\n timeout=request_timeout or None,\n show_progress_bar=show_progress_bar,\n model_kwargs=model_kwargs,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to initialize OpenAI embeddings: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n if provider == \"Ollama\":\n try:\n from langchain_ollama import OllamaEmbeddings\n except ImportError:\n try:\n from langchain_community.embeddings import OllamaEmbeddings\n except ImportError:\n msg = \"Please install langchain-ollama: pip install langchain-ollama\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ImportError(msg) from None\n\n try:\n transformed_base_url = transform_localhost_url(ollama_base_url)\n\n # Check if URL contains /v1 suffix (OpenAI-compatible mode)\n if transformed_base_url and transformed_base_url.rstrip(\"/\").endswith(\"/v1\"):\n # Strip /v1 suffix and log warning\n transformed_base_url = transformed_base_url.rstrip(\"/\").removesuffix(\"/v1\")\n logger.warning(\n \"Detected '/v1' suffix in base URL. The Ollama component uses the native Ollama API, \"\n \"not the OpenAI-compatible API. The '/v1' suffix has been automatically removed. \"\n \"If you want to use the OpenAI-compatible API, please use the OpenAI component instead. \"\n \"Learn more at https://docs.ollama.com/openai#openai-compatibility\"\n )\n\n final_base_url = transformed_base_url or \"http://localhost:11434\"\n\n # Create the primary embedding instance\n embeddings_instance = OllamaEmbeddings(\n model=model,\n base_url=final_base_url,\n **model_kwargs,\n )\n\n # Fetch available Ollama models\n available_model_names = await self.fetch_ollama_models()\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in available_model_names:\n available_models_dict[model_name] = OllamaEmbeddings(\n model=model_name,\n base_url=final_base_url,\n **model_kwargs,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to initialize Ollama embeddings: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n if provider == \"IBM watsonx.ai\":\n try:\n from langchain_ibm import WatsonxEmbeddings\n except ImportError:\n msg = \"Please install langchain-ibm: pip install langchain-ibm\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ImportError(msg) from None\n\n if not api_key:\n msg = \"IBM watsonx.ai API key is required when using IBM watsonx.ai provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n project_id = self.project_id\n\n if not project_id:\n msg = \"Project ID is required for IBM watsonx.ai provider\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n try:\n from ibm_watsonx_ai import APIClient, Credentials\n\n final_url = base_url_ibm_watsonx or \"https://us-south.ml.cloud.ibm.com\"\n\n credentials = Credentials(\n api_key=self.api_key,\n url=final_url,\n )\n\n api_client = APIClient(credentials)\n\n params = {\n EmbedTextParamsMetaNames.TRUNCATE_INPUT_TOKENS: self.truncate_input_tokens,\n EmbedTextParamsMetaNames.RETURN_OPTIONS: {\"input_text\": self.input_text},\n }\n\n # Create the primary embedding instance\n embeddings_instance = WatsonxEmbeddings(\n model_id=model,\n params=params,\n watsonx_client=api_client,\n project_id=project_id,\n )\n\n # Fetch available IBM watsonx.ai models\n available_model_names = self.fetch_ibm_models(final_url)\n\n # Create dedicated instances for each available model\n available_models_dict = {}\n for model_name in available_model_names:\n available_models_dict[model_name] = WatsonxEmbeddings(\n model_id=model_name,\n params=params,\n watsonx_client=api_client,\n project_id=project_id,\n )\n\n return EmbeddingsWithModels(\n embeddings=embeddings_instance,\n available_models=available_models_dict,\n )\n except Exception as e:\n msg = f\"Failed to authenticate with IBM watsonx.ai: {e}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise\n\n msg = f\"Unknown provider: {provider}\"\n if self.fail_safe_mode:\n logger.error(msg)\n return None\n raise ValueError(msg)\n\n async def update_build_config(\n self, build_config: dotdict, field_value: Any, field_name: str | None = None\n ) -> dotdict:\n # Handle fail_safe_mode changes first - set all required fields to False if enabled\n if field_name == \"fail_safe_mode\":\n if field_value: # If fail_safe_mode is enabled\n build_config[\"api_key\"][\"required\"] = False\n elif hasattr(self, \"provider\"):\n # If fail_safe_mode is disabled, restore required flags based on provider\n if self.provider in [\"OpenAI\", \"IBM watsonx.ai\"]:\n build_config[\"api_key\"][\"required\"] = True\n else: # Ollama\n build_config[\"api_key\"][\"required\"] = False\n\n if field_name == \"provider\":\n if field_value == \"OpenAI\":\n build_config[\"model\"][\"options\"] = OPENAI_EMBEDDING_MODEL_NAMES\n build_config[\"model\"][\"value\"] = OPENAI_EMBEDDING_MODEL_NAMES[0]\n build_config[\"api_key\"][\"display_name\"] = \"OpenAI API Key\"\n # Only set required=True if fail_safe_mode is not enabled\n build_config[\"api_key\"][\"required\"] = not (hasattr(self, \"fail_safe_mode\") and self.fail_safe_mode)\n build_config[\"api_key\"][\"show\"] = True\n build_config[\"api_base\"][\"display_name\"] = \"OpenAI API Base URL\"\n build_config[\"api_base\"][\"advanced\"] = True\n build_config[\"api_base\"][\"show\"] = True\n build_config[\"ollama_base_url\"][\"show\"] = False\n build_config[\"project_id\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = False\n build_config[\"truncate_input_tokens\"][\"show\"] = False\n build_config[\"input_text\"][\"show\"] = False\n elif field_value == \"Ollama\":\n build_config[\"ollama_base_url\"][\"show\"] = True\n\n if await is_valid_ollama_url(url=self.ollama_base_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n build_config[\"model\"][\"value\"] = models[0] if models else \"\"\n except ValueError:\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n else:\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n build_config[\"truncate_input_tokens\"][\"show\"] = False\n build_config[\"input_text\"][\"show\"] = False\n build_config[\"api_key\"][\"display_name\"] = \"API Key (Optional)\"\n build_config[\"api_key\"][\"required\"] = False\n build_config[\"api_key\"][\"show\"] = False\n build_config[\"api_base\"][\"show\"] = False\n build_config[\"project_id\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = False\n\n elif field_value == \"IBM watsonx.ai\":\n build_config[\"model\"][\"options\"] = self.fetch_ibm_models(base_url=self.base_url_ibm_watsonx)\n build_config[\"model\"][\"value\"] = self.fetch_ibm_models(base_url=self.base_url_ibm_watsonx)[0]\n build_config[\"api_key\"][\"display_name\"] = \"IBM watsonx.ai API Key\"\n # Only set required=True if fail_safe_mode is not enabled\n build_config[\"api_key\"][\"required\"] = not (hasattr(self, \"fail_safe_mode\") and self.fail_safe_mode)\n build_config[\"api_key\"][\"show\"] = True\n build_config[\"api_base\"][\"show\"] = False\n build_config[\"ollama_base_url\"][\"show\"] = False\n build_config[\"base_url_ibm_watsonx\"][\"show\"] = True\n build_config[\"project_id\"][\"show\"] = True\n build_config[\"truncate_input_tokens\"][\"show\"] = True\n build_config[\"input_text\"][\"show\"] = True\n elif field_name == \"base_url_ibm_watsonx\":\n build_config[\"model\"][\"options\"] = self.fetch_ibm_models(base_url=field_value)\n build_config[\"model\"][\"value\"] = self.fetch_ibm_models(base_url=field_value)[0]\n elif field_name == \"ollama_base_url\":\n # # Refresh Ollama models when base URL changes\n # if hasattr(self, \"provider\") and self.provider == \"Ollama\":\n # Use field_value if provided, otherwise fall back to instance attribute\n ollama_url = self.ollama_base_url\n if await is_valid_ollama_url(url=ollama_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n build_config[\"model\"][\"value\"] = models[0] if models else \"\"\n except ValueError:\n await logger.awarning(\"Failed to fetch Ollama embedding models.\")\n build_config[\"model\"][\"options\"] = []\n build_config[\"model\"][\"value\"] = \"\"\n\n elif field_name == \"model\" and self.provider == \"Ollama\":\n ollama_url = self.ollama_base_url\n if await is_valid_ollama_url(url=ollama_url):\n try:\n models = await self.fetch_ollama_models()\n build_config[\"model\"][\"options\"] = models\n except ValueError:\n await logger.awarning(\"Failed to refresh Ollama embedding models.\")\n build_config[\"model\"][\"options\"] = []\n\n return build_config\n" + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "dimensions": { - "_input_type": "IntInput", - "advanced": true, - "display_name": "Dimensions", + "input_value": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "info": "The number of dimensions the resulting output embeddings should have. Only supported by certain models.", + "info": "Text to be passed as input.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", - "name": "dimensions", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": "" + "track_in_telemetry": false, + "type": "str", + "value": "OWNER" }, - "fail_safe_mode": { + "is_refresh": false, + "use_global_variable": { "_input_type": "BoolInput", "advanced": true, - "display_name": "Fail-Safe Mode", + "display_name": "Use Global Variable", "dynamic": false, - "info": "When enabled, errors will be logged instead of raising exceptions. The component will return None on error.", + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", "list": false, "list_add_label": "Add More", - "load_from_db": false, - "name": "fail_safe_mode", + "name": "use_global_variable", "override_skip": false, "placeholder": "", "real_time_refresh": true, @@ -4941,274 +6003,489 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "bool", - "value": false - }, - "input_text": { - "_input_type": "BoolInput", - "advanced": true, - "display_name": "Include the original text in the output", - "dynamic": false, - "info": "", - "list": false, - "list_add_label": "Add More", - "name": "input_text", - "override_skip": false, - "placeholder": "", - "required": false, - "show": false, - "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", "value": true + } + }, + "tool_mode": false + }, + "showNode": false, + "type": "TextInput" + }, + "dragging": false, + "id": "TextInput-gRPNR", + "measured": { + "height": 52, + "width": 192 + }, + "position": { + "x": 1034.5767520283541, + "y": 1393.498573333566 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "TextInput-lTHSx", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, + "field_order": [ + "input_value", + "use_global_variable" + ], + "frozen": false, + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", + "legacy": false, + "metadata": { + "code_hash": "518f16485886", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 }, - "is_refresh": false, - "max_retries": { - "_input_type": "IntInput", + "module": "lfx.components.input_output.text.TextInputComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Text", + "group_outputs": false, + "loop_types": null, + "method": "text_response", + "name": "text", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" + }, + "_type": "Component", + "code": { "advanced": true, - "display_name": "Max Retries", - "dynamic": false, + "dynamic": true, + "fileTypes": [], + "file_path": "", "info": "", "list": false, - "list_add_label": "Add More", - "name": "max_retries", - "override_skip": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, "placeholder": "", - "required": false, + "required": true, "show": true, "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "int", - "value": 3 + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "model": { - "_input_type": "DropdownInput", + "input_value": { + "_input_type": "MultilineInput", "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Model Name", + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "external_options": {}, - "info": "Select the embedding model to use", - "name": "model", - "options": [ - "text-embedding-3-small", - "text-embedding-3-large", - "text-embedding-ada-002" + "info": "Text to be passed as input.", + "input_types": [ + "Message" ], - "options_metadata": [], + "list": false, + "list_add_label": "Add More", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", - "real_time_refresh": true, - "refresh_button": true, "required": false, "show": true, "title_case": false, - "toggle": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, - "track_in_telemetry": true, + "track_in_telemetry": false, "type": "str", - "value": "text-embedding-3-small" + "value": "OWNER_EMAIL" }, - "model_kwargs": { - "_input_type": "DictInput", + "is_refresh": false, + "use_global_variable": { + "_input_type": "BoolInput", "advanced": true, - "display_name": "Model Kwargs", + "display_name": "Use Global Variable", "dynamic": false, - "info": "Additional keyword arguments to pass to the model.", + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", "list": false, "list_add_label": "Add More", - "name": "model_kwargs", + "name": "use_global_variable", "override_skip": false, "placeholder": "", + "real_time_refresh": true, "required": false, "show": true, "title_case": false, "tool_mode": false, - "trace_as_input": true, - "track_in_telemetry": false, - "type": "dict", - "value": {} + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "showNode": false, + "type": "TextInput" + }, + "dragging": false, + "id": "TextInput-lTHSx", + "measured": { + "height": 52, + "width": 192 + }, + "position": { + "x": 1032.6873315933058, + "y": 1476.633072475687 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "TextInput-68n9L", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, + "field_order": [ + "input_value", + "use_global_variable" + ], + "frozen": false, + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", + "legacy": false, + "metadata": { + "code_hash": "518f16485886", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 }, - "ollama_base_url": { - "_input_type": "MessageTextInput", - "advanced": false, - "display_name": "Ollama API URL", - "dynamic": false, - "info": "Endpoint of the Ollama API (Ollama only). Defaults to http://localhost:11434", - "input_types": [ + "module": "lfx.components.input_output.text.TextInputComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Text", + "group_outputs": false, + "loop_types": null, + "method": "text_response", + "name": "text", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ "Message" ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" + }, + "_type": "Component", + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", "list": false, - "list_add_label": "Add More", "load_from_db": false, - "name": "ollama_base_url", - "override_skip": false, + "multiline": true, + "name": "code", + "password": false, "placeholder": "", - "real_time_refresh": true, - "required": false, - "show": false, + "required": true, + "show": true, "title_case": false, - "tool_mode": false, - "trace_as_input": true, - "trace_as_metadata": true, - "track_in_telemetry": false, - "type": "str", - "value": "" + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "project_id": { - "_input_type": "MessageTextInput", + "input_value": { + "_input_type": "MultilineInput", "advanced": false, - "display_name": "Project ID", + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "info": "IBM watsonx.ai Project ID (required for IBM watsonx.ai)", + "info": "Text to be passed as input.", "input_types": [ "Message" ], "list": false, "list_add_label": "Add More", - "load_from_db": false, - "name": "project_id", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "OWNER_NAME" }, - "provider": { - "_input_type": "DropdownInput", - "advanced": false, - "combobox": false, - "dialog_inputs": {}, - "display_name": "Model Provider", + "is_refresh": false, + "use_global_variable": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Use Global Variable", "dynamic": false, - "external_options": {}, - "info": "Select the embedding model provider", - "name": "provider", - "options": [ - "OpenAI", - "Ollama", - "IBM watsonx.ai" - ], - "options_metadata": [ - { - "icon": "OpenAI" - }, - { - "icon": "Ollama" - }, - { - "icon": "WatsonxAI" - } - ], + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", + "list": false, + "list_add_label": "Add More", + "name": "use_global_variable", "override_skip": false, "placeholder": "", "real_time_refresh": true, "required": false, - "selected_metadata": { - "icon": "OpenAI" - }, "show": true, "title_case": false, - "toggle": false, "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "str", - "value": "OpenAI" + "type": "bool", + "value": true + } + }, + "tool_mode": false + }, + "showNode": false, + "type": "TextInput" + }, + "dragging": false, + "id": "TextInput-68n9L", + "measured": { + "height": 52, + "width": 192 + }, + "position": { + "x": 1032.687331593306, + "y": 1552.2098898776146 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "TextInput-UZQ8v", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Get user text inputs.", + "display_name": "Text Input", + "documentation": "https://docs.langflow.org/text-input-and-output", + "edited": false, + "field_order": [ + "input_value", + "use_global_variable" + ], + "frozen": false, + "icon": "type", + "last_updated": "2026-02-27T18:37:07.463Z", + "legacy": false, + "metadata": { + "code_hash": "518f16485886", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 1 }, - "request_timeout": { - "_input_type": "FloatInput", + "module": "lfx.components.input_output.text.TextInputComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Output Text", + "group_outputs": false, + "loop_types": null, + "method": "text_response", + "name": "text", + "options": null, + "required_inputs": null, + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "5488df7c-b93f-4f87-a446-b67028bc0813" + }, + "_frontend_node_folder_id": { + "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e" + }, + "_type": "Component", + "code": { "advanced": true, - "display_name": "Request Timeout", - "dynamic": false, + "dynamic": true, + "fileTypes": [], + "file_path": "", "info": "", "list": false, - "list_add_label": "Add More", - "name": "request_timeout", - "override_skip": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, "placeholder": "", - "required": false, + "required": true, "show": true, "title_case": false, - "tool_mode": false, - "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "float", - "value": "" + "type": "code", + "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n" }, - "show_progress_bar": { - "_input_type": "BoolInput", - "advanced": true, - "display_name": "Show Progress Bar", + "input_value": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "Text", "dynamic": false, - "info": "", + "info": "Text to be passed as input.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", - "name": "show_progress_bar", + "load_from_db": true, + "multiline": false, + "name": "input_value", "override_skip": false, + "password": true, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, + "trace_as_input": true, "trace_as_metadata": true, - "track_in_telemetry": true, - "type": "bool", - "value": false + "track_in_telemetry": false, + "type": "str", + "value": "SOURCE_URL" }, - "truncate_input_tokens": { - "_input_type": "IntInput", + "is_refresh": false, + "use_global_variable": { + "_input_type": "BoolInput", "advanced": true, - "display_name": "Truncate Input Tokens", + "display_name": "Use Global Variable", "dynamic": false, - "info": "", + "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.", "list": false, "list_add_label": "Add More", - "name": "truncate_input_tokens", + "name": "use_global_variable", "override_skip": false, "placeholder": "", + "real_time_refresh": true, "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, - "type": "int", - "value": 200 + "type": "bool", + "value": true } }, "tool_mode": false }, - "showNode": true, - "type": "EmbeddingModel" + "showNode": false, + "type": "TextInput" }, "dragging": false, - "id": "EmbeddingModel-3LsIP", + "id": "TextInput-UZQ8v", "measured": { - "height": 369, - "width": 320 + "height": 52, + "width": 192 }, "position": { - "x": 1638.9179466145608, - "y": 2110.0422159522327 + "x": 1034.5767520283541, + "y": 1639.123229889832 }, "selected": false, "type": "genericNode" }, { "data": { - "id": "ChunkDoclingDocument-7Tsav", + "id": "ChunkDoclingDocument-DdOYd", "node": { "base_classes": [ "DataFrame" @@ -5227,15 +6504,15 @@ "hf_model_name", "openai_model_name", "max_tokens", + "merge_peers", + "always_emit_headings", "doc_key" ], "frozen": false, "icon": "Docling", - "last_updated": "2026-02-05T15:22:20.975Z", "legacy": false, - "lf_version": "1.7.0.dev21", "metadata": { - "code_hash": "397fa38f89d7", + "code_hash": "49d762d97039", "dependencies": { "dependencies": [ { @@ -5244,11 +6521,11 @@ }, { "name": "docling_core", - "version": "2.49.0" + "version": "2.60.1" }, { "name": "lfx", - "version": "0.2.0.dev21" + "version": null } ], "total_dependencies": 3 @@ -5263,11 +6540,8 @@ "cache": true, "display_name": "DataFrame", "group_outputs": false, - "loop_types": null, "method": "chunk_documents", "name": "dataframe", - "options": null, - "required_inputs": null, "selected": "DataFrame", "tool_mode": true, "types": [ @@ -5278,13 +6552,27 @@ ], "pinned": false, "template": { - "_frontend_node_flow_id": { - "value": "5488df7c-b93f-4f87-a446-b67028bc0813" - }, - "_frontend_node_folder_id": { - "value": "15acea35-7c03-488d-8702-13327d5a4cea" - }, "_type": "Component", + "always_emit_headings": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Always emit headings", + "dynamic": true, + "info": "Emit headings even for empty sections.", + "list": false, + "list_add_label": "Add More", + "name": "always_emit_headings", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": false + }, "chunker": { "_input_type": "DropdownInput", "advanced": false, @@ -5294,6 +6582,9 @@ "dynamic": false, "external_options": {}, "info": "Which chunker to use.", + "input_types": [ + "Message" + ], "name": "chunker", "options": [ "HybridChunker", @@ -5311,7 +6602,7 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "HierarchicalChunker" + "value": "HybridChunker" }, "code": { "advanced": true, @@ -5329,7 +6620,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import json\n\nimport tiktoken\nfrom docling_core.transforms.chunker import BaseChunker, DocMeta\nfrom docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ChunkDoclingDocumentComponent(Component):\n display_name: str = \"Chunk DoclingDocument\"\n description: str = \"Use the DocumentDocument chunkers to split the document into chunks.\"\n documentation = \"https://docling-project.github.io/docling/concepts/chunking/\"\n icon = \"Docling\"\n name = \"ChunkDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Data or DataFrame\",\n info=\"The data with documents to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\"],\n required=True,\n ),\n DropdownInput(\n name=\"chunker\",\n display_name=\"Chunker\",\n options=[\"HybridChunker\", \"HierarchicalChunker\"],\n info=(\"Which chunker to use.\"),\n value=\"HybridChunker\",\n real_time_refresh=True,\n ),\n DropdownInput(\n name=\"provider\",\n display_name=\"Provider\",\n options=[\"Hugging Face\", \"OpenAI\"],\n info=(\"Which tokenizer provider.\"),\n value=\"Hugging Face\",\n show=True,\n real_time_refresh=True,\n advanced=True,\n dynamic=True,\n ),\n StrInput(\n name=\"hf_model_name\",\n display_name=\"HF model name\",\n info=(\n \"Model name of the tokenizer to use with the HybridChunker when Hugging Face is chosen as a tokenizer.\"\n ),\n value=\"sentence-transformers/all-MiniLM-L6-v2\",\n show=True,\n advanced=True,\n dynamic=True,\n ),\n StrInput(\n name=\"openai_model_name\",\n display_name=\"OpenAI model name\",\n info=(\"Model name of the tokenizer to use with the HybridChunker when OpenAI is chosen as a tokenizer.\"),\n value=\"gpt-4o\",\n show=False,\n advanced=True,\n dynamic=True,\n ),\n IntInput(\n name=\"max_tokens\",\n display_name=\"Maximum tokens\",\n info=(\"Maximum number of tokens for the HybridChunker.\"),\n show=True,\n required=False,\n advanced=True,\n dynamic=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"DataFrame\", name=\"dataframe\", method=\"chunk_documents\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n if field_name == \"chunker\":\n provider_type = build_config[\"provider\"][\"value\"]\n is_hf = provider_type == \"Hugging Face\"\n is_openai = provider_type == \"OpenAI\"\n if field_value == \"HybridChunker\":\n build_config[\"provider\"][\"show\"] = True\n build_config[\"hf_model_name\"][\"show\"] = is_hf\n build_config[\"openai_model_name\"][\"show\"] = is_openai\n build_config[\"max_tokens\"][\"show\"] = True\n else:\n build_config[\"provider\"][\"show\"] = False\n build_config[\"hf_model_name\"][\"show\"] = False\n build_config[\"openai_model_name\"][\"show\"] = False\n build_config[\"max_tokens\"][\"show\"] = False\n elif field_name == \"provider\" and build_config[\"chunker\"][\"value\"] == \"HybridChunker\":\n if field_value == \"Hugging Face\":\n build_config[\"hf_model_name\"][\"show\"] = True\n build_config[\"openai_model_name\"][\"show\"] = False\n elif field_value == \"OpenAI\":\n build_config[\"hf_model_name\"][\"show\"] = False\n build_config[\"openai_model_name\"][\"show\"] = True\n\n return build_config\n\n def _docs_to_data(self, docs) -> list[Data]:\n return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n\n def chunk_documents(self) -> DataFrame:\n documents = extract_docling_documents(self.data_inputs, self.doc_key)\n\n chunker: BaseChunker\n if self.chunker == \"HybridChunker\":\n try:\n from docling_core.transforms.chunker.hybrid_chunker import HybridChunker\n except ImportError as e:\n msg = (\n \"HybridChunker is not installed. Please install it with `uv pip install docling-core[chunking] \"\n \"or `uv pip install transformers`\"\n )\n raise ImportError(msg) from e\n max_tokens: int | None = self.max_tokens if self.max_tokens else None\n if self.provider == \"Hugging Face\":\n try:\n from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer\n except ImportError as e:\n msg = (\n \"HuggingFaceTokenizer is not installed.\"\n \" Please install it with `uv pip install docling-core[chunking]`\"\n )\n raise ImportError(msg) from e\n tokenizer = HuggingFaceTokenizer.from_pretrained(\n model_name=self.hf_model_name,\n max_tokens=max_tokens,\n )\n elif self.provider == \"OpenAI\":\n try:\n from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer\n except ImportError as e:\n msg = (\n \"OpenAITokenizer is not installed.\"\n \" Please install it with `uv pip install docling-core[chunking]`\"\n \" or `uv pip install transformers`\"\n )\n raise ImportError(msg) from e\n if max_tokens is None:\n max_tokens = 128 * 1024 # context window length required for OpenAI tokenizers\n tokenizer = OpenAITokenizer(\n tokenizer=tiktoken.encoding_for_model(self.openai_model_name), max_tokens=max_tokens\n )\n chunker = HybridChunker(\n tokenizer=tokenizer,\n )\n elif self.chunker == \"HierarchicalChunker\":\n chunker = HierarchicalChunker()\n\n results: list[Data] = []\n try:\n for doc in documents:\n for chunk in chunker.chunk(dl_doc=doc):\n enriched_text = chunker.contextualize(chunk=chunk)\n meta = DocMeta.model_validate(chunk.meta)\n\n results.append(\n Data(\n data={\n \"text\": enriched_text,\n \"document_id\": f\"{doc.origin.binary_hash}\",\n \"doc_items\": json.dumps([item.self_ref for item in meta.doc_items]),\n }\n )\n )\n\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return DataFrame(results)\n" + "value": "import json\n\nimport tiktoken\nfrom docling_core.transforms.chunker import BaseChunker, DocMeta\nfrom docling_core.transforms.chunker.hierarchical_chunker import HierarchicalChunker\n\nfrom lfx.base.data.docling_utils import extract_docling_documents\nfrom lfx.custom import Component\nfrom lfx.io import BoolInput, DropdownInput, HandleInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.schema import Data, DataFrame\n\n\nclass ChunkDoclingDocumentComponent(Component):\n display_name: str = \"Chunk DoclingDocument\"\n description: str = \"Use the DocumentDocument chunkers to split the document into chunks.\"\n documentation = \"https://docling-project.github.io/docling/concepts/chunking/\"\n icon = \"Docling\"\n name = \"ChunkDoclingDocument\"\n\n inputs = [\n HandleInput(\n name=\"data_inputs\",\n display_name=\"Data or DataFrame\",\n info=\"The data with documents to split in chunks.\",\n input_types=[\"Data\", \"DataFrame\"],\n required=True,\n ),\n DropdownInput(\n name=\"chunker\",\n display_name=\"Chunker\",\n options=[\"HybridChunker\", \"HierarchicalChunker\"],\n info=(\"Which chunker to use.\"),\n value=\"HybridChunker\",\n real_time_refresh=True,\n input_types=[\"Message\"],\n ),\n DropdownInput(\n name=\"provider\",\n display_name=\"Provider\",\n options=[\"Hugging Face\", \"OpenAI\"],\n info=(\"Which tokenizer provider.\"),\n value=\"Hugging Face\",\n show=True,\n real_time_refresh=True,\n advanced=True,\n dynamic=True,\n ),\n StrInput(\n name=\"hf_model_name\",\n display_name=\"HF model name\",\n info=(\n \"Model name of the tokenizer to use with the HybridChunker when Hugging Face is chosen as a tokenizer.\"\n ),\n value=\"sentence-transformers/all-MiniLM-L6-v2\",\n show=True,\n advanced=True,\n dynamic=True,\n ),\n StrInput(\n name=\"openai_model_name\",\n display_name=\"OpenAI model name\",\n info=(\"Model name of the tokenizer to use with the HybridChunker when OpenAI is chosen as a tokenizer.\"),\n value=\"gpt-4o\",\n show=False,\n advanced=True,\n dynamic=True,\n ),\n IntInput(\n name=\"max_tokens\",\n display_name=\"Maximum tokens\",\n info=(\"Maximum number of tokens for the HybridChunker.\"),\n show=True,\n required=False,\n advanced=True,\n dynamic=True,\n input_types=[\"Message\"],\n ),\n BoolInput(\n name=\"merge_peers\",\n display_name=\"Merge peers\",\n info=\"Merge undersized chunks sharing the same relevant metadata.\",\n value=True,\n show=True,\n advanced=True,\n dynamic=True,\n ),\n BoolInput(\n name=\"always_emit_headings\",\n display_name=\"Always emit headings\",\n info=\"Emit headings even for empty sections.\",\n value=False,\n show=True,\n advanced=True,\n dynamic=True,\n ),\n MessageTextInput(\n name=\"doc_key\",\n display_name=\"Doc Key\",\n info=\"The key to use for the DoclingDocument column.\",\n value=\"doc\",\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"DataFrame\", name=\"dataframe\", method=\"chunk_documents\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Update build_config to show/hide fields based on chunker and provider selection.\"\"\"\n if field_name == \"chunker\":\n provider_type = build_config[\"provider\"][\"value\"]\n is_hf = provider_type == \"Hugging Face\"\n is_openai = provider_type == \"OpenAI\"\n if field_value == \"HybridChunker\":\n build_config[\"provider\"][\"show\"] = True\n build_config[\"hf_model_name\"][\"show\"] = is_hf\n build_config[\"openai_model_name\"][\"show\"] = is_openai\n build_config[\"max_tokens\"][\"show\"] = True\n build_config[\"merge_peers\"][\"show\"] = True\n build_config[\"always_emit_headings\"][\"show\"] = True\n else:\n build_config[\"provider\"][\"show\"] = False\n build_config[\"hf_model_name\"][\"show\"] = False\n build_config[\"openai_model_name\"][\"show\"] = False\n build_config[\"max_tokens\"][\"show\"] = False\n build_config[\"merge_peers\"][\"show\"] = False\n build_config[\"always_emit_headings\"][\"show\"] = False\n elif field_name == \"provider\" and build_config[\"chunker\"][\"value\"] == \"HybridChunker\":\n if field_value == \"Hugging Face\":\n build_config[\"hf_model_name\"][\"show\"] = True\n build_config[\"openai_model_name\"][\"show\"] = False\n elif field_value == \"OpenAI\":\n build_config[\"hf_model_name\"][\"show\"] = False\n build_config[\"openai_model_name\"][\"show\"] = True\n\n return build_config\n\n def _docs_to_data(self, docs) -> list[Data]:\n return [Data(text=doc.page_content, data=doc.metadata) for doc in docs]\n\n def chunk_documents(self) -> DataFrame:\n documents, warning = extract_docling_documents(self.data_inputs, self.doc_key)\n if warning:\n self.status = warning\n\n chunker: BaseChunker\n if self.chunker == \"HybridChunker\":\n try:\n from docling_core.transforms.chunker.hybrid_chunker import HybridChunker\n except ImportError as e:\n msg = (\n \"HybridChunker is not installed. Please install it with `uv pip install docling-core[chunking] \"\n \"or `uv pip install transformers`\"\n )\n raise ImportError(msg) from e\n max_tokens: int | None = self.max_tokens if self.max_tokens else None\n if self.provider == \"Hugging Face\":\n try:\n from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer\n except ImportError as e:\n msg = (\n \"HuggingFaceTokenizer is not installed.\"\n \" Please install it with `uv pip install docling-core[chunking]`\"\n )\n raise ImportError(msg) from e\n tokenizer = HuggingFaceTokenizer.from_pretrained(\n model_name=self.hf_model_name,\n max_tokens=max_tokens,\n )\n elif self.provider == \"OpenAI\":\n try:\n from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer\n except ImportError as e:\n msg = (\n \"OpenAITokenizer is not installed.\"\n \" Please install it with `uv pip install docling-core[chunking]`\"\n \" or `uv pip install transformers`\"\n )\n raise ImportError(msg) from e\n if max_tokens is None:\n max_tokens = 128 * 1024 # context window length required for OpenAI tokenizers\n tokenizer = OpenAITokenizer(\n tokenizer=tiktoken.encoding_for_model(self.openai_model_name), max_tokens=max_tokens\n )\n chunker = HybridChunker(\n tokenizer=tokenizer,\n merge_peers=bool(self.merge_peers),\n always_emit_headings=bool(self.always_emit_headings),\n )\n\n elif self.chunker == \"HierarchicalChunker\":\n chunker = HierarchicalChunker()\n else:\n msg = f\"Unknown chunker: {self.chunker}\"\n raise ValueError(msg)\n\n results: list[Data] = []\n try:\n for doc in documents:\n for chunk in chunker.chunk(dl_doc=doc):\n enriched_text = chunker.contextualize(chunk=chunk)\n meta = DocMeta.model_validate(chunk.meta)\n\n results.append(\n Data(\n data={\n \"text\": enriched_text,\n \"document_id\": f\"{doc.origin.binary_hash}\",\n \"doc_items\": json.dumps([item.self_ref for item in meta.doc_items]),\n }\n )\n )\n\n except Exception as e:\n msg = f\"Error splitting text: {e}\"\n raise TypeError(msg) from e\n\n return DataFrame(results)\n" }, "data_inputs": { "_input_type": "HandleInput", @@ -5356,7 +6647,7 @@ }, "doc_key": { "_input_type": "MessageTextInput", - "advanced": false, + "advanced": true, "display_name": "Doc Key", "dynamic": false, "info": "The key to use for the DoclingDocument column.", @@ -5392,7 +6683,7 @@ "override_skip": false, "placeholder": "", "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, @@ -5400,26 +6691,48 @@ "type": "str", "value": "sentence-transformers/all-MiniLM-L6-v2" }, - "is_refresh": false, "max_tokens": { "_input_type": "IntInput", "advanced": true, "display_name": "Maximum tokens", "dynamic": true, "info": "Maximum number of tokens for the HybridChunker.", + "input_types": [ + "Message" + ], "list": false, "list_add_label": "Add More", "name": "max_tokens", "override_skip": false, "placeholder": "", "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, "track_in_telemetry": true, "type": "int", - "value": "" + "value": 0 + }, + "merge_peers": { + "_input_type": "BoolInput", + "advanced": true, + "display_name": "Merge peers", + "dynamic": true, + "info": "Merge undersized chunks sharing the same relevant metadata.", + "list": false, + "list_add_label": "Add More", + "name": "merge_peers", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true }, "openai_model_name": { "_input_type": "StrInput", @@ -5461,7 +6774,7 @@ "placeholder": "", "real_time_refresh": true, "required": false, - "show": false, + "show": true, "title_case": false, "toggle": false, "tool_mode": false, @@ -5473,34 +6786,35 @@ }, "tool_mode": false }, - "showNode": true, + "showNode": false, "type": "ChunkDoclingDocument" }, "dragging": false, - "id": "ChunkDoclingDocument-7Tsav", + "id": "ChunkDoclingDocument-DdOYd", "measured": { - "height": 347, - "width": 320 + "height": 52, + "width": 192 }, "position": { - "x": 410.87455617068423, - "y": 1595.372832525497 + "x": 386.5505235051394, + "y": 1778.4620067663204 }, "selected": false, "type": "genericNode" } ], "viewport": { - "x": 492.2079532383307, - "y": -215.27371168994932, - "zoom": 0.5404962259634922 + "x": 214.73712842346947, + "y": -385.1444291751076, + "zoom": 0.49162047009596377 } }, "description": "Load your data for chat context with Retrieval Augmented Generation.", "endpoint_name": null, "id": "5488df7c-b93f-4f87-a446-b67028bc0813", "is_component": false, - "last_tested_version": "1.7.0.dev21", + "last_tested_version": "1.8.0", + "locked": true, "name": "OpenSearch Ingestion Flow", "tags": [ "openai", @@ -5508,4 +6822,4 @@ "rag", "q-a" ] -} \ No newline at end of file +}