From 5072399fef0cfcb3611f6eaca46d0cf7fafbc785 Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Wed, 28 Jan 2026 10:49:29 +0100 Subject: [PATCH 1/8] feature/output-cmd --- api/routes/pipelex/build/inputs.py | 5 +++++ api/routes/pipelex/build/output.py | 14 ++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/api/routes/pipelex/build/inputs.py b/api/routes/pipelex/build/inputs.py index 6ac98c6..9c5a593 100644 --- a/api/routes/pipelex/build/inputs.py +++ b/api/routes/pipelex/build/inputs.py @@ -1,8 +1,10 @@ import json +import traceback from typing import Any from fastapi import APIRouter, HTTPException from fastapi.responses import JSONResponse +from pipelex import log from pipelex.hub import get_library_manager, get_required_pipe, set_current_library from pipelex.pipeline.validate_bundle import validate_bundle from pydantic import BaseModel, Field @@ -63,6 +65,9 @@ async def generate_inputs_json(request_data: BuildInputsRequest): return JSONResponse(content=response_data.model_dump(serialize_as_any=True)) except Exception as exc: + log.error(f"Error generating inputs JSON for pipe '{request_data.pipe_code}':") + traceback.print_exc() + raise HTTPException( status_code=500, detail={ diff --git a/api/routes/pipelex/build/output.py b/api/routes/pipelex/build/output.py index eee10bc..e7dd3d9 100644 --- a/api/routes/pipelex/build/output.py +++ b/api/routes/pipelex/build/output.py @@ -1,8 +1,11 @@ +import json +import traceback from typing import Any from fastapi import APIRouter, HTTPException from fastapi.responses import JSONResponse -from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat +from pipelex import log +from pipelex.core.pipes.output.output_renderer import render_output from pipelex.hub import get_library_manager, get_required_pipe, set_current_library from pipelex.pipeline.validate_bundle import validate_bundle from pydantic import BaseModel, Field @@ -49,9 +52,9 @@ async def build_output(request_data: BuildOutputRequest): # Get the pipe the_pipe = get_required_pipe(pipe_code=request_data.pipe_code) - # Generate the output JSON (content only, no concept wrapper) - output_dict = the_pipe.output.render_stuff_spec(ConceptRepresentationFormat.JSON) - output_json = output_dict.get("content", output_dict) + # Generate the output JSON + output_json_str = render_output(the_pipe) + output_json = json.loads(output_json_str) response_data = BuildOutputResponse( output_json=output_json, @@ -63,6 +66,9 @@ async def build_output(request_data: BuildOutputRequest): return JSONResponse(content=response_data.model_dump(serialize_as_any=True)) except Exception as exc: + log.error(f"Error generating output JSON for pipe '{request_data.pipe_code}':") + traceback.print_exc() + raise HTTPException( status_code=500, detail={ From d10dd56c57bba575b1fe9c0bb58187b5e8557e0d Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Wed, 28 Jan 2026 11:48:44 +0100 Subject: [PATCH 2/8] is_normalize_data_urls_to_storage=false --- .pipelex/pipelex.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml index 5787db6..c99fd02 100644 --- a/.pipelex/pipelex.toml +++ b/.pipelex/pipelex.toml @@ -32,7 +32,7 @@ [pipelex.pipeline_execution_config] # Set to false to disable conversion of incoming data URLs to pipelex-storage:// URIs -is_normalize_data_urls_to_storage = true +is_normalize_data_urls_to_storage = false # Set to false to disable generation of execution graphs is_generate_graph = true From 4055b827b07a59c83bf65f9f49616c3c93a5cbc9 Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Wed, 28 Jan 2026 18:35:33 +0100 Subject: [PATCH 3/8] add some fixing --- api/routes/pipelex/build/inputs.py | 46 ++++-------------------------- api/routes/pipelex/build/output.py | 46 ++++-------------------------- api/routes/pipelex/pipeline.py | 1 - 3 files changed, 10 insertions(+), 83 deletions(-) diff --git a/api/routes/pipelex/build/inputs.py b/api/routes/pipelex/build/inputs.py index 9c5a593..dbc6e5f 100644 --- a/api/routes/pipelex/build/inputs.py +++ b/api/routes/pipelex/build/inputs.py @@ -3,7 +3,6 @@ from typing import Any from fastapi import APIRouter, HTTPException -from fastapi.responses import JSONResponse from pipelex import log from pipelex.hub import get_library_manager, get_required_pipe, set_current_library from pipelex.pipeline.validate_bundle import validate_bundle @@ -17,61 +16,26 @@ class BuildInputsRequest(BaseModel): pipe_code: str = Field(..., description="Pipe code to generate inputs JSON for") -class BuildInputsResponse(BaseModel): - inputs_json: dict[str, Any] = Field(..., description="Generated inputs JSON object") - pipe_code: str = Field(..., description="Pipe code that was used") - success: bool = Field(default=True, description="Whether the operation was successful") - message: str = Field(default="Inputs JSON generated successfully", description="Status message") - - -@router.post("/build/inputs", response_model=BuildInputsResponse) -async def generate_inputs_json(request_data: BuildInputsRequest): - """Generate example input JSON for a pipe. - - This endpoint generates a JSON object with example values for all pipe inputs - based on their concept types. - - It will: - 1. Parse and validate the PLX content - 2. Load pipes from the bundle - 3. Generate inputs JSON for the specified pipe - """ +@router.post("/build/inputs") +async def build_inputs(request_data: BuildInputsRequest) -> Any: + """Generate example input JSON for a pipe.""" library_manager = get_library_manager() try: - # Validate and load the PLX content validate_bundle_result = await validate_bundle(plx_content=request_data.plx_content) blueprint = validate_bundle_result.blueprints[0] library_id, _ = library_manager.open_library() set_current_library(library_id) - # Load pipes temporarily library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint]) - # Get the pipe the_pipe = get_required_pipe(pipe_code=request_data.pipe_code) - - # Generate the input JSON inputs_json_str = the_pipe.inputs.render_inputs(indent=2) - inputs_json = json.loads(inputs_json_str) - - response_data = BuildInputsResponse( - inputs_json=inputs_json, - pipe_code=request_data.pipe_code, - success=True, - message="Inputs JSON generated successfully", - ) - return JSONResponse(content=response_data.model_dump(serialize_as_any=True)) + return json.loads(inputs_json_str) except Exception as exc: log.error(f"Error generating inputs JSON for pipe '{request_data.pipe_code}':") traceback.print_exc() - raise HTTPException( - status_code=500, - detail={ - "error_type": type(exc).__name__, - "message": str(exc), - }, - ) from exc + raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/api/routes/pipelex/build/output.py b/api/routes/pipelex/build/output.py index e7dd3d9..362fbff 100644 --- a/api/routes/pipelex/build/output.py +++ b/api/routes/pipelex/build/output.py @@ -3,7 +3,6 @@ from typing import Any from fastapi import APIRouter, HTTPException -from fastapi.responses import JSONResponse from pipelex import log from pipelex.core.pipes.output.output_renderer import render_output from pipelex.hub import get_library_manager, get_required_pipe, set_current_library @@ -18,61 +17,26 @@ class BuildOutputRequest(BaseModel): pipe_code: str = Field(..., description="Pipe code to generate output JSON for") -class BuildOutputResponse(BaseModel): - output_json: dict[str, Any] = Field(..., description="Generated output JSON object") - pipe_code: str = Field(..., description="Pipe code that was used") - success: bool = Field(default=True, description="Whether the operation was successful") - message: str = Field(default="Output JSON generated successfully", description="Status message") - - -@router.post("/build/output", response_model=BuildOutputResponse) -async def build_output(request_data: BuildOutputRequest): - """Generate example output JSON for a pipe. - - This endpoint generates a JSON object showing the expected output structure - based on the pipe's output concept type. - - It will: - 1. Parse and validate the PLX content - 2. Load pipes from the bundle - 3. Generate output JSON for the specified pipe - """ +@router.post("/build/output") +async def build_output(request_data: BuildOutputRequest) -> Any: + """Generate example output JSON for a pipe.""" library_manager = get_library_manager() try: - # Validate and load the PLX content validate_bundle_result = await validate_bundle(plx_content=request_data.plx_content) blueprint = validate_bundle_result.blueprints[0] library_id, _ = library_manager.open_library() set_current_library(library_id) - # Load pipes temporarily library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint]) - # Get the pipe the_pipe = get_required_pipe(pipe_code=request_data.pipe_code) - - # Generate the output JSON output_json_str = render_output(the_pipe) - output_json = json.loads(output_json_str) - - response_data = BuildOutputResponse( - output_json=output_json, - pipe_code=request_data.pipe_code, - success=True, - message="Output JSON generated successfully", - ) - return JSONResponse(content=response_data.model_dump(serialize_as_any=True)) + return json.loads(output_json_str) except Exception as exc: log.error(f"Error generating output JSON for pipe '{request_data.pipe_code}':") traceback.print_exc() - raise HTTPException( - status_code=500, - detail={ - "error_type": type(exc).__name__, - "message": str(exc), - }, - ) from exc + raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/api/routes/pipelex/pipeline.py b/api/routes/pipelex/pipeline.py index 8fd62ad..e6e4145 100644 --- a/api/routes/pipelex/pipeline.py +++ b/api/routes/pipelex/pipeline.py @@ -46,7 +46,6 @@ async def execute( ) return PipelineResponseFactory.make_from_pipe_output( - status="success", pipeline_run_id=pipe_output.pipeline_run_id, pipeline_state=PipelineState.COMPLETED, created_at=created_at, From 4853cd6931f0d395e8415e292bf3ba98f29c2601 Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Fri, 30 Jan 2026 17:09:52 +0100 Subject: [PATCH 4/8] fix output --- api/routes/pipelex/build/output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/routes/pipelex/build/output.py b/api/routes/pipelex/build/output.py index 362fbff..99cb3e8 100644 --- a/api/routes/pipelex/build/output.py +++ b/api/routes/pipelex/build/output.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, HTTPException from pipelex import log +from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat from pipelex.core.pipes.output.output_renderer import render_output from pipelex.hub import get_library_manager, get_required_pipe, set_current_library from pipelex.pipeline.validate_bundle import validate_bundle @@ -15,6 +16,7 @@ class BuildOutputRequest(BaseModel): plx_content: str = Field(..., description="PLX content to load pipes from") pipe_code: str = Field(..., description="Pipe code to generate output JSON for") + format: ConceptRepresentationFormat = Field(default=ConceptRepresentationFormat.SCHEMA, description="Format to generate output in") @router.post("/build/output") @@ -31,7 +33,7 @@ async def build_output(request_data: BuildOutputRequest) -> Any: library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint]) the_pipe = get_required_pipe(pipe_code=request_data.pipe_code) - output_json_str = render_output(the_pipe) + output_json_str = render_output(the_pipe, output_format=request_data.format) return json.loads(output_json_str) From 6e6deda054844cd319e6d019dcc857ab2e390324 Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Fri, 30 Jan 2026 17:16:31 +0100 Subject: [PATCH 5/8] feature/output-model-schema --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b9352e8..890cd12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ ] [tool.uv.sources] -pipelex = { path = "../pipelex", editable = true} +pipelex = { git = "https://github.com/Pipelex/pipelex.git", rev = "feature/output-model-schema" } [build-system] From 81812025c654407da5fac490a5d9f4baece97ff4 Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Fri, 30 Jan 2026 18:04:26 +0100 Subject: [PATCH 6/8] moad --- pyproject.toml | 2 +- uv.lock | 73 ++------------------------------------------------ 2 files changed, 3 insertions(+), 72 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 890cd12..5d7863f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ ] [tool.uv.sources] -pipelex = { git = "https://github.com/Pipelex/pipelex.git", rev = "feature/output-model-schema" } +pipelex = { git = "https://github.com/Pipelex/pipelex.git", rev = "moad" } [build-system] diff --git a/uv.lock b/uv.lock index 15bc1cb..445ae03 100644 --- a/uv.lock +++ b/uv.lock @@ -1857,7 +1857,7 @@ wheels = [ [[package]] name = "pipelex" version = "0.18.0b2" -source = { editable = "../pipelex" } +source = { git = "https://github.com/Pipelex/pipelex.git?rev=moad#59a6f6b2033fea52ddc7e921392ca54d8d2471ca" } dependencies = [ { name = "aiofiles" }, { name = "filetype" }, @@ -1911,75 +1911,6 @@ mistralai = [ { name = "mistralai" }, ] -[package.metadata] -requires-dist = [ - { name = "aioboto3", marker = "extra == 'bedrock'", specifier = ">=13.4.0" }, - { name = "aioboto3", marker = "extra == 's3'", specifier = ">=13.4.0" }, - { name = "aiofiles", specifier = ">=23.2.1" }, - { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.60.0" }, - { name = "backports-strenum", marker = "python_full_version < '3.11'", specifier = ">=1.3.0" }, - { name = "boto3", marker = "extra == 'bedrock'", specifier = ">=1.34.131" }, - { name = "boto3", marker = "extra == 's3'", specifier = ">=1.34.131" }, - { name = "boto3-stubs", marker = "extra == 'dev'", specifier = ">=1.35.24" }, - { name = "docling", marker = "extra == 'docling'", specifier = ">=2.64.0" }, - { name = "fal-client", marker = "extra == 'fal'", specifier = ">=0.4.1" }, - { name = "filetype", specifier = ">=1.2.0" }, - { name = "google-auth-oauthlib", marker = "extra == 'google'", specifier = ">=1.2.1" }, - { name = "google-cloud-storage", marker = "extra == 'gcp-storage'", specifier = ">=2.10.0" }, - { name = "google-genai", marker = "extra == 'google-genai'" }, - { name = "httpx", specifier = ">=0.23.0,<1.0.0" }, - { name = "huggingface-hub", marker = "extra == 'huggingface'", specifier = ">=0.23,<1.0.0" }, - { name = "instructor", specifier = ">=1.8.3,!=1.11.*,!=1.12.*" }, - { name = "instructor", extras = ["google-genai"], marker = "extra == 'google-genai'" }, - { name = "jinja2", specifier = ">=3.1.4" }, - { name = "json2html", specifier = ">=1.3.0" }, - { name = "kajson", specifier = "==0.3.1" }, - { name = "markdown", specifier = ">=3.6" }, - { name = "mike", marker = "extra == 'docs'", specifier = ">=2.1.3" }, - { name = "mistralai", marker = "extra == 'mistralai'", specifier = "==1.5.2" }, - { name = "mkdocs", marker = "extra == 'docs'", specifier = ">=1.6.1" }, - { name = "mkdocs-glightbox", marker = "extra == 'docs'", specifier = ">=0.4.0" }, - { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.6.14" }, - { name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = ">=1.1.0" }, - { name = "moto", extras = ["s3"], marker = "extra == 'dev'", specifier = ">=5.0.0" }, - { name = "mypy", marker = "extra == 'dev'", specifier = "==1.19.1" }, - { name = "networkx", specifier = ">=3.4.2" }, - { name = "openai", specifier = ">=1.108.1" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "pillow", specifier = ">=11.2.1" }, - { name = "polyfactory", specifier = ">=2.21.0" }, - { name = "portkey-ai", specifier = ">=2.1.0" }, - { name = "posthog", specifier = ">=6.7.0" }, - { name = "pydantic", specifier = ">=2.10.6,<3.0.0" }, - { name = "pylint", marker = "extra == 'dev'", specifier = "==4.0.4" }, - { name = "pypdfium2", specifier = ">=4.30.0,!=4.30.1" }, - { name = "pyright", marker = "extra == 'dev'", specifier = "==1.1.408" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, - { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.1.1" }, - { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.14.0" }, - { name = "pytest-sugar", marker = "extra == 'dev'", specifier = ">=1.0.0" }, - { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.6.1" }, - { name = "python-dotenv", specifier = ">=1.0.1" }, - { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "rich", specifier = ">=13.8.1" }, - { name = "ruff", marker = "extra == 'dev'", specifier = "==0.14.13" }, - { name = "shortuuid", specifier = ">=1.0.13" }, - { name = "tomli", specifier = ">=2.3.0" }, - { name = "tomlkit", specifier = ">=0.13.2" }, - { name = "typer", specifier = ">=0.16.0" }, - { name = "types-aioboto3", extras = ["bedrock", "bedrock-runtime"], marker = "extra == 'dev'", specifier = ">=13.4.0" }, - { name = "types-aiofiles", marker = "extra == 'dev'", specifier = ">=24.1.0.20240626" }, - { name = "types-markdown", marker = "extra == 'dev'", specifier = ">=3.6.0.20240316" }, - { name = "types-networkx", marker = "extra == 'dev'", specifier = ">=3.3.0.20241020" }, - { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0.12.20250326" }, - { name = "typing-extensions", specifier = ">=4.13.2" }, -] -provides-extras = ["anthropic", "bedrock", "docling", "fal", "gcp-storage", "google", "google-genai", "huggingface", "mistralai", "s3", "docs", "dev"] - [[package]] name = "pipelex-api" version = "0.0.12" @@ -2032,7 +1963,7 @@ requires-dist = [ { name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = "==1.1.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.2" }, { name = "pandas-stubs", marker = "extra == 'dev'", specifier = ">=2.2.3.241126" }, - { name = "pipelex", extras = ["mistralai", "anthropic", "google", "google-genai", "bedrock", "fal"], editable = "../pipelex" }, + { name = "pipelex", extras = ["mistralai", "anthropic", "google", "google-genai", "bedrock", "fal"], git = "https://github.com/Pipelex/pipelex.git?rev=moad" }, { name = "pyjwt", specifier = ">=2.10.1" }, { name = "pylint", marker = "extra == 'dev'", specifier = ">=3.3.8" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.405" }, From c85d80f9be998c947daa80b072b8c0b9988e7606 Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Mon, 2 Feb 2026 11:05:36 +0100 Subject: [PATCH 7/8] remove doc-string --- api/routes/pipelex/build/output.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/routes/pipelex/build/output.py b/api/routes/pipelex/build/output.py index 99cb3e8..0160b9f 100644 --- a/api/routes/pipelex/build/output.py +++ b/api/routes/pipelex/build/output.py @@ -21,7 +21,6 @@ class BuildOutputRequest(BaseModel): @router.post("/build/output") async def build_output(request_data: BuildOutputRequest) -> Any: - """Generate example output JSON for a pipe.""" library_manager = get_library_manager() try: From 3a1385a727fd6059b194a9886128afc8ad41339d Mon Sep 17 00:00:00 2001 From: thomashebrard Date: Mon, 2 Feb 2026 11:35:13 +0100 Subject: [PATCH 8/8] add s3 is_fetch_remote_content_enabled --- .pipelex/pipelex.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml index c99fd02..df029c2 100644 --- a/.pipelex/pipelex.toml +++ b/.pipelex/pipelex.toml @@ -65,7 +65,7 @@ pan_to_top = true # Pan to show top of graph on load [pipelex.storage_config] # Storage method: "local", "in_memory" (default), "s3", or "gcp" -method = "in_memory" +method = "s3" # Whether to fetch remote HTTP URLs and store them locally is_fetch_remote_content_enabled = true @@ -81,8 +81,8 @@ uri_format = "{primary_id}/{secondary_id}/{hash}.{extension}" [pipelex.storage_config.s3] # AWS S3 storage settings (requires boto3: `pip install pipelex[s3]`) uri_format = "{primary_id}/{secondary_id}/{hash}.{extension}" -bucket_name = "" -region = "" +bucket_name = "pipelex-assets" +region = "eu-west-3" signed_urls_lifespan_seconds = 3600 # Set to "disabled" for public URLs [pipelex.storage_config.gcp]