diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml index 5787db6..df029c2 100644 --- a/.pipelex/pipelex.toml +++ b/.pipelex/pipelex.toml @@ -32,7 +32,7 @@ [pipelex.pipeline_execution_config] # Set to false to disable conversion of incoming data URLs to pipelex-storage:// URIs -is_normalize_data_urls_to_storage = true +is_normalize_data_urls_to_storage = false # Set to false to disable generation of execution graphs is_generate_graph = true @@ -65,7 +65,7 @@ pan_to_top = true # Pan to show top of graph on load [pipelex.storage_config] # Storage method: "local", "in_memory" (default), "s3", or "gcp" -method = "in_memory" +method = "s3" # Whether to fetch remote HTTP URLs and store them locally is_fetch_remote_content_enabled = true @@ -81,8 +81,8 @@ uri_format = "{primary_id}/{secondary_id}/{hash}.{extension}" [pipelex.storage_config.s3] # AWS S3 storage settings (requires boto3: `pip install pipelex[s3]`) uri_format = "{primary_id}/{secondary_id}/{hash}.{extension}" -bucket_name = "" -region = "" +bucket_name = "pipelex-assets" +region = "eu-west-3" signed_urls_lifespan_seconds = 3600 # Set to "disabled" for public URLs [pipelex.storage_config.gcp] diff --git a/api/routes/pipelex/build/inputs.py b/api/routes/pipelex/build/inputs.py index 6ac98c6..dbc6e5f 100644 --- a/api/routes/pipelex/build/inputs.py +++ b/api/routes/pipelex/build/inputs.py @@ -1,8 +1,9 @@ import json +import traceback from typing import Any from fastapi import APIRouter, HTTPException -from fastapi.responses import JSONResponse +from pipelex import log from pipelex.hub import get_library_manager, get_required_pipe, set_current_library from pipelex.pipeline.validate_bundle import validate_bundle from pydantic import BaseModel, Field @@ -15,58 +16,26 @@ class BuildInputsRequest(BaseModel): pipe_code: str = Field(..., description="Pipe code to generate inputs JSON for") -class BuildInputsResponse(BaseModel): - inputs_json: dict[str, Any] = Field(..., description="Generated inputs JSON object") - pipe_code: str = Field(..., description="Pipe code that was used") - success: bool = Field(default=True, description="Whether the operation was successful") - message: str = Field(default="Inputs JSON generated successfully", description="Status message") - - -@router.post("/build/inputs", response_model=BuildInputsResponse) -async def generate_inputs_json(request_data: BuildInputsRequest): - """Generate example input JSON for a pipe. - - This endpoint generates a JSON object with example values for all pipe inputs - based on their concept types. - - It will: - 1. Parse and validate the PLX content - 2. Load pipes from the bundle - 3. Generate inputs JSON for the specified pipe - """ +@router.post("/build/inputs") +async def build_inputs(request_data: BuildInputsRequest) -> Any: + """Generate example input JSON for a pipe.""" library_manager = get_library_manager() try: - # Validate and load the PLX content validate_bundle_result = await validate_bundle(plx_content=request_data.plx_content) blueprint = validate_bundle_result.blueprints[0] library_id, _ = library_manager.open_library() set_current_library(library_id) - # Load pipes temporarily library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint]) - # Get the pipe the_pipe = get_required_pipe(pipe_code=request_data.pipe_code) - - # Generate the input JSON inputs_json_str = the_pipe.inputs.render_inputs(indent=2) - inputs_json = json.loads(inputs_json_str) - - response_data = BuildInputsResponse( - inputs_json=inputs_json, - pipe_code=request_data.pipe_code, - success=True, - message="Inputs JSON generated successfully", - ) - return JSONResponse(content=response_data.model_dump(serialize_as_any=True)) + return json.loads(inputs_json_str) except Exception as exc: - raise HTTPException( - status_code=500, - detail={ - "error_type": type(exc).__name__, - "message": str(exc), - }, - ) from exc + log.error(f"Error generating inputs JSON for pipe '{request_data.pipe_code}':") + traceback.print_exc() + + raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/api/routes/pipelex/build/output.py b/api/routes/pipelex/build/output.py index eee10bc..0160b9f 100644 --- a/api/routes/pipelex/build/output.py +++ b/api/routes/pipelex/build/output.py @@ -1,8 +1,11 @@ +import json +import traceback from typing import Any from fastapi import APIRouter, HTTPException -from fastapi.responses import JSONResponse +from pipelex import log from pipelex.core.concepts.concept_representation_generator import ConceptRepresentationFormat +from pipelex.core.pipes.output.output_renderer import render_output from pipelex.hub import get_library_manager, get_required_pipe, set_current_library from pipelex.pipeline.validate_bundle import validate_bundle from pydantic import BaseModel, Field @@ -13,60 +16,28 @@ class BuildOutputRequest(BaseModel): plx_content: str = Field(..., description="PLX content to load pipes from") pipe_code: str = Field(..., description="Pipe code to generate output JSON for") + format: ConceptRepresentationFormat = Field(default=ConceptRepresentationFormat.SCHEMA, description="Format to generate output in") -class BuildOutputResponse(BaseModel): - output_json: dict[str, Any] = Field(..., description="Generated output JSON object") - pipe_code: str = Field(..., description="Pipe code that was used") - success: bool = Field(default=True, description="Whether the operation was successful") - message: str = Field(default="Output JSON generated successfully", description="Status message") - - -@router.post("/build/output", response_model=BuildOutputResponse) -async def build_output(request_data: BuildOutputRequest): - """Generate example output JSON for a pipe. - - This endpoint generates a JSON object showing the expected output structure - based on the pipe's output concept type. - - It will: - 1. Parse and validate the PLX content - 2. Load pipes from the bundle - 3. Generate output JSON for the specified pipe - """ +@router.post("/build/output") +async def build_output(request_data: BuildOutputRequest) -> Any: library_manager = get_library_manager() try: - # Validate and load the PLX content validate_bundle_result = await validate_bundle(plx_content=request_data.plx_content) blueprint = validate_bundle_result.blueprints[0] library_id, _ = library_manager.open_library() set_current_library(library_id) - # Load pipes temporarily library_manager.load_from_blueprints(library_id=library_id, blueprints=[blueprint]) - # Get the pipe the_pipe = get_required_pipe(pipe_code=request_data.pipe_code) + output_json_str = render_output(the_pipe, output_format=request_data.format) - # Generate the output JSON (content only, no concept wrapper) - output_dict = the_pipe.output.render_stuff_spec(ConceptRepresentationFormat.JSON) - output_json = output_dict.get("content", output_dict) - - response_data = BuildOutputResponse( - output_json=output_json, - pipe_code=request_data.pipe_code, - success=True, - message="Output JSON generated successfully", - ) - - return JSONResponse(content=response_data.model_dump(serialize_as_any=True)) + return json.loads(output_json_str) except Exception as exc: - raise HTTPException( - status_code=500, - detail={ - "error_type": type(exc).__name__, - "message": str(exc), - }, - ) from exc + log.error(f"Error generating output JSON for pipe '{request_data.pipe_code}':") + traceback.print_exc() + + raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/api/routes/pipelex/pipeline.py b/api/routes/pipelex/pipeline.py index 8fd62ad..e6e4145 100644 --- a/api/routes/pipelex/pipeline.py +++ b/api/routes/pipelex/pipeline.py @@ -46,7 +46,6 @@ async def execute( ) return PipelineResponseFactory.make_from_pipe_output( - status="success", pipeline_run_id=pipe_output.pipeline_run_id, pipeline_state=PipelineState.COMPLETED, created_at=created_at, diff --git a/pyproject.toml b/pyproject.toml index b9352e8..5d7863f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ ] [tool.uv.sources] -pipelex = { path = "../pipelex", editable = true} +pipelex = { git = "https://github.com/Pipelex/pipelex.git", rev = "moad" } [build-system] diff --git a/uv.lock b/uv.lock index 15bc1cb..445ae03 100644 --- a/uv.lock +++ b/uv.lock @@ -1857,7 +1857,7 @@ wheels = [ [[package]] name = "pipelex" version = "0.18.0b2" -source = { editable = "../pipelex" } +source = { git = "https://github.com/Pipelex/pipelex.git?rev=moad#59a6f6b2033fea52ddc7e921392ca54d8d2471ca" } dependencies = [ { name = "aiofiles" }, { name = "filetype" }, @@ -1911,75 +1911,6 @@ mistralai = [ { name = "mistralai" }, ] -[package.metadata] -requires-dist = [ - { name = "aioboto3", marker = "extra == 'bedrock'", specifier = ">=13.4.0" }, - { name = "aioboto3", marker = "extra == 's3'", specifier = ">=13.4.0" }, - { name = "aiofiles", specifier = ">=23.2.1" }, - { name = "anthropic", marker = "extra == 'anthropic'", specifier = ">=0.60.0" }, - { name = "backports-strenum", marker = "python_full_version < '3.11'", specifier = ">=1.3.0" }, - { name = "boto3", marker = "extra == 'bedrock'", specifier = ">=1.34.131" }, - { name = "boto3", marker = "extra == 's3'", specifier = ">=1.34.131" }, - { name = "boto3-stubs", marker = "extra == 'dev'", specifier = ">=1.35.24" }, - { name = "docling", marker = "extra == 'docling'", specifier = ">=2.64.0" }, - { name = "fal-client", marker = "extra == 'fal'", specifier = ">=0.4.1" }, - { name = "filetype", specifier = ">=1.2.0" }, - { name = "google-auth-oauthlib", marker = "extra == 'google'", specifier = ">=1.2.1" }, - { name = "google-cloud-storage", marker = "extra == 'gcp-storage'", specifier = ">=2.10.0" }, - { name = "google-genai", marker = "extra == 'google-genai'" }, - { name = "httpx", specifier = ">=0.23.0,<1.0.0" }, - { name = "huggingface-hub", marker = "extra == 'huggingface'", specifier = ">=0.23,<1.0.0" }, - { name = "instructor", specifier = ">=1.8.3,!=1.11.*,!=1.12.*" }, - { name = "instructor", extras = ["google-genai"], marker = "extra == 'google-genai'" }, - { name = "jinja2", specifier = ">=3.1.4" }, - { name = "json2html", specifier = ">=1.3.0" }, - { name = "kajson", specifier = "==0.3.1" }, - { name = "markdown", specifier = ">=3.6" }, - { name = "mike", marker = "extra == 'docs'", specifier = ">=2.1.3" }, - { name = "mistralai", marker = "extra == 'mistralai'", specifier = "==1.5.2" }, - { name = "mkdocs", marker = "extra == 'docs'", specifier = ">=1.6.1" }, - { name = "mkdocs-glightbox", marker = "extra == 'docs'", specifier = ">=0.4.0" }, - { name = "mkdocs-material", marker = "extra == 'docs'", specifier = ">=9.6.14" }, - { name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = ">=1.1.0" }, - { name = "moto", extras = ["s3"], marker = "extra == 'dev'", specifier = ">=5.0.0" }, - { name = "mypy", marker = "extra == 'dev'", specifier = "==1.19.1" }, - { name = "networkx", specifier = ">=3.4.2" }, - { name = "openai", specifier = ">=1.108.1" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "pillow", specifier = ">=11.2.1" }, - { name = "polyfactory", specifier = ">=2.21.0" }, - { name = "portkey-ai", specifier = ">=2.1.0" }, - { name = "posthog", specifier = ">=6.7.0" }, - { name = "pydantic", specifier = ">=2.10.6,<3.0.0" }, - { name = "pylint", marker = "extra == 'dev'", specifier = "==4.0.4" }, - { name = "pypdfium2", specifier = ">=4.30.0,!=4.30.1" }, - { name = "pyright", marker = "extra == 'dev'", specifier = "==1.1.408" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, - { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.1.1" }, - { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.14.0" }, - { name = "pytest-sugar", marker = "extra == 'dev'", specifier = ">=1.0.0" }, - { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.6.1" }, - { name = "python-dotenv", specifier = ">=1.0.1" }, - { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "rich", specifier = ">=13.8.1" }, - { name = "ruff", marker = "extra == 'dev'", specifier = "==0.14.13" }, - { name = "shortuuid", specifier = ">=1.0.13" }, - { name = "tomli", specifier = ">=2.3.0" }, - { name = "tomlkit", specifier = ">=0.13.2" }, - { name = "typer", specifier = ">=0.16.0" }, - { name = "types-aioboto3", extras = ["bedrock", "bedrock-runtime"], marker = "extra == 'dev'", specifier = ">=13.4.0" }, - { name = "types-aiofiles", marker = "extra == 'dev'", specifier = ">=24.1.0.20240626" }, - { name = "types-markdown", marker = "extra == 'dev'", specifier = ">=3.6.0.20240316" }, - { name = "types-networkx", marker = "extra == 'dev'", specifier = ">=3.3.0.20241020" }, - { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0.12.20250326" }, - { name = "typing-extensions", specifier = ">=4.13.2" }, -] -provides-extras = ["anthropic", "bedrock", "docling", "fal", "gcp-storage", "google", "google-genai", "huggingface", "mistralai", "s3", "docs", "dev"] - [[package]] name = "pipelex-api" version = "0.0.12" @@ -2032,7 +1963,7 @@ requires-dist = [ { name = "mkdocs-meta-manager", marker = "extra == 'docs'", specifier = "==1.1.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.11.2" }, { name = "pandas-stubs", marker = "extra == 'dev'", specifier = ">=2.2.3.241126" }, - { name = "pipelex", extras = ["mistralai", "anthropic", "google", "google-genai", "bedrock", "fal"], editable = "../pipelex" }, + { name = "pipelex", extras = ["mistralai", "anthropic", "google", "google-genai", "bedrock", "fal"], git = "https://github.com/Pipelex/pipelex.git?rev=moad" }, { name = "pyjwt", specifier = ">=2.10.1" }, { name = "pylint", marker = "extra == 'dev'", specifier = ">=3.3.8" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.405" },