diff --git a/core/services/workflow_service/controllers/compute_block_controller.py b/core/services/workflow_service/controllers/compute_block_controller.py index 5e783094..693ffee2 100644 --- a/core/services/workflow_service/controllers/compute_block_controller.py +++ b/core/services/workflow_service/controllers/compute_block_controller.py @@ -12,7 +12,8 @@ from utils.config.defaults import ( get_file_cfg_defaults_dict, get_pg_cfg_defaults_dict, - SETTINGS_CLASS, extract_default_keys_from_io + SETTINGS_CLASS, + extract_default_keys_from_io, ) import utils.data.file_handling as fh from utils.config.registry import RepoRegistry @@ -21,15 +22,13 @@ from services.workflow_service.models.input_output import ( DataType, InputOutput, - InputOutputType + InputOutputType, ) from services.workflow_service.schemas.compute_block import ( ConfigType, InputOutputDTO, ) -from services.workflow_service.schemas.compute_block import ( - BaseInputOutputDTO -) +from services.workflow_service.schemas.compute_block import BaseInputOutputDTO from scystream.sdk.config import load_config from scystream.sdk.config.models import ( ComputeBlock as SDKComputeBlock, @@ -45,13 +44,14 @@ def _get_cb_info_from_repo(repo_url: str) -> SDKComputeBlock: cbc_path = os.path.join(cached_path, CBC_FILE_IDENTIFIER) if not os.path.isfile(cbc_path): raise HTTPException( - status_code=422, - detail=f"Cached repo {repo_url} missing cbc.yaml" + status_code=422, detail=f"Cached repo {repo_url} missing cbc.yaml" ) return load_config(cbc_path) -def request_cb_info(repo_url: str) -> SDKComputeBlock: +def request_cb_info( + repo_url: str, project_name: str, compute_block_custom_name: str +) -> SDKComputeBlock: logging.debug(f"Requesting ComputeBlock info for: {repo_url}") cb = _get_cb_info_from_repo(repo_url) @@ -62,7 +62,9 @@ def request_cb_info(repo_url: str) -> SDKComputeBlock: default_values = ( get_file_cfg_defaults_dict(on) if o_type is DataType.FILE - else get_pg_cfg_defaults_dict(on) + else get_pg_cfg_defaults_dict( + project_name, on, compute_block_custom_name + ) ) # Update the config with default values @@ -74,9 +76,7 @@ def request_cb_info(repo_url: str) -> SDKComputeBlock: def updated_configs_with_values( - io: InputOutput, - default_values: dict, - type: DataType + io: InputOutput, default_values: dict, type: DataType ) -> dict: """ Returns a new config dict with default keys replaced by values from @@ -89,8 +89,10 @@ def updated_configs_with_values( Returns: - A new config dictionary with updated values. """ - logging.debug(f"Get update configs for source config { - io.config} and values {default_values}.") + logging.debug( + f"Get update configs for source config {io.config} and \ + values {default_values}." + ) new_config = io.config.copy() if io.config else {} settings_class = SETTINGS_CLASS.get(type) @@ -110,29 +112,23 @@ def updated_configs_with_values( return new_config -def _upload_file_to_bucket( - file_b64: str, - file_ext: str -): +def _upload_file_to_bucket(file_b64: str, file_ext: str): # TODO: Create Bucket if not avail file_uuid = uuid4() configs = get_file_cfg_defaults_dict(file_uuid) - target_file_name = f"{configs["FILE_NAME"]}.{file_ext}" + target_file_name = f"{configs['FILE_NAME']}.{file_ext}" - s3_url = fh.get_minio_url( - configs["S3_HOST"], configs["S3_PORT"]) + s3_url = fh.get_minio_url(configs["S3_HOST"], configs["S3_PORT"]) client = fh.get_s3_client( s3_url=s3_url, access_key=configs["S3_ACCESS_KEY"], - secret_key=configs["S3_SECRET_KEY"] + secret_key=configs["S3_SECRET_KEY"], ) # Decode and upload file_bytes = base64.b64decode(file_b64) client.put_object( - Bucket=configs["BUCKET_NAME"], - Key=target_file_name, - Body=file_bytes + Bucket=configs["BUCKET_NAME"], Key=target_file_name, Body=file_bytes ) # Add uploaded file ext to config @@ -141,17 +137,15 @@ def _upload_file_to_bucket( return configs -def bulk_upload_files( - data: list[InputOutputDTO] -) -> list[InputOutput]: +def bulk_upload_files(data: list[InputOutputDTO]) -> list[InputOutput]: for inp in data: if inp.selected_file_b64 and inp.selected_file_type: file_loc_desc = _upload_file_to_bucket( - inp.selected_file_b64, - inp.selected_file_type.lstrip(".") + inp.selected_file_b64, inp.selected_file_type.lstrip(".") ) updated_cfgs = updated_configs_with_values( - inp, file_loc_desc, DataType.FILE) + inp, file_loc_desc, DataType.FILE + ) inp.config = updated_cfgs return data @@ -171,21 +165,21 @@ def bulk_query_blocks(repo_urls: list[str]) -> dict[str, SDKComputeBlock]: def create_compute_block( - db: Session, - name: str, - description: str, - author: str, - docker_image: str, - repo_url: str, - custom_name: str, - x_pos: float, - y_pos: float, - entry_name: str, - entry_description: str, - envs: ConfigType, - inputs: list[InputOutput], - outputs: list[InputOutput], - project_id: str + db: Session, + name: str, + description: str, + author: str, + docker_image: str, + repo_url: str, + custom_name: str, + x_pos: float, + y_pos: float, + entry_name: str, + entry_description: str, + envs: ConfigType, + inputs: list[InputOutput], + outputs: list[InputOutput], + project_id: str, ) -> Block: """ Creates a Compute Block. @@ -224,7 +218,7 @@ def create_compute_block( cbc_url=repo_url, x_pos=x_pos, y_pos=y_pos, - selected_entrypoint_uuid=entry.uuid + selected_entrypoint_uuid=entry.uuid, ) db.add(cb) db.flush() @@ -248,23 +242,20 @@ def get_envs_for_entrypoint(e_id: UUID) -> ConfigType | None: return e.envs -def get_ios_by_ids( - ids: list[UUID], - db: Session -) -> list[InputOutput]: +def get_ios_by_ids(ids: list[UUID], db: Session) -> list[InputOutput]: return db.query(InputOutput).filter(InputOutput.uuid.in_(ids)).all() def get_io_for_entrypoint( - e_id: UUID, - io_type: InputOutputType | None + e_id: UUID, io_type: InputOutputType | None ) -> list[InputOutput]: db: Session = next(get_database()) - return db.query(InputOutput).filter_by( - entrypoint_uuid=e_id, - type=io_type - ).all() + return ( + db.query(InputOutput) + .filter_by(entrypoint_uuid=e_id, type=io_type) + .all() + ) def get_compute_blocks_by_project(project_id: UUID) -> list[Block]: @@ -274,7 +265,7 @@ def get_compute_blocks_by_project(project_id: UUID) -> list[Block]: (InputOutput.data_type == DataType.FILE, 1), (InputOutput.data_type == DataType.PGTABLE, 2), (InputOutput.data_type == DataType.CUSTOM, 3), - else_=4 + else_=4, ) blocks = ( @@ -284,8 +275,9 @@ def get_compute_blocks_by_project(project_id: UUID) -> list[Block]: .filter(Block.project_uuid == project_id) .order_by(order_case, asc(InputOutput.name)) .options( - contains_eager(Block.selected_entrypoint) - .contains_eager(Entrypoint.input_outputs) + contains_eager(Block.selected_entrypoint).contains_eager( + Entrypoint.input_outputs + ) ) .all() ) @@ -293,19 +285,17 @@ def get_compute_blocks_by_project(project_id: UUID) -> list[Block]: return blocks -def get_block_dependencies_for_blocks( - block_ids: list[UUID] -) -> list: +def get_block_dependencies_for_blocks(block_ids: list[UUID]) -> list: db: Session = next(get_database()) query = select( block_dependencies.c.upstream_block_uuid, block_dependencies.c.upstream_output_uuid, block_dependencies.c.downstream_block_uuid, - block_dependencies.c.downstream_input_uuid + block_dependencies.c.downstream_input_uuid, ).where( - block_dependencies.c.upstream_block_uuid.in_(block_ids) | - block_dependencies.c.downstream_block_uuid.in_(block_ids) + block_dependencies.c.upstream_block_uuid.in_(block_ids) + | block_dependencies.c.downstream_block_uuid.in_(block_ids) ) # Fetch the dependencies @@ -327,8 +317,7 @@ def do_config_keys_match( # Check if all update keys are in the original config keys invalid_keys = update_keys - original_keys if invalid_keys: - logging.debug(f"Invalid keys found for { - config_type}:") + logging.debug(f"Invalid keys found for {config_type}:") logging.debug(f"Original {config_type} keys: {original_keys}") logging.debug(f"Updated {config_type} keys: {update_keys}") logging.debug(f"Invalid keys: {invalid_keys}") @@ -337,9 +326,7 @@ def do_config_keys_match( def _update_io( - db: Session, - io: InputOutput, - new_config: ConfigType + db: Session, io: InputOutput, new_config: ConfigType ) -> list[UUID]: """ Returns a list of IO uuids that were automatically updated (downstreams). @@ -351,20 +338,17 @@ def _update_io( else: raise HTTPException( status_code=422, - detail=f"Config Keys of io with id {io.uuid} do not match." + detail=f"Config Keys of io with id {io.uuid} do not match.", ) # Only Update Outputs of type File & PgTable - if ( - io.type != InputOutputType.OUTPUT and - (io.data_type != DataType.FILE - or io.data_type != DataType.PGTABLE) + if io.type != InputOutputType.OUTPUT and ( + io.data_type != DataType.FILE or io.data_type != DataType.PGTABLE ): return [] dep = db.execute( - select(block_dependencies.c.downstream_input_uuid) - .where( + select(block_dependencies.c.downstream_input_uuid).where( block_dependencies.c.upstream_output_uuid == io.uuid ) ).fetchall() @@ -372,9 +356,11 @@ def _update_io( if not downstream_inputs: return [] - downstream_ios = db.query(InputOutput).filter( - InputOutput.uuid.in_(downstream_inputs) - ).all() + downstream_ios = ( + db.query(InputOutput) + .filter(InputOutput.uuid.in_(downstream_inputs)) + .all() + ) logging.debug(f"Updating connected input configs: {downstream_ios}.") # Apply the same config update to connected inputs @@ -386,15 +372,13 @@ def _update_io( downstream_io, update_dict, downstream_io.data_type ) updated_downstream_ids.append(downstream_io.uuid) - logging.debug(f"Updated input config with id { - downstream_io.uuid}") + logging.debug(f"Updated input config with id {downstream_io.uuid}") return updated_downstream_ids def update_ios( - update_dict: dict[UUID, ConfigType], - db: Session + update_dict: dict[UUID, ConfigType], db: Session ) -> list[InputOutput]: logging.debug("Updating input/outputs.") @@ -403,32 +387,29 @@ def update_ios( if len(ios) == 0: raise HTTPException( - status_code=400, detail="Provided Inputs do not exist.") + status_code=400, detail="Provided Inputs do not exist." + ) for io in ios: - updated_downstreams = _update_io( - db, io, update_dict.get(io.uuid)) + updated_downstreams = _update_io(db, io, update_dict.get(io.uuid)) ids.extend(updated_downstreams) - updated = db.query(InputOutput).filter( - InputOutput.uuid.in_(set(ids))).all() + updated = ( + db.query(InputOutput).filter(InputOutput.uuid.in_(set(ids))).all() + ) return updated def update_ios_with_uploads( - data: list[BaseInputOutputDTO], - db: Session + data: list[BaseInputOutputDTO], db: Session ) -> list[InputOutput]: upload_candidates = [ d for d in data if d.selected_file_b64 and d.selected_file_type ] if upload_candidates: - db_ios = get_ios_by_ids( - db=db, - ids=[d.id for d in upload_candidates] - ) + db_ios = get_ios_by_ids(db=db, ids=[d.id for d in upload_candidates]) db_io_map = {io.uuid: io for io in db_ios} # Inject existing configs into upload candidates @@ -440,10 +421,7 @@ def update_ios_with_uploads( bulk_upload_files(upload_candidates) # Update all configs - updated = update_ios( - update_dict={d.id: d.config for d in data}, - db=db - ) + updated = update_ios(update_dict={d.id: d.config for d in data}, db=db) return updated @@ -452,10 +430,7 @@ class BulkBlockEnvsUpdate(BaseModel): envs: ConfigType -def bulk_update_block_envs( - updates: list[BulkBlockEnvsUpdate], - db: Session -): +def bulk_update_block_envs(updates: list[BulkBlockEnvsUpdate], db: Session): block_ids = [item.block_id for item in updates] blocks = db.query(Block).filter(Block.uuid.in_(block_ids)).all() @@ -469,25 +444,23 @@ def bulk_update_block_envs( if not block: raise HTTPException( - status_code=404, - detail=f"Block not found: {block_id}" + status_code=404, detail=f"Block not found: {block_id}" ) envs = item.envs if envs: if do_config_keys_match( - "envs", - block.selected_entrypoint.envs, - envs + "envs", block.selected_entrypoint.envs, envs ): block.selected_entrypoint.envs = { - **block.selected_entrypoint.envs, **envs + **block.selected_entrypoint.envs, + **envs, } else: raise HTTPException( status_code=422, - detail=f"Env-Keys do not match for block: {block_id}" + detail=f"Env-Keys do not match for block: {block_id}", ) updated_blocks.append(block) @@ -500,7 +473,7 @@ def update_block( envs: ConfigType | None, custom_name: str | None, x_pos: float | None, - y_pos: float | None + y_pos: float | None, ) -> Block: db: Session = next(get_database()) @@ -512,17 +485,14 @@ def update_block( block.custom_name = custom_name if envs: - if do_config_keys_match( - "envs", - block.selected_entrypoint.envs, - envs - ): + if do_config_keys_match("envs", block.selected_entrypoint.envs, envs): block.selected_entrypoint.envs = { - **block.selected_entrypoint.envs, **envs} + **block.selected_entrypoint.envs, + **envs, + } else: raise HTTPException( - status_code=422, - detail="Env-Keys do not match." + status_code=422, detail="Env-Keys do not match." ) if x_pos is not None: @@ -537,9 +507,7 @@ def update_block( return block -def delete_block( - id: UUID -): +def delete_block(id: UUID): logging.debug(f"Deleting Compute Block with id: {id}") db: Session = next(get_database()) @@ -576,35 +544,35 @@ def create_stream_and_update_target_cfg( db.execute(block_dependencies.insert().values(dependency)) # Compare the cfgs, overwrite the cfgs - target_io = db.query(InputOutput).filter_by( - uuid=to_input_uuid).one_or_none() - source_io = db.query(InputOutput).filter_by( - uuid=from_output_uuid).one_or_none() + target_io = ( + db.query(InputOutput).filter_by(uuid=to_input_uuid).one_or_none() + ) + source_io = ( + db.query(InputOutput).filter_by(uuid=from_output_uuid).one_or_none() + ) if target_io.data_type != source_io.data_type: # Data types do not match, dont allow connection - logging.error(f"Input datatype { - target_io.data_type} does not match with \ - output type {target_io.data_type}") + logging.error( + f"Input datatype {target_io.data_type} does not match with \ + output type {target_io.data_type}" + ) raise HTTPException( - status_code=400, - detail="Source & Target types do not match" + status_code=400, detail="Source & Target types do not match" ) # Custom inputs are not overwritten - if ( - (target_io.data_type != source_io.data_type) or - (target_io.data_type is DataType.CUSTOM) + if (target_io.data_type != source_io.data_type) or ( + target_io.data_type is DataType.CUSTOM ): logging.info("Edge from custom output to input created.") return target_io.uuid logging.debug(f"Updating Input {to_input_uuid} configs.") - extracted_defaults = extract_default_keys_from_io( - source_io - ) + extracted_defaults = extract_default_keys_from_io(source_io) target_io.config = updated_configs_with_values( - target_io, extracted_defaults, target_io.data_type) + target_io, extracted_defaults, target_io.data_type + ) return target_io.entrypoint_uuid @@ -615,9 +583,10 @@ def delete_edge( to_block_uuid: UUID, to_input_uuid: UUID, ): - logging.debug(f"Deleting Edge from block { - from_block_uuid} output {from_output_uuid} to \ - {to_block_uuid} with input {to_input_uuid}.") + logging.debug( + f"Deleting Edge from block {from_block_uuid} output {from_output_uuid}\ + to {to_block_uuid} with input {to_input_uuid}." + ) db: Session = next(get_database()) stmt = delete(block_dependencies).where( diff --git a/core/services/workflow_service/controllers/project_controller.py b/core/services/workflow_service/controllers/project_controller.py index dd550754..4dcafcaa 100644 --- a/core/services/workflow_service/controllers/project_controller.py +++ b/core/services/workflow_service/controllers/project_controller.py @@ -6,18 +6,17 @@ from fastapi import HTTPException from services.workflow_service.models.project import Project -import services.workflow_service.controllers.compute_block_controller as \ - compute_block_controller -import services.workflow_service.controllers.template_controller as \ - template_controller -from services.workflow_service.schemas.workflow import ( - WorkflowTemplate +from services.workflow_service.controllers import ( + compute_block_controller, + template_controller, ) +from services.workflow_service.schemas.workflow import WorkflowTemplate def create_project(db: Session, name: str, current_user_uuid: UUID) -> UUID: - logging.debug(f"Creating project with name: { - name} for user: {current_user_uuid}") + logging.debug( + f"Creating project with name: {name} for user: {current_user_uuid}" + ) project: Project = Project() @@ -33,9 +32,10 @@ def create_project(db: Session, name: str, current_user_uuid: UUID) -> UUID: def create_project_from_template( - name: str, - template_identifier: str, - current_user_uuid: UUID + name: str, + template_identifier: str, + current_user_uuid: UUID, + project_name: str, ) -> UUID: """ This method will handle the creation of project, blocks and edges as @@ -43,10 +43,11 @@ def create_project_from_template( """ db: Session = next(get_database()) - template: WorkflowTemplate =\ + template: WorkflowTemplate = ( template_controller.get_workflow_template_by_identifier( template_identifier ) + ) required_blocks = template_controller.extract_block_urls_from_template( template ) @@ -59,16 +60,19 @@ def create_project_from_template( try: with db.begin(): project_id = create_project(db, name, current_user_uuid) - block_name_to_model, block_outputs_by_name, block_inputs_by_name =\ - template_controller.configure_and_create_blocks( - G, db, unconfigured_blocks, project_id - ) + ( + block_name_to_model, + block_outputs_by_name, + block_inputs_by_name, + ) = template_controller.configure_and_create_blocks( + G, db, unconfigured_blocks, project_id, project_name + ) template_controller.create_edges_from_template( G, db, block_name_to_model, block_outputs_by_name, - block_inputs_by_name + block_inputs_by_name, ) return project_id except Exception as e: @@ -183,9 +187,7 @@ def read_projects_by_user_uuid(user_uuid: UUID) -> list[Project]: db: Session = next(get_database()) projects = ( - db.query(Project) - .filter(Project.users.contains([user_uuid])) - .all() + db.query(Project).filter(Project.users.contains([user_uuid])).all() ) if not projects: diff --git a/core/services/workflow_service/controllers/template_controller.py b/core/services/workflow_service/controllers/template_controller.py index 14ff9f7b..910cc2e1 100644 --- a/core/services/workflow_service/controllers/template_controller.py +++ b/core/services/workflow_service/controllers/template_controller.py @@ -21,17 +21,19 @@ from scystream.sdk.config.models import ( ComputeBlock, Entrypoint as SDKEntrypoint, - InputOutputModel -) -from services.workflow_service.models.block import ( - Block + InputOutputModel, ) +from services.workflow_service.models.block import Block from services.workflow_service.models.input_output import ( - InputOutput, InputOutputType, DataType + InputOutput, + InputOutputType, + DataType, ) from services.workflow_service.controllers.compute_block_controller import ( - updated_configs_with_values, do_config_keys_match, create_compute_block, - create_stream_and_update_target_cfg + updated_configs_with_values, + do_config_keys_match, + create_compute_block, + create_stream_and_update_target_cfg, ) from utils.config.defaults import ( get_file_cfg_defaults_dict, @@ -47,16 +49,15 @@ def get_workflow_template_by_identifier(identifier: str) -> WorkflowTemplate: """ try: registry = RepoRegistry() - repo_path = registry.get_repo( - ENV.WORKFLOW_TEMPLATE_REPO - ) + repo_path = registry.get_repo(ENV.WORKFLOW_TEMPLATE_REPO) file_path = os.path.join(repo_path, identifier) if not os.path.isfile(file_path): raise HTTPException( status_code=404, - detail=f"Template file '{ - identifier}' not found in repository." + detail=( + f"Template file '{identifier}' not found in repository." + ), ) with open(file_path, "r") as f: @@ -66,8 +67,7 @@ def get_workflow_template_by_identifier(identifier: str) -> WorkflowTemplate: except ValidationError as ve: logging.warning(f"Validation failed for {identifier}: {ve}") raise HTTPException( - status_code=422, - detail=f"Template validation failed: {ve}" + status_code=422, detail=f"Template validation failed: {ve}" ) @@ -75,9 +75,7 @@ def get_workflow_templates() -> list[WorkflowTemplate]: templates: WorkflowTemplate = [] registry = RepoRegistry() - repo_path = registry.get_repo( - ENV.WORKFLOW_TEMPLATE_REPO - ) + repo_path = registry.get_repo(ENV.WORKFLOW_TEMPLATE_REPO) for file in os.listdir(repo_path): if not file.endswith((".yaml", ".yml")): @@ -134,11 +132,12 @@ def build_workflow_graph(template: WorkflowTemplate): from_block, to_block, input_identifier=inp.identifier, - output_identifier=inp.depends_on.output + output_identifier=inp.depends_on.output, ) if not nx.is_directed_acyclic_graph(G): raise HTTPException( - status_code=422, detail="Template defines a cyclic dependency.") + status_code=422, detail="Template defines a cyclic dependency." + ) # Assigning Positions level_map = {} @@ -167,7 +166,9 @@ def _build_io( data_type: DataType, description: str, config: dict, - template_settings: dict | None = None + project_name: str, + block_name: str, + template_settings: dict | None = None, ) -> InputOutput: """ Constructs an InputOutput object, applying default values for outputs @@ -179,14 +180,14 @@ def _build_io( name=identifier, data_type=data_type, description=description, - config=config + config=config, ) if io_type is InputOutputType.OUTPUT: default_values = ( get_file_cfg_defaults_dict(identifier) if data_type is DataType.FILE - else get_pg_cfg_defaults_dict(identifier) + else get_pg_cfg_defaults_dict(project_name, identifier, block_name) ) io.config = updated_configs_with_values(io, default_values, data_type) @@ -199,7 +200,9 @@ def _build_io( def _configure_io_items( template_ios: list[InputTemplate] | list[OutputTemplate], unconfigured_ios: dict[str, InputOutputModel], - io_type: InputOutputType + io_type: InputOutputType, + project_name: str, + block_name: str, ) -> list[InputOutput]: """ Iterates over the compute blocks ios. @@ -229,9 +232,10 @@ def _configure_io_items( status_code=421, detail=( f"The keys used in the template to configure IO '{ - template.identifier}' " + template.identifier + }' " f"do not match those in the compute block definition." - ) + ), ) configured.append( @@ -241,7 +245,9 @@ def _configure_io_items( data_type=data_type, description=unconfigured_io.description, config=unconfigured_io.config, - template_settings=template.settings + template_settings=template.settings, + project_name=project_name, + block_name=block_name, ) ) else: @@ -251,7 +257,9 @@ def _configure_io_items( io_type=io_type, data_type=data_type, description=unconfigured_io.description, - config=unconfigured_io.config + config=unconfigured_io.config, + project_name=project_name, + block_name=block_name, ) ) @@ -260,12 +268,10 @@ def _configure_io_items( def _configure_block( block_template: BlockTemplate, - unconfigured_entry: SDKEntrypoint -) -> ( - ConfigType, - list[InputOutput], - list[InputOutput] -): + unconfigured_entry: SDKEntrypoint, + project_name: str, + block_name: str, +) -> (ConfigType, list[InputOutput], list[InputOutput]): """ This method returns: :dict: the configuration from the template applied to the configuration @@ -291,19 +297,23 @@ def _configure_block( The Config-Keys provided by the template do not match with the configs that the block {block_template.name} offers. - """ + """, ) configured_inputs: list[InputOutput] = _configure_io_items( block_template.inputs or [], unconfigured_entry.inputs or {}, - InputOutputType.INPUT + InputOutputType.INPUT, + project_name, + block_name, ) configured_outputs: list[InputOutput] = _configure_io_items( block_template.outputs or [], unconfigured_entry.outputs or {}, - InputOutputType.OUTPUT + InputOutputType.OUTPUT, + project_name, + block_name, ) return (configured_envs, configured_inputs, configured_outputs) @@ -313,11 +323,10 @@ def configure_and_create_blocks( G: nx.DiGraph, db: Session, unconfigured_blocks: dict[str, ComputeBlock], - project_id: UUID + project_id: UUID, + project_name: str, ) -> tuple[ - dict[str, Block], - dict[str, dict[str, UUID]], - dict[str, dict[str, UUID]] + dict[str, Block], dict[str, dict[str, UUID]], dict[str, dict[str, UUID]] ]: """ Configures and creates blocks defined in the template graph. @@ -344,13 +353,11 @@ def configure_and_create_blocks( for block_name in nx.topological_sort(G): block_template = G.nodes[block_name]["block"] # 1. Validate wether Template Definition of Compute Block is correct - compute_block = unconfigured_blocks.get( - block_template.repo_url - ) + compute_block = unconfigured_blocks.get(block_template.repo_url) if compute_block is None: raise HTTPException( status_code=422, - detail=f"Block repo '{block_template.repo_url}' not found." + detail=f"Block repo '{block_template.repo_url}' not found.", ) entrypoint = compute_block.entrypoints.get(block_template.entrypoint) @@ -358,12 +365,12 @@ def configure_and_create_blocks( raise HTTPException( status_code=422, detail=f"Entrypoint '{block_template.entrypoint}' not found in\ - block '{block_template.name}'." + block '{block_template.name}'.", ) # 2. Configure the Block configured_envs, inputs, outputs = _configure_block( - block_template, entrypoint + block_template, entrypoint, project_name, compute_block.name ) # 3. Create the Block @@ -383,7 +390,7 @@ def configure_and_create_blocks( envs=configured_envs, inputs=inputs, outputs=outputs, - project_id=project_id + project_id=project_id, ) # 4. Create the maps that "connect" template to database representation @@ -426,7 +433,7 @@ def create_edges_from_template( Dependency resolution failed for edge: "{from_block} -> {to_block} "({output_identifier}-> {input_identifier}) - """ + """, ) create_stream_and_update_target_cfg( @@ -434,5 +441,5 @@ def create_edges_from_template( upstream_block.uuid, output_uuid, downstream_block.uuid, - input_uuid + input_uuid, ) diff --git a/core/services/workflow_service/schemas/compute_block.py b/core/services/workflow_service/schemas/compute_block.py index 122bd7df..6351b404 100644 --- a/core/services/workflow_service/schemas/compute_block.py +++ b/core/services/workflow_service/schemas/compute_block.py @@ -2,13 +2,13 @@ from enum import Enum from typing import Literal -from pydantic import BaseModel, validator, model_validator +from pydantic import BaseModel, validator, model_validator, Field from urllib.parse import urlparse from services.workflow_service.models.input_output import ( DataType, InputOutput, - InputOutputType + InputOutputType, ) from utils.config.environment import ENV from utils.config.defaults import get_file_cfg_defaults_dict @@ -44,7 +44,8 @@ def _validate_url(url: str): parsed = urlparse(url) if parsed.scheme != "https": raise ValueError( - "Insecure URL! Only HTTPS or SSH git URLs are allowed.") + "Insecure URL! Only HTTPS or SSH git URLs are allowed." + ) def _get_io_data_type(type: str) -> str: @@ -58,8 +59,9 @@ def _get_io_data_type(type: str) -> str: def replace_minio_host(url: str | None) -> str | None: if url: defaults = get_file_cfg_defaults_dict("placeholder") - default_minio_url = f"{defaults.get("S3_HOST")}:{ - defaults.get("S3_PORT")}" + default_minio_url = ( + f"{defaults.get('S3_HOST')}:{defaults.get('S3_PORT')}" + ) """ The client can never use the presigned url with the default minio host. Therefore we replace the default minio host, if it exists in @@ -69,6 +71,7 @@ def replace_minio_host(url: str | None) -> str | None: return url.replace(default_minio_url, ENV.EXTERNAL_URL_DATA_S3) return url + # Inputs & Outputs @@ -79,11 +82,7 @@ class BaseIODTO(BaseModel): @classmethod def from_input_output(cls, io): - return cls( - id=io.uuid, - name=io.name, - data_type=io.data_type - ) + return cls(id=io.uuid, name=io.name, data_type=io.data_type) class InputOutputDTO(BaseIODTO): @@ -107,10 +106,10 @@ def validate_selected_file_fields(self): @classmethod def from_input_output( - cls, - name: str, - input_output, - presigned_url: str | None = None, + cls, + name: str, + input_output, + presigned_url: str | None = None, ): return cls( id=getattr(input_output, "uuid", None), @@ -129,27 +128,27 @@ def from_sdk_input_output(cls, name: str, input_output): name=name, data_type=_get_io_data_type(input_output.type), description=input_output.description or "", - config=input_output.config or {} + config=input_output.config or {}, ) @classmethod - def to_input_output( - cls, - input_output, - type: Literal["Input", "Output"] - ): + def to_input_output(cls, input_output, type: Literal["Input", "Output"]): return InputOutput( - type=(InputOutputType.INPUT if type == - "Input" else InputOutputType.OUTPUT), + type=( + InputOutputType.INPUT + if type == "Input" + else InputOutputType.OUTPUT + ), name=input_output.name, data_type=input_output.data_type, description=input_output.description, - config=input_output.config + config=input_output.config, ) # Entrypoint + class BaseEntrypointDTO(BaseModel): id: UUID | None = None name: str @@ -184,6 +183,7 @@ def from_sdk_entrypoint(cls, name: str, entrypoint): # Node: + class PositionDTO(BaseModel): x: float y: float @@ -221,11 +221,7 @@ class SimpleNodeDTO(BaseNodeDTO): data: SimpleNodeDataDTO @classmethod - def from_compute_block( - cls, - cb, - status: BlockStatus = BlockStatus.IDLE - ): + def from_compute_block(cls, cb, status: BlockStatus = BlockStatus.IDLE): return cls( id=cb.uuid, position=PositionDTO( @@ -252,9 +248,9 @@ def from_compute_block( BaseIODTO.from_input_output(io) for io in cb.selected_entrypoint.input_outputs if io.type == InputOutputType.OUTPUT - ] + ], ), - status=status + status=status, ), ) @@ -296,14 +292,15 @@ def from_compute_block(cls, cb): description=cb.selected_entrypoint.description, envs=cb.selected_entrypoint.envs, inputs=inputs, - outputs=outputs + outputs=outputs, ), - ) + ), ) # Edge: + class EdgeDTO(BaseModel): id: str | None = None source: UUID @@ -324,8 +321,11 @@ def from_block_dependencies(cls, bd): # Requests & Responses: + class ComputeBlockInformationRequest(BaseModel): + compute_block_custom_name: str = Field(..., max_length=15) cbc_url: str + project_name: str @validator("cbc_url") def validate_cbc_url(cls, v): @@ -350,7 +350,7 @@ def from_sdk_compute_block(cls, cb): entrypoints=[ EntrypointDTO.from_sdk_entrypoint(name, entrypoint) for name, entrypoint in cb.entrypoints.items() - ] + ], ) @@ -410,7 +410,7 @@ def from_input_output(cls, input_output, presigned_url: str | None = None): type=input_output.type, entrypoint_id=input_output.entrypoint_uuid, config=input_output.config or {}, - presigned_url=replace_minio_host(url=presigned_url) + presigned_url=replace_minio_host(url=presigned_url), ) diff --git a/core/services/workflow_service/schemas/project.py b/core/services/workflow_service/schemas/project.py index ba172ab2..145984fc 100644 --- a/core/services/workflow_service/schemas/project.py +++ b/core/services/workflow_service/schemas/project.py @@ -1,5 +1,5 @@ from uuid import UUID -from pydantic import BaseModel +from pydantic import BaseModel, Field from datetime import datetime @@ -15,7 +15,7 @@ class Config: class CreateProjectRequest(BaseModel): - name: str + name: str = Field(..., max_length=30) class CreateProjectResponse(BaseModel): @@ -23,7 +23,7 @@ class CreateProjectResponse(BaseModel): class CreateProjectFromTemplateRequest(BaseModel): - name: str + name: str = Field(..., max_length=30) template_identifier: str # File Name of yaml definition of DAG-Template diff --git a/core/services/workflow_service/views/compute_block.py b/core/services/workflow_service/views/compute_block.py index 9bbbf6c7..ca794a0a 100644 --- a/core/services/workflow_service/views/compute_block.py +++ b/core/services/workflow_service/views/compute_block.py @@ -5,16 +5,21 @@ from utils.database.session_injector import get_database from utils.errors.error import handle_error from utils.data.file_handling import bulk_presigned_urls_from_ios -from services.workflow_service.models.input_output import ( - InputOutputType -) +from services.workflow_service.models.input_output import InputOutputType from services.workflow_service.schemas.compute_block import ( - ComputeBlockInformationRequest, ComputeBlockInformationResponse, - CreateComputeBlockRequest, IDResponse, + ComputeBlockInformationRequest, + ComputeBlockInformationResponse, + CreateComputeBlockRequest, + IDResponse, GetNodesByProjectResponse, - EdgeDTO, SimpleNodeDTO, InputOutputDTO, BaseInputOutputDTO, - UpdateInputOutputResponseDTO, UpdateComputeBlockDTO, ConfigType, - BlockStatus + EdgeDTO, + SimpleNodeDTO, + InputOutputDTO, + BaseInputOutputDTO, + UpdateInputOutputResponseDTO, + UpdateComputeBlockDTO, + ConfigType, + BlockStatus, ) from fastapi import APIRouter, Depends, HTTPException from services.workflow_service.controllers import workflow_controller @@ -30,7 +35,7 @@ get_io_for_entrypoint, request_cb_info, update_block, - update_ios_with_uploads + update_ios_with_uploads, ) from utils.security.token import User, get_user @@ -44,7 +49,7 @@ async def cb_information( ): try: cb = request_cb_info( - data.cbc_url, + data.cbc_url, data.project_name, data.compute_block_custom_name ) return ComputeBlockInformationResponse.from_sdk_compute_block(cb) except Exception as e: @@ -56,7 +61,7 @@ async def cb_information( async def create( data: CreateComputeBlockRequest, _: User = Depends(get_user), - db: Session = Depends(get_database) + db: Session = Depends(get_database), ): try: """ @@ -81,10 +86,14 @@ async def create( data.selected_entrypoint.name, data.selected_entrypoint.description, data.selected_entrypoint.envs, - [input.to_input_output(input, "Input") - for input in updated_is], - [output.to_input_output(output, "Output") - for output in data.selected_entrypoint.outputs], + [ + input.to_input_output(input, "Input") + for input in updated_is + ], + [ + output.to_input_output(output, "Output") + for output in data.selected_entrypoint.outputs + ], data.project_id, ) return SimpleNodeDTO.from_compute_block(cb) @@ -183,10 +192,11 @@ async def get_io( try: ios = get_io_for_entrypoint(entry_id, io_type) presigned_urls = bulk_presigned_urls_from_ios(ios) - return [InputOutputDTO.from_input_output( - io.name, - io, - presigned_urls.get(io.uuid, None)) for io in ios + return [ + InputOutputDTO.from_input_output( + io.name, io, presigned_urls.get(io.uuid, None) + ) + for io in ios ] except Exception as e: logging.exception( @@ -195,8 +205,9 @@ async def get_io( raise handle_error(e) -@router.put("/entrypoint/io/", - response_model=list[UpdateInputOutputResponseDTO]) +@router.put( + "/entrypoint/io/", response_model=list[UpdateInputOutputResponseDTO] +) async def update_io(data: list[BaseInputOutputDTO]): db = next(get_database()) try: @@ -252,7 +263,7 @@ def create_io_stream_and_update_io_cfg( data.source, data.sourceHandle, data.target, - data.targetHandle + data.targetHandle, ) return IDResponse(id=id) except Exception as e: diff --git a/core/services/workflow_service/views/project.py b/core/services/workflow_service/views/project.py index 0ef2beda..57e127f0 100644 --- a/core/services/workflow_service/views/project.py +++ b/core/services/workflow_service/views/project.py @@ -28,7 +28,7 @@ async def create_project( data: CreateProjectRequest, user: User = Depends(get_user), - db: Session = Depends(get_database) + db: Session = Depends(get_database), ): try: with db.begin(): @@ -42,18 +42,14 @@ async def create_project( @router.post( - "/from_template", - response_model=CreateProjectFromTemplateResponse + "/from_template", response_model=CreateProjectFromTemplateResponse ) async def create_project_from_template( - data: CreateProjectFromTemplateRequest, - user: User = Depends(get_user) + data: CreateProjectFromTemplateRequest, user: User = Depends(get_user) ): try: id = project_controller.create_project_from_template( - data.name, - data.template_identifier, - user.uuid, + data.name, data.template_identifier, user.uuid, data.name ) return CreateProjectResponse(project_uuid=id) except Exception as e: @@ -85,7 +81,7 @@ async def read_projects_by_user( @router.get("/{project_id}", response_model=Project) async def read_project( - project_id: UUID | None = None, + project_id: UUID | None = None, ): try: if project_id is None: @@ -100,8 +96,7 @@ async def read_project( @router.put("/", response_model=Project) async def rename_project( - data: RenameProjectRequest, - db: Session = Depends(get_database) + data: RenameProjectRequest, db: Session = Depends(get_database) ): try: with db.begin(): diff --git a/core/utils/config/defaults.py b/core/utils/config/defaults.py index 2653252f..99369258 100644 --- a/core/utils/config/defaults.py +++ b/core/utils/config/defaults.py @@ -1,3 +1,4 @@ +import re from uuid import uuid4 from utils.config.environment import ENV @@ -5,10 +6,53 @@ from scystream.sdk.env.settings import PostgresSettings, FileSettings -SETTINGS_CLASS = { - DataType.FILE: FileSettings, - DataType.PGTABLE: PostgresSettings -} +SETTINGS_CLASS = {DataType.FILE: FileSettings, + DataType.PGTABLE: PostgresSettings} + + +def _normalize_identifier(value: str) -> str: + value = value.lower() + + # replace spaces and hyphens with underscore + value = re.sub(r"[ \-]+", "_", value) + + # remove invalid characters (keep a-z, 0-9, _) + value = re.sub(r"[^a-z0-9_]", "", value) + + # collapse multiple underscores + value = re.sub(r"_+", "_", value) + + # strip leading/trailing underscores + value = value.strip("_") + + # ensure it doesn't start with a digit + if value and value[0].isdigit(): + value = f"t_{value}" + + return value + + +def _normalize_table_name( + project_name: str, io_name: str, compute_block_custom_name: str +) -> str: + max_length = 63 + sep_count = 2 + remaining = max_length - sep_count + + project_name = _normalize_identifier(project_name) + io_name = _normalize_identifier(io_name) + compute_block_custom_name = _normalize_identifier( + compute_block_custom_name) + + io_len = remaining - (len(compute_block_custom_name) + len(project_name)) + + # prevent negative slicing + if io_len < 0: + io_len = 0 + + io_part = io_name[:io_len] + + return f"{project_name}_{io_part}_{compute_block_custom_name}" def get_file_cfg_defaults_dict(io_name: str) -> dict: @@ -24,19 +68,21 @@ def get_file_cfg_defaults_dict(io_name: str) -> dict: } -def get_pg_cfg_defaults_dict(io_name: str) -> dict: +def get_pg_cfg_defaults_dict( + project_name: str, io_name: str, compute_block_custom_name: str +) -> dict: return { "PG_USER": ENV.DEFAULT_CB_CONFIG_PG_USER, "PG_PASS": ENV.DEFAULT_CB_CONFIG_PG_PASS, "PG_HOST": ENV.DEFAULT_CB_CONFIG_PG_HOST, "PG_PORT": ENV.DEFAULT_CB_CONFIG_PG_PORT, - "DB_TABLE": f"table_{io_name}_{uuid4()}", + "DB_TABLE": _normalize_table_name( + project_name, io_name, compute_block_custom_name + ), } -def extract_default_keys_from_io( - io: InputOutput -): +def extract_default_keys_from_io(io: InputOutput): """ This class returns a dict that maps the previously prefixed default keys values to their default keys. @@ -53,10 +99,9 @@ def extract_default_keys_from_io( } """ settings_class = SETTINGS_CLASS.get(io.data_type) - default_keys = set( - settings_class.__annotations__.keys() - ) + default_keys = set(settings_class.__annotations__.keys()) return { - dk: value for key, value in io.config.items() + dk: value + for key, value in io.config.items() if (dk := next((d for d in default_keys if d in key), None)) } diff --git a/core/utils/config/environment.py b/core/utils/config/environment.py index 3b096f62..5267d600 100644 --- a/core/utils/config/environment.py +++ b/core/utils/config/environment.py @@ -30,7 +30,7 @@ class Settings(BaseSettings): CB_NETWORK_MODE: str = "scystream_data_processing" DEFAULT_CB_CONFIG_S3_HOST: str = "http://data-minio" - DEFAULT_CB_CONFIG_S3_PORT: str = "9000" + DEFAULT_CB_CONFIG_S3_PORT: int = 9000 DEFAULT_CB_CONFIG_S3_ACCESS_KEY: str = "minioadmin" DEFAULT_CB_CONFIG_S3_SECRET_KEY: str = "minioadmin" DEFAULT_CB_CONFIG_S3_BUCKET_NAME: str = "data" @@ -39,7 +39,7 @@ class Settings(BaseSettings): DEFAULT_CB_CONFIG_PG_USER: str = "postgres" DEFAULT_CB_CONFIG_PG_PASS: str = "postgres" DEFAULT_CB_CONFIG_PG_HOST: str = "data-postgres" - DEFAULT_CB_CONFIG_PG_PORT: str = "5432" + DEFAULT_CB_CONFIG_PG_PORT: int = 5432 AIRFLOW_HOST: str = "http://localhost:8080" AIRFLOW_USER: str = "airflow" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..b12621c8 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,313 @@ +networks: + core: + airflow: + data_processing: + +volumes: + postgres-airflow-volume: + airflow-plugins: + airflow-dags: + airflow-logs: + airflow-config: + airflow-sources: + core-postgres: + minio-data: + keycloak-postgres-data: + +x-airflow-common: + &airflow-common + image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:3.1.6} + environment: + &airflow-common-env + AIRFLOW__CORE__EXECUTOR: CeleryExecutor + AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager + AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-airflow/airflow + AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres-airflow/airflow + AIRFLOW__CELERY__BROKER_URL: redis://:@redis-airflow:6379/0 + AIRFLOW__CORE__FERNET_KEY: '' + AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' + AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 0 + AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL: 0 + AIRFLOW__API_AUTH__JWT_SECRET: 'TqR6DjTN+WZw/yg3cRR1gA==' + _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} + volumes: + - ./airflow-dags:/opt/airflow/dags + - airflow-logs:/opt/airflow/logs + - airflow-config:/opt/airflow/config + - airflow-plugins:/opt/airflow/plugins + - /var/run/docker.sock:/var/run/docker.sock + - ~/.docker:/root/.docker:ro + user: "0:0" + depends_on: + &airflow-common-depends-on + redis-airflow: + condition: service_healthy + postgres-airflow: + condition: service_healthy + +services: + redis-airflow: + image: redis:7.2-bookworm + expose: + - 6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 30s + retries: 50 + start_period: 30s + restart: always + networks: + - airflow + + postgres-airflow: + image: postgres:14 + environment: + POSTGRES_USER: airflow + POSTGRES_PASSWORD: airflow + POSTGRES_DB: airflow + volumes: + - postgres-airflow-volume:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "airflow"] + interval: 10s + retries: 5 + start_period: 5s + restart: always + networks: + - airflow + + airflow-init: + <<: *airflow-common + entrypoint: /bin/bash + command: + - -c + - | + mkdir -p /sources/logs /sources/dags /sources/plugins + chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} + exec /entrypoint airflow version + environment: + <<: *airflow-common-env + _AIRFLOW_DB_MIGRATE: 'true' + _AIRFLOW_WWW_USER_CREATE: 'true' + _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} + _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} + user: "0:0" + volumes: + - airflow-sources:/sources + networks: + - airflow + + airflow-apiserver: + <<: *airflow-common + command: api-server + ports: + - "8080:8080" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/version"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + networks: + - airflow + - core + + airflow-scheduler: + <<: *airflow-common + command: scheduler + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + networks: + - airflow + + airflow-dag-processor: + <<: *airflow-common + command: dag-processor + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + networks: + - airflow + + airflow-worker: + <<: *airflow-common + command: celery worker + healthcheck: + test: + - "CMD-SHELL" + - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + environment: + <<: *airflow-common-env + DUMB_INIT_SETSID: "0" + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-apiserver: + condition: service_healthy + airflow-init: + condition: service_completed_successfully + networks: + - data_processing + - airflow + + airflow-triggerer: + <<: *airflow-common + command: triggerer + healthcheck: + test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + networks: + - airflow + + airflow-cli: + <<: *airflow-common + profiles: + - debug + environment: + <<: *airflow-common-env + CONNECTION_CHECK_MAX_COUNT: "0" + command: + - bash + - -c + - airflow + networks: + - airflow + + flower: + <<: *airflow-common + command: celery flower + profiles: + - flower + ports: + - "5555:5555" + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:5555/"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: always + depends_on: + <<: *airflow-common-depends-on + airflow-init: + condition: service_completed_successfully + networks: + - airflow + + core-postgres: + image: postgres:17 + environment: + POSTGRES_USER: core + POSTGRES_PASSWORD: core + POSTGRES_DB: core + volumes: + - core-postgres:/var/lib/postgresql/data + restart: always + healthcheck: + test: ["CMD-SHELL", "pg_isready -U core -d core -q"] + interval: 10s + timeout: 2s + retries: 5 + ports: + - "5432:5432" + networks: + - core + + data-postgres: + image: postgres + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + networks: + - data_processing + + data-minio: + image: quay.io/minio/minio + restart: always + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + volumes: + - minio-data:/data + ports: + - "9000:9000" + - "9001:9001" + command: server /data --console-address ":9001" + networks: + - data_processing + + keycloak-postgres: + image: postgres:17 + volumes: + - "keycloak-postgres-data:/var/lib/postgresql/data" + environment: + POSTGRES_DB: "keycloak" + POSTGRES_USER: "keycloak" + POSTGRES_PASSWORD: "password" + networks: + - core + + keycloak: + image: quay.io/keycloak/keycloak + environment: + KC_DB: "postgres" + KC_DB_URL: "jdbc:postgresql://keycloak-postgres:5432/keycloak" + KC_DB_USERNAME: "keycloak" + KC_DB_PASSWORD: "password" + KC_HOSTNAME: "localhost" + KC_HOSTNAME_PORT: 8080 + KC_HOSTNAME_STRICT: false + KC_HOSTNAME_STRICT_HTTPS: false + KC_LOG_LEVEL: "info" + KC_METRICS_ENABLED: true + KC_HEALTH_ENABLED: true + KEYCLOAK_ADMIN: "admin" + KEYCLOAK_ADMIN_PASSWORD: "admin" + command: start-dev --import-realm + ports: + - "8090:8080" + depends_on: + - keycloak-postgres + volumes: + - ./.keycloak-config:/opt/keycloak/data/import + networks: + - core diff --git a/frontend/components/CreateProjectModal.tsx b/frontend/components/CreateProjectModal.tsx index 3272cf10..98d3f89e 100644 --- a/frontend/components/CreateProjectModal.tsx +++ b/frontend/components/CreateProjectModal.tsx @@ -43,7 +43,7 @@ export default function CreateProjectModal({ diff --git a/frontend/components/steps/CreateComputeBlockConfigurationStep.tsx b/frontend/components/steps/CreateComputeBlockConfigurationStep.tsx index 0ea26470..70fa0678 100644 --- a/frontend/components/steps/CreateComputeBlockConfigurationStep.tsx +++ b/frontend/components/steps/CreateComputeBlockConfigurationStep.tsx @@ -27,7 +27,6 @@ export default function CreateComputeBlockConfigurationStep({ onPrev, selectedEntrypoint, computeBlock, - setComputeBlock, setSelectedEntrypoint, loading }: PageProps) { @@ -83,12 +82,6 @@ export default function CreateComputeBlockConfigurationStep({ }) } - function handleCustomNameChange(value: string) { - if (!setComputeBlock) return - - setComputeBlock((prev) => ({ ...prev, custom_name: value })) - } - useEffect(() => { function validateForm() { if (!computeBlock?.custom_name.trim()) { @@ -108,17 +101,6 @@ export default function CreateComputeBlockConfigurationStep({ return (