From fae2ce0d465e6b5ed24fd441bba1c8c6e610a7c1 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 27 Jan 2026 18:20:12 +0100 Subject: [PATCH 01/41] refactor: move functionality from main file --- cloudos_cli/__main__.py | 4353 +--------------------- cloudos_cli/__main__.py.backup | 4367 +++++++++++++++++++++++ cloudos_cli/__main__.py.before_refactor | 156 + cloudos_cli/__main__.py.old | 4367 +++++++++++++++++++++++ cloudos_cli/bash/__init__.py | 1 + cloudos_cli/bash/cli.py | 564 +++ cloudos_cli/configure/cli.py | 48 + cloudos_cli/cromwell/__init__.py | 1 + cloudos_cli/cromwell/cli.py | 218 ++ cloudos_cli/datasets/cli.py | 849 +++++ cloudos_cli/jobs/cli.py | 1753 +++++++++ cloudos_cli/procurement/cli.py | 71 + cloudos_cli/projects/__init__.py | 1 + cloudos_cli/projects/cli.py | 174 + cloudos_cli/queue/cli.py | 83 + cloudos_cli/utils/cli_helpers.py | 87 + cloudos_cli/workflows/__init__.py | 1 + cloudos_cli/workflows/cli.py | 153 + 18 files changed, 12926 insertions(+), 4321 deletions(-) create mode 100644 cloudos_cli/__main__.py.backup create mode 100644 cloudos_cli/__main__.py.before_refactor create mode 100644 cloudos_cli/__main__.py.old create mode 100644 cloudos_cli/bash/__init__.py create mode 100644 cloudos_cli/bash/cli.py create mode 100644 cloudos_cli/configure/cli.py create mode 100644 cloudos_cli/cromwell/__init__.py create mode 100644 cloudos_cli/cromwell/cli.py create mode 100644 cloudos_cli/datasets/cli.py create mode 100644 cloudos_cli/jobs/cli.py create mode 100644 cloudos_cli/procurement/cli.py create mode 100644 cloudos_cli/projects/__init__.py create mode 100644 cloudos_cli/projects/cli.py create mode 100644 cloudos_cli/queue/cli.py create mode 100644 cloudos_cli/utils/cli_helpers.py create mode 100644 cloudos_cli/workflows/__init__.py create mode 100644 cloudos_cli/workflows/cli.py diff --git a/cloudos_cli/__main__.py b/cloudos_cli/__main__.py index 1eab8bfd..676bc4aa 100644 --- a/cloudos_cli/__main__.py +++ b/cloudos_cli/__main__.py @@ -1,40 +1,32 @@ #!/usr/bin/env python3 import rich_click as click -import cloudos_cli.jobs.job as jb -from cloudos_cli.clos import Cloudos -from cloudos_cli.import_wf.import_wf import ImportWorflow -from cloudos_cli.queue.queue import Queue -from cloudos_cli.utils.errors import BadRequestException -import json -import time import sys -import traceback -import copy from ._version import __version__ -from cloudos_cli.configure.configure import ConfigurationProfile -from rich.console import Console -from rich.table import Table -from cloudos_cli.datasets import Datasets -from cloudos_cli.procurement import Images -from cloudos_cli.utils.resources import ssl_selector, format_bytes -from rich.style import Style -from cloudos_cli.utils.array_job import generate_datasets_for_project -from cloudos_cli.utils.details import create_job_details, create_job_list_table -from cloudos_cli.link import Link -from cloudos_cli.cost.cost import CostViewer -from cloudos_cli.logging.logger import setup_logging, update_command_context_from_click -import logging +from cloudos_cli.logging.logger import update_command_context_from_click from cloudos_cli.configure.configure import ( - with_profile_config, build_default_map_for_group, - get_shared_config, - CLOUDOS_URL + get_shared_config ) -from cloudos_cli.related_analyses.related_analyses import related_analyses +from cloudos_cli.utils.cli_helpers import ( + custom_exception_handler, + pass_debug_to_subcommands, + setup_debug +) + +# Import all command groups from their cli modules +from cloudos_cli.jobs.cli import job +from cloudos_cli.workflows.cli import workflow +from cloudos_cli.projects.cli import project +from cloudos_cli.cromwell.cli import cromwell +from cloudos_cli.queue.cli import queue +from cloudos_cli.bash.cli import bash +from cloudos_cli.procurement.cli import procurement +from cloudos_cli.datasets.cli import datasets +from cloudos_cli.configure.cli import configure -# GLOBAL VARS +# GLOBAL CONSTANTS - Keep these for backward compatibility JOB_COMPLETED = 'completed' REQUEST_INTERVAL_CROMWELL = 30 AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] @@ -45,91 +37,13 @@ HPC_NEXTFLOW_LATEST = '22.10.8' ABORT_JOB_STATES = ['running', 'initializing'] - -def custom_exception_handler(exc_type, exc_value, exc_traceback): - """Custom exception handler that respects debug mode""" - console = Console(stderr=True) - # Initialise logger - debug_mode = '--debug' in sys.argv - setup_logging(debug_mode) - logger = logging.getLogger("CloudOS") - if get_debug_mode(): - logger.error(exc_value, exc_info=exc_value) - console.print("[yellow]Debug mode: showing full traceback[/yellow]") - sys.__excepthook__(exc_type, exc_value, exc_traceback) - else: - # Extract a clean error message - if hasattr(exc_value, 'message'): - error_msg = exc_value.message - elif str(exc_value): - error_msg = str(exc_value) - else: - error_msg = f"{exc_type.__name__}" - logger.error(exc_value) - console.print(f"[bold red]Error: {error_msg}[/bold red]") - - # For network errors, give helpful context - if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): - console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") - # Install the custom exception handler sys.excepthook = custom_exception_handler -def pass_debug_to_subcommands(group_cls=click.RichGroup): - """Custom Group class that passes --debug option to all subcommands""" - - class DebugGroup(group_cls): - def add_command(self, cmd, name=None): - # Add debug option to the command if it doesn't already have it - if isinstance(cmd, (click.Command, click.Group)): - has_debug = any(param.name == 'debug' for param in cmd.params) - if not has_debug: - debug_option = click.Option( - ['--debug'], - is_flag=True, - help='Show detailed error information and tracebacks', - is_eager=True, - expose_value=False, - callback=self._debug_callback - ) - cmd.params.insert(-1, debug_option) # Insert at the end for precedence - - super().add_command(cmd, name) - - def _debug_callback(self, ctx, param, value): - """Callback to handle debug flag""" - global _global_debug - if value: - _global_debug = True - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - return DebugGroup - - -def get_debug_mode(): - """Get current debug mode state""" - return _global_debug - - -# Helper function for debug setup -def _setup_debug(ctx, param, value): - """Setup debug mode globally and in context""" - global _global_debug - _global_debug = value - if value: - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - @click.group(cls=pass_debug_to_subcommands()) @click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', - is_eager=True, expose_value=False, callback=_setup_debug) + is_eager=True, expose_value=False, callback=setup_debug) @click.version_option(__version__) @click.pass_context def run_cloudos_cli(ctx): @@ -148,4220 +62,17 @@ def run_cloudos_cli(ctx): ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def job(): - """CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS.""" - print(job.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def workflow(): - """CloudOS workflow functionality: list and import workflows.""" - print(workflow.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def project(): - """CloudOS project functionality: list and create projects in CloudOS.""" - print(project.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def cromwell(): - """Cromwell server functionality: check status, start and stop.""" - print(cromwell.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def queue(): - """CloudOS job queue functionality.""" - print(queue.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def bash(): - """CloudOS bash functionality.""" - print(bash.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def procurement(): - """CloudOS procurement functionality.""" - print(procurement.__doc__ + '\n') - - -@procurement.group(cls=pass_debug_to_subcommands()) -def images(): - """CloudOS procurement images functionality.""" - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -@click.pass_context -def datasets(ctx): - """CloudOS datasets functionality.""" - update_command_context_from_click(ctx) - if ctx.args and ctx.args[0] != 'ls': - print(datasets.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands(), invoke_without_command=True) -@click.option('--profile', help='Profile to use from the config file', default='default') -@click.option('--make-default', - is_flag=True, - help='Make the profile the default one.') -@click.pass_context -def configure(ctx, profile, make_default): - """CloudOS configuration.""" - print(configure.__doc__ + '\n') - update_command_context_from_click(ctx) - profile = profile or ctx.obj['profile'] - config_manager = ConfigurationProfile() - - if ctx.invoked_subcommand is None and profile == "default" and not make_default: - config_manager.create_profile_from_input(profile_name="default") - - if profile != "default" and not make_default: - config_manager.create_profile_from_input(profile_name=profile) - if make_default: - config_manager.make_default_profile(profile_name=profile) - - -@job.command('run', cls=click.RichCommand) -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('--job-config', - help=('A config file similar to a nextflow.config file, ' + - 'but only with the parameters to use with your job.')) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p input=s3://path_to_my_file. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--nextflow-profile', - help=('A comma separated string indicating the nextflow profile/s ' + - 'to use with your job.')) -@click.option('--nextflow-version', - help=('Nextflow version to use when executing the workflow in CloudOS. ' + - 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']), - default='22.10.8') -@click.option('--git-commit', - help=('The git commit hash to run for ' + - 'the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--git-tag', - help=('The tag to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--git-branch', - help=('The branch to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--resumable', - help='Whether to make the job able to be resumed or not.', - is_flag=True) -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--wdl-mainfile', - help='For WDL workflows, which mainFile (.wdl) is configured to use.',) -@click.option('--wdl-importsfile', - help='For WDL workflows, which importsFile (.zip) is configured to use.',) -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. Currently, not necessary ' + - 'as apikey can be used instead, but maintained for backwards compatibility.')) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - type=click.Choice(['aws', 'azure', 'hpc']), - default='aws') -@click.option('--hpc-id', - help=('ID of your HPC, only applicable when --execution-platform=hpc. ' + - 'Default=660fae20f93358ad61e0104b'), - default='660fae20f93358ad61e0104b') -@click.option('--azure-worker-instance-type', - help=('The worker node instance type to be used in azure. ' + - 'Default=Standard_D4as_v4'), - default='Standard_D4as_v4') -@click.option('--azure-worker-instance-disk', - help='The disk size in GB for the worker node to be used in azure. Default=100', - type=int, - default=100) -@click.option('--azure-worker-instance-spot', - help='Whether the azure worker nodes have to be spot instances or not.', - is_flag=True) -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-file-staging', - help='Enables AWS S3 mountpoint for quicker file staging.', - is_flag=True) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--use-private-docker-repository', - help=('Allows to use private docker repository for running jobs. The Docker user ' + - 'account has to be already linked to CloudOS.'), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run(ctx, - apikey, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - job_config, - parameter, - git_commit, - git_tag, - git_branch, - job_name, - resumable, - do_not_save_logs, - job_queue, - nextflow_profile, - nextflow_version, - instance_type, - instance_disk, - storage_mode, - lustre_size, - wait_completion, - wait_time, - wdl_mainfile, - wdl_importsfile, - cromwell_token, - repository_platform, - execution_platform, - hpc_id, - azure_worker_instance_type, - azure_worker_instance_disk, - azure_worker_instance_spot, - cost_limit, - accelerate_file_staging, - accelerate_saving_results, - use_private_docker_repository, - verbose, - request_interval, - disable_ssl_verification, - ssl_cert, - profile): - """Submit a job to CloudOS.""" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if do_not_save_logs: - save_logs = False - else: - save_logs = True - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - if execution_platform == 'azure' or execution_platform == 'hpc': - batch = False - else: - batch = True - if execution_platform == 'hpc': - print('\nHPC execution platform selected') - if hpc_id is None: - raise ValueError('Please, specify your HPC ID using --hpc parameter') - print('Please, take into account that HPC execution do not support ' + - 'the following parameters and all of them will be ignored:\n' + - '\t--job-queue\n' + - '\t--resumable | --do-not-save-logs\n' + - '\t--instance-type | --instance-disk | --cost-limit\n' + - '\t--storage-mode | --lustre-size\n' + - '\t--wdl-mainfile | --wdl-importsfile | --cromwell-token\n') - wdl_mainfile = None - wdl_importsfile = None - storage_mode = 'regular' - save_logs = False - if accelerate_file_staging: - if execution_platform != 'aws': - print('You have selected accelerate file staging, but this function is ' + - 'only available when execution platform is AWS. The accelerate file staging ' + - 'will not be applied') - use_mountpoints = False - else: - use_mountpoints = True - print('Enabling AWS S3 mountpoint for accelerated file staging. ' + - 'Please, take into consideration the following:\n' + - '\t- It significantly reduces runtime and compute costs but may increase network costs.\n' + - '\t- Requires extra memory. Adjust process memory or optimise resource usage if necessary.\n' + - '\t- This is still a CloudOS BETA feature.\n') - else: - use_mountpoints = False - if verbose: - print('\t...Detecting workflow type') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - workflow_type = cl.detect_workflow(workflow_name, workspace_id, verify_ssl, last) - is_module = cl.is_module(workflow_name, workspace_id, verify_ssl, last) - if execution_platform == 'hpc' and workflow_type == 'wdl': - raise ValueError(f'The workflow {workflow_name} is a WDL workflow. ' + - 'WDL is not supported on HPC execution platform.') - if workflow_type == 'wdl': - print('WDL workflow detected') - if wdl_mainfile is None: - raise ValueError('Please, specify WDL mainFile using --wdl-mainfile .') - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h == 'Stopped': - print('\tStarting Cromwell server...\n') - cl.cromwell_switch(workspace_id, 'restart', verify_ssl) - elapsed = 0 - while elapsed < 300 and c_status_h != 'Running': - c_status_old = c_status_h - time.sleep(REQUEST_INTERVAL_CROMWELL) - elapsed += REQUEST_INTERVAL_CROMWELL - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - if c_status_h != c_status_old: - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h != 'Running': - raise Exception('Cromwell server did not restarted properly.') - cromwell_id = json.loads(c_status.content)["_id"] - click.secho('\t' + ('*' * 80) + '\n' + - '\tCromwell server is now running. Please, remember to stop it when ' + - 'your\n' + '\tjob finishes. You can use the following command:\n' + - '\tcloudos cromwell stop \\\n' + - '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + - f'\t\t--cloudos-url {cloudos_url} \\\n' + - f'\t\t--workspace-id {workspace_id}\n' + - '\t' + ('*' * 80) + '\n', fg='yellow', bold=True) - else: - cromwell_id = None - if verbose: - print('\t...Preparing objects') - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=wdl_mainfile, importsfile=wdl_importsfile, - repository_platform=repository_platform, verify=verify_ssl, last=last) - if verbose: - print('\tThe following Job object was created:') - print('\t' + str(j)) - print('\t...Sending job to CloudOS\n') - if is_module: - if job_queue is not None: - print(f'Ignoring job queue "{job_queue}" for ' + - f'Platform Workflow "{workflow_name}". Platform Workflows ' + - 'use their own predetermined queues.') - job_queue_id = None - if nextflow_version != '22.10.8': - print(f'The selected worflow \'{workflow_name}\' ' + - 'is a CloudOS module. CloudOS modules only work with ' + - 'Nextflow version 22.10.8. Switching to use 22.10.8') - nextflow_version = '22.10.8' - if execution_platform == 'azure': - print(f'The selected worflow \'{workflow_name}\' ' + - 'is a CloudOS module. For these workflows, worker nodes ' + - 'are managed internally. For this reason, the options ' + - 'azure-worker-instance-type, azure-worker-instance-disk and ' + - 'azure-worker-instance-spot are not taking effect.') - else: - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=cromwell_token, - workspace_id=workspace_id, verify=verify_ssl) - job_queue_id = queue.fetch_job_queue_id(workflow_type=workflow_type, batch=batch, - job_queue=job_queue) - if use_private_docker_repository: - if is_module: - print(f'Workflow "{workflow_name}" is a CloudOS module. ' + - 'Option --use-private-docker-repository will be ignored.') - docker_login = False - else: - me = j.get_user_info(verify=verify_ssl)['dockerRegistriesCredentials'] - if len(me) == 0: - raise Exception('User private Docker repository has been selected but your user ' + - 'credentials have not been configured yet. Please, link your ' + - 'Docker account to CloudOS before using ' + - '--use-private-docker-repository option.') - print('Use private Docker repository has been selected. A custom job ' + - 'queue to support private Docker containers and/or Lustre FSx will be created for ' + - 'your job. The selected job queue will serve as a template.') - docker_login = True - else: - docker_login = False - if nextflow_version == 'latest': - if execution_platform == 'aws': - nextflow_version = AWS_NEXTFLOW_LATEST - elif execution_platform == 'azure': - nextflow_version = AZURE_NEXTFLOW_LATEST - else: - nextflow_version = HPC_NEXTFLOW_LATEST - print('You have specified Nextflow version \'latest\' for execution platform ' + - f'\'{execution_platform}\'. The workflow will use the ' + - f'latest version available on CloudOS: {nextflow_version}.') - if execution_platform == 'aws': - if nextflow_version not in AWS_NEXTFLOW_VERSIONS: - print('For execution platform \'aws\', the workflow will use the default ' + - '\'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' - if execution_platform == 'azure': - if nextflow_version not in AZURE_NEXTFLOW_VERSIONS: - print('For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' ' + - 'version on CloudOS.') - nextflow_version = '22.11.1-edge' - if execution_platform == 'hpc': - if nextflow_version not in HPC_NEXTFLOW_VERSIONS: - print('For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' - if nextflow_version != '22.10.8' and nextflow_version != '22.11.1-edge': - click.secho(f'You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' + - 'to be written in DSL2 and does not support DSL1.', fg='yellow', bold=True) - print('\nExecuting run...') - if workflow_type == 'nextflow': - print(f'\tNextflow version: {nextflow_version}') - j_id = j.send_job(job_config=job_config, - parameter=parameter, - is_module=is_module, - git_commit=git_commit, - git_tag=git_tag, - git_branch=git_branch, - job_name=job_name, - resumable=resumable, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - nextflow_profile=nextflow_profile, - nextflow_version=nextflow_version, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=hpc_id, - workflow_type=workflow_type, - cromwell_id=cromwell_id, - azure_worker_instance_type=azure_worker_instance_type, - azure_worker_instance_disk=azure_worker_instance_disk, - azure_worker_instance_spot=azure_worker_instance_spot, - cost_limit=cost_limit, - use_mountpoints=use_mountpoints, - accelerate_saving_results=accelerate_saving_results, - docker_login=docker_login, - verify=verify_ssl) - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=verbose, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@job.command('status') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_status(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Check job status in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - print('Executing status...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' - print(f'\tTo further check your job status you can either go to {j_url} ' + - 'or repeat the command you just used.') - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") - - -@job.command('workdir') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the working directory to an interactive session.', - is_flag=True) -@click.option('--delete', - help='Delete the results directory of a CloudOS job.', - is_flag=True) -@click.option('-y', '--yes', - help='Skip confirmation prompt when deleting results.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--status', - help='Check the deletion status of the working directory.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_workdir(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - delete, - yes, - session_id, - status, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the working directory of a specified job or check deletion status.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Handle --status flag - if status: - console = Console() - - if verbose: - console.print('[bold cyan]Checking deletion status of job working directory...[/bold cyan]') - console.print('\t[dim]...Preparing objects[/dim]') - console.print('\t[bold]Using the following parameters:[/bold]') - console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') - console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') - console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - - # Use Cloudos object to access the deletion status method - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - console.print('\t[dim]The following Cloudos object was created:[/dim]') - console.print('\t' + str(cl) + '\n') - - try: - deletion_status = cl.get_workdir_deletion_status( - job_id=job_id, - workspace_id=workspace_id, - verify=verify_ssl - ) - - # Convert API status to user-friendly terminology with color - status_config = { - "ready": ("available", "green"), - "deleting": ("deleting", "yellow"), - "scheduledForDeletion": ("scheduled for deletion", "yellow"), - "deleted": ("deleted", "red"), - "failedToDelete": ("failed to delete", "red") - } - - # Get the status of the workdir folder itself and convert it - api_status = deletion_status.get("status", "unknown") - folder_status, status_color = status_config.get(api_status, (api_status, "white")) - folder_info = deletion_status.get("items", {}) - - # Display results in a clear, styled format with human-readable sentence - console.print(f'The working directory of job [cyan]{deletion_status["job_id"]}[/cyan] is in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - - # For non-available statuses, always show update time and user info - if folder_status != "available": - if folder_info.get("updatedAt"): - console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - - # Show user information - prefer deletedBy over user field - user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) - if user_info: - user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() - user_email = user_info.get('email', '') - if user_name or user_email: - user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) - console.print(f'[blue]User:[/blue] {user_display}') - - # Display detailed information if verbose - if verbose: - console.print(f'\n[bold]Additional information:[/bold]') - console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') - console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') - console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') - - # Show folder metadata if available - if folder_info.get("createdAt"): - console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') - if folder_info.get("updatedAt"): - console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') - if folder_info.get("folderType"): - console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - - except ValueError as e: - raise click.ClickException(str(e)) - except Exception as e: - raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - - return - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Finding working directory path...') - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl) - print(f"Working directory for job {job_id}: {workdir}") - - # Link to interactive session if requested - if link: - if verbose: - print(f'\tLinking working directory to interactive session {session_id}...') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - link_client.link_folder(workdir.strip(), session_id) - - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") - - # Delete workdir directory if requested - if delete: - try: - # Ask for confirmation unless --yes flag is provided - if not yes: - confirmation_message = ( - "\n⚠️ Deleting intermediate results is permanent and cannot be undone. " - "All associated data will be permanently removed and cannot be recovered. " - "The current job, as well as any other jobs sharing the same working directory, " - "will no longer be resumable. This action will be logged in the audit trail " - "(if auditing is enabled for your organisation), and you will be recorded as " - "the user who performed the deletion. You can skip this confirmation step by " - "providing -y or --yes flag to cloudos job workdir --delete. Please confirm " - "that you want to delete intermediate results of this analysis? [y/n] " - ) - click.secho(confirmation_message, fg='black', bg='yellow') - user_input = input().strip().lower() - if user_input != 'y': - print('\nDeletion cancelled.') - return - # Proceed with deletion - job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - job.delete_job_results(job_id, "workDirectory", verify=verify_ssl) - click.secho('\nIntermediate results directories deleted successfully.', fg='green', bold=True) - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve intermediate results for job '{job_id}'. {str(e)}") - else: - if yes: - click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) - - -@job.command('logs') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the logs directories to an interactive session.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_logs(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - session_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the logs of a specified job.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Executing logs...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - logs = cl.get_job_logs(job_id, workspace_id, verify_ssl) - for name, path in logs.items(): - print(f"{name}: {path}") - - # Link to interactive session if requested - if link: - if logs: - # Extract the parent logs directory from any log file path - # All log files should be in the same logs directory - first_log_path = next(iter(logs.values())) - # Remove the filename to get the logs directory - # e.g., "s3://bucket/path/to/logs/filename.txt" -> "s3://bucket/path/to/logs" - logs_dir = '/'.join(first_log_path.split('/')[:-1]) - - if verbose: - print(f'\tLinking logs directory to interactive session {session_id}...') - print(f'\t\tLogs directory: {logs_dir}') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - link_client.link_folder(logs_dir, session_id) - else: - if verbose: - print('\tNo logs found to link.') - - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve logs for job '{job_id}'. {str(e)}") - - -@job.command('results') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the results directories to an interactive session.', - is_flag=True) -@click.option('--delete', - help='Delete the results directory of a CloudOS job.', - is_flag=True) -@click.option('-y', '--yes', - help='Skip confirmation prompt when deleting results.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--status', - help='Check the deletion status of the job results.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_results(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - delete, - yes, - session_id, - status, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the results of a specified job or check deletion status.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Handle --status flag - if status: - console = Console() - - if verbose: - console.print('[bold cyan]Checking deletion status of job results...[/bold cyan]') - console.print('\t[dim]...Preparing objects[/dim]') - console.print('\t[bold]Using the following parameters:[/bold]') - console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') - console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') - console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - - # Use Cloudos object to access the deletion status method - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - console.print('\t[dim]The following Cloudos object was created:[/dim]') - console.print('\t' + str(cl) + '\n') - - try: - deletion_status = cl.get_results_deletion_status( - job_id=job_id, - workspace_id=workspace_id, - verify=verify_ssl - ) - - # Convert API status to user-friendly terminology with color - status_config = { - "ready": ("available", "green"), - "deleting": ("deleting", "yellow"), - "scheduledForDeletion": ("scheduled for deletion", "yellow"), - "deleted": ("deleted", "red"), - "failedToDelete": ("failed to delete", "red") - } - - # Get the status of the results folder itself and convert it - api_status = deletion_status.get("status", "unknown") - folder_status, status_color = status_config.get(api_status, (api_status, "white")) - folder_info = deletion_status.get("items", {}) - - # Display results in a clear, styled format with human-readable sentence - console.print(f'The results of job [cyan]{deletion_status["job_id"]}[/cyan] are in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - - # For non-available statuses, always show update time and user info - if folder_status != "available": - if folder_info.get("updatedAt"): - console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - - # Show user information - prefer deletedBy over user field - user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) - if user_info: - user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() - user_email = user_info.get('email', '') - if user_name or user_email: - user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) - console.print(f'[blue]User:[/blue] {user_display}') - - # Display detailed information if verbose - if verbose: - console.print(f'\n[bold]Additional information:[/bold]') - console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') - console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') - console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') - - # Show folder metadata if available - if folder_info.get("createdAt"): - console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') - if folder_info.get("updatedAt"): - console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') - if folder_info.get("folderType"): - console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - - except ValueError as e: - raise click.ClickException(str(e)) - except Exception as e: - raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - - return - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Executing results...') - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) - print(f"results: {results_path}") - - # Link to interactive session if requested - if link: - if verbose: - print(f'\tLinking results directory to interactive session {session_id}...') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - if verbose: - print(f'\t\tLinking results ({results_path})...') - - link_client.link_folder(results_path, session_id) - - # Delete results directory if requested - if delete: - # Ask for confirmation unless --yes flag is provided - if not yes: - confirmation_message = ( - "\n⚠️ Deleting final analysis results is irreversible. " - "All data and backups will be permanently removed and cannot be recovered. " - "You can skip this confirmation step by providing '-y' or '--yes' flag to " - "'cloudos job results --delete'. " - "Please confirm that you want to delete final results of this analysis? [y/n] " - ) - click.secho(confirmation_message, fg='black', bg='yellow') - user_input = input().strip().lower() - if user_input != 'y': - print('\nDeletion cancelled.') - return - if verbose: - print(f'\nDeleting result directories from CloudOS...') - # Proceed with deletion - job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - job.delete_job_results(job_id, "analysisResults", verify=verify_ssl) - click.secho('\nResults directories deleted successfully.', fg='green', bold=True) - else: - if yes: - click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve results for job '{job_id}'. {str(e)}") - - -@job.command('details') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--output-format', - help=('The desired display for the output, either directly in standard output or saved as file. ' + - 'Default=stdout.'), - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--output-basename', - help=('Output file base name to save jobs details. ' + - 'Default={job_id}_details'), - required=False) -@click.option('--parameters', - help=('Whether to generate a ".config" file that can be used as input for --job-config parameter. ' + - 'It will have the same basename as defined in "--output-basename". '), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_details(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - output_basename, - parameters, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve job details in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if ctx.get_parameter_source('output_basename') == click.core.ParameterSource.DEFAULT: - output_basename = f"{job_id}_details" - - print('Executing details...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - - # check if the API gives a 403 error/forbidden error - try: - j_details = cl.get_job_status(job_id, workspace_id, verify_ssl) - except BadRequestException as e: - if '403' in str(e) or 'Forbidden' in str(e): - raise ValueError("API can only show job details of your own jobs, cannot see other user's job details.") - else: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve details for job '{job_id}'. {str(e)}") - create_job_details(json.loads(j_details.content), job_id, output_format, output_basename, parameters, cloudos_url) - - -@job.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save jobs list. ' + - 'Default=joblist'), - default='joblist', - required=False) -@click.option('--output-format', - help='The desired output format. For json option --all-fields will be automatically set to True. Default=stdout.', - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--table-columns', - help=('Comma-separated list of columns to display in the table. Only applicable when --output-format=stdout. ' + - 'Available columns: status,name,project,owner,pipeline,id,submit_time,end_time,run_time,commit,cost,resources,storage_type. ' + - 'Default: responsive (auto-selects columns based on terminal width)'), - default=None) -@click.option('--all-fields', - help=('Whether to collect all available fields from jobs or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv. Automatically enabled for json output.'), - is_flag=True) -@click.option('--last-n-jobs', - help=("The number of last workspace jobs to retrieve. You can use 'all' to " + - "retrieve all workspace jobs. When adding this option, options " + - "'--page' and '--page-size' are ignored.")) -@click.option('--page', - help=('Page number to fetch from the API. Used with --page-size to control jobs ' + - 'per page (e.g. --page=4 --page-size=20). Default=1.'), - type=int, - default=1) -@click.option('--page-size', - help=('Page size to retrieve from API, corresponds to the number of jobs per page. ' + - 'Maximum allowed integer is 100. Default=10.'), - type=int, - default=10) -@click.option('--archived', - help=('When this flag is used, only archived jobs list is collected.'), - is_flag=True) -@click.option('--filter-status', - help='Filter jobs by status (e.g., completed, running, failed, aborted).') -@click.option('--filter-job-name', - help='Filter jobs by job name ( case insensitive ).') -@click.option('--filter-project', - help='Filter jobs by project name.') -@click.option('--filter-workflow', - help='Filter jobs by workflow/pipeline name.') -@click.option('--last', - help=('When workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('--filter-job-id', - help='Filter jobs by specific job ID.') -@click.option('--filter-only-mine', - help='Filter to show only jobs belonging to the current user.', - is_flag=True) -@click.option('--filter-queue', - help='Filter jobs by queue name. Only applies to jobs running in batch environment. Non-batch jobs are preserved in results.') -@click.option('--filter-owner', - help='Filter jobs by owner username.') -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - table_columns, - all_fields, - last_n_jobs, - page, - page_size, - archived, - filter_status, - filter_job_name, - filter_project, - filter_workflow, - last, - filter_job_id, - filter_only_mine, - filter_owner, - filter_queue, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect and display workspace jobs from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Pass table_columns directly to create_job_list_table for validation and processing - selected_columns = table_columns - # Only set outfile if not using stdout - if output_format != 'stdout': - outfile = output_basename + '.' + output_format - - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for jobs in the following workspace: ' + - f'{workspace_id}') - # Check if the user provided the --page option - ctx = click.get_current_context() - if not isinstance(page, int) or page < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') - - if not isinstance(page_size, int) or page_size < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page-size parameter') - - # Validate page_size limit - must be done before API call - if page_size > 100: - click.secho('Error: Page size cannot exceed 100. Please use --page-size with a value <= 100', fg='red', err=True) - raise SystemExit(1) - - result = cl.get_job_list(workspace_id, last_n_jobs, page, page_size, archived, verify_ssl, - filter_status=filter_status, - filter_job_name=filter_job_name, - filter_project=filter_project, - filter_workflow=filter_workflow, - filter_job_id=filter_job_id, - filter_only_mine=filter_only_mine, - filter_owner=filter_owner, - filter_queue=filter_queue, - last=last) - - # Extract jobs and pagination metadata from result - my_jobs_r = result['jobs'] - pagination_metadata = result['pagination_metadata'] - - # Validate requested page exists - if pagination_metadata: - total_jobs = pagination_metadata.get('Pagination-Count', 0) - current_page_size = pagination_metadata.get('Pagination-Limit', page_size) - - if total_jobs > 0: - total_pages = (total_jobs + current_page_size - 1) // current_page_size - if page > total_pages: - click.secho(f'Error: Page {page} does not exist. There are only {total_pages} page(s) available with {total_jobs} total job(s). ' - f'Please use --page with a value between 1 and {total_pages}', fg='red', err=True) - raise SystemExit(1) - - if len(my_jobs_r) == 0: - # Check if any filtering options are being used - filters_used = any([ - filter_status, - filter_job_name, - filter_project, - filter_workflow, - filter_job_id, - filter_only_mine, - filter_owner, - filter_queue - ]) - if output_format == 'stdout': - # For stdout, always show a user-friendly message - create_job_list_table([], cloudos_url, pagination_metadata, selected_columns) - else: - if filters_used: - print('A total of 0 jobs collected.') - elif ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - print('A total of 0 jobs collected. This is likely because your workspace ' + - 'has no jobs created yet.') - else: - print('A total of 0 jobs collected. This is likely because the --page you requested ' + - 'does not exist. Please, try a smaller number for --page or collect all the jobs by not ' + - 'using --page parameter.') - elif output_format == 'stdout': - # Display as table - create_job_list_table(my_jobs_r, cloudos_url, pagination_metadata, selected_columns) - elif output_format == 'csv': - my_jobs = cl.process_job_list(my_jobs_r, all_fields) - cl.save_job_list_to_csv(my_jobs, outfile) - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_jobs_r)) - print(f'\tJob list collected with a total of {len(my_jobs_r)} jobs.') - print(f'\tJob list saved to {outfile}') - else: - raise ValueError('Unrecognised output format. Please use one of [stdout|csv|json]') - - -@job.command('abort') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-ids', - help=('One or more job ids to abort. If more than ' + - 'one is provided, they must be provided as ' + - 'a comma separated list of ids. E.g. id1,id2,id3'), - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--force', - help='Force abort the job even if it is not in a running or initializing state.', - is_flag=True) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def abort_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - job_ids, - verbose, - disable_ssl_verification, - ssl_cert, - profile, - force): - """Abort all specified jobs from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - print('Aborting jobs...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for jobs in the following workspace: ' + - f'{workspace_id}') - # check if the user provided an empty job list - jobs = job_ids.replace(' ', '') - if not jobs: - raise ValueError('No job IDs provided. Please specify at least one job ID to abort.') - jobs = jobs.split(',') - - # Issue warning if using --force flag - if force: - click.secho(f"Warning: Using --force to abort jobs. Some data might be lost.", fg='yellow', bold=True) - - for job in jobs: - try: - j_status = cl.get_job_status(job, workspace_id, verify_ssl) - except Exception as e: - click.secho(f"Failed to get status for job {job}, please make sure it exists in the workspace: {e}", fg='yellow', bold=True) - continue - - j_status_content = json.loads(j_status.content) - job_status = j_status_content['status'] - - # Check if job is in a state that normally allows abortion - is_abortable = job_status in ABORT_JOB_STATES - - # Issue warning if job is in initializing state and not using force - if job_status == 'initializing' and not force: - click.secho(f"Warning: Job {job} is in initializing state.", fg='yellow', bold=True) - - # Check if job can be aborted - if not is_abortable: - click.secho(f"Job {job} is not in a state that can be aborted and is ignored. " + - f"Current status: {job_status}", fg='yellow', bold=True) - else: - try: - cl.abort_job(job, workspace_id, verify_ssl, force) - click.secho(f"Job '{job}' aborted successfully.", fg='green', bold=True) - except Exception as e: - click.secho(f"Failed to abort job {job}. Error: {e}", fg='red', bold=True) - - -@job.command('cost') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to get costs for.', - required=True) -@click.option('--output-format', - help='The desired file format (file extension) for the output. For json option --all-fields will be automatically set to True. Default=csv.', - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_cost(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve job cost information in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - print('Retrieving cost information...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cost_viewer = CostViewer(cloudos_url, apikey) - if verbose: - print(f'\tSearching for cost data for job id: {job_id}') - # Display costs with pagination - cost_viewer.display_costs(job_id, workspace_id, output_format, verify_ssl) - - -@job.command('related') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to get costs for.', - required=True) -@click.option('--output-format', - help='The desired output format. Default=stdout.', - type=click.Choice(['stdout', 'json'], case_sensitive=False), - default='stdout') -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def related(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve related job analyses in CloudOS.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - related_analyses(cloudos_url, apikey, job_id, workspace_id, output_format, verify_ssl) - - -@click.command() -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-ids', - help=('One or more job ids to archive/unarchive. If more than ' + - 'one is provided, they must be provided as ' + - 'a comma separated list of ids. E.g. id1,id2,id3'), - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def archive_unarchive_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - job_ids, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Archive or unarchive specified jobs in a CloudOS workspace.""" - # Determine operation based on the command name used - target_archived_state = ctx.info_name == "archive" - action = "archive" if target_archived_state else "unarchive" - action_past = "archived" if target_archived_state else "unarchived" - action_ing = "archiving" if target_archived_state else "unarchiving" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - print(f'{action_ing.capitalize()} jobs...') - - if verbose: - print('\t...Preparing objects') - - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\t{action_ing.capitalize()} jobs in the following workspace: {workspace_id}') - - # check if the user provided an empty job list - jobs = job_ids.replace(' ', '') - if not jobs: - raise ValueError(f'No job IDs provided. Please specify at least one job ID to {action}.') - jobs_list = [job for job in jobs.split(',') if job] # Filter out empty strings - - # Check for duplicate job IDs - duplicates = [job_id for job_id in set(jobs_list) if jobs_list.count(job_id) > 1] - if duplicates: - dup_str = ', '.join(duplicates) - click.secho(f'Warning: Duplicate job IDs detected and will be processed only once: {dup_str}', fg='yellow', bold=True) - # Remove duplicates while preserving order - jobs_list = list(dict.fromkeys(jobs_list)) - if verbose: - print(f'\tDuplicate job IDs removed. Processing {len(jobs_list)} unique job(s).') - - # Check archive status for all jobs - status_check = cl.check_jobs_archive_status(jobs_list, workspace_id, target_archived_state=target_archived_state, verify=verify_ssl, verbose=verbose) - valid_jobs = status_check['valid_jobs'] - already_processed = status_check['already_processed'] - invalid_jobs = status_check['invalid_jobs'] - - # Report invalid jobs (but continue processing valid ones) - for job_id, error_msg in invalid_jobs.items(): - click.secho(f"Failed to get status for job {job_id}, please make sure it exists in the workspace: {error_msg}", fg='yellow', bold=True) - - if not valid_jobs and not already_processed: - # All jobs were invalid - exit gracefully - click.secho('No valid job IDs found. Please check that the job IDs exist and are accessible.', fg='yellow', bold=True) - return - - if not valid_jobs: - if len(already_processed) == 1: - click.secho(f"Job '{already_processed[0]}' is already {action_past}. No action needed.", fg='cyan', bold=True) - else: - click.secho(f"All {len(already_processed)} jobs are already {action_past}. No action needed.", fg='cyan', bold=True) - return - - try: - # Call the appropriate action method - if target_archived_state: - cl.archive_jobs(valid_jobs, workspace_id, verify_ssl) - else: - cl.unarchive_jobs(valid_jobs, workspace_id, verify_ssl) - - success_msg = [] - if len(valid_jobs) == 1: - success_msg.append(f"Job '{valid_jobs[0]}' {action_past} successfully.") - else: - success_msg.append(f"{len(valid_jobs)} jobs {action_past} successfully: {', '.join(valid_jobs)}") - - if already_processed: - if len(already_processed) == 1: - success_msg.append(f"Job '{already_processed[0]}' was already {action_past}.") - else: - success_msg.append(f"{len(already_processed)} jobs were already {action_past}: {', '.join(already_processed)}") - - click.secho('\n'.join(success_msg), fg='green', bold=True) - except Exception as e: - raise ValueError(f"Failed to {action} jobs: {str(e)}") - - -@click.command(help='Clone or resume a job with modified parameters') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.') -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p input=s3://path_to_my_file. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--nextflow-profile', - help=('A comma separated string indicating the nextflow profile/s ' + - 'to use with your job.')) -@click.option('--nextflow-version', - help=('Nextflow version to use when executing the workflow in CloudOS. ' + - 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest'])) -@click.option('--git-branch', - help=('The branch to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--job-name', - help='The name of the job. If not set, will take the name of the cloned job.') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help=('Name of the job queue to use with a batch job. ' + - 'In Azure workspaces, this option is ignored.')) -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).')) -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float) -@click.option('--job-id', - help='The CloudOS job id of the job to be cloned.', - required=True) -@click.option('--accelerate-file-staging', - help='Enables AWS S3 mountpoint for quicker file staging.', - is_flag=True) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--resumable', - help='Whether to make the job able to be resumed or not.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', - help='Profile to use from the config file', - default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def clone_resume(ctx, - apikey, - cloudos_url, - workspace_id, - project_name, - parameter, - nextflow_profile, - nextflow_version, - git_branch, - repository_platform, - job_name, - do_not_save_logs, - job_queue, - instance_type, - cost_limit, - job_id, - accelerate_file_staging, - accelerate_saving_results, - resumable, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - if ctx.info_name == "clone": - mode, action = "clone", "cloning" - elif ctx.info_name == "resume": - mode, action = "resume", "resuming" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - print(f'{action.capitalize()} job...') - if verbose: - print('\t...Preparing objects') - - # Create Job object (set dummy values for project_name and workflow_name, since they come from the cloned job) - job_obj = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - - if verbose: - print('\tThe following Job object was created:') - print('\t' + str(job_obj) + '\n') - print(f'\t{action.capitalize()} job {job_id} in workspace: {workspace_id}') - - try: - - # Clone/resume the job with provided overrides - cloned_resumed_job_id = job_obj.clone_or_resume_job( - source_job_id=job_id, - queue_name=job_queue, - cost_limit=cost_limit, - master_instance=instance_type, - job_name=job_name, - nextflow_version=nextflow_version, - branch=git_branch, - repository_platform=repository_platform, - profile=nextflow_profile, - do_not_save_logs=do_not_save_logs, - use_fusion=accelerate_file_staging, - accelerate_saving_results=accelerate_saving_results, - resumable=resumable, - # only when explicitly setting --project-name will be overridden, else using the original project - project_name=project_name if ctx.get_parameter_source("project_name") == click.core.ParameterSource.COMMANDLINE else None, - parameters=list(parameter) if parameter else None, - verify=verify_ssl, - mode=mode - ) - - if verbose: - print(f'\t{mode.capitalize()}d job ID: {cloned_resumed_job_id}') - - print(f"Job successfully {mode}d. New job ID: {cloned_resumed_job_id}") - - except BadRequestException as e: - raise ValueError(f"Failed to {mode} job. Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") - - -# Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) -archive_unarchive_jobs.help = 'Archive specified jobs in a CloudOS workspace.' -job.add_command(archive_unarchive_jobs, "archive") - -# Create a copy with different help text for unarchive -archive_unarchive_jobs_copy = copy.deepcopy(archive_unarchive_jobs) -archive_unarchive_jobs_copy.help = 'Unarchive specified jobs in a CloudOS workspace.' -job.add_command(archive_unarchive_jobs_copy, "unarchive") - - -# Apply the best Click solution: Set specific help text for each command registration -clone_resume.help = 'Clone a job with modified parameters' -job.add_command(clone_resume, "clone") - -# Create a copy with different help text for resume -clone_resume_copy = copy.deepcopy(clone_resume) -clone_resume_copy.help = 'Resume a job with modified parameters' -job.add_command(clone_resume_copy, "resume") - - -@workflow.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save workflow list. ' + - 'Default=workflow_list'), - default='workflow_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from workflows or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_workflows(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all workflows from a CloudOS workspace in CSV format.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for workflows in the following workspace: ' + - f'{workspace_id}') - my_workflows_r = cl.get_workflow_list(workspace_id, verify=verify_ssl) - if output_format == 'csv': - my_workflows = cl.process_workflow_list(my_workflows_r, all_fields) - my_workflows.to_csv(outfile, index=False) - print(f'\tWorkflow list collected with a total of {my_workflows.shape[0]} workflows.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_workflows_r)) - print(f'\tWorkflow list collected with a total of {len(my_workflows_r)} workflows.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tWorkflow list saved to {outfile}') - - -@workflow.command('import') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=('The CloudOS url you are trying to access to. ' + - f'Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option("--workflow-name", help="The name that the workflow will have in CloudOS.", required=True) -@click.option("-w", "--workflow-url", help="URL of the workflow repository.", required=True) -@click.option("-d", "--workflow-docs-link", help="URL to the documentation of the workflow.", default='') -@click.option("--cost-limit", help="Cost limit for the workflow. Default: $30 USD.", default=30) -@click.option("--workflow-description", help="Workflow description", default="") -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name']) -def import_wf(ctx, - apikey, - cloudos_url, - workspace_id, - workflow_name, - workflow_url, - workflow_docs_link, - cost_limit, - workflow_description, - repository_platform, - disable_ssl_verification, - ssl_cert, - profile): - """ - Import workflows from supported repository providers. - """ - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - repo_import = ImportWorflow( - cloudos_url=cloudos_url, cloudos_apikey=apikey, workspace_id=workspace_id, platform=repository_platform, - workflow_name=workflow_name, workflow_url=workflow_url, workflow_docs_link=workflow_docs_link, - cost_limit=cost_limit, workflow_description=workflow_description, verify=verify_ssl - ) - workflow_id = repo_import.import_workflow() - print(f'\tWorkflow {workflow_name} was imported successfully with the ' + - f'following ID: {workflow_id}') - - -@project.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save project list. ' + - 'Default=project_list'), - default='project_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from projects or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--page', - help=('Response page to retrieve. Default=1.'), - type=int, - default=1) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_projects(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - page, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all projects from a CloudOS workspace in CSV format.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for projects in the following workspace: ' + - f'{workspace_id}') - # Check if the user provided the --page option - ctx = click.get_current_context() - if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - get_all = True - else: - get_all = False - if not isinstance(page, int) or page < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') - my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all) - if len(my_projects_r) == 0: - if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - print('A total of 0 projects collected. This is likely because your workspace ' + - 'has no projects created yet.') - else: - print('A total of 0 projects collected. This is likely because the --page you ' + - 'requested does not exist. Please, try a smaller number for --page or collect all the ' + - 'projects by not using --page parameter.') - elif output_format == 'csv': - my_projects = cl.process_project_list(my_projects_r, all_fields) - my_projects.to_csv(outfile, index=False) - print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_projects_r)) - print(f'\tProject list collected with a total of {len(my_projects_r)} projects.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tProject list saved to {outfile}') - - -@project.command('create') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--new-project', - help='The name for the new project.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def create_project(ctx, - apikey, - cloudos_url, - workspace_id, - new_project, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Create a new project in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - # verify ssl configuration - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Print basic output - if verbose: - print(f'\tUsing CloudOS URL: {cloudos_url}') - print(f'\tUsing workspace: {workspace_id}') - print(f'\tProject name: {new_project}') - - cl = Cloudos(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None) - - try: - project_id = cl.create_project(workspace_id, new_project, verify_ssl) - print(f'\tProject "{new_project}" created successfully with ID: {project_id}') - if verbose: - print(f'\tProject URL: {cloudos_url}/app/projects/{project_id}') - except Exception as e: - print(f'\tError creating project: {str(e)}') - sys.exit(1) - - -@cromwell.command('status') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_status(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Check Cromwell server status in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - print('Executing status...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tChecking Cromwell status in {workspace_id} workspace') - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - - -@cromwell.command('start') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to Cromwell restart. ' + - 'Default=300.'), - default=300) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_restart(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - wait_time, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Restart Cromwell server in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - action = 'restart' - print('Starting Cromwell server...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tStarting Cromwell server in {workspace_id} workspace') - cl.cromwell_switch(workspace_id, action, verify_ssl) - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - elapsed = 0 - while elapsed < wait_time and c_status_h != 'Running': - c_status_old = c_status_h - time.sleep(REQUEST_INTERVAL_CROMWELL) - elapsed += REQUEST_INTERVAL_CROMWELL - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - if c_status_h != c_status_old: - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h != 'Running': - print(f'\tYour current Cromwell status is: {c_status_h}. The ' + - f'selected wait-time of {wait_time} was exceeded. Please, ' + - 'consider to set a longer wait-time.') - print('\tTo further check your Cromwell status you can either go to ' + - f'{cloudos_url} or use the following command:\n' + - '\tcloudos cromwell status \\\n' + - f'\t\t--cloudos-url {cloudos_url} \\\n' + - '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + - f'\t\t--workspace-id {workspace_id}') - sys.exit(1) - - -@cromwell.command('stop') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_stop(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Stop Cromwell server in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - action = 'stop' - print('Stopping Cromwell server...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tStopping Cromwell server in {workspace_id} workspace') - cl.cromwell_switch(workspace_id, action, verify_ssl) - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - - -@queue.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save job queue list. ' + - 'Default=job_queue_list'), - default='job_queue_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from workflows or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_queues(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all available job queues from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - j_queue = Queue(cloudos_url, apikey, None, workspace_id, verify=verify_ssl) - my_queues = j_queue.get_job_queues() - if len(my_queues) == 0: - raise ValueError('No AWS batch queues found. Please, make sure that your CloudOS supports AWS bath queues') - if output_format == 'csv': - queues_processed = j_queue.process_queue_list(my_queues, all_fields) - queues_processed.to_csv(outfile, index=False) - print(f'\tJob queue list collected with a total of {queues_processed.shape[0]} queues.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_queues)) - print(f'\tJob queue list collected with a total of {len(my_queues)} queues.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tJob queue list saved to {outfile}') - - -@configure.command('list-profiles') -def list_profiles(): - config_manager = ConfigurationProfile() - config_manager.list_profiles() - - -@configure.command('remove-profile') -@click.option('--profile', - help='Name of the profile. Not using this option will lead to profile named "deafults" being generated', - required=True) -@click.pass_context -def remove_profile(ctx, profile): - update_command_context_from_click(ctx) - profile = profile or ctx.obj['profile'] - config_manager = ConfigurationProfile() - config_manager.remove_profile(profile) - - -@bash.command('job') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('--command', - help='The command to run in the bash job.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--cpus', - help='The number of CPUs to use for the task\'s master node. Default=1.', - type=int, - default=1) -@click.option('--memory', - help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', - type=int, - default=4) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - default='aws') -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run_bash_job(ctx, - apikey, - command, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - parameter, - job_name, - do_not_save_logs, - job_queue, - instance_type, - instance_disk, - cpus, - memory, - storage_mode, - lustre_size, - wait_completion, - wait_time, - repository_platform, - execution_platform, - cost_limit, - accelerate_saving_results, - request_interval, - disable_ssl_verification, - ssl_cert, - profile): - """Run a bash job in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - - if do_not_save_logs: - save_logs = False - else: - save_logs = True - - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=None, importsfile=None, - repository_platform=repository_platform, verify=verify_ssl, last=last) - - if job_queue is not None: - batch = True - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, - workspace_id=workspace_id, verify=verify_ssl) - # I have to add 'nextflow', other wise the job queue id is not found - job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, - job_queue=job_queue) - else: - job_queue_id = None - batch = False - j_id = j.send_job(job_config=None, - parameter=parameter, - git_commit=None, - git_tag=None, - git_branch=None, - job_name=job_name, - resumable=False, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - workflow_type='docker', - nextflow_profile=None, - nextflow_version=None, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=None, - cost_limit=cost_limit, - accelerate_saving_results=accelerate_saving_results, - verify=verify_ssl, - command={"command": command}, - cpus=cpus, - memory=memory) - - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=False, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@bash.command('array-job') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('--command', - help='The command to run in the bash job.') -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + - 'times as parameters you want to include. ' + - 'For parameters pointing to a file, the format expected is ' + - 'parameter_name=/Data/parameter_value. The parameter value must be a ' + - 'file located in the `Data` subfolder. If no is specified, it defaults to ' + - 'the project specified by the profile or --project-name parameter. ' + - 'E.g.: -p "--file=Data/file.txt" or "--file=/Data/folder/file.txt"')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--cpus', - help='The number of CPUs to use for the task\'s master node. Default=1.', - type=int, - default=1) -@click.option('--memory', - help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', - type=int, - default=4) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - type=click.Choice(['aws', 'azure', 'hpc']), - default='aws') -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--array-file', - help=('Path to a file containing an array of commands to run in the bash job.'), - default=None, - required=True) -@click.option('--separator', - help=('Separator to use in the array file. Default=",".'), - type=click.Choice([',', ';', 'tab', 'space', '|']), - default=",", - required=True) -@click.option('--list-columns', - help=('List columns present in the array file. ' + - 'This option will not run any job.'), - is_flag=True) -@click.option('--array-file-project', - help=('Name of the project in which the array file is placed, if different from --project-name.'), - default=None) -@click.option('--disable-column-check', - help=('Disable the check for the columns in the array file. ' + - 'This option is only used when --array-file is provided.'), - is_flag=True) -@click.option('-a', '--array-parameter', - multiple=True, - help=('A single parameter to pass to the job call only for specifying array columns. ' + - 'It should be in the following form: parameter_name=array_file_column_name. E.g.: ' + - '-a --test=value or -a -test=value or -a test=value or -a =value (for no prefix). ' + - 'You can use this option as many times as parameters you want to include.')) -@click.option('--custom-script-path', - help=('Path of a custom script to run in the bash array job instead of a command.'), - default=None) -@click.option('--custom-script-project', - help=('Name of the project to use when running the custom command script, if ' + - 'different than --project-name.'), - default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run_bash_array_job(ctx, - apikey, - command, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - parameter, - job_name, - do_not_save_logs, - job_queue, - instance_type, - instance_disk, - cpus, - memory, - storage_mode, - lustre_size, - wait_completion, - wait_time, - repository_platform, - execution_platform, - cost_limit, - accelerate_saving_results, - request_interval, - disable_ssl_verification, - ssl_cert, - profile, - array_file, - separator, - list_columns, - array_file_project, - disable_column_check, - array_parameter, - custom_script_path, - custom_script_project): - """Run a bash array job in CloudOS.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - if not list_columns and not (command or custom_script_path): - raise click.UsageError("Must provide --command or --custom-script-path if --list-columns is not set.") - - # when not set, use the global project name - if array_file_project is None: - array_file_project = project_name - - # this needs to be in another call to datasets, by default it uses the global project name - if custom_script_project is None: - custom_script_project = project_name - - # setup separators for API and array file (the're different) - separators = { - ",": {"api": ",", "file": ","}, - ";": {"api": "%3B", "file": ";"}, - "space": {"api": "+", "file": " "}, - "tab": {"api": "tab", "file": "tab"}, - "|": {"api": "%7C", "file": "|"} - } - - # setup important options for the job - if do_not_save_logs: - save_logs = False - else: - save_logs = True - - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=None, importsfile=None, - repository_platform=repository_platform, verify=verify_ssl, last=last) - - # retrieve columns - r = j.retrieve_cols_from_array_file( - array_file, - generate_datasets_for_project(cloudos_url, apikey, workspace_id, array_file_project, verify_ssl), - separators[separator]['api'], - verify_ssl - ) - - if not disable_column_check: - columns = json.loads(r.content).get("headers", None) - # pass this to the SEND JOB API call - # b'{"headers":[{"index":0,"name":"id"},{"index":1,"name":"title"},{"index":2,"name":"filename"},{"index":3,"name":"file2name"}]}' - if columns is None: - raise ValueError("No columns found in the array file metadata.") - if list_columns: - print("Columns: ") - for col in columns: - print(f"\t- {col['name']}") - return - else: - columns = [] - - # setup parameters for the job - cmd = j.setup_params_array_file( - custom_script_path, - generate_datasets_for_project(cloudos_url, apikey, workspace_id, custom_script_project, verify_ssl), - command, - separators[separator]['file'] - ) - - # check columns in the array file vs parameters added - if not disable_column_check and array_parameter: - print("\nChecking columns in the array file vs parameters added...\n") - for ap in array_parameter: - ap_split = ap.split('=') - ap_value = '='.join(ap_split[1:]) - for col in columns: - if col['name'] == ap_value: - print(f"Found column '{ap_value}' in the array file.") - break - else: - raise ValueError(f"Column '{ap_value}' not found in the array file. " + \ - f"Columns in array-file: {separator.join([col['name'] for col in columns])}") - - if job_queue is not None: - batch = True - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, - workspace_id=workspace_id, verify=verify_ssl) - # I have to add 'nextflow', other wise the job queue id is not found - job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, - job_queue=job_queue) - else: - job_queue_id = None - batch = False - - # send job - j_id = j.send_job(job_config=None, - parameter=parameter, - array_parameter=array_parameter, - array_file_header=columns, - git_commit=None, - git_tag=None, - git_branch=None, - job_name=job_name, - resumable=False, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - workflow_type='docker', - nextflow_profile=None, - nextflow_version=None, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=None, - cost_limit=cost_limit, - accelerate_saving_results=accelerate_saving_results, - verify=verify_ssl, - command=cmd, - cpus=cpus, - memory=memory) - - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=False, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@datasets.command(name="ls") -@click.argument("path", required=False, nargs=1) -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--details', - help=('When selected, it prints the details of the listed files. ' + - 'Details contains "Type", "Owner", "Size", "Last Updated", ' + - '"Virtual Name", "Storage Path".'), - is_flag=True) -@click.option('--output-format', - help=('The desired display for the output, either directly in standard output or saved as file. ' + - 'Default=stdout.'), - type=click.Choice(['stdout', 'csv'], case_sensitive=False), - default='stdout') -@click.option('--output-basename', - help=('Output file base name to save jobs details. ' + - 'Default=datasets_ls'), - default='datasets_ls', - required=False) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def list_files(ctx, - apikey, - cloudos_url, - workspace_id, - disable_ssl_verification, - ssl_cert, - project_name, - profile, - path, - details, - output_format, - output_basename): - """List contents of a path within a CloudOS workspace dataset.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - datasets = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = datasets.list_folder_content(path) - contents = result.get("contents") or result.get("datasets", []) - - if not contents: - contents = result.get("files", []) + result.get("folders", []) - - # Process items to extract data - processed_items = [] - for item in contents: - is_folder = "folderType" in item or item.get("isDir", False) - type_ = "folder" if is_folder else "file" - - # Enhanced type information - if is_folder: - folder_type = item.get("folderType") - if folder_type == "VirtualFolder": - type_ = "virtual folder" - elif folder_type == "S3Folder": - type_ = "s3 folder" - elif folder_type == "AzureBlobFolder": - type_ = "azure folder" - else: - type_ = "folder" - else: - # Check if file is managed by Lifebit (user uploaded) - is_managed_by_lifebit = item.get("isManagedByLifebit", False) - if is_managed_by_lifebit: - type_ = "file (user uploaded)" - else: - type_ = "file (virtual copy)" - - user = item.get("user", {}) - if isinstance(user, dict): - name = user.get("name", "").strip() - surname = user.get("surname", "").strip() - else: - name = surname = "" - if name and surname: - owner = f"{name} {surname}" - elif name: - owner = name - elif surname: - owner = surname - else: - owner = "-" - - raw_size = item.get("sizeInBytes", item.get("size")) - size = format_bytes(raw_size) if not is_folder and raw_size is not None else "-" - - updated = item.get("updatedAt") or item.get("lastModified", "-") - filepath = item.get("name", "-") - - if item.get("fileType") == "S3File" or item.get("folderType") == "S3Folder": - bucket = item.get("s3BucketName") - key = item.get("s3ObjectKey") or item.get("s3Prefix") - storage_path = f"s3://{bucket}/{key}" if bucket and key else "-" - elif item.get("fileType") == "AzureBlobFile" or item.get("folderType") == "AzureBlobFolder": - account = item.get("blobStorageAccountName") - container = item.get("blobContainerName") - key = item.get("blobName") if item.get("fileType") == "AzureBlobFile" else item.get("blobPrefix") - storage_path = f"az://{account}.blob.core.windows.net/{container}/{key}" if account and container and key else "-" - else: - storage_path = "-" - - processed_items.append({ - 'type': type_, - 'owner': owner, - 'size': size, - 'raw_size': raw_size, - 'updated': updated, - 'name': filepath, - 'storage_path': storage_path, - 'is_folder': is_folder - }) - - # Output handling - if output_format == 'csv': - import csv - - csv_filename = f'{output_basename}.csv' - - if details: - # CSV with all details - with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: - fieldnames = ['Type', 'Owner', 'Size', 'Size (bytes)', 'Last Updated', 'Virtual Name', 'Storage Path'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - - for item in processed_items: - writer.writerow({ - 'Type': item['type'], - 'Owner': item['owner'], - 'Size': item['size'], - 'Size (bytes)': item['raw_size'] if item['raw_size'] is not None else '', - 'Last Updated': item['updated'], - 'Virtual Name': item['name'], - 'Storage Path': item['storage_path'] - }) - else: - # CSV with just names - with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: - writer = csv.writer(csvfile) - writer.writerow(['Name', 'Storage Path']) - for item in processed_items: - writer.writerow([item['name'], item['storage_path']]) - - click.secho(f'\nDatasets list saved to: {csv_filename}', fg='green', bold=True) - - else: # stdout - if details: - console = Console(width=None) - table = Table(show_header=True, header_style="bold white") - table.add_column("Type", style="cyan", no_wrap=True) - table.add_column("Owner", style="white") - table.add_column("Size", style="magenta") - table.add_column("Last Updated", style="green") - table.add_column("Virtual Name", style="bold", overflow="fold") - table.add_column("Storage Path", style="dim", no_wrap=False, overflow="fold", ratio=2) - - for item in processed_items: - style = Style(color="blue", underline=True) if item['is_folder'] else None - table.add_row( - item['type'], - item['owner'], - item['size'], - item['updated'], - item['name'], - item['storage_path'], - style=style - ) - - console.print(table) - - else: - console = Console() - for item in processed_items: - if item['is_folder']: - console.print(f"[blue underline]{item['name']}[/]") - else: - console.print(item['name']) - - except Exception as e: - raise ValueError(f"Failed to list files for project '{project_name}'. {str(e)}") - - -@datasets.command(name="mv") -@click.argument("source_path", required=True) -@click.argument("destination_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The source project name.') -@click.option('--destination-project-name', required=False, - help='The destination project name. Defaults to the source project.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def move_files(ctx, source_path, destination_path, apikey, cloudos_url, workspace_id, - project_name, destination_project_name, - disable_ssl_verification, ssl_cert, profile): - """ - Move a file or folder from a source path to a destination path within or across CloudOS projects. - - SOURCE_PATH [path]: the full path to the file or folder to move. It must be a 'Data' folder path. - E.g.: 'Data/folderA/file.txt'\n - DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. - E.g.: 'Data/folderB' - """ - # Validate destination constraint - if not destination_path.strip("/").startswith("Data/") and destination_path.strip("/") != "Data": - raise ValueError("Destination path must begin with 'Data/' or be 'Data'.") - if not source_path.strip("/").startswith("Data/") and source_path.strip("/") != "Data": - raise ValueError("SOURCE_PATH must start with 'Data/' or be 'Data'.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - destination_project_name = destination_project_name or project_name - # Initialize Datasets clients - source_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - dest_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=destination_project_name, - verify=verify_ssl, - cromwell_token=None - ) - print('Checking source path') - # === Resolve Source Item === - source_parts = source_path.strip("/").split("/") - source_parent_path = "/".join(source_parts[:-1]) if len(source_parts) > 1 else None - source_item_name = source_parts[-1] - - try: - source_contents = source_client.list_folder_content(source_parent_path) - except Exception as e: - raise ValueError(f"Could not resolve source path '{source_path}'. {str(e)}") - - found_source = None - for collection in ["files", "folders"]: - for item in source_contents.get(collection, []): - if item.get("name") == source_item_name: - found_source = item - break - if found_source: - break - if not found_source: - raise ValueError(f"Item '{source_item_name}' not found in '{source_parent_path or '[project root]'}'") - - source_id = found_source["_id"] - source_kind = "Folder" if "folderType" in found_source else "File" - print("Checking destination path") - # === Resolve Destination Folder === - dest_parts = destination_path.strip("/").split("/") - dest_folder_name = dest_parts[-1] - dest_parent_path = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else None - - try: - dest_contents = dest_client.list_folder_content(dest_parent_path) - match = next((f for f in dest_contents.get("folders", []) if f.get("name") == dest_folder_name), None) - if not match: - raise ValueError(f"Could not resolve destination folder '{destination_path}'") - - target_id = match["_id"] - folder_type = match.get("folderType") - # Normalize kind: top-level datasets are kind=Dataset, all other folders are kind=Folder - if folder_type in ("VirtualFolder", "Folder"): - target_kind = "Folder" - elif folder_type == "S3Folder": - raise ValueError(f"Unable to move item '{source_item_name}' to '{destination_path}'. " + - "The destination is an S3 folder, and only virtual folders can be selected as valid move destinations.") - elif isinstance(folder_type, bool) and folder_type: # legacy dataset structure - target_kind = "Dataset" - else: - raise ValueError(f"Unrecognized folderType '{folder_type}' for destination '{destination_path}'") - - except Exception as e: - raise ValueError(f"Could not resolve destination path '{destination_path}'. {str(e)}") - print(f"Moving {source_kind} '{source_item_name}' to '{destination_path}' " + - f"in project '{destination_project_name} ...") - # === Perform Move === - try: - response = source_client.move_files_and_folders( - source_id=source_id, - source_kind=source_kind, - target_id=target_id, - target_kind=target_kind - ) - if response.ok: - click.secho(f"{source_kind} '{source_item_name}' moved to '{destination_path}' " + - f"in project '{destination_project_name}'.", fg="green", bold=True) - else: - raise ValueError(f"Move failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Move operation failed. {str(e)}") - - -@datasets.command(name="rename") -@click.argument("source_path", required=True) -@click.argument("new_name", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def renaming_item(ctx, - source_path, - new_name, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Rename a file or folder in a CloudOS project. - - SOURCE_PATH [path]: the full path to the file or folder to rename. It must be a 'Data' folder path. - E.g.: 'Data/folderA/old_name.txt'\n - NEW_NAME [name]: the new name to assign to the file or folder. E.g.: 'new_name.txt' - """ - if not source_path.strip("/").startswith("Data/"): - raise ValueError("SOURCE_PATH must start with 'Data/', pointing to a file or folder in that dataset.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) +# Register all command groups +run_cloudos_cli.add_command(job) +run_cloudos_cli.add_command(workflow) +run_cloudos_cli.add_command(project) +run_cloudos_cli.add_command(cromwell) +run_cloudos_cli.add_command(queue) +run_cloudos_cli.add_command(bash) +run_cloudos_cli.add_command(procurement) +run_cloudos_cli.add_command(datasets) +run_cloudos_cli.add_command(configure) - parts = source_path.strip("/").split("/") - - parent_path = "/".join(parts[:-1]) - target_name = parts[-1] - - try: - contents = client.list_folder_content(parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") - - # Search for file/folder - found_item = None - for category in ["files", "folders"]: - for item in contents.get(category, []): - if item.get("name") == target_name: - found_item = item - break - if found_item: - break - - if not found_item: - raise ValueError(f"Item '{target_name}' not found in '{parent_path or '[project root]'}'") - - item_id = found_item["_id"] - kind = "Folder" if "folderType" in found_item else "File" - - print(f"Renaming {kind} '{target_name}' to '{new_name}'...") - try: - response = client.rename_item(item_id=item_id, new_name=new_name, kind=kind) - if response.ok: - click.secho( - f"{kind} '{target_name}' renamed to '{new_name}' in folder '{parent_path}'.", - fg="green", - bold=True - ) - else: - raise ValueError(f"Rename failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Rename operation failed. {str(e)}") - - -@datasets.command(name="cp") -@click.argument("source_path", required=True) -@click.argument("destination_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The source project name.') -@click.option('--destination-project-name', required=False, help='The destination project name. Defaults to the source project.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def copy_item_cli(ctx, - source_path, - destination_path, - apikey, - cloudos_url, - workspace_id, - project_name, - destination_project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Copy a file or folder (S3 or virtual) from SOURCE_PATH to DESTINATION_PATH. - - SOURCE_PATH [path]: the full path to the file or folder to copy. - E.g.: AnalysesResults/my_analysis/results/my_plot.png\n - DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. - E.g.: Data/plots - """ - destination_project_name = destination_project_name or project_name - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - # Initialize clients - source_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - dest_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=destination_project_name, - verify=verify_ssl, - cromwell_token=None - ) - # Validate paths - dest_parts = destination_path.strip("/").split("/") - if not dest_parts or dest_parts[0] != "Data": - raise ValueError("DESTINATION_PATH must start with 'Data/'.") - # Parse source and destination - source_parts = source_path.strip("/").split("/") - source_parent = "/".join(source_parts[:-1]) if len(source_parts) > 1 else "" - source_name = source_parts[-1] - dest_folder_name = dest_parts[-1] - dest_parent = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else "" - try: - source_content = source_client.list_folder_content(source_parent) - dest_content = dest_client.list_folder_content(dest_parent) - except Exception as e: - raise ValueError(f"Could not access paths. {str(e)}") - # Find the source item - source_item = None - for item in source_content.get('files', []) + source_content.get('folders', []): - if item.get("name") == source_name: - source_item = item - break - if not source_item: - raise ValueError(f"Item '{source_name}' not found in '{source_parent or '[project root]'}'") - # Find the destination folder - destination_folder = None - for folder in dest_content.get("folders", []): - if folder.get("name") == dest_folder_name: - destination_folder = folder - break - if not destination_folder: - raise ValueError(f"Destination folder '{destination_path}' not found.") - try: - # Determine item type - if "fileType" in source_item: - item_type = "file" - elif source_item.get("folderType") == "VirtualFolder": - item_type = "virtual_folder" - elif "s3BucketName" in source_item and source_item.get("folderType") == "S3Folder": - item_type = "s3_folder" - else: - raise ValueError("Could not determine item type.") - print(f"Copying {item_type.replace('_', ' ')} '{source_name}' to '{destination_path}'...") - if destination_folder.get("folderType") is True and destination_folder.get("kind") in ("Data", "Cohorts", "AnalysesResults"): - destination_kind = "Dataset" - elif destination_folder.get("folderType") == "S3Folder": - raise ValueError(f"Unable to copy item '{source_name}' to '{destination_path}'. The destination is an S3 folder, and only virtual folders can be selected as valid copy destinations.") - else: - destination_kind = "Folder" - response = source_client.copy_item( - item=source_item, - destination_id=destination_folder["_id"], - destination_kind=destination_kind - ) - if response.ok: - click.secho("Item copied successfully.", fg="green", bold=True) - else: - raise ValueError(f"Copy failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Copy operation failed. {str(e)}") - - -@datasets.command(name="mkdir") -@click.argument("new_folder_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def mkdir_item(ctx, - new_folder_path, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Create a virtual folder in a CloudOS project. - - NEW_FOLDER_PATH [path]: Full path to the new folder including its name. Must start with 'Data'. - """ - new_folder_path = new_folder_path.strip("/") - if not new_folder_path.startswith("Data"): - raise ValueError("NEW_FOLDER_PATH must start with 'Data'.") - - path_parts = new_folder_path.split("/") - if len(path_parts) < 2: - raise ValueError("NEW_FOLDER_PATH must include at least a parent folder and the new folder name.") - - parent_path = "/".join(path_parts[:-1]) - folder_name = path_parts[-1] - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - # Split parent path to get its parent + name - parent_parts = parent_path.split("/") - parent_name = parent_parts[-1] - parent_of_parent_path = "/".join(parent_parts[:-1]) - - # List the parent of the parent - try: - contents = client.list_folder_content(parent_of_parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_of_parent_path}'. {str(e)}") - - # Find the parent folder in the contents - folder_info = next( - (f for f in contents.get("folders", []) if f.get("name") == parent_name), - None - ) - - if not folder_info: - raise ValueError(f"Could not find folder '{parent_name}' in '{parent_of_parent_path}'.") - - parent_id = folder_info.get("_id") - folder_type = folder_info.get("folderType") - - if folder_type is True: - parent_kind = "Dataset" - elif isinstance(folder_type, str): - parent_kind = "Folder" - else: - raise ValueError(f"Unrecognized folderType for '{parent_path}'.") - - # Create the folder - print(f"Creating folder '{folder_name}' under '{parent_path}' ({parent_kind})...") - try: - response = client.create_virtual_folder(name=folder_name, parent_id=parent_id, parent_kind=parent_kind) - if response.ok: - click.secho(f"Folder '{folder_name}' created under '{parent_path}'", fg="green", bold=True) - else: - raise ValueError(f"Folder creation failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Folder creation failed. {str(e)}") - - -@datasets.command(name="rm") -@click.argument("target_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.option('--force', is_flag=True, help='Force delete files. Required when deleting user uploaded files. This may also delete the file from the cloud provider storage.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def rm_item(ctx, - target_path, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile, - force): - """ - Delete a file or folder in a CloudOS project. - - TARGET_PATH [path]: the full path to the file or folder to delete. Must start with 'Data'. \n - E.g.: 'Data/folderA/file.txt' or 'Data/my_analysis/results/folderB' - """ - if not target_path.strip("/").startswith("Data/"): - raise ValueError("TARGET_PATH must start with 'Data/', pointing to a file or folder.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - parts = target_path.strip("/").split("/") - parent_path = "/".join(parts[:-1]) - item_name = parts[-1] - - try: - contents = client.list_folder_content(parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") - - found_item = None - for item in contents.get('files', []) + contents.get('folders', []): - if item.get("name") == item_name: - found_item = item - break - - if not found_item: - raise ValueError(f"Item '{item_name}' not found in '{parent_path or '[project root]'}'") - - item_id = found_item.get("_id", '') - kind = "Folder" if "folderType" in found_item else "File" - if item_id == '': - raise ValueError(f"Item '{item_name}' could not be removed as the parent folder is an s3 folder and their content cannot be modified.") - # Check if the item is managed by Lifebit - is_managed_by_lifebit = found_item.get("isManagedByLifebit", False) - if is_managed_by_lifebit and not force: - raise ValueError("By removing this file, it will be permanently deleted. If you want to go forward, please use the --force flag.") - print(f"Removing {kind} '{item_name}' from '{parent_path or '[root]'}'...") - try: - response = client.delete_item(item_id=item_id, kind=kind) - if response.ok: - if is_managed_by_lifebit: - click.secho( - f"{kind} '{item_name}' was permanently deleted from '{parent_path or '[root]'}'.", - fg="green", bold=True - ) - else: - click.secho( - f"{kind} '{item_name}' was removed from '{parent_path or '[root]'}'.", - fg="green", bold=True - ) - click.secho("This item will still be available on your Cloud Provider.", fg="yellow") - else: - raise ValueError(f"Removal failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Remove operation failed. {str(e)}") - - -@datasets.command(name="link") -@click.argument("path", required=True) -@click.option('-k', '--apikey', help='Your CloudOS API key', required=True) -@click.option('-c', '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=False) -@click.option('--workspace-id', help='The specific CloudOS workspace id.', required=True) -@click.option('--session-id', help='The specific CloudOS interactive session id.', required=True) -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default='default') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) -def link(ctx, - path, - apikey, - cloudos_url, - project_name, - workspace_id, - session_id, - disable_ssl_verification, - ssl_cert, - profile): - """ - Link a folder (S3 or File Explorer) to an active interactive analysis. - - PATH [path]: the full path to the S3 folder to link or relative to File Explorer. - E.g.: 's3://bucket-name/folder/subfolder', 'Data/Downloads' or 'Data'. - """ - if not path.startswith("s3://") and project_name is None: - # for non-s3 paths we need the project, for S3 we don't - raise click.UsageError("When using File Explorer paths '--project-name' needs to be defined") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - link_p = Link( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - cromwell_token=None, - project_name=project_name, - verify=verify_ssl - ) - - # Minimal folder validation and improved error messages - is_s3 = path.startswith("s3://") - is_folder = True - if is_s3: - # S3 path validation - use heuristics to determine if it's likely a folder - try: - # If path ends with '/', it's likely a folder - if path.endswith('/'): - is_folder = True - else: - # Check the last part of the path - path_parts = path.rstrip("/").split("/") - if path_parts: - last_part = path_parts[-1] - # If the last part has no dot, it's likely a folder - if '.' not in last_part: - is_folder = True - else: - # If it has a dot, it might be a file - set to None for warning - is_folder = None - else: - # Empty path parts, set to None for uncertainty - is_folder = None - except Exception: - # If we can't parse the S3 path, set to None for uncertainty - is_folder = None - else: - # File Explorer path validation (existing logic) - try: - datasets = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - parts = path.strip("/").split("/") - parent_path = "/".join(parts[:-1]) if len(parts) > 1 else "" - item_name = parts[-1] - contents = datasets.list_folder_content(parent_path) - found = None - for item in contents.get("folders", []): - if item.get("name") == item_name: - found = item - break - if not found: - for item in contents.get("files", []): - if item.get("name") == item_name: - found = item - break - if found and ("folderType" not in found): - is_folder = False - except Exception: - is_folder = None - - if is_folder is False: - if is_s3: - raise ValueError("The S3 path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.") - else: - raise ValueError("Linking files or virtual folders is not supported. Link the S3 parent folder instead.", err=True) - return - elif is_folder is None and is_s3: - click.secho("Unable to verify whether the S3 path is a folder. Proceeding with linking; " + - "however, if the operation fails, please confirm that you are linking a folder rather than a file.", fg='yellow', bold=True) - - try: - link_p.link_folder(path, session_id) - except Exception as e: - if is_s3: - print("If you are linking an S3 path, please ensure it is a folder.") - raise ValueError(f"Could not link folder. {e}") - - -@images.command(name="ls") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--page', help='The response page. Defaults to 1.', required=False, default=1) -@click.option('--limit', help='The page size limit. Defaults to 10', required=False, default=10) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def list_images(ctx, - apikey, - cloudos_url, - procurement_id, - disable_ssl_verification, - ssl_cert, - profile, - page, - limit): - """List images associated with organisations of a given procurement.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None, - page=page, - limit=limit - ) - - try: - result = procurement_images.list_procurement_images() - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - - -@images.command(name="set") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) -@click.option('--image-type', help='The CloudOS resource image type.', required=True, - type=click.Choice([ - 'RegularInteractiveSessions', - 'SparkInteractiveSessions', - 'RStudioInteractiveSessions', - 'JupyterInteractiveSessions', - 'JobDefault', - 'NextflowBatchComputeEnvironment'])) -@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') -@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) -@click.option('--image-id', help='The new image id value.', required=True) -@click.option('--image-name', help='The new image name value.', required=False) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def set_organisation_image(ctx, - apikey, - cloudos_url, - procurement_id, - organisation_id, - image_type, - provider, - region, - image_id, - image_name, - disable_ssl_verification, - ssl_cert, - profile): - """Set a new image id or name to image associated with an organisations of a given procurement.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = procurement_images.set_procurement_organisation_image( - organisation_id, - image_type, - provider, - region, - image_id, - image_name - ) - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - - -@images.command(name="reset") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) -@click.option('--image-type', help='The CloudOS resource image type.', required=True, - type=click.Choice([ - 'RegularInteractiveSessions', - 'SparkInteractiveSessions', - 'RStudioInteractiveSessions', - 'JupyterInteractiveSessions', - 'JobDefault', - 'NextflowBatchComputeEnvironment'])) -@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') -@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def reset_organisation_image(ctx, - apikey, - cloudos_url, - procurement_id, - organisation_id, - image_type, - provider, - region, - disable_ssl_verification, - ssl_cert, - profile): - """Reset image associated with an organisations of a given procurement to CloudOS defaults.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = procurement_images.reset_procurement_organisation_image( - organisation_id, - image_type, - provider, - region - ) - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - -@run_cloudos_cli.command('link') -@click.argument('path', required=False) -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS. When provided, links results, workdir and logs by default.', - required=False) -@click.option('--project-name', - help='The name of a CloudOS project. Required for File Explorer paths.', - required=False) -@click.option('--results', - help='Link only results folder (only works with --job-id).', - is_flag=True) -@click.option('--workdir', - help='Link only working directory (only works with --job-id).', - is_flag=True) -@click.option('--logs', - help='Link only logs folder (only works with --job-id).', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) -def link_command(ctx, - path, - apikey, - cloudos_url, - workspace_id, - session_id, - job_id, - project_name, - results, - workdir, - logs, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """ - Link folders to an interactive analysis session. - - This command is used to link folders - to an active interactive analysis session for direct access to data. - - PATH: Optional path to link (S3). - Required if --job-id is not provided. - - Two modes of operation: - - 1. Job-based linking (--job-id): Links job-related folders. - By default, links results, workdir, and logs folders. - Use --results, --workdir, or --logs flags to link only specific folders. - - 2. Direct path linking (PATH argument): Links a specific S3 path. - - Examples: - - # Link all job folders (results, workdir, logs) - cloudos link --job-id 12345 --session-id abc123 - - # Link only results from a job - cloudos link --job-id 12345 --session-id abc123 --results - - # Link a specific S3 path - cloudos link s3://bucket/folder --session-id abc123 - - """ - print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Validate input parameters - if not job_id and not path: - raise click.UsageError("Either --job-id or PATH argument must be provided.") - - if job_id and path: - raise click.UsageError("Cannot use both --job-id and PATH argument. Please provide only one.") - - # Validate folder-specific flags only work with --job-id - if (results or workdir or logs) and not job_id: - raise click.UsageError("--results, --workdir, and --logs flags can only be used with --job-id.") - - # If no specific folders are selected with job-id, link all by default - if job_id and not (results or workdir or logs): - results = True - workdir = True - logs = True - - if verbose: - print('Using the following parameters:') - print(f'\tCloudOS url: {cloudos_url}') - print(f'\tWorkspace ID: {workspace_id}') - print(f'\tSession ID: {session_id}') - if job_id: - print(f'\tJob ID: {job_id}') - print(f'\tLink results: {results}') - print(f'\tLink workdir: {workdir}') - print(f'\tLink logs: {logs}') - else: - print(f'\tPath: {path}') - - # Initialize Link client - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl - ) - - try: - if job_id: - # Job-based linking - print(f'Linking folders from job {job_id} to interactive session {session_id}...\n') - - # Link results - if results: - link_client.link_job_results(job_id, workspace_id, session_id, verify_ssl, verbose) - - # Link workdir - if workdir: - link_client.link_job_workdir(job_id, workspace_id, session_id, verify_ssl, verbose) - - # Link logs - if logs: - link_client.link_job_logs(job_id, workspace_id, session_id, verify_ssl, verbose) - - - else: - # Direct path linking - print(f'Linking path to interactive session {session_id}...\n') - - # Link path with validation - link_client.link_path_with_validation(path, session_id, verify_ssl, project_name, verbose) - - print('\nLinking operation completed.') - - except BadRequestException as e: - raise ValueError(f"Request failed: {str(e)}") - except Exception as e: - raise ValueError(f"Failed to link folder(s): {str(e)}") -if __name__ == "__main__": - # Setup logging - debug_mode = '--debug' in sys.argv - setup_logging(debug_mode) - logger = logging.getLogger("CloudOS") - # Check if debug flag was passed (fallback for cases where Click doesn't handle it) - try: - run_cloudos_cli() - except Exception as e: - if debug_mode: - logger.error(e, exc_info=True) - traceback.print_exc() - else: - logger.error(e) - click.echo(click.style(f"Error: {e}", fg='red'), err=True) - sys.exit(1) \ No newline at end of file +if __name__ == '__main__': + run_cloudos_cli() diff --git a/cloudos_cli/__main__.py.backup b/cloudos_cli/__main__.py.backup new file mode 100644 index 00000000..1eab8bfd --- /dev/null +++ b/cloudos_cli/__main__.py.backup @@ -0,0 +1,4367 @@ +#!/usr/bin/env python3 + +import rich_click as click +import cloudos_cli.jobs.job as jb +from cloudos_cli.clos import Cloudos +from cloudos_cli.import_wf.import_wf import ImportWorflow +from cloudos_cli.queue.queue import Queue +from cloudos_cli.utils.errors import BadRequestException +import json +import time +import sys +import traceback +import copy +from ._version import __version__ +from cloudos_cli.configure.configure import ConfigurationProfile +from rich.console import Console +from rich.table import Table +from cloudos_cli.datasets import Datasets +from cloudos_cli.procurement import Images +from cloudos_cli.utils.resources import ssl_selector, format_bytes +from rich.style import Style +from cloudos_cli.utils.array_job import generate_datasets_for_project +from cloudos_cli.utils.details import create_job_details, create_job_list_table +from cloudos_cli.link import Link +from cloudos_cli.cost.cost import CostViewer +from cloudos_cli.logging.logger import setup_logging, update_command_context_from_click +import logging +from cloudos_cli.configure.configure import ( + with_profile_config, + build_default_map_for_group, + get_shared_config, + CLOUDOS_URL +) +from cloudos_cli.related_analyses.related_analyses import related_analyses + + +# GLOBAL VARS +JOB_COMPLETED = 'completed' +REQUEST_INTERVAL_CROMWELL = 30 +AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] +AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] +HPC_NEXTFLOW_VERSIONS = ['22.10.8'] +AWS_NEXTFLOW_LATEST = '24.04.4' +AZURE_NEXTFLOW_LATEST = '22.11.1-edge' +HPC_NEXTFLOW_LATEST = '22.10.8' +ABORT_JOB_STATES = ['running', 'initializing'] + + +def custom_exception_handler(exc_type, exc_value, exc_traceback): + """Custom exception handler that respects debug mode""" + console = Console(stderr=True) + # Initialise logger + debug_mode = '--debug' in sys.argv + setup_logging(debug_mode) + logger = logging.getLogger("CloudOS") + if get_debug_mode(): + logger.error(exc_value, exc_info=exc_value) + console.print("[yellow]Debug mode: showing full traceback[/yellow]") + sys.__excepthook__(exc_type, exc_value, exc_traceback) + else: + # Extract a clean error message + if hasattr(exc_value, 'message'): + error_msg = exc_value.message + elif str(exc_value): + error_msg = str(exc_value) + else: + error_msg = f"{exc_type.__name__}" + logger.error(exc_value) + console.print(f"[bold red]Error: {error_msg}[/bold red]") + + # For network errors, give helpful context + if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): + console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") + +# Install the custom exception handler +sys.excepthook = custom_exception_handler + + +def pass_debug_to_subcommands(group_cls=click.RichGroup): + """Custom Group class that passes --debug option to all subcommands""" + + class DebugGroup(group_cls): + def add_command(self, cmd, name=None): + # Add debug option to the command if it doesn't already have it + if isinstance(cmd, (click.Command, click.Group)): + has_debug = any(param.name == 'debug' for param in cmd.params) + if not has_debug: + debug_option = click.Option( + ['--debug'], + is_flag=True, + help='Show detailed error information and tracebacks', + is_eager=True, + expose_value=False, + callback=self._debug_callback + ) + cmd.params.insert(-1, debug_option) # Insert at the end for precedence + + super().add_command(cmd, name) + + def _debug_callback(self, ctx, param, value): + """Callback to handle debug flag""" + global _global_debug + if value: + _global_debug = True + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value + + return DebugGroup + + +def get_debug_mode(): + """Get current debug mode state""" + return _global_debug + + +# Helper function for debug setup +def _setup_debug(ctx, param, value): + """Setup debug mode globally and in context""" + global _global_debug + _global_debug = value + if value: + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value + + +@click.group(cls=pass_debug_to_subcommands()) +@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', + is_eager=True, expose_value=False, callback=_setup_debug) +@click.version_option(__version__) +@click.pass_context +def run_cloudos_cli(ctx): + """CloudOS python package: a package for interacting with CloudOS.""" + update_command_context_from_click(ctx) + ctx.ensure_object(dict) + + if ctx.invoked_subcommand not in ['datasets']: + print(run_cloudos_cli.__doc__ + '\n') + print('Version: ' + __version__ + '\n') + + # Load shared configuration (handles missing profiles and fields gracefully) + shared_config = get_shared_config() + + # Automatically build default_map from registered commands + ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def job(): + """CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS.""" + print(job.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def workflow(): + """CloudOS workflow functionality: list and import workflows.""" + print(workflow.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def project(): + """CloudOS project functionality: list and create projects in CloudOS.""" + print(project.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def cromwell(): + """Cromwell server functionality: check status, start and stop.""" + print(cromwell.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def queue(): + """CloudOS job queue functionality.""" + print(queue.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def bash(): + """CloudOS bash functionality.""" + print(bash.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def procurement(): + """CloudOS procurement functionality.""" + print(procurement.__doc__ + '\n') + + +@procurement.group(cls=pass_debug_to_subcommands()) +def images(): + """CloudOS procurement images functionality.""" + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +@click.pass_context +def datasets(ctx): + """CloudOS datasets functionality.""" + update_command_context_from_click(ctx) + if ctx.args and ctx.args[0] != 'ls': + print(datasets.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands(), invoke_without_command=True) +@click.option('--profile', help='Profile to use from the config file', default='default') +@click.option('--make-default', + is_flag=True, + help='Make the profile the default one.') +@click.pass_context +def configure(ctx, profile, make_default): + """CloudOS configuration.""" + print(configure.__doc__ + '\n') + update_command_context_from_click(ctx) + profile = profile or ctx.obj['profile'] + config_manager = ConfigurationProfile() + + if ctx.invoked_subcommand is None and profile == "default" and not make_default: + config_manager.create_profile_from_input(profile_name="default") + + if profile != "default" and not make_default: + config_manager.create_profile_from_input(profile_name=profile) + if make_default: + config_manager.make_default_profile(profile_name=profile) + + +@job.command('run', cls=click.RichCommand) +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('--job-config', + help=('A config file similar to a nextflow.config file, ' + + 'but only with the parameters to use with your job.')) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p input=s3://path_to_my_file. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--nextflow-profile', + help=('A comma separated string indicating the nextflow profile/s ' + + 'to use with your job.')) +@click.option('--nextflow-version', + help=('Nextflow version to use when executing the workflow in CloudOS. ' + + 'Default=22.10.8.'), + type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']), + default='22.10.8') +@click.option('--git-commit', + help=('The git commit hash to run for ' + + 'the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--git-tag', + help=('The tag to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--git-branch', + help=('The branch to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--resumable', + help='Whether to make the job able to be resumed or not.', + is_flag=True) +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--wdl-mainfile', + help='For WDL workflows, which mainFile (.wdl) is configured to use.',) +@click.option('--wdl-importsfile', + help='For WDL workflows, which importsFile (.zip) is configured to use.',) +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. Currently, not necessary ' + + 'as apikey can be used instead, but maintained for backwards compatibility.')) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + type=click.Choice(['aws', 'azure', 'hpc']), + default='aws') +@click.option('--hpc-id', + help=('ID of your HPC, only applicable when --execution-platform=hpc. ' + + 'Default=660fae20f93358ad61e0104b'), + default='660fae20f93358ad61e0104b') +@click.option('--azure-worker-instance-type', + help=('The worker node instance type to be used in azure. ' + + 'Default=Standard_D4as_v4'), + default='Standard_D4as_v4') +@click.option('--azure-worker-instance-disk', + help='The disk size in GB for the worker node to be used in azure. Default=100', + type=int, + default=100) +@click.option('--azure-worker-instance-spot', + help='Whether the azure worker nodes have to be spot instances or not.', + is_flag=True) +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-file-staging', + help='Enables AWS S3 mountpoint for quicker file staging.', + is_flag=True) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--use-private-docker-repository', + help=('Allows to use private docker repository for running jobs. The Docker user ' + + 'account has to be already linked to CloudOS.'), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run(ctx, + apikey, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + job_config, + parameter, + git_commit, + git_tag, + git_branch, + job_name, + resumable, + do_not_save_logs, + job_queue, + nextflow_profile, + nextflow_version, + instance_type, + instance_disk, + storage_mode, + lustre_size, + wait_completion, + wait_time, + wdl_mainfile, + wdl_importsfile, + cromwell_token, + repository_platform, + execution_platform, + hpc_id, + azure_worker_instance_type, + azure_worker_instance_disk, + azure_worker_instance_spot, + cost_limit, + accelerate_file_staging, + accelerate_saving_results, + use_private_docker_repository, + verbose, + request_interval, + disable_ssl_verification, + ssl_cert, + profile): + """Submit a job to CloudOS.""" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if do_not_save_logs: + save_logs = False + else: + save_logs = True + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + if execution_platform == 'azure' or execution_platform == 'hpc': + batch = False + else: + batch = True + if execution_platform == 'hpc': + print('\nHPC execution platform selected') + if hpc_id is None: + raise ValueError('Please, specify your HPC ID using --hpc parameter') + print('Please, take into account that HPC execution do not support ' + + 'the following parameters and all of them will be ignored:\n' + + '\t--job-queue\n' + + '\t--resumable | --do-not-save-logs\n' + + '\t--instance-type | --instance-disk | --cost-limit\n' + + '\t--storage-mode | --lustre-size\n' + + '\t--wdl-mainfile | --wdl-importsfile | --cromwell-token\n') + wdl_mainfile = None + wdl_importsfile = None + storage_mode = 'regular' + save_logs = False + if accelerate_file_staging: + if execution_platform != 'aws': + print('You have selected accelerate file staging, but this function is ' + + 'only available when execution platform is AWS. The accelerate file staging ' + + 'will not be applied') + use_mountpoints = False + else: + use_mountpoints = True + print('Enabling AWS S3 mountpoint for accelerated file staging. ' + + 'Please, take into consideration the following:\n' + + '\t- It significantly reduces runtime and compute costs but may increase network costs.\n' + + '\t- Requires extra memory. Adjust process memory or optimise resource usage if necessary.\n' + + '\t- This is still a CloudOS BETA feature.\n') + else: + use_mountpoints = False + if verbose: + print('\t...Detecting workflow type') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + workflow_type = cl.detect_workflow(workflow_name, workspace_id, verify_ssl, last) + is_module = cl.is_module(workflow_name, workspace_id, verify_ssl, last) + if execution_platform == 'hpc' and workflow_type == 'wdl': + raise ValueError(f'The workflow {workflow_name} is a WDL workflow. ' + + 'WDL is not supported on HPC execution platform.') + if workflow_type == 'wdl': + print('WDL workflow detected') + if wdl_mainfile is None: + raise ValueError('Please, specify WDL mainFile using --wdl-mainfile .') + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h == 'Stopped': + print('\tStarting Cromwell server...\n') + cl.cromwell_switch(workspace_id, 'restart', verify_ssl) + elapsed = 0 + while elapsed < 300 and c_status_h != 'Running': + c_status_old = c_status_h + time.sleep(REQUEST_INTERVAL_CROMWELL) + elapsed += REQUEST_INTERVAL_CROMWELL + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + if c_status_h != c_status_old: + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h != 'Running': + raise Exception('Cromwell server did not restarted properly.') + cromwell_id = json.loads(c_status.content)["_id"] + click.secho('\t' + ('*' * 80) + '\n' + + '\tCromwell server is now running. Please, remember to stop it when ' + + 'your\n' + '\tjob finishes. You can use the following command:\n' + + '\tcloudos cromwell stop \\\n' + + '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + + f'\t\t--cloudos-url {cloudos_url} \\\n' + + f'\t\t--workspace-id {workspace_id}\n' + + '\t' + ('*' * 80) + '\n', fg='yellow', bold=True) + else: + cromwell_id = None + if verbose: + print('\t...Preparing objects') + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=wdl_mainfile, importsfile=wdl_importsfile, + repository_platform=repository_platform, verify=verify_ssl, last=last) + if verbose: + print('\tThe following Job object was created:') + print('\t' + str(j)) + print('\t...Sending job to CloudOS\n') + if is_module: + if job_queue is not None: + print(f'Ignoring job queue "{job_queue}" for ' + + f'Platform Workflow "{workflow_name}". Platform Workflows ' + + 'use their own predetermined queues.') + job_queue_id = None + if nextflow_version != '22.10.8': + print(f'The selected worflow \'{workflow_name}\' ' + + 'is a CloudOS module. CloudOS modules only work with ' + + 'Nextflow version 22.10.8. Switching to use 22.10.8') + nextflow_version = '22.10.8' + if execution_platform == 'azure': + print(f'The selected worflow \'{workflow_name}\' ' + + 'is a CloudOS module. For these workflows, worker nodes ' + + 'are managed internally. For this reason, the options ' + + 'azure-worker-instance-type, azure-worker-instance-disk and ' + + 'azure-worker-instance-spot are not taking effect.') + else: + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=cromwell_token, + workspace_id=workspace_id, verify=verify_ssl) + job_queue_id = queue.fetch_job_queue_id(workflow_type=workflow_type, batch=batch, + job_queue=job_queue) + if use_private_docker_repository: + if is_module: + print(f'Workflow "{workflow_name}" is a CloudOS module. ' + + 'Option --use-private-docker-repository will be ignored.') + docker_login = False + else: + me = j.get_user_info(verify=verify_ssl)['dockerRegistriesCredentials'] + if len(me) == 0: + raise Exception('User private Docker repository has been selected but your user ' + + 'credentials have not been configured yet. Please, link your ' + + 'Docker account to CloudOS before using ' + + '--use-private-docker-repository option.') + print('Use private Docker repository has been selected. A custom job ' + + 'queue to support private Docker containers and/or Lustre FSx will be created for ' + + 'your job. The selected job queue will serve as a template.') + docker_login = True + else: + docker_login = False + if nextflow_version == 'latest': + if execution_platform == 'aws': + nextflow_version = AWS_NEXTFLOW_LATEST + elif execution_platform == 'azure': + nextflow_version = AZURE_NEXTFLOW_LATEST + else: + nextflow_version = HPC_NEXTFLOW_LATEST + print('You have specified Nextflow version \'latest\' for execution platform ' + + f'\'{execution_platform}\'. The workflow will use the ' + + f'latest version available on CloudOS: {nextflow_version}.') + if execution_platform == 'aws': + if nextflow_version not in AWS_NEXTFLOW_VERSIONS: + print('For execution platform \'aws\', the workflow will use the default ' + + '\'22.10.8\' version on CloudOS.') + nextflow_version = '22.10.8' + if execution_platform == 'azure': + if nextflow_version not in AZURE_NEXTFLOW_VERSIONS: + print('For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' ' + + 'version on CloudOS.') + nextflow_version = '22.11.1-edge' + if execution_platform == 'hpc': + if nextflow_version not in HPC_NEXTFLOW_VERSIONS: + print('For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.') + nextflow_version = '22.10.8' + if nextflow_version != '22.10.8' and nextflow_version != '22.11.1-edge': + click.secho(f'You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' + + 'to be written in DSL2 and does not support DSL1.', fg='yellow', bold=True) + print('\nExecuting run...') + if workflow_type == 'nextflow': + print(f'\tNextflow version: {nextflow_version}') + j_id = j.send_job(job_config=job_config, + parameter=parameter, + is_module=is_module, + git_commit=git_commit, + git_tag=git_tag, + git_branch=git_branch, + job_name=job_name, + resumable=resumable, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + nextflow_profile=nextflow_profile, + nextflow_version=nextflow_version, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=hpc_id, + workflow_type=workflow_type, + cromwell_id=cromwell_id, + azure_worker_instance_type=azure_worker_instance_type, + azure_worker_instance_disk=azure_worker_instance_disk, + azure_worker_instance_spot=azure_worker_instance_spot, + cost_limit=cost_limit, + use_mountpoints=use_mountpoints, + accelerate_saving_results=accelerate_saving_results, + docker_login=docker_login, + verify=verify_ssl) + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=verbose, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') + + +@job.command('status') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_status(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Check job status in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + print('Executing status...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' + print(f'\tTo further check your job status you can either go to {j_url} ' + + 'or repeat the command you just used.') + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") + + +@job.command('workdir') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the working directory to an interactive session.', + is_flag=True) +@click.option('--delete', + help='Delete the results directory of a CloudOS job.', + is_flag=True) +@click.option('-y', '--yes', + help='Skip confirmation prompt when deleting results.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--status', + help='Check the deletion status of the working directory.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_workdir(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + delete, + yes, + session_id, + status, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the working directory of a specified job or check deletion status.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Handle --status flag + if status: + console = Console() + + if verbose: + console.print('[bold cyan]Checking deletion status of job working directory...[/bold cyan]') + console.print('\t[dim]...Preparing objects[/dim]') + console.print('\t[bold]Using the following parameters:[/bold]') + console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') + console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') + console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') + + # Use Cloudos object to access the deletion status method + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + console.print('\t[dim]The following Cloudos object was created:[/dim]') + console.print('\t' + str(cl) + '\n') + + try: + deletion_status = cl.get_workdir_deletion_status( + job_id=job_id, + workspace_id=workspace_id, + verify=verify_ssl + ) + + # Convert API status to user-friendly terminology with color + status_config = { + "ready": ("available", "green"), + "deleting": ("deleting", "yellow"), + "scheduledForDeletion": ("scheduled for deletion", "yellow"), + "deleted": ("deleted", "red"), + "failedToDelete": ("failed to delete", "red") + } + + # Get the status of the workdir folder itself and convert it + api_status = deletion_status.get("status", "unknown") + folder_status, status_color = status_config.get(api_status, (api_status, "white")) + folder_info = deletion_status.get("items", {}) + + # Display results in a clear, styled format with human-readable sentence + console.print(f'The working directory of job [cyan]{deletion_status["job_id"]}[/cyan] is in status: [bold {status_color}]{folder_status}[/bold {status_color}]') + + # For non-available statuses, always show update time and user info + if folder_status != "available": + if folder_info.get("updatedAt"): + console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') + + # Show user information - prefer deletedBy over user field + user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) + if user_info: + user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() + user_email = user_info.get('email', '') + if user_name or user_email: + user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) + console.print(f'[blue]User:[/blue] {user_display}') + + # Display detailed information if verbose + if verbose: + console.print(f'\n[bold]Additional information:[/bold]') + console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') + console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') + console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') + + # Show folder metadata if available + if folder_info.get("createdAt"): + console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') + if folder_info.get("updatedAt"): + console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') + if folder_info.get("folderType"): + console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') + + except ValueError as e: + raise click.ClickException(str(e)) + except Exception as e: + raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") + + return + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Finding working directory path...') + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl) + print(f"Working directory for job {job_id}: {workdir}") + + # Link to interactive session if requested + if link: + if verbose: + print(f'\tLinking working directory to interactive session {session_id}...') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + link_client.link_folder(workdir.strip(), session_id) + + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") + + # Delete workdir directory if requested + if delete: + try: + # Ask for confirmation unless --yes flag is provided + if not yes: + confirmation_message = ( + "\n⚠️ Deleting intermediate results is permanent and cannot be undone. " + "All associated data will be permanently removed and cannot be recovered. " + "The current job, as well as any other jobs sharing the same working directory, " + "will no longer be resumable. This action will be logged in the audit trail " + "(if auditing is enabled for your organisation), and you will be recorded as " + "the user who performed the deletion. You can skip this confirmation step by " + "providing -y or --yes flag to cloudos job workdir --delete. Please confirm " + "that you want to delete intermediate results of this analysis? [y/n] " + ) + click.secho(confirmation_message, fg='black', bg='yellow') + user_input = input().strip().lower() + if user_input != 'y': + print('\nDeletion cancelled.') + return + # Proceed with deletion + job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + job.delete_job_results(job_id, "workDirectory", verify=verify_ssl) + click.secho('\nIntermediate results directories deleted successfully.', fg='green', bold=True) + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve intermediate results for job '{job_id}'. {str(e)}") + else: + if yes: + click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) + + +@job.command('logs') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the logs directories to an interactive session.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_logs(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + session_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the logs of a specified job.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Executing logs...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + logs = cl.get_job_logs(job_id, workspace_id, verify_ssl) + for name, path in logs.items(): + print(f"{name}: {path}") + + # Link to interactive session if requested + if link: + if logs: + # Extract the parent logs directory from any log file path + # All log files should be in the same logs directory + first_log_path = next(iter(logs.values())) + # Remove the filename to get the logs directory + # e.g., "s3://bucket/path/to/logs/filename.txt" -> "s3://bucket/path/to/logs" + logs_dir = '/'.join(first_log_path.split('/')[:-1]) + + if verbose: + print(f'\tLinking logs directory to interactive session {session_id}...') + print(f'\t\tLogs directory: {logs_dir}') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + link_client.link_folder(logs_dir, session_id) + else: + if verbose: + print('\tNo logs found to link.') + + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve logs for job '{job_id}'. {str(e)}") + + +@job.command('results') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the results directories to an interactive session.', + is_flag=True) +@click.option('--delete', + help='Delete the results directory of a CloudOS job.', + is_flag=True) +@click.option('-y', '--yes', + help='Skip confirmation prompt when deleting results.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--status', + help='Check the deletion status of the job results.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_results(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + delete, + yes, + session_id, + status, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the results of a specified job or check deletion status.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Handle --status flag + if status: + console = Console() + + if verbose: + console.print('[bold cyan]Checking deletion status of job results...[/bold cyan]') + console.print('\t[dim]...Preparing objects[/dim]') + console.print('\t[bold]Using the following parameters:[/bold]') + console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') + console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') + console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') + + # Use Cloudos object to access the deletion status method + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + console.print('\t[dim]The following Cloudos object was created:[/dim]') + console.print('\t' + str(cl) + '\n') + + try: + deletion_status = cl.get_results_deletion_status( + job_id=job_id, + workspace_id=workspace_id, + verify=verify_ssl + ) + + # Convert API status to user-friendly terminology with color + status_config = { + "ready": ("available", "green"), + "deleting": ("deleting", "yellow"), + "scheduledForDeletion": ("scheduled for deletion", "yellow"), + "deleted": ("deleted", "red"), + "failedToDelete": ("failed to delete", "red") + } + + # Get the status of the results folder itself and convert it + api_status = deletion_status.get("status", "unknown") + folder_status, status_color = status_config.get(api_status, (api_status, "white")) + folder_info = deletion_status.get("items", {}) + + # Display results in a clear, styled format with human-readable sentence + console.print(f'The results of job [cyan]{deletion_status["job_id"]}[/cyan] are in status: [bold {status_color}]{folder_status}[/bold {status_color}]') + + # For non-available statuses, always show update time and user info + if folder_status != "available": + if folder_info.get("updatedAt"): + console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') + + # Show user information - prefer deletedBy over user field + user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) + if user_info: + user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() + user_email = user_info.get('email', '') + if user_name or user_email: + user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) + console.print(f'[blue]User:[/blue] {user_display}') + + # Display detailed information if verbose + if verbose: + console.print(f'\n[bold]Additional information:[/bold]') + console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') + console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') + console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') + + # Show folder metadata if available + if folder_info.get("createdAt"): + console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') + if folder_info.get("updatedAt"): + console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') + if folder_info.get("folderType"): + console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') + + except ValueError as e: + raise click.ClickException(str(e)) + except Exception as e: + raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") + + return + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Executing results...') + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) + print(f"results: {results_path}") + + # Link to interactive session if requested + if link: + if verbose: + print(f'\tLinking results directory to interactive session {session_id}...') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + if verbose: + print(f'\t\tLinking results ({results_path})...') + + link_client.link_folder(results_path, session_id) + + # Delete results directory if requested + if delete: + # Ask for confirmation unless --yes flag is provided + if not yes: + confirmation_message = ( + "\n⚠️ Deleting final analysis results is irreversible. " + "All data and backups will be permanently removed and cannot be recovered. " + "You can skip this confirmation step by providing '-y' or '--yes' flag to " + "'cloudos job results --delete'. " + "Please confirm that you want to delete final results of this analysis? [y/n] " + ) + click.secho(confirmation_message, fg='black', bg='yellow') + user_input = input().strip().lower() + if user_input != 'y': + print('\nDeletion cancelled.') + return + if verbose: + print(f'\nDeleting result directories from CloudOS...') + # Proceed with deletion + job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + job.delete_job_results(job_id, "analysisResults", verify=verify_ssl) + click.secho('\nResults directories deleted successfully.', fg='green', bold=True) + else: + if yes: + click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve results for job '{job_id}'. {str(e)}") + + +@job.command('details') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--output-format', + help=('The desired display for the output, either directly in standard output or saved as file. ' + + 'Default=stdout.'), + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--output-basename', + help=('Output file base name to save jobs details. ' + + 'Default={job_id}_details'), + required=False) +@click.option('--parameters', + help=('Whether to generate a ".config" file that can be used as input for --job-config parameter. ' + + 'It will have the same basename as defined in "--output-basename". '), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_details(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + output_basename, + parameters, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve job details in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if ctx.get_parameter_source('output_basename') == click.core.ParameterSource.DEFAULT: + output_basename = f"{job_id}_details" + + print('Executing details...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + + # check if the API gives a 403 error/forbidden error + try: + j_details = cl.get_job_status(job_id, workspace_id, verify_ssl) + except BadRequestException as e: + if '403' in str(e) or 'Forbidden' in str(e): + raise ValueError("API can only show job details of your own jobs, cannot see other user's job details.") + else: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve details for job '{job_id}'. {str(e)}") + create_job_details(json.loads(j_details.content), job_id, output_format, output_basename, parameters, cloudos_url) + + +@job.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save jobs list. ' + + 'Default=joblist'), + default='joblist', + required=False) +@click.option('--output-format', + help='The desired output format. For json option --all-fields will be automatically set to True. Default=stdout.', + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--table-columns', + help=('Comma-separated list of columns to display in the table. Only applicable when --output-format=stdout. ' + + 'Available columns: status,name,project,owner,pipeline,id,submit_time,end_time,run_time,commit,cost,resources,storage_type. ' + + 'Default: responsive (auto-selects columns based on terminal width)'), + default=None) +@click.option('--all-fields', + help=('Whether to collect all available fields from jobs or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv. Automatically enabled for json output.'), + is_flag=True) +@click.option('--last-n-jobs', + help=("The number of last workspace jobs to retrieve. You can use 'all' to " + + "retrieve all workspace jobs. When adding this option, options " + + "'--page' and '--page-size' are ignored.")) +@click.option('--page', + help=('Page number to fetch from the API. Used with --page-size to control jobs ' + + 'per page (e.g. --page=4 --page-size=20). Default=1.'), + type=int, + default=1) +@click.option('--page-size', + help=('Page size to retrieve from API, corresponds to the number of jobs per page. ' + + 'Maximum allowed integer is 100. Default=10.'), + type=int, + default=10) +@click.option('--archived', + help=('When this flag is used, only archived jobs list is collected.'), + is_flag=True) +@click.option('--filter-status', + help='Filter jobs by status (e.g., completed, running, failed, aborted).') +@click.option('--filter-job-name', + help='Filter jobs by job name ( case insensitive ).') +@click.option('--filter-project', + help='Filter jobs by project name.') +@click.option('--filter-workflow', + help='Filter jobs by workflow/pipeline name.') +@click.option('--last', + help=('When workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('--filter-job-id', + help='Filter jobs by specific job ID.') +@click.option('--filter-only-mine', + help='Filter to show only jobs belonging to the current user.', + is_flag=True) +@click.option('--filter-queue', + help='Filter jobs by queue name. Only applies to jobs running in batch environment. Non-batch jobs are preserved in results.') +@click.option('--filter-owner', + help='Filter jobs by owner username.') +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + table_columns, + all_fields, + last_n_jobs, + page, + page_size, + archived, + filter_status, + filter_job_name, + filter_project, + filter_workflow, + last, + filter_job_id, + filter_only_mine, + filter_owner, + filter_queue, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect and display workspace jobs from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Pass table_columns directly to create_job_list_table for validation and processing + selected_columns = table_columns + # Only set outfile if not using stdout + if output_format != 'stdout': + outfile = output_basename + '.' + output_format + + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for jobs in the following workspace: ' + + f'{workspace_id}') + # Check if the user provided the --page option + ctx = click.get_current_context() + if not isinstance(page, int) or page < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') + + if not isinstance(page_size, int) or page_size < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page-size parameter') + + # Validate page_size limit - must be done before API call + if page_size > 100: + click.secho('Error: Page size cannot exceed 100. Please use --page-size with a value <= 100', fg='red', err=True) + raise SystemExit(1) + + result = cl.get_job_list(workspace_id, last_n_jobs, page, page_size, archived, verify_ssl, + filter_status=filter_status, + filter_job_name=filter_job_name, + filter_project=filter_project, + filter_workflow=filter_workflow, + filter_job_id=filter_job_id, + filter_only_mine=filter_only_mine, + filter_owner=filter_owner, + filter_queue=filter_queue, + last=last) + + # Extract jobs and pagination metadata from result + my_jobs_r = result['jobs'] + pagination_metadata = result['pagination_metadata'] + + # Validate requested page exists + if pagination_metadata: + total_jobs = pagination_metadata.get('Pagination-Count', 0) + current_page_size = pagination_metadata.get('Pagination-Limit', page_size) + + if total_jobs > 0: + total_pages = (total_jobs + current_page_size - 1) // current_page_size + if page > total_pages: + click.secho(f'Error: Page {page} does not exist. There are only {total_pages} page(s) available with {total_jobs} total job(s). ' + f'Please use --page with a value between 1 and {total_pages}', fg='red', err=True) + raise SystemExit(1) + + if len(my_jobs_r) == 0: + # Check if any filtering options are being used + filters_used = any([ + filter_status, + filter_job_name, + filter_project, + filter_workflow, + filter_job_id, + filter_only_mine, + filter_owner, + filter_queue + ]) + if output_format == 'stdout': + # For stdout, always show a user-friendly message + create_job_list_table([], cloudos_url, pagination_metadata, selected_columns) + else: + if filters_used: + print('A total of 0 jobs collected.') + elif ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + print('A total of 0 jobs collected. This is likely because your workspace ' + + 'has no jobs created yet.') + else: + print('A total of 0 jobs collected. This is likely because the --page you requested ' + + 'does not exist. Please, try a smaller number for --page or collect all the jobs by not ' + + 'using --page parameter.') + elif output_format == 'stdout': + # Display as table + create_job_list_table(my_jobs_r, cloudos_url, pagination_metadata, selected_columns) + elif output_format == 'csv': + my_jobs = cl.process_job_list(my_jobs_r, all_fields) + cl.save_job_list_to_csv(my_jobs, outfile) + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_jobs_r)) + print(f'\tJob list collected with a total of {len(my_jobs_r)} jobs.') + print(f'\tJob list saved to {outfile}') + else: + raise ValueError('Unrecognised output format. Please use one of [stdout|csv|json]') + + +@job.command('abort') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-ids', + help=('One or more job ids to abort. If more than ' + + 'one is provided, they must be provided as ' + + 'a comma separated list of ids. E.g. id1,id2,id3'), + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--force', + help='Force abort the job even if it is not in a running or initializing state.', + is_flag=True) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def abort_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + job_ids, + verbose, + disable_ssl_verification, + ssl_cert, + profile, + force): + """Abort all specified jobs from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + print('Aborting jobs...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for jobs in the following workspace: ' + + f'{workspace_id}') + # check if the user provided an empty job list + jobs = job_ids.replace(' ', '') + if not jobs: + raise ValueError('No job IDs provided. Please specify at least one job ID to abort.') + jobs = jobs.split(',') + + # Issue warning if using --force flag + if force: + click.secho(f"Warning: Using --force to abort jobs. Some data might be lost.", fg='yellow', bold=True) + + for job in jobs: + try: + j_status = cl.get_job_status(job, workspace_id, verify_ssl) + except Exception as e: + click.secho(f"Failed to get status for job {job}, please make sure it exists in the workspace: {e}", fg='yellow', bold=True) + continue + + j_status_content = json.loads(j_status.content) + job_status = j_status_content['status'] + + # Check if job is in a state that normally allows abortion + is_abortable = job_status in ABORT_JOB_STATES + + # Issue warning if job is in initializing state and not using force + if job_status == 'initializing' and not force: + click.secho(f"Warning: Job {job} is in initializing state.", fg='yellow', bold=True) + + # Check if job can be aborted + if not is_abortable: + click.secho(f"Job {job} is not in a state that can be aborted and is ignored. " + + f"Current status: {job_status}", fg='yellow', bold=True) + else: + try: + cl.abort_job(job, workspace_id, verify_ssl, force) + click.secho(f"Job '{job}' aborted successfully.", fg='green', bold=True) + except Exception as e: + click.secho(f"Failed to abort job {job}. Error: {e}", fg='red', bold=True) + + +@job.command('cost') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to get costs for.', + required=True) +@click.option('--output-format', + help='The desired file format (file extension) for the output. For json option --all-fields will be automatically set to True. Default=csv.', + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_cost(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve job cost information in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + print('Retrieving cost information...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cost_viewer = CostViewer(cloudos_url, apikey) + if verbose: + print(f'\tSearching for cost data for job id: {job_id}') + # Display costs with pagination + cost_viewer.display_costs(job_id, workspace_id, output_format, verify_ssl) + + +@job.command('related') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to get costs for.', + required=True) +@click.option('--output-format', + help='The desired output format. Default=stdout.', + type=click.Choice(['stdout', 'json'], case_sensitive=False), + default='stdout') +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def related(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve related job analyses in CloudOS.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + related_analyses(cloudos_url, apikey, job_id, workspace_id, output_format, verify_ssl) + + +@click.command() +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-ids', + help=('One or more job ids to archive/unarchive. If more than ' + + 'one is provided, they must be provided as ' + + 'a comma separated list of ids. E.g. id1,id2,id3'), + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def archive_unarchive_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + job_ids, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Archive or unarchive specified jobs in a CloudOS workspace.""" + # Determine operation based on the command name used + target_archived_state = ctx.info_name == "archive" + action = "archive" if target_archived_state else "unarchive" + action_past = "archived" if target_archived_state else "unarchived" + action_ing = "archiving" if target_archived_state else "unarchiving" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + print(f'{action_ing.capitalize()} jobs...') + + if verbose: + print('\t...Preparing objects') + + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\t{action_ing.capitalize()} jobs in the following workspace: {workspace_id}') + + # check if the user provided an empty job list + jobs = job_ids.replace(' ', '') + if not jobs: + raise ValueError(f'No job IDs provided. Please specify at least one job ID to {action}.') + jobs_list = [job for job in jobs.split(',') if job] # Filter out empty strings + + # Check for duplicate job IDs + duplicates = [job_id for job_id in set(jobs_list) if jobs_list.count(job_id) > 1] + if duplicates: + dup_str = ', '.join(duplicates) + click.secho(f'Warning: Duplicate job IDs detected and will be processed only once: {dup_str}', fg='yellow', bold=True) + # Remove duplicates while preserving order + jobs_list = list(dict.fromkeys(jobs_list)) + if verbose: + print(f'\tDuplicate job IDs removed. Processing {len(jobs_list)} unique job(s).') + + # Check archive status for all jobs + status_check = cl.check_jobs_archive_status(jobs_list, workspace_id, target_archived_state=target_archived_state, verify=verify_ssl, verbose=verbose) + valid_jobs = status_check['valid_jobs'] + already_processed = status_check['already_processed'] + invalid_jobs = status_check['invalid_jobs'] + + # Report invalid jobs (but continue processing valid ones) + for job_id, error_msg in invalid_jobs.items(): + click.secho(f"Failed to get status for job {job_id}, please make sure it exists in the workspace: {error_msg}", fg='yellow', bold=True) + + if not valid_jobs and not already_processed: + # All jobs were invalid - exit gracefully + click.secho('No valid job IDs found. Please check that the job IDs exist and are accessible.', fg='yellow', bold=True) + return + + if not valid_jobs: + if len(already_processed) == 1: + click.secho(f"Job '{already_processed[0]}' is already {action_past}. No action needed.", fg='cyan', bold=True) + else: + click.secho(f"All {len(already_processed)} jobs are already {action_past}. No action needed.", fg='cyan', bold=True) + return + + try: + # Call the appropriate action method + if target_archived_state: + cl.archive_jobs(valid_jobs, workspace_id, verify_ssl) + else: + cl.unarchive_jobs(valid_jobs, workspace_id, verify_ssl) + + success_msg = [] + if len(valid_jobs) == 1: + success_msg.append(f"Job '{valid_jobs[0]}' {action_past} successfully.") + else: + success_msg.append(f"{len(valid_jobs)} jobs {action_past} successfully: {', '.join(valid_jobs)}") + + if already_processed: + if len(already_processed) == 1: + success_msg.append(f"Job '{already_processed[0]}' was already {action_past}.") + else: + success_msg.append(f"{len(already_processed)} jobs were already {action_past}: {', '.join(already_processed)}") + + click.secho('\n'.join(success_msg), fg='green', bold=True) + except Exception as e: + raise ValueError(f"Failed to {action} jobs: {str(e)}") + + +@click.command(help='Clone or resume a job with modified parameters') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.') +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p input=s3://path_to_my_file. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--nextflow-profile', + help=('A comma separated string indicating the nextflow profile/s ' + + 'to use with your job.')) +@click.option('--nextflow-version', + help=('Nextflow version to use when executing the workflow in CloudOS. ' + + 'Default=22.10.8.'), + type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest'])) +@click.option('--git-branch', + help=('The branch to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--job-name', + help='The name of the job. If not set, will take the name of the cloned job.') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help=('Name of the job queue to use with a batch job. ' + + 'In Azure workspaces, this option is ignored.')) +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).')) +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float) +@click.option('--job-id', + help='The CloudOS job id of the job to be cloned.', + required=True) +@click.option('--accelerate-file-staging', + help='Enables AWS S3 mountpoint for quicker file staging.', + is_flag=True) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--resumable', + help='Whether to make the job able to be resumed or not.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', + help='Profile to use from the config file', + default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def clone_resume(ctx, + apikey, + cloudos_url, + workspace_id, + project_name, + parameter, + nextflow_profile, + nextflow_version, + git_branch, + repository_platform, + job_name, + do_not_save_logs, + job_queue, + instance_type, + cost_limit, + job_id, + accelerate_file_staging, + accelerate_saving_results, + resumable, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + if ctx.info_name == "clone": + mode, action = "clone", "cloning" + elif ctx.info_name == "resume": + mode, action = "resume", "resuming" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + print(f'{action.capitalize()} job...') + if verbose: + print('\t...Preparing objects') + + # Create Job object (set dummy values for project_name and workflow_name, since they come from the cloned job) + job_obj = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + + if verbose: + print('\tThe following Job object was created:') + print('\t' + str(job_obj) + '\n') + print(f'\t{action.capitalize()} job {job_id} in workspace: {workspace_id}') + + try: + + # Clone/resume the job with provided overrides + cloned_resumed_job_id = job_obj.clone_or_resume_job( + source_job_id=job_id, + queue_name=job_queue, + cost_limit=cost_limit, + master_instance=instance_type, + job_name=job_name, + nextflow_version=nextflow_version, + branch=git_branch, + repository_platform=repository_platform, + profile=nextflow_profile, + do_not_save_logs=do_not_save_logs, + use_fusion=accelerate_file_staging, + accelerate_saving_results=accelerate_saving_results, + resumable=resumable, + # only when explicitly setting --project-name will be overridden, else using the original project + project_name=project_name if ctx.get_parameter_source("project_name") == click.core.ParameterSource.COMMANDLINE else None, + parameters=list(parameter) if parameter else None, + verify=verify_ssl, + mode=mode + ) + + if verbose: + print(f'\t{mode.capitalize()}d job ID: {cloned_resumed_job_id}') + + print(f"Job successfully {mode}d. New job ID: {cloned_resumed_job_id}") + + except BadRequestException as e: + raise ValueError(f"Failed to {mode} job. Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") + + +# Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) +archive_unarchive_jobs.help = 'Archive specified jobs in a CloudOS workspace.' +job.add_command(archive_unarchive_jobs, "archive") + +# Create a copy with different help text for unarchive +archive_unarchive_jobs_copy = copy.deepcopy(archive_unarchive_jobs) +archive_unarchive_jobs_copy.help = 'Unarchive specified jobs in a CloudOS workspace.' +job.add_command(archive_unarchive_jobs_copy, "unarchive") + + +# Apply the best Click solution: Set specific help text for each command registration +clone_resume.help = 'Clone a job with modified parameters' +job.add_command(clone_resume, "clone") + +# Create a copy with different help text for resume +clone_resume_copy = copy.deepcopy(clone_resume) +clone_resume_copy.help = 'Resume a job with modified parameters' +job.add_command(clone_resume_copy, "resume") + + +@workflow.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save workflow list. ' + + 'Default=workflow_list'), + default='workflow_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from workflows or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_workflows(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all workflows from a CloudOS workspace in CSV format.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for workflows in the following workspace: ' + + f'{workspace_id}') + my_workflows_r = cl.get_workflow_list(workspace_id, verify=verify_ssl) + if output_format == 'csv': + my_workflows = cl.process_workflow_list(my_workflows_r, all_fields) + my_workflows.to_csv(outfile, index=False) + print(f'\tWorkflow list collected with a total of {my_workflows.shape[0]} workflows.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_workflows_r)) + print(f'\tWorkflow list collected with a total of {len(my_workflows_r)} workflows.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tWorkflow list saved to {outfile}') + + +@workflow.command('import') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=('The CloudOS url you are trying to access to. ' + + f'Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option("--workflow-name", help="The name that the workflow will have in CloudOS.", required=True) +@click.option("-w", "--workflow-url", help="URL of the workflow repository.", required=True) +@click.option("-d", "--workflow-docs-link", help="URL to the documentation of the workflow.", default='') +@click.option("--cost-limit", help="Cost limit for the workflow. Default: $30 USD.", default=30) +@click.option("--workflow-description", help="Workflow description", default="") +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name']) +def import_wf(ctx, + apikey, + cloudos_url, + workspace_id, + workflow_name, + workflow_url, + workflow_docs_link, + cost_limit, + workflow_description, + repository_platform, + disable_ssl_verification, + ssl_cert, + profile): + """ + Import workflows from supported repository providers. + """ + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + repo_import = ImportWorflow( + cloudos_url=cloudos_url, cloudos_apikey=apikey, workspace_id=workspace_id, platform=repository_platform, + workflow_name=workflow_name, workflow_url=workflow_url, workflow_docs_link=workflow_docs_link, + cost_limit=cost_limit, workflow_description=workflow_description, verify=verify_ssl + ) + workflow_id = repo_import.import_workflow() + print(f'\tWorkflow {workflow_name} was imported successfully with the ' + + f'following ID: {workflow_id}') + + +@project.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save project list. ' + + 'Default=project_list'), + default='project_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from projects or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--page', + help=('Response page to retrieve. Default=1.'), + type=int, + default=1) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_projects(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + page, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all projects from a CloudOS workspace in CSV format.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for projects in the following workspace: ' + + f'{workspace_id}') + # Check if the user provided the --page option + ctx = click.get_current_context() + if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + get_all = True + else: + get_all = False + if not isinstance(page, int) or page < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') + my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all) + if len(my_projects_r) == 0: + if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + print('A total of 0 projects collected. This is likely because your workspace ' + + 'has no projects created yet.') + else: + print('A total of 0 projects collected. This is likely because the --page you ' + + 'requested does not exist. Please, try a smaller number for --page or collect all the ' + + 'projects by not using --page parameter.') + elif output_format == 'csv': + my_projects = cl.process_project_list(my_projects_r, all_fields) + my_projects.to_csv(outfile, index=False) + print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_projects_r)) + print(f'\tProject list collected with a total of {len(my_projects_r)} projects.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tProject list saved to {outfile}') + + +@project.command('create') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--new-project', + help='The name for the new project.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def create_project(ctx, + apikey, + cloudos_url, + workspace_id, + new_project, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Create a new project in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + # verify ssl configuration + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Print basic output + if verbose: + print(f'\tUsing CloudOS URL: {cloudos_url}') + print(f'\tUsing workspace: {workspace_id}') + print(f'\tProject name: {new_project}') + + cl = Cloudos(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None) + + try: + project_id = cl.create_project(workspace_id, new_project, verify_ssl) + print(f'\tProject "{new_project}" created successfully with ID: {project_id}') + if verbose: + print(f'\tProject URL: {cloudos_url}/app/projects/{project_id}') + except Exception as e: + print(f'\tError creating project: {str(e)}') + sys.exit(1) + + +@cromwell.command('status') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_status(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Check Cromwell server status in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + print('Executing status...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tChecking Cromwell status in {workspace_id} workspace') + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + + +@cromwell.command('start') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to Cromwell restart. ' + + 'Default=300.'), + default=300) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_restart(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + wait_time, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Restart Cromwell server in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + action = 'restart' + print('Starting Cromwell server...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tStarting Cromwell server in {workspace_id} workspace') + cl.cromwell_switch(workspace_id, action, verify_ssl) + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + elapsed = 0 + while elapsed < wait_time and c_status_h != 'Running': + c_status_old = c_status_h + time.sleep(REQUEST_INTERVAL_CROMWELL) + elapsed += REQUEST_INTERVAL_CROMWELL + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + if c_status_h != c_status_old: + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h != 'Running': + print(f'\tYour current Cromwell status is: {c_status_h}. The ' + + f'selected wait-time of {wait_time} was exceeded. Please, ' + + 'consider to set a longer wait-time.') + print('\tTo further check your Cromwell status you can either go to ' + + f'{cloudos_url} or use the following command:\n' + + '\tcloudos cromwell status \\\n' + + f'\t\t--cloudos-url {cloudos_url} \\\n' + + '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + + f'\t\t--workspace-id {workspace_id}') + sys.exit(1) + + +@cromwell.command('stop') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_stop(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Stop Cromwell server in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + action = 'stop' + print('Stopping Cromwell server...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tStopping Cromwell server in {workspace_id} workspace') + cl.cromwell_switch(workspace_id, action, verify_ssl) + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + + +@queue.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save job queue list. ' + + 'Default=job_queue_list'), + default='job_queue_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from workflows or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_queues(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all available job queues from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + j_queue = Queue(cloudos_url, apikey, None, workspace_id, verify=verify_ssl) + my_queues = j_queue.get_job_queues() + if len(my_queues) == 0: + raise ValueError('No AWS batch queues found. Please, make sure that your CloudOS supports AWS bath queues') + if output_format == 'csv': + queues_processed = j_queue.process_queue_list(my_queues, all_fields) + queues_processed.to_csv(outfile, index=False) + print(f'\tJob queue list collected with a total of {queues_processed.shape[0]} queues.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_queues)) + print(f'\tJob queue list collected with a total of {len(my_queues)} queues.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tJob queue list saved to {outfile}') + + +@configure.command('list-profiles') +def list_profiles(): + config_manager = ConfigurationProfile() + config_manager.list_profiles() + + +@configure.command('remove-profile') +@click.option('--profile', + help='Name of the profile. Not using this option will lead to profile named "deafults" being generated', + required=True) +@click.pass_context +def remove_profile(ctx, profile): + update_command_context_from_click(ctx) + profile = profile or ctx.obj['profile'] + config_manager = ConfigurationProfile() + config_manager.remove_profile(profile) + + +@bash.command('job') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('--command', + help='The command to run in the bash job.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--cpus', + help='The number of CPUs to use for the task\'s master node. Default=1.', + type=int, + default=1) +@click.option('--memory', + help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', + type=int, + default=4) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + default='aws') +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run_bash_job(ctx, + apikey, + command, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + parameter, + job_name, + do_not_save_logs, + job_queue, + instance_type, + instance_disk, + cpus, + memory, + storage_mode, + lustre_size, + wait_completion, + wait_time, + repository_platform, + execution_platform, + cost_limit, + accelerate_saving_results, + request_interval, + disable_ssl_verification, + ssl_cert, + profile): + """Run a bash job in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + + if do_not_save_logs: + save_logs = False + else: + save_logs = True + + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=None, importsfile=None, + repository_platform=repository_platform, verify=verify_ssl, last=last) + + if job_queue is not None: + batch = True + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, + workspace_id=workspace_id, verify=verify_ssl) + # I have to add 'nextflow', other wise the job queue id is not found + job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, + job_queue=job_queue) + else: + job_queue_id = None + batch = False + j_id = j.send_job(job_config=None, + parameter=parameter, + git_commit=None, + git_tag=None, + git_branch=None, + job_name=job_name, + resumable=False, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + workflow_type='docker', + nextflow_profile=None, + nextflow_version=None, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=None, + cost_limit=cost_limit, + accelerate_saving_results=accelerate_saving_results, + verify=verify_ssl, + command={"command": command}, + cpus=cpus, + memory=memory) + + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=False, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') + + +@bash.command('array-job') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('--command', + help='The command to run in the bash job.') +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + + 'times as parameters you want to include. ' + + 'For parameters pointing to a file, the format expected is ' + + 'parameter_name=/Data/parameter_value. The parameter value must be a ' + + 'file located in the `Data` subfolder. If no is specified, it defaults to ' + + 'the project specified by the profile or --project-name parameter. ' + + 'E.g.: -p "--file=Data/file.txt" or "--file=/Data/folder/file.txt"')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--cpus', + help='The number of CPUs to use for the task\'s master node. Default=1.', + type=int, + default=1) +@click.option('--memory', + help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', + type=int, + default=4) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + type=click.Choice(['aws', 'azure', 'hpc']), + default='aws') +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--array-file', + help=('Path to a file containing an array of commands to run in the bash job.'), + default=None, + required=True) +@click.option('--separator', + help=('Separator to use in the array file. Default=",".'), + type=click.Choice([',', ';', 'tab', 'space', '|']), + default=",", + required=True) +@click.option('--list-columns', + help=('List columns present in the array file. ' + + 'This option will not run any job.'), + is_flag=True) +@click.option('--array-file-project', + help=('Name of the project in which the array file is placed, if different from --project-name.'), + default=None) +@click.option('--disable-column-check', + help=('Disable the check for the columns in the array file. ' + + 'This option is only used when --array-file is provided.'), + is_flag=True) +@click.option('-a', '--array-parameter', + multiple=True, + help=('A single parameter to pass to the job call only for specifying array columns. ' + + 'It should be in the following form: parameter_name=array_file_column_name. E.g.: ' + + '-a --test=value or -a -test=value or -a test=value or -a =value (for no prefix). ' + + 'You can use this option as many times as parameters you want to include.')) +@click.option('--custom-script-path', + help=('Path of a custom script to run in the bash array job instead of a command.'), + default=None) +@click.option('--custom-script-project', + help=('Name of the project to use when running the custom command script, if ' + + 'different than --project-name.'), + default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run_bash_array_job(ctx, + apikey, + command, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + parameter, + job_name, + do_not_save_logs, + job_queue, + instance_type, + instance_disk, + cpus, + memory, + storage_mode, + lustre_size, + wait_completion, + wait_time, + repository_platform, + execution_platform, + cost_limit, + accelerate_saving_results, + request_interval, + disable_ssl_verification, + ssl_cert, + profile, + array_file, + separator, + list_columns, + array_file_project, + disable_column_check, + array_parameter, + custom_script_path, + custom_script_project): + """Run a bash array job in CloudOS.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + if not list_columns and not (command or custom_script_path): + raise click.UsageError("Must provide --command or --custom-script-path if --list-columns is not set.") + + # when not set, use the global project name + if array_file_project is None: + array_file_project = project_name + + # this needs to be in another call to datasets, by default it uses the global project name + if custom_script_project is None: + custom_script_project = project_name + + # setup separators for API and array file (the're different) + separators = { + ",": {"api": ",", "file": ","}, + ";": {"api": "%3B", "file": ";"}, + "space": {"api": "+", "file": " "}, + "tab": {"api": "tab", "file": "tab"}, + "|": {"api": "%7C", "file": "|"} + } + + # setup important options for the job + if do_not_save_logs: + save_logs = False + else: + save_logs = True + + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=None, importsfile=None, + repository_platform=repository_platform, verify=verify_ssl, last=last) + + # retrieve columns + r = j.retrieve_cols_from_array_file( + array_file, + generate_datasets_for_project(cloudos_url, apikey, workspace_id, array_file_project, verify_ssl), + separators[separator]['api'], + verify_ssl + ) + + if not disable_column_check: + columns = json.loads(r.content).get("headers", None) + # pass this to the SEND JOB API call + # b'{"headers":[{"index":0,"name":"id"},{"index":1,"name":"title"},{"index":2,"name":"filename"},{"index":3,"name":"file2name"}]}' + if columns is None: + raise ValueError("No columns found in the array file metadata.") + if list_columns: + print("Columns: ") + for col in columns: + print(f"\t- {col['name']}") + return + else: + columns = [] + + # setup parameters for the job + cmd = j.setup_params_array_file( + custom_script_path, + generate_datasets_for_project(cloudos_url, apikey, workspace_id, custom_script_project, verify_ssl), + command, + separators[separator]['file'] + ) + + # check columns in the array file vs parameters added + if not disable_column_check and array_parameter: + print("\nChecking columns in the array file vs parameters added...\n") + for ap in array_parameter: + ap_split = ap.split('=') + ap_value = '='.join(ap_split[1:]) + for col in columns: + if col['name'] == ap_value: + print(f"Found column '{ap_value}' in the array file.") + break + else: + raise ValueError(f"Column '{ap_value}' not found in the array file. " + \ + f"Columns in array-file: {separator.join([col['name'] for col in columns])}") + + if job_queue is not None: + batch = True + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, + workspace_id=workspace_id, verify=verify_ssl) + # I have to add 'nextflow', other wise the job queue id is not found + job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, + job_queue=job_queue) + else: + job_queue_id = None + batch = False + + # send job + j_id = j.send_job(job_config=None, + parameter=parameter, + array_parameter=array_parameter, + array_file_header=columns, + git_commit=None, + git_tag=None, + git_branch=None, + job_name=job_name, + resumable=False, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + workflow_type='docker', + nextflow_profile=None, + nextflow_version=None, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=None, + cost_limit=cost_limit, + accelerate_saving_results=accelerate_saving_results, + verify=verify_ssl, + command=cmd, + cpus=cpus, + memory=memory) + + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=False, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') + + +@datasets.command(name="ls") +@click.argument("path", required=False, nargs=1) +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--details', + help=('When selected, it prints the details of the listed files. ' + + 'Details contains "Type", "Owner", "Size", "Last Updated", ' + + '"Virtual Name", "Storage Path".'), + is_flag=True) +@click.option('--output-format', + help=('The desired display for the output, either directly in standard output or saved as file. ' + + 'Default=stdout.'), + type=click.Choice(['stdout', 'csv'], case_sensitive=False), + default='stdout') +@click.option('--output-basename', + help=('Output file base name to save jobs details. ' + + 'Default=datasets_ls'), + default='datasets_ls', + required=False) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def list_files(ctx, + apikey, + cloudos_url, + workspace_id, + disable_ssl_verification, + ssl_cert, + project_name, + profile, + path, + details, + output_format, + output_basename): + """List contents of a path within a CloudOS workspace dataset.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + datasets = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = datasets.list_folder_content(path) + contents = result.get("contents") or result.get("datasets", []) + + if not contents: + contents = result.get("files", []) + result.get("folders", []) + + # Process items to extract data + processed_items = [] + for item in contents: + is_folder = "folderType" in item or item.get("isDir", False) + type_ = "folder" if is_folder else "file" + + # Enhanced type information + if is_folder: + folder_type = item.get("folderType") + if folder_type == "VirtualFolder": + type_ = "virtual folder" + elif folder_type == "S3Folder": + type_ = "s3 folder" + elif folder_type == "AzureBlobFolder": + type_ = "azure folder" + else: + type_ = "folder" + else: + # Check if file is managed by Lifebit (user uploaded) + is_managed_by_lifebit = item.get("isManagedByLifebit", False) + if is_managed_by_lifebit: + type_ = "file (user uploaded)" + else: + type_ = "file (virtual copy)" + + user = item.get("user", {}) + if isinstance(user, dict): + name = user.get("name", "").strip() + surname = user.get("surname", "").strip() + else: + name = surname = "" + if name and surname: + owner = f"{name} {surname}" + elif name: + owner = name + elif surname: + owner = surname + else: + owner = "-" + + raw_size = item.get("sizeInBytes", item.get("size")) + size = format_bytes(raw_size) if not is_folder and raw_size is not None else "-" + + updated = item.get("updatedAt") or item.get("lastModified", "-") + filepath = item.get("name", "-") + + if item.get("fileType") == "S3File" or item.get("folderType") == "S3Folder": + bucket = item.get("s3BucketName") + key = item.get("s3ObjectKey") or item.get("s3Prefix") + storage_path = f"s3://{bucket}/{key}" if bucket and key else "-" + elif item.get("fileType") == "AzureBlobFile" or item.get("folderType") == "AzureBlobFolder": + account = item.get("blobStorageAccountName") + container = item.get("blobContainerName") + key = item.get("blobName") if item.get("fileType") == "AzureBlobFile" else item.get("blobPrefix") + storage_path = f"az://{account}.blob.core.windows.net/{container}/{key}" if account and container and key else "-" + else: + storage_path = "-" + + processed_items.append({ + 'type': type_, + 'owner': owner, + 'size': size, + 'raw_size': raw_size, + 'updated': updated, + 'name': filepath, + 'storage_path': storage_path, + 'is_folder': is_folder + }) + + # Output handling + if output_format == 'csv': + import csv + + csv_filename = f'{output_basename}.csv' + + if details: + # CSV with all details + with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: + fieldnames = ['Type', 'Owner', 'Size', 'Size (bytes)', 'Last Updated', 'Virtual Name', 'Storage Path'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for item in processed_items: + writer.writerow({ + 'Type': item['type'], + 'Owner': item['owner'], + 'Size': item['size'], + 'Size (bytes)': item['raw_size'] if item['raw_size'] is not None else '', + 'Last Updated': item['updated'], + 'Virtual Name': item['name'], + 'Storage Path': item['storage_path'] + }) + else: + # CSV with just names + with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(['Name', 'Storage Path']) + for item in processed_items: + writer.writerow([item['name'], item['storage_path']]) + + click.secho(f'\nDatasets list saved to: {csv_filename}', fg='green', bold=True) + + else: # stdout + if details: + console = Console(width=None) + table = Table(show_header=True, header_style="bold white") + table.add_column("Type", style="cyan", no_wrap=True) + table.add_column("Owner", style="white") + table.add_column("Size", style="magenta") + table.add_column("Last Updated", style="green") + table.add_column("Virtual Name", style="bold", overflow="fold") + table.add_column("Storage Path", style="dim", no_wrap=False, overflow="fold", ratio=2) + + for item in processed_items: + style = Style(color="blue", underline=True) if item['is_folder'] else None + table.add_row( + item['type'], + item['owner'], + item['size'], + item['updated'], + item['name'], + item['storage_path'], + style=style + ) + + console.print(table) + + else: + console = Console() + for item in processed_items: + if item['is_folder']: + console.print(f"[blue underline]{item['name']}[/]") + else: + console.print(item['name']) + + except Exception as e: + raise ValueError(f"Failed to list files for project '{project_name}'. {str(e)}") + + +@datasets.command(name="mv") +@click.argument("source_path", required=True) +@click.argument("destination_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The source project name.') +@click.option('--destination-project-name', required=False, + help='The destination project name. Defaults to the source project.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def move_files(ctx, source_path, destination_path, apikey, cloudos_url, workspace_id, + project_name, destination_project_name, + disable_ssl_verification, ssl_cert, profile): + """ + Move a file or folder from a source path to a destination path within or across CloudOS projects. + + SOURCE_PATH [path]: the full path to the file or folder to move. It must be a 'Data' folder path. + E.g.: 'Data/folderA/file.txt'\n + DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. + E.g.: 'Data/folderB' + """ + # Validate destination constraint + if not destination_path.strip("/").startswith("Data/") and destination_path.strip("/") != "Data": + raise ValueError("Destination path must begin with 'Data/' or be 'Data'.") + if not source_path.strip("/").startswith("Data/") and source_path.strip("/") != "Data": + raise ValueError("SOURCE_PATH must start with 'Data/' or be 'Data'.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + destination_project_name = destination_project_name or project_name + # Initialize Datasets clients + source_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + dest_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=destination_project_name, + verify=verify_ssl, + cromwell_token=None + ) + print('Checking source path') + # === Resolve Source Item === + source_parts = source_path.strip("/").split("/") + source_parent_path = "/".join(source_parts[:-1]) if len(source_parts) > 1 else None + source_item_name = source_parts[-1] + + try: + source_contents = source_client.list_folder_content(source_parent_path) + except Exception as e: + raise ValueError(f"Could not resolve source path '{source_path}'. {str(e)}") + + found_source = None + for collection in ["files", "folders"]: + for item in source_contents.get(collection, []): + if item.get("name") == source_item_name: + found_source = item + break + if found_source: + break + if not found_source: + raise ValueError(f"Item '{source_item_name}' not found in '{source_parent_path or '[project root]'}'") + + source_id = found_source["_id"] + source_kind = "Folder" if "folderType" in found_source else "File" + print("Checking destination path") + # === Resolve Destination Folder === + dest_parts = destination_path.strip("/").split("/") + dest_folder_name = dest_parts[-1] + dest_parent_path = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else None + + try: + dest_contents = dest_client.list_folder_content(dest_parent_path) + match = next((f for f in dest_contents.get("folders", []) if f.get("name") == dest_folder_name), None) + if not match: + raise ValueError(f"Could not resolve destination folder '{destination_path}'") + + target_id = match["_id"] + folder_type = match.get("folderType") + # Normalize kind: top-level datasets are kind=Dataset, all other folders are kind=Folder + if folder_type in ("VirtualFolder", "Folder"): + target_kind = "Folder" + elif folder_type == "S3Folder": + raise ValueError(f"Unable to move item '{source_item_name}' to '{destination_path}'. " + + "The destination is an S3 folder, and only virtual folders can be selected as valid move destinations.") + elif isinstance(folder_type, bool) and folder_type: # legacy dataset structure + target_kind = "Dataset" + else: + raise ValueError(f"Unrecognized folderType '{folder_type}' for destination '{destination_path}'") + + except Exception as e: + raise ValueError(f"Could not resolve destination path '{destination_path}'. {str(e)}") + print(f"Moving {source_kind} '{source_item_name}' to '{destination_path}' " + + f"in project '{destination_project_name} ...") + # === Perform Move === + try: + response = source_client.move_files_and_folders( + source_id=source_id, + source_kind=source_kind, + target_id=target_id, + target_kind=target_kind + ) + if response.ok: + click.secho(f"{source_kind} '{source_item_name}' moved to '{destination_path}' " + + f"in project '{destination_project_name}'.", fg="green", bold=True) + else: + raise ValueError(f"Move failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Move operation failed. {str(e)}") + + +@datasets.command(name="rename") +@click.argument("source_path", required=True) +@click.argument("new_name", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def renaming_item(ctx, + source_path, + new_name, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Rename a file or folder in a CloudOS project. + + SOURCE_PATH [path]: the full path to the file or folder to rename. It must be a 'Data' folder path. + E.g.: 'Data/folderA/old_name.txt'\n + NEW_NAME [name]: the new name to assign to the file or folder. E.g.: 'new_name.txt' + """ + if not source_path.strip("/").startswith("Data/"): + raise ValueError("SOURCE_PATH must start with 'Data/', pointing to a file or folder in that dataset.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + parts = source_path.strip("/").split("/") + + parent_path = "/".join(parts[:-1]) + target_name = parts[-1] + + try: + contents = client.list_folder_content(parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") + + # Search for file/folder + found_item = None + for category in ["files", "folders"]: + for item in contents.get(category, []): + if item.get("name") == target_name: + found_item = item + break + if found_item: + break + + if not found_item: + raise ValueError(f"Item '{target_name}' not found in '{parent_path or '[project root]'}'") + + item_id = found_item["_id"] + kind = "Folder" if "folderType" in found_item else "File" + + print(f"Renaming {kind} '{target_name}' to '{new_name}'...") + try: + response = client.rename_item(item_id=item_id, new_name=new_name, kind=kind) + if response.ok: + click.secho( + f"{kind} '{target_name}' renamed to '{new_name}' in folder '{parent_path}'.", + fg="green", + bold=True + ) + else: + raise ValueError(f"Rename failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Rename operation failed. {str(e)}") + + +@datasets.command(name="cp") +@click.argument("source_path", required=True) +@click.argument("destination_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The source project name.') +@click.option('--destination-project-name', required=False, help='The destination project name. Defaults to the source project.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def copy_item_cli(ctx, + source_path, + destination_path, + apikey, + cloudos_url, + workspace_id, + project_name, + destination_project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Copy a file or folder (S3 or virtual) from SOURCE_PATH to DESTINATION_PATH. + + SOURCE_PATH [path]: the full path to the file or folder to copy. + E.g.: AnalysesResults/my_analysis/results/my_plot.png\n + DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. + E.g.: Data/plots + """ + destination_project_name = destination_project_name or project_name + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + # Initialize clients + source_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + dest_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=destination_project_name, + verify=verify_ssl, + cromwell_token=None + ) + # Validate paths + dest_parts = destination_path.strip("/").split("/") + if not dest_parts or dest_parts[0] != "Data": + raise ValueError("DESTINATION_PATH must start with 'Data/'.") + # Parse source and destination + source_parts = source_path.strip("/").split("/") + source_parent = "/".join(source_parts[:-1]) if len(source_parts) > 1 else "" + source_name = source_parts[-1] + dest_folder_name = dest_parts[-1] + dest_parent = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else "" + try: + source_content = source_client.list_folder_content(source_parent) + dest_content = dest_client.list_folder_content(dest_parent) + except Exception as e: + raise ValueError(f"Could not access paths. {str(e)}") + # Find the source item + source_item = None + for item in source_content.get('files', []) + source_content.get('folders', []): + if item.get("name") == source_name: + source_item = item + break + if not source_item: + raise ValueError(f"Item '{source_name}' not found in '{source_parent or '[project root]'}'") + # Find the destination folder + destination_folder = None + for folder in dest_content.get("folders", []): + if folder.get("name") == dest_folder_name: + destination_folder = folder + break + if not destination_folder: + raise ValueError(f"Destination folder '{destination_path}' not found.") + try: + # Determine item type + if "fileType" in source_item: + item_type = "file" + elif source_item.get("folderType") == "VirtualFolder": + item_type = "virtual_folder" + elif "s3BucketName" in source_item and source_item.get("folderType") == "S3Folder": + item_type = "s3_folder" + else: + raise ValueError("Could not determine item type.") + print(f"Copying {item_type.replace('_', ' ')} '{source_name}' to '{destination_path}'...") + if destination_folder.get("folderType") is True and destination_folder.get("kind") in ("Data", "Cohorts", "AnalysesResults"): + destination_kind = "Dataset" + elif destination_folder.get("folderType") == "S3Folder": + raise ValueError(f"Unable to copy item '{source_name}' to '{destination_path}'. The destination is an S3 folder, and only virtual folders can be selected as valid copy destinations.") + else: + destination_kind = "Folder" + response = source_client.copy_item( + item=source_item, + destination_id=destination_folder["_id"], + destination_kind=destination_kind + ) + if response.ok: + click.secho("Item copied successfully.", fg="green", bold=True) + else: + raise ValueError(f"Copy failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Copy operation failed. {str(e)}") + + +@datasets.command(name="mkdir") +@click.argument("new_folder_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def mkdir_item(ctx, + new_folder_path, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Create a virtual folder in a CloudOS project. + + NEW_FOLDER_PATH [path]: Full path to the new folder including its name. Must start with 'Data'. + """ + new_folder_path = new_folder_path.strip("/") + if not new_folder_path.startswith("Data"): + raise ValueError("NEW_FOLDER_PATH must start with 'Data'.") + + path_parts = new_folder_path.split("/") + if len(path_parts) < 2: + raise ValueError("NEW_FOLDER_PATH must include at least a parent folder and the new folder name.") + + parent_path = "/".join(path_parts[:-1]) + folder_name = path_parts[-1] + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + # Split parent path to get its parent + name + parent_parts = parent_path.split("/") + parent_name = parent_parts[-1] + parent_of_parent_path = "/".join(parent_parts[:-1]) + + # List the parent of the parent + try: + contents = client.list_folder_content(parent_of_parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_of_parent_path}'. {str(e)}") + + # Find the parent folder in the contents + folder_info = next( + (f for f in contents.get("folders", []) if f.get("name") == parent_name), + None + ) + + if not folder_info: + raise ValueError(f"Could not find folder '{parent_name}' in '{parent_of_parent_path}'.") + + parent_id = folder_info.get("_id") + folder_type = folder_info.get("folderType") + + if folder_type is True: + parent_kind = "Dataset" + elif isinstance(folder_type, str): + parent_kind = "Folder" + else: + raise ValueError(f"Unrecognized folderType for '{parent_path}'.") + + # Create the folder + print(f"Creating folder '{folder_name}' under '{parent_path}' ({parent_kind})...") + try: + response = client.create_virtual_folder(name=folder_name, parent_id=parent_id, parent_kind=parent_kind) + if response.ok: + click.secho(f"Folder '{folder_name}' created under '{parent_path}'", fg="green", bold=True) + else: + raise ValueError(f"Folder creation failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Folder creation failed. {str(e)}") + + +@datasets.command(name="rm") +@click.argument("target_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.option('--force', is_flag=True, help='Force delete files. Required when deleting user uploaded files. This may also delete the file from the cloud provider storage.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def rm_item(ctx, + target_path, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile, + force): + """ + Delete a file or folder in a CloudOS project. + + TARGET_PATH [path]: the full path to the file or folder to delete. Must start with 'Data'. \n + E.g.: 'Data/folderA/file.txt' or 'Data/my_analysis/results/folderB' + """ + if not target_path.strip("/").startswith("Data/"): + raise ValueError("TARGET_PATH must start with 'Data/', pointing to a file or folder.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + parts = target_path.strip("/").split("/") + parent_path = "/".join(parts[:-1]) + item_name = parts[-1] + + try: + contents = client.list_folder_content(parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") + + found_item = None + for item in contents.get('files', []) + contents.get('folders', []): + if item.get("name") == item_name: + found_item = item + break + + if not found_item: + raise ValueError(f"Item '{item_name}' not found in '{parent_path or '[project root]'}'") + + item_id = found_item.get("_id", '') + kind = "Folder" if "folderType" in found_item else "File" + if item_id == '': + raise ValueError(f"Item '{item_name}' could not be removed as the parent folder is an s3 folder and their content cannot be modified.") + # Check if the item is managed by Lifebit + is_managed_by_lifebit = found_item.get("isManagedByLifebit", False) + if is_managed_by_lifebit and not force: + raise ValueError("By removing this file, it will be permanently deleted. If you want to go forward, please use the --force flag.") + print(f"Removing {kind} '{item_name}' from '{parent_path or '[root]'}'...") + try: + response = client.delete_item(item_id=item_id, kind=kind) + if response.ok: + if is_managed_by_lifebit: + click.secho( + f"{kind} '{item_name}' was permanently deleted from '{parent_path or '[root]'}'.", + fg="green", bold=True + ) + else: + click.secho( + f"{kind} '{item_name}' was removed from '{parent_path or '[root]'}'.", + fg="green", bold=True + ) + click.secho("This item will still be available on your Cloud Provider.", fg="yellow") + else: + raise ValueError(f"Removal failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Remove operation failed. {str(e)}") + + +@datasets.command(name="link") +@click.argument("path", required=True) +@click.option('-k', '--apikey', help='Your CloudOS API key', required=True) +@click.option('-c', '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=False) +@click.option('--workspace-id', help='The specific CloudOS workspace id.', required=True) +@click.option('--session-id', help='The specific CloudOS interactive session id.', required=True) +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default='default') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) +def link(ctx, + path, + apikey, + cloudos_url, + project_name, + workspace_id, + session_id, + disable_ssl_verification, + ssl_cert, + profile): + """ + Link a folder (S3 or File Explorer) to an active interactive analysis. + + PATH [path]: the full path to the S3 folder to link or relative to File Explorer. + E.g.: 's3://bucket-name/folder/subfolder', 'Data/Downloads' or 'Data'. + """ + if not path.startswith("s3://") and project_name is None: + # for non-s3 paths we need the project, for S3 we don't + raise click.UsageError("When using File Explorer paths '--project-name' needs to be defined") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + link_p = Link( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + cromwell_token=None, + project_name=project_name, + verify=verify_ssl + ) + + # Minimal folder validation and improved error messages + is_s3 = path.startswith("s3://") + is_folder = True + if is_s3: + # S3 path validation - use heuristics to determine if it's likely a folder + try: + # If path ends with '/', it's likely a folder + if path.endswith('/'): + is_folder = True + else: + # Check the last part of the path + path_parts = path.rstrip("/").split("/") + if path_parts: + last_part = path_parts[-1] + # If the last part has no dot, it's likely a folder + if '.' not in last_part: + is_folder = True + else: + # If it has a dot, it might be a file - set to None for warning + is_folder = None + else: + # Empty path parts, set to None for uncertainty + is_folder = None + except Exception: + # If we can't parse the S3 path, set to None for uncertainty + is_folder = None + else: + # File Explorer path validation (existing logic) + try: + datasets = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + parts = path.strip("/").split("/") + parent_path = "/".join(parts[:-1]) if len(parts) > 1 else "" + item_name = parts[-1] + contents = datasets.list_folder_content(parent_path) + found = None + for item in contents.get("folders", []): + if item.get("name") == item_name: + found = item + break + if not found: + for item in contents.get("files", []): + if item.get("name") == item_name: + found = item + break + if found and ("folderType" not in found): + is_folder = False + except Exception: + is_folder = None + + if is_folder is False: + if is_s3: + raise ValueError("The S3 path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.") + else: + raise ValueError("Linking files or virtual folders is not supported. Link the S3 parent folder instead.", err=True) + return + elif is_folder is None and is_s3: + click.secho("Unable to verify whether the S3 path is a folder. Proceeding with linking; " + + "however, if the operation fails, please confirm that you are linking a folder rather than a file.", fg='yellow', bold=True) + + try: + link_p.link_folder(path, session_id) + except Exception as e: + if is_s3: + print("If you are linking an S3 path, please ensure it is a folder.") + raise ValueError(f"Could not link folder. {e}") + + +@images.command(name="ls") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--page', help='The response page. Defaults to 1.', required=False, default=1) +@click.option('--limit', help='The page size limit. Defaults to 10', required=False, default=10) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def list_images(ctx, + apikey, + cloudos_url, + procurement_id, + disable_ssl_verification, + ssl_cert, + profile, + page, + limit): + """List images associated with organisations of a given procurement.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None, + page=page, + limit=limit + ) + + try: + result = procurement_images.list_procurement_images() + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") + + +@images.command(name="set") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) +@click.option('--image-type', help='The CloudOS resource image type.', required=True, + type=click.Choice([ + 'RegularInteractiveSessions', + 'SparkInteractiveSessions', + 'RStudioInteractiveSessions', + 'JupyterInteractiveSessions', + 'JobDefault', + 'NextflowBatchComputeEnvironment'])) +@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') +@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) +@click.option('--image-id', help='The new image id value.', required=True) +@click.option('--image-name', help='The new image name value.', required=False) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def set_organisation_image(ctx, + apikey, + cloudos_url, + procurement_id, + organisation_id, + image_type, + provider, + region, + image_id, + image_name, + disable_ssl_verification, + ssl_cert, + profile): + """Set a new image id or name to image associated with an organisations of a given procurement.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = procurement_images.set_procurement_organisation_image( + organisation_id, + image_type, + provider, + region, + image_id, + image_name + ) + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") + + +@images.command(name="reset") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) +@click.option('--image-type', help='The CloudOS resource image type.', required=True, + type=click.Choice([ + 'RegularInteractiveSessions', + 'SparkInteractiveSessions', + 'RStudioInteractiveSessions', + 'JupyterInteractiveSessions', + 'JobDefault', + 'NextflowBatchComputeEnvironment'])) +@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') +@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def reset_organisation_image(ctx, + apikey, + cloudos_url, + procurement_id, + organisation_id, + image_type, + provider, + region, + disable_ssl_verification, + ssl_cert, + profile): + """Reset image associated with an organisations of a given procurement to CloudOS defaults.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = procurement_images.reset_procurement_organisation_image( + organisation_id, + image_type, + provider, + region + ) + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") + +@run_cloudos_cli.command('link') +@click.argument('path', required=False) +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS. When provided, links results, workdir and logs by default.', + required=False) +@click.option('--project-name', + help='The name of a CloudOS project. Required for File Explorer paths.', + required=False) +@click.option('--results', + help='Link only results folder (only works with --job-id).', + is_flag=True) +@click.option('--workdir', + help='Link only working directory (only works with --job-id).', + is_flag=True) +@click.option('--logs', + help='Link only logs folder (only works with --job-id).', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) +def link_command(ctx, + path, + apikey, + cloudos_url, + workspace_id, + session_id, + job_id, + project_name, + results, + workdir, + logs, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """ + Link folders to an interactive analysis session. + + This command is used to link folders + to an active interactive analysis session for direct access to data. + + PATH: Optional path to link (S3). + Required if --job-id is not provided. + + Two modes of operation: + + 1. Job-based linking (--job-id): Links job-related folders. + By default, links results, workdir, and logs folders. + Use --results, --workdir, or --logs flags to link only specific folders. + + 2. Direct path linking (PATH argument): Links a specific S3 path. + + Examples: + + # Link all job folders (results, workdir, logs) + cloudos link --job-id 12345 --session-id abc123 + + # Link only results from a job + cloudos link --job-id 12345 --session-id abc123 --results + + # Link a specific S3 path + cloudos link s3://bucket/folder --session-id abc123 + + """ + print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Validate input parameters + if not job_id and not path: + raise click.UsageError("Either --job-id or PATH argument must be provided.") + + if job_id and path: + raise click.UsageError("Cannot use both --job-id and PATH argument. Please provide only one.") + + # Validate folder-specific flags only work with --job-id + if (results or workdir or logs) and not job_id: + raise click.UsageError("--results, --workdir, and --logs flags can only be used with --job-id.") + + # If no specific folders are selected with job-id, link all by default + if job_id and not (results or workdir or logs): + results = True + workdir = True + logs = True + + if verbose: + print('Using the following parameters:') + print(f'\tCloudOS url: {cloudos_url}') + print(f'\tWorkspace ID: {workspace_id}') + print(f'\tSession ID: {session_id}') + if job_id: + print(f'\tJob ID: {job_id}') + print(f'\tLink results: {results}') + print(f'\tLink workdir: {workdir}') + print(f'\tLink logs: {logs}') + else: + print(f'\tPath: {path}') + + # Initialize Link client + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl + ) + + try: + if job_id: + # Job-based linking + print(f'Linking folders from job {job_id} to interactive session {session_id}...\n') + + # Link results + if results: + link_client.link_job_results(job_id, workspace_id, session_id, verify_ssl, verbose) + + # Link workdir + if workdir: + link_client.link_job_workdir(job_id, workspace_id, session_id, verify_ssl, verbose) + + # Link logs + if logs: + link_client.link_job_logs(job_id, workspace_id, session_id, verify_ssl, verbose) + + + else: + # Direct path linking + print(f'Linking path to interactive session {session_id}...\n') + + # Link path with validation + link_client.link_path_with_validation(path, session_id, verify_ssl, project_name, verbose) + + print('\nLinking operation completed.') + + except BadRequestException as e: + raise ValueError(f"Request failed: {str(e)}") + except Exception as e: + raise ValueError(f"Failed to link folder(s): {str(e)}") + +if __name__ == "__main__": + # Setup logging + debug_mode = '--debug' in sys.argv + setup_logging(debug_mode) + logger = logging.getLogger("CloudOS") + # Check if debug flag was passed (fallback for cases where Click doesn't handle it) + try: + run_cloudos_cli() + except Exception as e: + if debug_mode: + logger.error(e, exc_info=True) + traceback.print_exc() + else: + logger.error(e) + click.echo(click.style(f"Error: {e}", fg='red'), err=True) + sys.exit(1) \ No newline at end of file diff --git a/cloudos_cli/__main__.py.before_refactor b/cloudos_cli/__main__.py.before_refactor new file mode 100644 index 00000000..c9804d3e --- /dev/null +++ b/cloudos_cli/__main__.py.before_refactor @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + +import rich_click as click +import sys +import logging +from ._version import __version__ +from rich.console import Console +from cloudos_cli.logging.logger import setup_logging, update_command_context_from_click +from cloudos_cli.configure.configure import ( + build_default_map_for_group, + get_shared_config +) + +# Import all command groups from their cli modules +from cloudos_cli.jobs.cli import job +from cloudos_cli.workflows.cli import workflow +from cloudos_cli.projects.cli import project +from cloudos_cli.cromwell.cli import cromwell +from cloudos_cli.queue.cli import queue +from cloudos_cli.bash.cli import bash +from cloudos_cli.procurement.cli import procurement +from cloudos_cli.datasets.cli import datasets +from cloudos_cli.configure.cli import configure + + +# GLOBAL VARS - Keep these for backward compatibility +JOB_COMPLETED = 'completed' +REQUEST_INTERVAL_CROMWELL = 30 +AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] +AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] +HPC_NEXTFLOW_VERSIONS = ['22.10.8'] +AWS_NEXTFLOW_LATEST = '24.04.4' +AZURE_NEXTFLOW_LATEST = '22.11.1-edge' +HPC_NEXTFLOW_LATEST = '22.10.8' +ABORT_JOB_STATES = ['running', 'initializing'] + +# Global debug state +_global_debug = False + + +def custom_exception_handler(exc_type, exc_value, exc_traceback): + """Custom exception handler that respects debug mode""" + console = Console(stderr=True) + # Initialise logger + debug_mode = '--debug' in sys.argv + setup_logging(debug_mode) + logger = logging.getLogger("CloudOS") + if get_debug_mode(): + logger.error(exc_value, exc_info=exc_value) + console.print("[yellow]Debug mode: showing full traceback[/yellow]") + sys.__excepthook__(exc_type, exc_value, exc_traceback) + else: + # Extract a clean error message + if hasattr(exc_value, 'message'): + error_msg = exc_value.message + elif str(exc_value): + error_msg = str(exc_value) + else: + error_msg = f"{exc_type.__name__}" + logger.error(exc_value) + console.print(f"[bold red]Error: {error_msg}[/bold red]") + + # For network errors, give helpful context + if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): + console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") + +# Install the custom exception handler +sys.excepthook = custom_exception_handler + + +def pass_debug_to_subcommands(group_cls=click.RichGroup): + """Custom Group class that passes --debug option to all subcommands""" + + class DebugGroup(group_cls): + def add_command(self, cmd, name=None): + # Add debug option to the command if it doesn't already have it + if isinstance(cmd, (click.Command, click.Group)): + has_debug = any(param.name == 'debug' for param in cmd.params) + if not has_debug: + debug_option = click.Option( + ['--debug'], + is_flag=True, + help='Show detailed error information and tracebacks', + is_eager=True, + expose_value=False, + callback=self._debug_callback + ) + cmd.params.insert(-1, debug_option) # Insert at the end for precedence + + super().add_command(cmd, name) + + def _debug_callback(self, ctx, param, value): + """Callback to handle debug flag""" + global _global_debug + if value: + _global_debug = True + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value + + return DebugGroup + + +def get_debug_mode(): + """Get current debug mode state""" + return _global_debug + + +# Helper function for debug setup +def _setup_debug(ctx, param, value): + """Setup debug mode globally and in context""" + global _global_debug + _global_debug = value + if value: + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value + + +@click.group(cls=pass_debug_to_subcommands()) +@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', + is_eager=True, expose_value=False, callback=_setup_debug) +@click.version_option(__version__) +@click.pass_context +def run_cloudos_cli(ctx): + """CloudOS python package: a package for interacting with CloudOS.""" + update_command_context_from_click(ctx) + ctx.ensure_object(dict) + + if ctx.invoked_subcommand not in ['datasets']: + print(run_cloudos_cli.__doc__ + '\n') + print('Version: ' + __version__ + '\n') + + # Load shared configuration (handles missing profiles and fields gracefully) + shared_config = get_shared_config() + + # Automatically build default_map from registered commands + ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) + + +# Register all command groups +run_cloudos_cli.add_command(job) +run_cloudos_cli.add_command(workflow) +run_cloudos_cli.add_command(project) +run_cloudos_cli.add_command(cromwell) +run_cloudos_cli.add_command(queue) +run_cloudos_cli.add_command(bash) +run_cloudos_cli.add_command(procurement) +run_cloudos_cli.add_command(datasets) +run_cloudos_cli.add_command(configure) + + +if __name__ == '__main__': + run_cloudos_cli() diff --git a/cloudos_cli/__main__.py.old b/cloudos_cli/__main__.py.old new file mode 100644 index 00000000..1eab8bfd --- /dev/null +++ b/cloudos_cli/__main__.py.old @@ -0,0 +1,4367 @@ +#!/usr/bin/env python3 + +import rich_click as click +import cloudos_cli.jobs.job as jb +from cloudos_cli.clos import Cloudos +from cloudos_cli.import_wf.import_wf import ImportWorflow +from cloudos_cli.queue.queue import Queue +from cloudos_cli.utils.errors import BadRequestException +import json +import time +import sys +import traceback +import copy +from ._version import __version__ +from cloudos_cli.configure.configure import ConfigurationProfile +from rich.console import Console +from rich.table import Table +from cloudos_cli.datasets import Datasets +from cloudos_cli.procurement import Images +from cloudos_cli.utils.resources import ssl_selector, format_bytes +from rich.style import Style +from cloudos_cli.utils.array_job import generate_datasets_for_project +from cloudos_cli.utils.details import create_job_details, create_job_list_table +from cloudos_cli.link import Link +from cloudos_cli.cost.cost import CostViewer +from cloudos_cli.logging.logger import setup_logging, update_command_context_from_click +import logging +from cloudos_cli.configure.configure import ( + with_profile_config, + build_default_map_for_group, + get_shared_config, + CLOUDOS_URL +) +from cloudos_cli.related_analyses.related_analyses import related_analyses + + +# GLOBAL VARS +JOB_COMPLETED = 'completed' +REQUEST_INTERVAL_CROMWELL = 30 +AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] +AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] +HPC_NEXTFLOW_VERSIONS = ['22.10.8'] +AWS_NEXTFLOW_LATEST = '24.04.4' +AZURE_NEXTFLOW_LATEST = '22.11.1-edge' +HPC_NEXTFLOW_LATEST = '22.10.8' +ABORT_JOB_STATES = ['running', 'initializing'] + + +def custom_exception_handler(exc_type, exc_value, exc_traceback): + """Custom exception handler that respects debug mode""" + console = Console(stderr=True) + # Initialise logger + debug_mode = '--debug' in sys.argv + setup_logging(debug_mode) + logger = logging.getLogger("CloudOS") + if get_debug_mode(): + logger.error(exc_value, exc_info=exc_value) + console.print("[yellow]Debug mode: showing full traceback[/yellow]") + sys.__excepthook__(exc_type, exc_value, exc_traceback) + else: + # Extract a clean error message + if hasattr(exc_value, 'message'): + error_msg = exc_value.message + elif str(exc_value): + error_msg = str(exc_value) + else: + error_msg = f"{exc_type.__name__}" + logger.error(exc_value) + console.print(f"[bold red]Error: {error_msg}[/bold red]") + + # For network errors, give helpful context + if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): + console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") + +# Install the custom exception handler +sys.excepthook = custom_exception_handler + + +def pass_debug_to_subcommands(group_cls=click.RichGroup): + """Custom Group class that passes --debug option to all subcommands""" + + class DebugGroup(group_cls): + def add_command(self, cmd, name=None): + # Add debug option to the command if it doesn't already have it + if isinstance(cmd, (click.Command, click.Group)): + has_debug = any(param.name == 'debug' for param in cmd.params) + if not has_debug: + debug_option = click.Option( + ['--debug'], + is_flag=True, + help='Show detailed error information and tracebacks', + is_eager=True, + expose_value=False, + callback=self._debug_callback + ) + cmd.params.insert(-1, debug_option) # Insert at the end for precedence + + super().add_command(cmd, name) + + def _debug_callback(self, ctx, param, value): + """Callback to handle debug flag""" + global _global_debug + if value: + _global_debug = True + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value + + return DebugGroup + + +def get_debug_mode(): + """Get current debug mode state""" + return _global_debug + + +# Helper function for debug setup +def _setup_debug(ctx, param, value): + """Setup debug mode globally and in context""" + global _global_debug + _global_debug = value + if value: + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value + + +@click.group(cls=pass_debug_to_subcommands()) +@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', + is_eager=True, expose_value=False, callback=_setup_debug) +@click.version_option(__version__) +@click.pass_context +def run_cloudos_cli(ctx): + """CloudOS python package: a package for interacting with CloudOS.""" + update_command_context_from_click(ctx) + ctx.ensure_object(dict) + + if ctx.invoked_subcommand not in ['datasets']: + print(run_cloudos_cli.__doc__ + '\n') + print('Version: ' + __version__ + '\n') + + # Load shared configuration (handles missing profiles and fields gracefully) + shared_config = get_shared_config() + + # Automatically build default_map from registered commands + ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def job(): + """CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS.""" + print(job.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def workflow(): + """CloudOS workflow functionality: list and import workflows.""" + print(workflow.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def project(): + """CloudOS project functionality: list and create projects in CloudOS.""" + print(project.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def cromwell(): + """Cromwell server functionality: check status, start and stop.""" + print(cromwell.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def queue(): + """CloudOS job queue functionality.""" + print(queue.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def bash(): + """CloudOS bash functionality.""" + print(bash.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +def procurement(): + """CloudOS procurement functionality.""" + print(procurement.__doc__ + '\n') + + +@procurement.group(cls=pass_debug_to_subcommands()) +def images(): + """CloudOS procurement images functionality.""" + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) +@click.pass_context +def datasets(ctx): + """CloudOS datasets functionality.""" + update_command_context_from_click(ctx) + if ctx.args and ctx.args[0] != 'ls': + print(datasets.__doc__ + '\n') + + +@run_cloudos_cli.group(cls=pass_debug_to_subcommands(), invoke_without_command=True) +@click.option('--profile', help='Profile to use from the config file', default='default') +@click.option('--make-default', + is_flag=True, + help='Make the profile the default one.') +@click.pass_context +def configure(ctx, profile, make_default): + """CloudOS configuration.""" + print(configure.__doc__ + '\n') + update_command_context_from_click(ctx) + profile = profile or ctx.obj['profile'] + config_manager = ConfigurationProfile() + + if ctx.invoked_subcommand is None and profile == "default" and not make_default: + config_manager.create_profile_from_input(profile_name="default") + + if profile != "default" and not make_default: + config_manager.create_profile_from_input(profile_name=profile) + if make_default: + config_manager.make_default_profile(profile_name=profile) + + +@job.command('run', cls=click.RichCommand) +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('--job-config', + help=('A config file similar to a nextflow.config file, ' + + 'but only with the parameters to use with your job.')) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p input=s3://path_to_my_file. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--nextflow-profile', + help=('A comma separated string indicating the nextflow profile/s ' + + 'to use with your job.')) +@click.option('--nextflow-version', + help=('Nextflow version to use when executing the workflow in CloudOS. ' + + 'Default=22.10.8.'), + type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']), + default='22.10.8') +@click.option('--git-commit', + help=('The git commit hash to run for ' + + 'the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--git-tag', + help=('The tag to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--git-branch', + help=('The branch to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--resumable', + help='Whether to make the job able to be resumed or not.', + is_flag=True) +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--wdl-mainfile', + help='For WDL workflows, which mainFile (.wdl) is configured to use.',) +@click.option('--wdl-importsfile', + help='For WDL workflows, which importsFile (.zip) is configured to use.',) +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. Currently, not necessary ' + + 'as apikey can be used instead, but maintained for backwards compatibility.')) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + type=click.Choice(['aws', 'azure', 'hpc']), + default='aws') +@click.option('--hpc-id', + help=('ID of your HPC, only applicable when --execution-platform=hpc. ' + + 'Default=660fae20f93358ad61e0104b'), + default='660fae20f93358ad61e0104b') +@click.option('--azure-worker-instance-type', + help=('The worker node instance type to be used in azure. ' + + 'Default=Standard_D4as_v4'), + default='Standard_D4as_v4') +@click.option('--azure-worker-instance-disk', + help='The disk size in GB for the worker node to be used in azure. Default=100', + type=int, + default=100) +@click.option('--azure-worker-instance-spot', + help='Whether the azure worker nodes have to be spot instances or not.', + is_flag=True) +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-file-staging', + help='Enables AWS S3 mountpoint for quicker file staging.', + is_flag=True) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--use-private-docker-repository', + help=('Allows to use private docker repository for running jobs. The Docker user ' + + 'account has to be already linked to CloudOS.'), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run(ctx, + apikey, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + job_config, + parameter, + git_commit, + git_tag, + git_branch, + job_name, + resumable, + do_not_save_logs, + job_queue, + nextflow_profile, + nextflow_version, + instance_type, + instance_disk, + storage_mode, + lustre_size, + wait_completion, + wait_time, + wdl_mainfile, + wdl_importsfile, + cromwell_token, + repository_platform, + execution_platform, + hpc_id, + azure_worker_instance_type, + azure_worker_instance_disk, + azure_worker_instance_spot, + cost_limit, + accelerate_file_staging, + accelerate_saving_results, + use_private_docker_repository, + verbose, + request_interval, + disable_ssl_verification, + ssl_cert, + profile): + """Submit a job to CloudOS.""" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if do_not_save_logs: + save_logs = False + else: + save_logs = True + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + if execution_platform == 'azure' or execution_platform == 'hpc': + batch = False + else: + batch = True + if execution_platform == 'hpc': + print('\nHPC execution platform selected') + if hpc_id is None: + raise ValueError('Please, specify your HPC ID using --hpc parameter') + print('Please, take into account that HPC execution do not support ' + + 'the following parameters and all of them will be ignored:\n' + + '\t--job-queue\n' + + '\t--resumable | --do-not-save-logs\n' + + '\t--instance-type | --instance-disk | --cost-limit\n' + + '\t--storage-mode | --lustre-size\n' + + '\t--wdl-mainfile | --wdl-importsfile | --cromwell-token\n') + wdl_mainfile = None + wdl_importsfile = None + storage_mode = 'regular' + save_logs = False + if accelerate_file_staging: + if execution_platform != 'aws': + print('You have selected accelerate file staging, but this function is ' + + 'only available when execution platform is AWS. The accelerate file staging ' + + 'will not be applied') + use_mountpoints = False + else: + use_mountpoints = True + print('Enabling AWS S3 mountpoint for accelerated file staging. ' + + 'Please, take into consideration the following:\n' + + '\t- It significantly reduces runtime and compute costs but may increase network costs.\n' + + '\t- Requires extra memory. Adjust process memory or optimise resource usage if necessary.\n' + + '\t- This is still a CloudOS BETA feature.\n') + else: + use_mountpoints = False + if verbose: + print('\t...Detecting workflow type') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + workflow_type = cl.detect_workflow(workflow_name, workspace_id, verify_ssl, last) + is_module = cl.is_module(workflow_name, workspace_id, verify_ssl, last) + if execution_platform == 'hpc' and workflow_type == 'wdl': + raise ValueError(f'The workflow {workflow_name} is a WDL workflow. ' + + 'WDL is not supported on HPC execution platform.') + if workflow_type == 'wdl': + print('WDL workflow detected') + if wdl_mainfile is None: + raise ValueError('Please, specify WDL mainFile using --wdl-mainfile .') + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h == 'Stopped': + print('\tStarting Cromwell server...\n') + cl.cromwell_switch(workspace_id, 'restart', verify_ssl) + elapsed = 0 + while elapsed < 300 and c_status_h != 'Running': + c_status_old = c_status_h + time.sleep(REQUEST_INTERVAL_CROMWELL) + elapsed += REQUEST_INTERVAL_CROMWELL + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + if c_status_h != c_status_old: + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h != 'Running': + raise Exception('Cromwell server did not restarted properly.') + cromwell_id = json.loads(c_status.content)["_id"] + click.secho('\t' + ('*' * 80) + '\n' + + '\tCromwell server is now running. Please, remember to stop it when ' + + 'your\n' + '\tjob finishes. You can use the following command:\n' + + '\tcloudos cromwell stop \\\n' + + '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + + f'\t\t--cloudos-url {cloudos_url} \\\n' + + f'\t\t--workspace-id {workspace_id}\n' + + '\t' + ('*' * 80) + '\n', fg='yellow', bold=True) + else: + cromwell_id = None + if verbose: + print('\t...Preparing objects') + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=wdl_mainfile, importsfile=wdl_importsfile, + repository_platform=repository_platform, verify=verify_ssl, last=last) + if verbose: + print('\tThe following Job object was created:') + print('\t' + str(j)) + print('\t...Sending job to CloudOS\n') + if is_module: + if job_queue is not None: + print(f'Ignoring job queue "{job_queue}" for ' + + f'Platform Workflow "{workflow_name}". Platform Workflows ' + + 'use their own predetermined queues.') + job_queue_id = None + if nextflow_version != '22.10.8': + print(f'The selected worflow \'{workflow_name}\' ' + + 'is a CloudOS module. CloudOS modules only work with ' + + 'Nextflow version 22.10.8. Switching to use 22.10.8') + nextflow_version = '22.10.8' + if execution_platform == 'azure': + print(f'The selected worflow \'{workflow_name}\' ' + + 'is a CloudOS module. For these workflows, worker nodes ' + + 'are managed internally. For this reason, the options ' + + 'azure-worker-instance-type, azure-worker-instance-disk and ' + + 'azure-worker-instance-spot are not taking effect.') + else: + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=cromwell_token, + workspace_id=workspace_id, verify=verify_ssl) + job_queue_id = queue.fetch_job_queue_id(workflow_type=workflow_type, batch=batch, + job_queue=job_queue) + if use_private_docker_repository: + if is_module: + print(f'Workflow "{workflow_name}" is a CloudOS module. ' + + 'Option --use-private-docker-repository will be ignored.') + docker_login = False + else: + me = j.get_user_info(verify=verify_ssl)['dockerRegistriesCredentials'] + if len(me) == 0: + raise Exception('User private Docker repository has been selected but your user ' + + 'credentials have not been configured yet. Please, link your ' + + 'Docker account to CloudOS before using ' + + '--use-private-docker-repository option.') + print('Use private Docker repository has been selected. A custom job ' + + 'queue to support private Docker containers and/or Lustre FSx will be created for ' + + 'your job. The selected job queue will serve as a template.') + docker_login = True + else: + docker_login = False + if nextflow_version == 'latest': + if execution_platform == 'aws': + nextflow_version = AWS_NEXTFLOW_LATEST + elif execution_platform == 'azure': + nextflow_version = AZURE_NEXTFLOW_LATEST + else: + nextflow_version = HPC_NEXTFLOW_LATEST + print('You have specified Nextflow version \'latest\' for execution platform ' + + f'\'{execution_platform}\'. The workflow will use the ' + + f'latest version available on CloudOS: {nextflow_version}.') + if execution_platform == 'aws': + if nextflow_version not in AWS_NEXTFLOW_VERSIONS: + print('For execution platform \'aws\', the workflow will use the default ' + + '\'22.10.8\' version on CloudOS.') + nextflow_version = '22.10.8' + if execution_platform == 'azure': + if nextflow_version not in AZURE_NEXTFLOW_VERSIONS: + print('For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' ' + + 'version on CloudOS.') + nextflow_version = '22.11.1-edge' + if execution_platform == 'hpc': + if nextflow_version not in HPC_NEXTFLOW_VERSIONS: + print('For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.') + nextflow_version = '22.10.8' + if nextflow_version != '22.10.8' and nextflow_version != '22.11.1-edge': + click.secho(f'You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' + + 'to be written in DSL2 and does not support DSL1.', fg='yellow', bold=True) + print('\nExecuting run...') + if workflow_type == 'nextflow': + print(f'\tNextflow version: {nextflow_version}') + j_id = j.send_job(job_config=job_config, + parameter=parameter, + is_module=is_module, + git_commit=git_commit, + git_tag=git_tag, + git_branch=git_branch, + job_name=job_name, + resumable=resumable, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + nextflow_profile=nextflow_profile, + nextflow_version=nextflow_version, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=hpc_id, + workflow_type=workflow_type, + cromwell_id=cromwell_id, + azure_worker_instance_type=azure_worker_instance_type, + azure_worker_instance_disk=azure_worker_instance_disk, + azure_worker_instance_spot=azure_worker_instance_spot, + cost_limit=cost_limit, + use_mountpoints=use_mountpoints, + accelerate_saving_results=accelerate_saving_results, + docker_login=docker_login, + verify=verify_ssl) + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=verbose, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') + + +@job.command('status') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_status(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Check job status in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + print('Executing status...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' + print(f'\tTo further check your job status you can either go to {j_url} ' + + 'or repeat the command you just used.') + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") + + +@job.command('workdir') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the working directory to an interactive session.', + is_flag=True) +@click.option('--delete', + help='Delete the results directory of a CloudOS job.', + is_flag=True) +@click.option('-y', '--yes', + help='Skip confirmation prompt when deleting results.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--status', + help='Check the deletion status of the working directory.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_workdir(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + delete, + yes, + session_id, + status, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the working directory of a specified job or check deletion status.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Handle --status flag + if status: + console = Console() + + if verbose: + console.print('[bold cyan]Checking deletion status of job working directory...[/bold cyan]') + console.print('\t[dim]...Preparing objects[/dim]') + console.print('\t[bold]Using the following parameters:[/bold]') + console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') + console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') + console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') + + # Use Cloudos object to access the deletion status method + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + console.print('\t[dim]The following Cloudos object was created:[/dim]') + console.print('\t' + str(cl) + '\n') + + try: + deletion_status = cl.get_workdir_deletion_status( + job_id=job_id, + workspace_id=workspace_id, + verify=verify_ssl + ) + + # Convert API status to user-friendly terminology with color + status_config = { + "ready": ("available", "green"), + "deleting": ("deleting", "yellow"), + "scheduledForDeletion": ("scheduled for deletion", "yellow"), + "deleted": ("deleted", "red"), + "failedToDelete": ("failed to delete", "red") + } + + # Get the status of the workdir folder itself and convert it + api_status = deletion_status.get("status", "unknown") + folder_status, status_color = status_config.get(api_status, (api_status, "white")) + folder_info = deletion_status.get("items", {}) + + # Display results in a clear, styled format with human-readable sentence + console.print(f'The working directory of job [cyan]{deletion_status["job_id"]}[/cyan] is in status: [bold {status_color}]{folder_status}[/bold {status_color}]') + + # For non-available statuses, always show update time and user info + if folder_status != "available": + if folder_info.get("updatedAt"): + console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') + + # Show user information - prefer deletedBy over user field + user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) + if user_info: + user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() + user_email = user_info.get('email', '') + if user_name or user_email: + user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) + console.print(f'[blue]User:[/blue] {user_display}') + + # Display detailed information if verbose + if verbose: + console.print(f'\n[bold]Additional information:[/bold]') + console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') + console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') + console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') + + # Show folder metadata if available + if folder_info.get("createdAt"): + console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') + if folder_info.get("updatedAt"): + console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') + if folder_info.get("folderType"): + console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') + + except ValueError as e: + raise click.ClickException(str(e)) + except Exception as e: + raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") + + return + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Finding working directory path...') + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl) + print(f"Working directory for job {job_id}: {workdir}") + + # Link to interactive session if requested + if link: + if verbose: + print(f'\tLinking working directory to interactive session {session_id}...') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + link_client.link_folder(workdir.strip(), session_id) + + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") + + # Delete workdir directory if requested + if delete: + try: + # Ask for confirmation unless --yes flag is provided + if not yes: + confirmation_message = ( + "\n⚠️ Deleting intermediate results is permanent and cannot be undone. " + "All associated data will be permanently removed and cannot be recovered. " + "The current job, as well as any other jobs sharing the same working directory, " + "will no longer be resumable. This action will be logged in the audit trail " + "(if auditing is enabled for your organisation), and you will be recorded as " + "the user who performed the deletion. You can skip this confirmation step by " + "providing -y or --yes flag to cloudos job workdir --delete. Please confirm " + "that you want to delete intermediate results of this analysis? [y/n] " + ) + click.secho(confirmation_message, fg='black', bg='yellow') + user_input = input().strip().lower() + if user_input != 'y': + print('\nDeletion cancelled.') + return + # Proceed with deletion + job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + job.delete_job_results(job_id, "workDirectory", verify=verify_ssl) + click.secho('\nIntermediate results directories deleted successfully.', fg='green', bold=True) + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve intermediate results for job '{job_id}'. {str(e)}") + else: + if yes: + click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) + + +@job.command('logs') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the logs directories to an interactive session.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_logs(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + session_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the logs of a specified job.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Executing logs...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + logs = cl.get_job_logs(job_id, workspace_id, verify_ssl) + for name, path in logs.items(): + print(f"{name}: {path}") + + # Link to interactive session if requested + if link: + if logs: + # Extract the parent logs directory from any log file path + # All log files should be in the same logs directory + first_log_path = next(iter(logs.values())) + # Remove the filename to get the logs directory + # e.g., "s3://bucket/path/to/logs/filename.txt" -> "s3://bucket/path/to/logs" + logs_dir = '/'.join(first_log_path.split('/')[:-1]) + + if verbose: + print(f'\tLinking logs directory to interactive session {session_id}...') + print(f'\t\tLogs directory: {logs_dir}') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + link_client.link_folder(logs_dir, session_id) + else: + if verbose: + print('\tNo logs found to link.') + + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve logs for job '{job_id}'. {str(e)}") + + +@job.command('results') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the results directories to an interactive session.', + is_flag=True) +@click.option('--delete', + help='Delete the results directory of a CloudOS job.', + is_flag=True) +@click.option('-y', '--yes', + help='Skip confirmation prompt when deleting results.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--status', + help='Check the deletion status of the job results.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_results(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + delete, + yes, + session_id, + status, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the results of a specified job or check deletion status.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Handle --status flag + if status: + console = Console() + + if verbose: + console.print('[bold cyan]Checking deletion status of job results...[/bold cyan]') + console.print('\t[dim]...Preparing objects[/dim]') + console.print('\t[bold]Using the following parameters:[/bold]') + console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') + console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') + console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') + + # Use Cloudos object to access the deletion status method + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + console.print('\t[dim]The following Cloudos object was created:[/dim]') + console.print('\t' + str(cl) + '\n') + + try: + deletion_status = cl.get_results_deletion_status( + job_id=job_id, + workspace_id=workspace_id, + verify=verify_ssl + ) + + # Convert API status to user-friendly terminology with color + status_config = { + "ready": ("available", "green"), + "deleting": ("deleting", "yellow"), + "scheduledForDeletion": ("scheduled for deletion", "yellow"), + "deleted": ("deleted", "red"), + "failedToDelete": ("failed to delete", "red") + } + + # Get the status of the results folder itself and convert it + api_status = deletion_status.get("status", "unknown") + folder_status, status_color = status_config.get(api_status, (api_status, "white")) + folder_info = deletion_status.get("items", {}) + + # Display results in a clear, styled format with human-readable sentence + console.print(f'The results of job [cyan]{deletion_status["job_id"]}[/cyan] are in status: [bold {status_color}]{folder_status}[/bold {status_color}]') + + # For non-available statuses, always show update time and user info + if folder_status != "available": + if folder_info.get("updatedAt"): + console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') + + # Show user information - prefer deletedBy over user field + user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) + if user_info: + user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() + user_email = user_info.get('email', '') + if user_name or user_email: + user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) + console.print(f'[blue]User:[/blue] {user_display}') + + # Display detailed information if verbose + if verbose: + console.print(f'\n[bold]Additional information:[/bold]') + console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') + console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') + console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') + + # Show folder metadata if available + if folder_info.get("createdAt"): + console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') + if folder_info.get("updatedAt"): + console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') + if folder_info.get("folderType"): + console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') + + except ValueError as e: + raise click.ClickException(str(e)) + except Exception as e: + raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") + + return + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Executing results...') + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) + print(f"results: {results_path}") + + # Link to interactive session if requested + if link: + if verbose: + print(f'\tLinking results directory to interactive session {session_id}...') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + if verbose: + print(f'\t\tLinking results ({results_path})...') + + link_client.link_folder(results_path, session_id) + + # Delete results directory if requested + if delete: + # Ask for confirmation unless --yes flag is provided + if not yes: + confirmation_message = ( + "\n⚠️ Deleting final analysis results is irreversible. " + "All data and backups will be permanently removed and cannot be recovered. " + "You can skip this confirmation step by providing '-y' or '--yes' flag to " + "'cloudos job results --delete'. " + "Please confirm that you want to delete final results of this analysis? [y/n] " + ) + click.secho(confirmation_message, fg='black', bg='yellow') + user_input = input().strip().lower() + if user_input != 'y': + print('\nDeletion cancelled.') + return + if verbose: + print(f'\nDeleting result directories from CloudOS...') + # Proceed with deletion + job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + job.delete_job_results(job_id, "analysisResults", verify=verify_ssl) + click.secho('\nResults directories deleted successfully.', fg='green', bold=True) + else: + if yes: + click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve results for job '{job_id}'. {str(e)}") + + +@job.command('details') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--output-format', + help=('The desired display for the output, either directly in standard output or saved as file. ' + + 'Default=stdout.'), + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--output-basename', + help=('Output file base name to save jobs details. ' + + 'Default={job_id}_details'), + required=False) +@click.option('--parameters', + help=('Whether to generate a ".config" file that can be used as input for --job-config parameter. ' + + 'It will have the same basename as defined in "--output-basename". '), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_details(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + output_basename, + parameters, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve job details in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if ctx.get_parameter_source('output_basename') == click.core.ParameterSource.DEFAULT: + output_basename = f"{job_id}_details" + + print('Executing details...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + + # check if the API gives a 403 error/forbidden error + try: + j_details = cl.get_job_status(job_id, workspace_id, verify_ssl) + except BadRequestException as e: + if '403' in str(e) or 'Forbidden' in str(e): + raise ValueError("API can only show job details of your own jobs, cannot see other user's job details.") + else: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve details for job '{job_id}'. {str(e)}") + create_job_details(json.loads(j_details.content), job_id, output_format, output_basename, parameters, cloudos_url) + + +@job.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save jobs list. ' + + 'Default=joblist'), + default='joblist', + required=False) +@click.option('--output-format', + help='The desired output format. For json option --all-fields will be automatically set to True. Default=stdout.', + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--table-columns', + help=('Comma-separated list of columns to display in the table. Only applicable when --output-format=stdout. ' + + 'Available columns: status,name,project,owner,pipeline,id,submit_time,end_time,run_time,commit,cost,resources,storage_type. ' + + 'Default: responsive (auto-selects columns based on terminal width)'), + default=None) +@click.option('--all-fields', + help=('Whether to collect all available fields from jobs or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv. Automatically enabled for json output.'), + is_flag=True) +@click.option('--last-n-jobs', + help=("The number of last workspace jobs to retrieve. You can use 'all' to " + + "retrieve all workspace jobs. When adding this option, options " + + "'--page' and '--page-size' are ignored.")) +@click.option('--page', + help=('Page number to fetch from the API. Used with --page-size to control jobs ' + + 'per page (e.g. --page=4 --page-size=20). Default=1.'), + type=int, + default=1) +@click.option('--page-size', + help=('Page size to retrieve from API, corresponds to the number of jobs per page. ' + + 'Maximum allowed integer is 100. Default=10.'), + type=int, + default=10) +@click.option('--archived', + help=('When this flag is used, only archived jobs list is collected.'), + is_flag=True) +@click.option('--filter-status', + help='Filter jobs by status (e.g., completed, running, failed, aborted).') +@click.option('--filter-job-name', + help='Filter jobs by job name ( case insensitive ).') +@click.option('--filter-project', + help='Filter jobs by project name.') +@click.option('--filter-workflow', + help='Filter jobs by workflow/pipeline name.') +@click.option('--last', + help=('When workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('--filter-job-id', + help='Filter jobs by specific job ID.') +@click.option('--filter-only-mine', + help='Filter to show only jobs belonging to the current user.', + is_flag=True) +@click.option('--filter-queue', + help='Filter jobs by queue name. Only applies to jobs running in batch environment. Non-batch jobs are preserved in results.') +@click.option('--filter-owner', + help='Filter jobs by owner username.') +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + table_columns, + all_fields, + last_n_jobs, + page, + page_size, + archived, + filter_status, + filter_job_name, + filter_project, + filter_workflow, + last, + filter_job_id, + filter_only_mine, + filter_owner, + filter_queue, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect and display workspace jobs from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Pass table_columns directly to create_job_list_table for validation and processing + selected_columns = table_columns + # Only set outfile if not using stdout + if output_format != 'stdout': + outfile = output_basename + '.' + output_format + + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for jobs in the following workspace: ' + + f'{workspace_id}') + # Check if the user provided the --page option + ctx = click.get_current_context() + if not isinstance(page, int) or page < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') + + if not isinstance(page_size, int) or page_size < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page-size parameter') + + # Validate page_size limit - must be done before API call + if page_size > 100: + click.secho('Error: Page size cannot exceed 100. Please use --page-size with a value <= 100', fg='red', err=True) + raise SystemExit(1) + + result = cl.get_job_list(workspace_id, last_n_jobs, page, page_size, archived, verify_ssl, + filter_status=filter_status, + filter_job_name=filter_job_name, + filter_project=filter_project, + filter_workflow=filter_workflow, + filter_job_id=filter_job_id, + filter_only_mine=filter_only_mine, + filter_owner=filter_owner, + filter_queue=filter_queue, + last=last) + + # Extract jobs and pagination metadata from result + my_jobs_r = result['jobs'] + pagination_metadata = result['pagination_metadata'] + + # Validate requested page exists + if pagination_metadata: + total_jobs = pagination_metadata.get('Pagination-Count', 0) + current_page_size = pagination_metadata.get('Pagination-Limit', page_size) + + if total_jobs > 0: + total_pages = (total_jobs + current_page_size - 1) // current_page_size + if page > total_pages: + click.secho(f'Error: Page {page} does not exist. There are only {total_pages} page(s) available with {total_jobs} total job(s). ' + f'Please use --page with a value between 1 and {total_pages}', fg='red', err=True) + raise SystemExit(1) + + if len(my_jobs_r) == 0: + # Check if any filtering options are being used + filters_used = any([ + filter_status, + filter_job_name, + filter_project, + filter_workflow, + filter_job_id, + filter_only_mine, + filter_owner, + filter_queue + ]) + if output_format == 'stdout': + # For stdout, always show a user-friendly message + create_job_list_table([], cloudos_url, pagination_metadata, selected_columns) + else: + if filters_used: + print('A total of 0 jobs collected.') + elif ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + print('A total of 0 jobs collected. This is likely because your workspace ' + + 'has no jobs created yet.') + else: + print('A total of 0 jobs collected. This is likely because the --page you requested ' + + 'does not exist. Please, try a smaller number for --page or collect all the jobs by not ' + + 'using --page parameter.') + elif output_format == 'stdout': + # Display as table + create_job_list_table(my_jobs_r, cloudos_url, pagination_metadata, selected_columns) + elif output_format == 'csv': + my_jobs = cl.process_job_list(my_jobs_r, all_fields) + cl.save_job_list_to_csv(my_jobs, outfile) + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_jobs_r)) + print(f'\tJob list collected with a total of {len(my_jobs_r)} jobs.') + print(f'\tJob list saved to {outfile}') + else: + raise ValueError('Unrecognised output format. Please use one of [stdout|csv|json]') + + +@job.command('abort') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-ids', + help=('One or more job ids to abort. If more than ' + + 'one is provided, they must be provided as ' + + 'a comma separated list of ids. E.g. id1,id2,id3'), + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--force', + help='Force abort the job even if it is not in a running or initializing state.', + is_flag=True) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def abort_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + job_ids, + verbose, + disable_ssl_verification, + ssl_cert, + profile, + force): + """Abort all specified jobs from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + print('Aborting jobs...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for jobs in the following workspace: ' + + f'{workspace_id}') + # check if the user provided an empty job list + jobs = job_ids.replace(' ', '') + if not jobs: + raise ValueError('No job IDs provided. Please specify at least one job ID to abort.') + jobs = jobs.split(',') + + # Issue warning if using --force flag + if force: + click.secho(f"Warning: Using --force to abort jobs. Some data might be lost.", fg='yellow', bold=True) + + for job in jobs: + try: + j_status = cl.get_job_status(job, workspace_id, verify_ssl) + except Exception as e: + click.secho(f"Failed to get status for job {job}, please make sure it exists in the workspace: {e}", fg='yellow', bold=True) + continue + + j_status_content = json.loads(j_status.content) + job_status = j_status_content['status'] + + # Check if job is in a state that normally allows abortion + is_abortable = job_status in ABORT_JOB_STATES + + # Issue warning if job is in initializing state and not using force + if job_status == 'initializing' and not force: + click.secho(f"Warning: Job {job} is in initializing state.", fg='yellow', bold=True) + + # Check if job can be aborted + if not is_abortable: + click.secho(f"Job {job} is not in a state that can be aborted and is ignored. " + + f"Current status: {job_status}", fg='yellow', bold=True) + else: + try: + cl.abort_job(job, workspace_id, verify_ssl, force) + click.secho(f"Job '{job}' aborted successfully.", fg='green', bold=True) + except Exception as e: + click.secho(f"Failed to abort job {job}. Error: {e}", fg='red', bold=True) + + +@job.command('cost') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to get costs for.', + required=True) +@click.option('--output-format', + help='The desired file format (file extension) for the output. For json option --all-fields will be automatically set to True. Default=csv.', + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_cost(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve job cost information in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + print('Retrieving cost information...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cost_viewer = CostViewer(cloudos_url, apikey) + if verbose: + print(f'\tSearching for cost data for job id: {job_id}') + # Display costs with pagination + cost_viewer.display_costs(job_id, workspace_id, output_format, verify_ssl) + + +@job.command('related') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to get costs for.', + required=True) +@click.option('--output-format', + help='The desired output format. Default=stdout.', + type=click.Choice(['stdout', 'json'], case_sensitive=False), + default='stdout') +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def related(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve related job analyses in CloudOS.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + related_analyses(cloudos_url, apikey, job_id, workspace_id, output_format, verify_ssl) + + +@click.command() +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-ids', + help=('One or more job ids to archive/unarchive. If more than ' + + 'one is provided, they must be provided as ' + + 'a comma separated list of ids. E.g. id1,id2,id3'), + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def archive_unarchive_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + job_ids, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Archive or unarchive specified jobs in a CloudOS workspace.""" + # Determine operation based on the command name used + target_archived_state = ctx.info_name == "archive" + action = "archive" if target_archived_state else "unarchive" + action_past = "archived" if target_archived_state else "unarchived" + action_ing = "archiving" if target_archived_state else "unarchiving" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + print(f'{action_ing.capitalize()} jobs...') + + if verbose: + print('\t...Preparing objects') + + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\t{action_ing.capitalize()} jobs in the following workspace: {workspace_id}') + + # check if the user provided an empty job list + jobs = job_ids.replace(' ', '') + if not jobs: + raise ValueError(f'No job IDs provided. Please specify at least one job ID to {action}.') + jobs_list = [job for job in jobs.split(',') if job] # Filter out empty strings + + # Check for duplicate job IDs + duplicates = [job_id for job_id in set(jobs_list) if jobs_list.count(job_id) > 1] + if duplicates: + dup_str = ', '.join(duplicates) + click.secho(f'Warning: Duplicate job IDs detected and will be processed only once: {dup_str}', fg='yellow', bold=True) + # Remove duplicates while preserving order + jobs_list = list(dict.fromkeys(jobs_list)) + if verbose: + print(f'\tDuplicate job IDs removed. Processing {len(jobs_list)} unique job(s).') + + # Check archive status for all jobs + status_check = cl.check_jobs_archive_status(jobs_list, workspace_id, target_archived_state=target_archived_state, verify=verify_ssl, verbose=verbose) + valid_jobs = status_check['valid_jobs'] + already_processed = status_check['already_processed'] + invalid_jobs = status_check['invalid_jobs'] + + # Report invalid jobs (but continue processing valid ones) + for job_id, error_msg in invalid_jobs.items(): + click.secho(f"Failed to get status for job {job_id}, please make sure it exists in the workspace: {error_msg}", fg='yellow', bold=True) + + if not valid_jobs and not already_processed: + # All jobs were invalid - exit gracefully + click.secho('No valid job IDs found. Please check that the job IDs exist and are accessible.', fg='yellow', bold=True) + return + + if not valid_jobs: + if len(already_processed) == 1: + click.secho(f"Job '{already_processed[0]}' is already {action_past}. No action needed.", fg='cyan', bold=True) + else: + click.secho(f"All {len(already_processed)} jobs are already {action_past}. No action needed.", fg='cyan', bold=True) + return + + try: + # Call the appropriate action method + if target_archived_state: + cl.archive_jobs(valid_jobs, workspace_id, verify_ssl) + else: + cl.unarchive_jobs(valid_jobs, workspace_id, verify_ssl) + + success_msg = [] + if len(valid_jobs) == 1: + success_msg.append(f"Job '{valid_jobs[0]}' {action_past} successfully.") + else: + success_msg.append(f"{len(valid_jobs)} jobs {action_past} successfully: {', '.join(valid_jobs)}") + + if already_processed: + if len(already_processed) == 1: + success_msg.append(f"Job '{already_processed[0]}' was already {action_past}.") + else: + success_msg.append(f"{len(already_processed)} jobs were already {action_past}: {', '.join(already_processed)}") + + click.secho('\n'.join(success_msg), fg='green', bold=True) + except Exception as e: + raise ValueError(f"Failed to {action} jobs: {str(e)}") + + +@click.command(help='Clone or resume a job with modified parameters') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.') +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p input=s3://path_to_my_file. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--nextflow-profile', + help=('A comma separated string indicating the nextflow profile/s ' + + 'to use with your job.')) +@click.option('--nextflow-version', + help=('Nextflow version to use when executing the workflow in CloudOS. ' + + 'Default=22.10.8.'), + type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest'])) +@click.option('--git-branch', + help=('The branch to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--job-name', + help='The name of the job. If not set, will take the name of the cloned job.') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help=('Name of the job queue to use with a batch job. ' + + 'In Azure workspaces, this option is ignored.')) +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).')) +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float) +@click.option('--job-id', + help='The CloudOS job id of the job to be cloned.', + required=True) +@click.option('--accelerate-file-staging', + help='Enables AWS S3 mountpoint for quicker file staging.', + is_flag=True) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--resumable', + help='Whether to make the job able to be resumed or not.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', + help='Profile to use from the config file', + default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def clone_resume(ctx, + apikey, + cloudos_url, + workspace_id, + project_name, + parameter, + nextflow_profile, + nextflow_version, + git_branch, + repository_platform, + job_name, + do_not_save_logs, + job_queue, + instance_type, + cost_limit, + job_id, + accelerate_file_staging, + accelerate_saving_results, + resumable, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + if ctx.info_name == "clone": + mode, action = "clone", "cloning" + elif ctx.info_name == "resume": + mode, action = "resume", "resuming" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + print(f'{action.capitalize()} job...') + if verbose: + print('\t...Preparing objects') + + # Create Job object (set dummy values for project_name and workflow_name, since they come from the cloned job) + job_obj = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + + if verbose: + print('\tThe following Job object was created:') + print('\t' + str(job_obj) + '\n') + print(f'\t{action.capitalize()} job {job_id} in workspace: {workspace_id}') + + try: + + # Clone/resume the job with provided overrides + cloned_resumed_job_id = job_obj.clone_or_resume_job( + source_job_id=job_id, + queue_name=job_queue, + cost_limit=cost_limit, + master_instance=instance_type, + job_name=job_name, + nextflow_version=nextflow_version, + branch=git_branch, + repository_platform=repository_platform, + profile=nextflow_profile, + do_not_save_logs=do_not_save_logs, + use_fusion=accelerate_file_staging, + accelerate_saving_results=accelerate_saving_results, + resumable=resumable, + # only when explicitly setting --project-name will be overridden, else using the original project + project_name=project_name if ctx.get_parameter_source("project_name") == click.core.ParameterSource.COMMANDLINE else None, + parameters=list(parameter) if parameter else None, + verify=verify_ssl, + mode=mode + ) + + if verbose: + print(f'\t{mode.capitalize()}d job ID: {cloned_resumed_job_id}') + + print(f"Job successfully {mode}d. New job ID: {cloned_resumed_job_id}") + + except BadRequestException as e: + raise ValueError(f"Failed to {mode} job. Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") + + +# Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) +archive_unarchive_jobs.help = 'Archive specified jobs in a CloudOS workspace.' +job.add_command(archive_unarchive_jobs, "archive") + +# Create a copy with different help text for unarchive +archive_unarchive_jobs_copy = copy.deepcopy(archive_unarchive_jobs) +archive_unarchive_jobs_copy.help = 'Unarchive specified jobs in a CloudOS workspace.' +job.add_command(archive_unarchive_jobs_copy, "unarchive") + + +# Apply the best Click solution: Set specific help text for each command registration +clone_resume.help = 'Clone a job with modified parameters' +job.add_command(clone_resume, "clone") + +# Create a copy with different help text for resume +clone_resume_copy = copy.deepcopy(clone_resume) +clone_resume_copy.help = 'Resume a job with modified parameters' +job.add_command(clone_resume_copy, "resume") + + +@workflow.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save workflow list. ' + + 'Default=workflow_list'), + default='workflow_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from workflows or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_workflows(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all workflows from a CloudOS workspace in CSV format.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for workflows in the following workspace: ' + + f'{workspace_id}') + my_workflows_r = cl.get_workflow_list(workspace_id, verify=verify_ssl) + if output_format == 'csv': + my_workflows = cl.process_workflow_list(my_workflows_r, all_fields) + my_workflows.to_csv(outfile, index=False) + print(f'\tWorkflow list collected with a total of {my_workflows.shape[0]} workflows.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_workflows_r)) + print(f'\tWorkflow list collected with a total of {len(my_workflows_r)} workflows.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tWorkflow list saved to {outfile}') + + +@workflow.command('import') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=('The CloudOS url you are trying to access to. ' + + f'Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option("--workflow-name", help="The name that the workflow will have in CloudOS.", required=True) +@click.option("-w", "--workflow-url", help="URL of the workflow repository.", required=True) +@click.option("-d", "--workflow-docs-link", help="URL to the documentation of the workflow.", default='') +@click.option("--cost-limit", help="Cost limit for the workflow. Default: $30 USD.", default=30) +@click.option("--workflow-description", help="Workflow description", default="") +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name']) +def import_wf(ctx, + apikey, + cloudos_url, + workspace_id, + workflow_name, + workflow_url, + workflow_docs_link, + cost_limit, + workflow_description, + repository_platform, + disable_ssl_verification, + ssl_cert, + profile): + """ + Import workflows from supported repository providers. + """ + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + repo_import = ImportWorflow( + cloudos_url=cloudos_url, cloudos_apikey=apikey, workspace_id=workspace_id, platform=repository_platform, + workflow_name=workflow_name, workflow_url=workflow_url, workflow_docs_link=workflow_docs_link, + cost_limit=cost_limit, workflow_description=workflow_description, verify=verify_ssl + ) + workflow_id = repo_import.import_workflow() + print(f'\tWorkflow {workflow_name} was imported successfully with the ' + + f'following ID: {workflow_id}') + + +@project.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save project list. ' + + 'Default=project_list'), + default='project_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from projects or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--page', + help=('Response page to retrieve. Default=1.'), + type=int, + default=1) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_projects(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + page, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all projects from a CloudOS workspace in CSV format.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for projects in the following workspace: ' + + f'{workspace_id}') + # Check if the user provided the --page option + ctx = click.get_current_context() + if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + get_all = True + else: + get_all = False + if not isinstance(page, int) or page < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') + my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all) + if len(my_projects_r) == 0: + if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + print('A total of 0 projects collected. This is likely because your workspace ' + + 'has no projects created yet.') + else: + print('A total of 0 projects collected. This is likely because the --page you ' + + 'requested does not exist. Please, try a smaller number for --page or collect all the ' + + 'projects by not using --page parameter.') + elif output_format == 'csv': + my_projects = cl.process_project_list(my_projects_r, all_fields) + my_projects.to_csv(outfile, index=False) + print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_projects_r)) + print(f'\tProject list collected with a total of {len(my_projects_r)} projects.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tProject list saved to {outfile}') + + +@project.command('create') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--new-project', + help='The name for the new project.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def create_project(ctx, + apikey, + cloudos_url, + workspace_id, + new_project, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Create a new project in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + # verify ssl configuration + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Print basic output + if verbose: + print(f'\tUsing CloudOS URL: {cloudos_url}') + print(f'\tUsing workspace: {workspace_id}') + print(f'\tProject name: {new_project}') + + cl = Cloudos(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None) + + try: + project_id = cl.create_project(workspace_id, new_project, verify_ssl) + print(f'\tProject "{new_project}" created successfully with ID: {project_id}') + if verbose: + print(f'\tProject URL: {cloudos_url}/app/projects/{project_id}') + except Exception as e: + print(f'\tError creating project: {str(e)}') + sys.exit(1) + + +@cromwell.command('status') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_status(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Check Cromwell server status in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + print('Executing status...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tChecking Cromwell status in {workspace_id} workspace') + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + + +@cromwell.command('start') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to Cromwell restart. ' + + 'Default=300.'), + default=300) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_restart(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + wait_time, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Restart Cromwell server in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + action = 'restart' + print('Starting Cromwell server...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tStarting Cromwell server in {workspace_id} workspace') + cl.cromwell_switch(workspace_id, action, verify_ssl) + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + elapsed = 0 + while elapsed < wait_time and c_status_h != 'Running': + c_status_old = c_status_h + time.sleep(REQUEST_INTERVAL_CROMWELL) + elapsed += REQUEST_INTERVAL_CROMWELL + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + if c_status_h != c_status_old: + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h != 'Running': + print(f'\tYour current Cromwell status is: {c_status_h}. The ' + + f'selected wait-time of {wait_time} was exceeded. Please, ' + + 'consider to set a longer wait-time.') + print('\tTo further check your Cromwell status you can either go to ' + + f'{cloudos_url} or use the following command:\n' + + '\tcloudos cromwell status \\\n' + + f'\t\t--cloudos-url {cloudos_url} \\\n' + + '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + + f'\t\t--workspace-id {workspace_id}') + sys.exit(1) + + +@cromwell.command('stop') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_stop(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Stop Cromwell server in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + action = 'stop' + print('Stopping Cromwell server...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tStopping Cromwell server in {workspace_id} workspace') + cl.cromwell_switch(workspace_id, action, verify_ssl) + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + + +@queue.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save job queue list. ' + + 'Default=job_queue_list'), + default='job_queue_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from workflows or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_queues(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all available job queues from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + j_queue = Queue(cloudos_url, apikey, None, workspace_id, verify=verify_ssl) + my_queues = j_queue.get_job_queues() + if len(my_queues) == 0: + raise ValueError('No AWS batch queues found. Please, make sure that your CloudOS supports AWS bath queues') + if output_format == 'csv': + queues_processed = j_queue.process_queue_list(my_queues, all_fields) + queues_processed.to_csv(outfile, index=False) + print(f'\tJob queue list collected with a total of {queues_processed.shape[0]} queues.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_queues)) + print(f'\tJob queue list collected with a total of {len(my_queues)} queues.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tJob queue list saved to {outfile}') + + +@configure.command('list-profiles') +def list_profiles(): + config_manager = ConfigurationProfile() + config_manager.list_profiles() + + +@configure.command('remove-profile') +@click.option('--profile', + help='Name of the profile. Not using this option will lead to profile named "deafults" being generated', + required=True) +@click.pass_context +def remove_profile(ctx, profile): + update_command_context_from_click(ctx) + profile = profile or ctx.obj['profile'] + config_manager = ConfigurationProfile() + config_manager.remove_profile(profile) + + +@bash.command('job') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('--command', + help='The command to run in the bash job.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--cpus', + help='The number of CPUs to use for the task\'s master node. Default=1.', + type=int, + default=1) +@click.option('--memory', + help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', + type=int, + default=4) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + default='aws') +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run_bash_job(ctx, + apikey, + command, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + parameter, + job_name, + do_not_save_logs, + job_queue, + instance_type, + instance_disk, + cpus, + memory, + storage_mode, + lustre_size, + wait_completion, + wait_time, + repository_platform, + execution_platform, + cost_limit, + accelerate_saving_results, + request_interval, + disable_ssl_verification, + ssl_cert, + profile): + """Run a bash job in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + + if do_not_save_logs: + save_logs = False + else: + save_logs = True + + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=None, importsfile=None, + repository_platform=repository_platform, verify=verify_ssl, last=last) + + if job_queue is not None: + batch = True + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, + workspace_id=workspace_id, verify=verify_ssl) + # I have to add 'nextflow', other wise the job queue id is not found + job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, + job_queue=job_queue) + else: + job_queue_id = None + batch = False + j_id = j.send_job(job_config=None, + parameter=parameter, + git_commit=None, + git_tag=None, + git_branch=None, + job_name=job_name, + resumable=False, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + workflow_type='docker', + nextflow_profile=None, + nextflow_version=None, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=None, + cost_limit=cost_limit, + accelerate_saving_results=accelerate_saving_results, + verify=verify_ssl, + command={"command": command}, + cpus=cpus, + memory=memory) + + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=False, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') + + +@bash.command('array-job') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('--command', + help='The command to run in the bash job.') +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + + 'times as parameters you want to include. ' + + 'For parameters pointing to a file, the format expected is ' + + 'parameter_name=/Data/parameter_value. The parameter value must be a ' + + 'file located in the `Data` subfolder. If no is specified, it defaults to ' + + 'the project specified by the profile or --project-name parameter. ' + + 'E.g.: -p "--file=Data/file.txt" or "--file=/Data/folder/file.txt"')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--cpus', + help='The number of CPUs to use for the task\'s master node. Default=1.', + type=int, + default=1) +@click.option('--memory', + help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', + type=int, + default=4) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + type=click.Choice(['aws', 'azure', 'hpc']), + default='aws') +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--array-file', + help=('Path to a file containing an array of commands to run in the bash job.'), + default=None, + required=True) +@click.option('--separator', + help=('Separator to use in the array file. Default=",".'), + type=click.Choice([',', ';', 'tab', 'space', '|']), + default=",", + required=True) +@click.option('--list-columns', + help=('List columns present in the array file. ' + + 'This option will not run any job.'), + is_flag=True) +@click.option('--array-file-project', + help=('Name of the project in which the array file is placed, if different from --project-name.'), + default=None) +@click.option('--disable-column-check', + help=('Disable the check for the columns in the array file. ' + + 'This option is only used when --array-file is provided.'), + is_flag=True) +@click.option('-a', '--array-parameter', + multiple=True, + help=('A single parameter to pass to the job call only for specifying array columns. ' + + 'It should be in the following form: parameter_name=array_file_column_name. E.g.: ' + + '-a --test=value or -a -test=value or -a test=value or -a =value (for no prefix). ' + + 'You can use this option as many times as parameters you want to include.')) +@click.option('--custom-script-path', + help=('Path of a custom script to run in the bash array job instead of a command.'), + default=None) +@click.option('--custom-script-project', + help=('Name of the project to use when running the custom command script, if ' + + 'different than --project-name.'), + default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run_bash_array_job(ctx, + apikey, + command, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + parameter, + job_name, + do_not_save_logs, + job_queue, + instance_type, + instance_disk, + cpus, + memory, + storage_mode, + lustre_size, + wait_completion, + wait_time, + repository_platform, + execution_platform, + cost_limit, + accelerate_saving_results, + request_interval, + disable_ssl_verification, + ssl_cert, + profile, + array_file, + separator, + list_columns, + array_file_project, + disable_column_check, + array_parameter, + custom_script_path, + custom_script_project): + """Run a bash array job in CloudOS.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + if not list_columns and not (command or custom_script_path): + raise click.UsageError("Must provide --command or --custom-script-path if --list-columns is not set.") + + # when not set, use the global project name + if array_file_project is None: + array_file_project = project_name + + # this needs to be in another call to datasets, by default it uses the global project name + if custom_script_project is None: + custom_script_project = project_name + + # setup separators for API and array file (the're different) + separators = { + ",": {"api": ",", "file": ","}, + ";": {"api": "%3B", "file": ";"}, + "space": {"api": "+", "file": " "}, + "tab": {"api": "tab", "file": "tab"}, + "|": {"api": "%7C", "file": "|"} + } + + # setup important options for the job + if do_not_save_logs: + save_logs = False + else: + save_logs = True + + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=None, importsfile=None, + repository_platform=repository_platform, verify=verify_ssl, last=last) + + # retrieve columns + r = j.retrieve_cols_from_array_file( + array_file, + generate_datasets_for_project(cloudos_url, apikey, workspace_id, array_file_project, verify_ssl), + separators[separator]['api'], + verify_ssl + ) + + if not disable_column_check: + columns = json.loads(r.content).get("headers", None) + # pass this to the SEND JOB API call + # b'{"headers":[{"index":0,"name":"id"},{"index":1,"name":"title"},{"index":2,"name":"filename"},{"index":3,"name":"file2name"}]}' + if columns is None: + raise ValueError("No columns found in the array file metadata.") + if list_columns: + print("Columns: ") + for col in columns: + print(f"\t- {col['name']}") + return + else: + columns = [] + + # setup parameters for the job + cmd = j.setup_params_array_file( + custom_script_path, + generate_datasets_for_project(cloudos_url, apikey, workspace_id, custom_script_project, verify_ssl), + command, + separators[separator]['file'] + ) + + # check columns in the array file vs parameters added + if not disable_column_check and array_parameter: + print("\nChecking columns in the array file vs parameters added...\n") + for ap in array_parameter: + ap_split = ap.split('=') + ap_value = '='.join(ap_split[1:]) + for col in columns: + if col['name'] == ap_value: + print(f"Found column '{ap_value}' in the array file.") + break + else: + raise ValueError(f"Column '{ap_value}' not found in the array file. " + \ + f"Columns in array-file: {separator.join([col['name'] for col in columns])}") + + if job_queue is not None: + batch = True + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, + workspace_id=workspace_id, verify=verify_ssl) + # I have to add 'nextflow', other wise the job queue id is not found + job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, + job_queue=job_queue) + else: + job_queue_id = None + batch = False + + # send job + j_id = j.send_job(job_config=None, + parameter=parameter, + array_parameter=array_parameter, + array_file_header=columns, + git_commit=None, + git_tag=None, + git_branch=None, + job_name=job_name, + resumable=False, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + workflow_type='docker', + nextflow_profile=None, + nextflow_version=None, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=None, + cost_limit=cost_limit, + accelerate_saving_results=accelerate_saving_results, + verify=verify_ssl, + command=cmd, + cpus=cpus, + memory=memory) + + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=False, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') + + +@datasets.command(name="ls") +@click.argument("path", required=False, nargs=1) +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--details', + help=('When selected, it prints the details of the listed files. ' + + 'Details contains "Type", "Owner", "Size", "Last Updated", ' + + '"Virtual Name", "Storage Path".'), + is_flag=True) +@click.option('--output-format', + help=('The desired display for the output, either directly in standard output or saved as file. ' + + 'Default=stdout.'), + type=click.Choice(['stdout', 'csv'], case_sensitive=False), + default='stdout') +@click.option('--output-basename', + help=('Output file base name to save jobs details. ' + + 'Default=datasets_ls'), + default='datasets_ls', + required=False) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def list_files(ctx, + apikey, + cloudos_url, + workspace_id, + disable_ssl_verification, + ssl_cert, + project_name, + profile, + path, + details, + output_format, + output_basename): + """List contents of a path within a CloudOS workspace dataset.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + datasets = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = datasets.list_folder_content(path) + contents = result.get("contents") or result.get("datasets", []) + + if not contents: + contents = result.get("files", []) + result.get("folders", []) + + # Process items to extract data + processed_items = [] + for item in contents: + is_folder = "folderType" in item or item.get("isDir", False) + type_ = "folder" if is_folder else "file" + + # Enhanced type information + if is_folder: + folder_type = item.get("folderType") + if folder_type == "VirtualFolder": + type_ = "virtual folder" + elif folder_type == "S3Folder": + type_ = "s3 folder" + elif folder_type == "AzureBlobFolder": + type_ = "azure folder" + else: + type_ = "folder" + else: + # Check if file is managed by Lifebit (user uploaded) + is_managed_by_lifebit = item.get("isManagedByLifebit", False) + if is_managed_by_lifebit: + type_ = "file (user uploaded)" + else: + type_ = "file (virtual copy)" + + user = item.get("user", {}) + if isinstance(user, dict): + name = user.get("name", "").strip() + surname = user.get("surname", "").strip() + else: + name = surname = "" + if name and surname: + owner = f"{name} {surname}" + elif name: + owner = name + elif surname: + owner = surname + else: + owner = "-" + + raw_size = item.get("sizeInBytes", item.get("size")) + size = format_bytes(raw_size) if not is_folder and raw_size is not None else "-" + + updated = item.get("updatedAt") or item.get("lastModified", "-") + filepath = item.get("name", "-") + + if item.get("fileType") == "S3File" or item.get("folderType") == "S3Folder": + bucket = item.get("s3BucketName") + key = item.get("s3ObjectKey") or item.get("s3Prefix") + storage_path = f"s3://{bucket}/{key}" if bucket and key else "-" + elif item.get("fileType") == "AzureBlobFile" or item.get("folderType") == "AzureBlobFolder": + account = item.get("blobStorageAccountName") + container = item.get("blobContainerName") + key = item.get("blobName") if item.get("fileType") == "AzureBlobFile" else item.get("blobPrefix") + storage_path = f"az://{account}.blob.core.windows.net/{container}/{key}" if account and container and key else "-" + else: + storage_path = "-" + + processed_items.append({ + 'type': type_, + 'owner': owner, + 'size': size, + 'raw_size': raw_size, + 'updated': updated, + 'name': filepath, + 'storage_path': storage_path, + 'is_folder': is_folder + }) + + # Output handling + if output_format == 'csv': + import csv + + csv_filename = f'{output_basename}.csv' + + if details: + # CSV with all details + with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: + fieldnames = ['Type', 'Owner', 'Size', 'Size (bytes)', 'Last Updated', 'Virtual Name', 'Storage Path'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for item in processed_items: + writer.writerow({ + 'Type': item['type'], + 'Owner': item['owner'], + 'Size': item['size'], + 'Size (bytes)': item['raw_size'] if item['raw_size'] is not None else '', + 'Last Updated': item['updated'], + 'Virtual Name': item['name'], + 'Storage Path': item['storage_path'] + }) + else: + # CSV with just names + with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(['Name', 'Storage Path']) + for item in processed_items: + writer.writerow([item['name'], item['storage_path']]) + + click.secho(f'\nDatasets list saved to: {csv_filename}', fg='green', bold=True) + + else: # stdout + if details: + console = Console(width=None) + table = Table(show_header=True, header_style="bold white") + table.add_column("Type", style="cyan", no_wrap=True) + table.add_column("Owner", style="white") + table.add_column("Size", style="magenta") + table.add_column("Last Updated", style="green") + table.add_column("Virtual Name", style="bold", overflow="fold") + table.add_column("Storage Path", style="dim", no_wrap=False, overflow="fold", ratio=2) + + for item in processed_items: + style = Style(color="blue", underline=True) if item['is_folder'] else None + table.add_row( + item['type'], + item['owner'], + item['size'], + item['updated'], + item['name'], + item['storage_path'], + style=style + ) + + console.print(table) + + else: + console = Console() + for item in processed_items: + if item['is_folder']: + console.print(f"[blue underline]{item['name']}[/]") + else: + console.print(item['name']) + + except Exception as e: + raise ValueError(f"Failed to list files for project '{project_name}'. {str(e)}") + + +@datasets.command(name="mv") +@click.argument("source_path", required=True) +@click.argument("destination_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The source project name.') +@click.option('--destination-project-name', required=False, + help='The destination project name. Defaults to the source project.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def move_files(ctx, source_path, destination_path, apikey, cloudos_url, workspace_id, + project_name, destination_project_name, + disable_ssl_verification, ssl_cert, profile): + """ + Move a file or folder from a source path to a destination path within or across CloudOS projects. + + SOURCE_PATH [path]: the full path to the file or folder to move. It must be a 'Data' folder path. + E.g.: 'Data/folderA/file.txt'\n + DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. + E.g.: 'Data/folderB' + """ + # Validate destination constraint + if not destination_path.strip("/").startswith("Data/") and destination_path.strip("/") != "Data": + raise ValueError("Destination path must begin with 'Data/' or be 'Data'.") + if not source_path.strip("/").startswith("Data/") and source_path.strip("/") != "Data": + raise ValueError("SOURCE_PATH must start with 'Data/' or be 'Data'.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + destination_project_name = destination_project_name or project_name + # Initialize Datasets clients + source_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + dest_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=destination_project_name, + verify=verify_ssl, + cromwell_token=None + ) + print('Checking source path') + # === Resolve Source Item === + source_parts = source_path.strip("/").split("/") + source_parent_path = "/".join(source_parts[:-1]) if len(source_parts) > 1 else None + source_item_name = source_parts[-1] + + try: + source_contents = source_client.list_folder_content(source_parent_path) + except Exception as e: + raise ValueError(f"Could not resolve source path '{source_path}'. {str(e)}") + + found_source = None + for collection in ["files", "folders"]: + for item in source_contents.get(collection, []): + if item.get("name") == source_item_name: + found_source = item + break + if found_source: + break + if not found_source: + raise ValueError(f"Item '{source_item_name}' not found in '{source_parent_path or '[project root]'}'") + + source_id = found_source["_id"] + source_kind = "Folder" if "folderType" in found_source else "File" + print("Checking destination path") + # === Resolve Destination Folder === + dest_parts = destination_path.strip("/").split("/") + dest_folder_name = dest_parts[-1] + dest_parent_path = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else None + + try: + dest_contents = dest_client.list_folder_content(dest_parent_path) + match = next((f for f in dest_contents.get("folders", []) if f.get("name") == dest_folder_name), None) + if not match: + raise ValueError(f"Could not resolve destination folder '{destination_path}'") + + target_id = match["_id"] + folder_type = match.get("folderType") + # Normalize kind: top-level datasets are kind=Dataset, all other folders are kind=Folder + if folder_type in ("VirtualFolder", "Folder"): + target_kind = "Folder" + elif folder_type == "S3Folder": + raise ValueError(f"Unable to move item '{source_item_name}' to '{destination_path}'. " + + "The destination is an S3 folder, and only virtual folders can be selected as valid move destinations.") + elif isinstance(folder_type, bool) and folder_type: # legacy dataset structure + target_kind = "Dataset" + else: + raise ValueError(f"Unrecognized folderType '{folder_type}' for destination '{destination_path}'") + + except Exception as e: + raise ValueError(f"Could not resolve destination path '{destination_path}'. {str(e)}") + print(f"Moving {source_kind} '{source_item_name}' to '{destination_path}' " + + f"in project '{destination_project_name} ...") + # === Perform Move === + try: + response = source_client.move_files_and_folders( + source_id=source_id, + source_kind=source_kind, + target_id=target_id, + target_kind=target_kind + ) + if response.ok: + click.secho(f"{source_kind} '{source_item_name}' moved to '{destination_path}' " + + f"in project '{destination_project_name}'.", fg="green", bold=True) + else: + raise ValueError(f"Move failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Move operation failed. {str(e)}") + + +@datasets.command(name="rename") +@click.argument("source_path", required=True) +@click.argument("new_name", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def renaming_item(ctx, + source_path, + new_name, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Rename a file or folder in a CloudOS project. + + SOURCE_PATH [path]: the full path to the file or folder to rename. It must be a 'Data' folder path. + E.g.: 'Data/folderA/old_name.txt'\n + NEW_NAME [name]: the new name to assign to the file or folder. E.g.: 'new_name.txt' + """ + if not source_path.strip("/").startswith("Data/"): + raise ValueError("SOURCE_PATH must start with 'Data/', pointing to a file or folder in that dataset.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + parts = source_path.strip("/").split("/") + + parent_path = "/".join(parts[:-1]) + target_name = parts[-1] + + try: + contents = client.list_folder_content(parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") + + # Search for file/folder + found_item = None + for category in ["files", "folders"]: + for item in contents.get(category, []): + if item.get("name") == target_name: + found_item = item + break + if found_item: + break + + if not found_item: + raise ValueError(f"Item '{target_name}' not found in '{parent_path or '[project root]'}'") + + item_id = found_item["_id"] + kind = "Folder" if "folderType" in found_item else "File" + + print(f"Renaming {kind} '{target_name}' to '{new_name}'...") + try: + response = client.rename_item(item_id=item_id, new_name=new_name, kind=kind) + if response.ok: + click.secho( + f"{kind} '{target_name}' renamed to '{new_name}' in folder '{parent_path}'.", + fg="green", + bold=True + ) + else: + raise ValueError(f"Rename failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Rename operation failed. {str(e)}") + + +@datasets.command(name="cp") +@click.argument("source_path", required=True) +@click.argument("destination_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The source project name.') +@click.option('--destination-project-name', required=False, help='The destination project name. Defaults to the source project.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def copy_item_cli(ctx, + source_path, + destination_path, + apikey, + cloudos_url, + workspace_id, + project_name, + destination_project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Copy a file or folder (S3 or virtual) from SOURCE_PATH to DESTINATION_PATH. + + SOURCE_PATH [path]: the full path to the file or folder to copy. + E.g.: AnalysesResults/my_analysis/results/my_plot.png\n + DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. + E.g.: Data/plots + """ + destination_project_name = destination_project_name or project_name + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + # Initialize clients + source_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + dest_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=destination_project_name, + verify=verify_ssl, + cromwell_token=None + ) + # Validate paths + dest_parts = destination_path.strip("/").split("/") + if not dest_parts or dest_parts[0] != "Data": + raise ValueError("DESTINATION_PATH must start with 'Data/'.") + # Parse source and destination + source_parts = source_path.strip("/").split("/") + source_parent = "/".join(source_parts[:-1]) if len(source_parts) > 1 else "" + source_name = source_parts[-1] + dest_folder_name = dest_parts[-1] + dest_parent = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else "" + try: + source_content = source_client.list_folder_content(source_parent) + dest_content = dest_client.list_folder_content(dest_parent) + except Exception as e: + raise ValueError(f"Could not access paths. {str(e)}") + # Find the source item + source_item = None + for item in source_content.get('files', []) + source_content.get('folders', []): + if item.get("name") == source_name: + source_item = item + break + if not source_item: + raise ValueError(f"Item '{source_name}' not found in '{source_parent or '[project root]'}'") + # Find the destination folder + destination_folder = None + for folder in dest_content.get("folders", []): + if folder.get("name") == dest_folder_name: + destination_folder = folder + break + if not destination_folder: + raise ValueError(f"Destination folder '{destination_path}' not found.") + try: + # Determine item type + if "fileType" in source_item: + item_type = "file" + elif source_item.get("folderType") == "VirtualFolder": + item_type = "virtual_folder" + elif "s3BucketName" in source_item and source_item.get("folderType") == "S3Folder": + item_type = "s3_folder" + else: + raise ValueError("Could not determine item type.") + print(f"Copying {item_type.replace('_', ' ')} '{source_name}' to '{destination_path}'...") + if destination_folder.get("folderType") is True and destination_folder.get("kind") in ("Data", "Cohorts", "AnalysesResults"): + destination_kind = "Dataset" + elif destination_folder.get("folderType") == "S3Folder": + raise ValueError(f"Unable to copy item '{source_name}' to '{destination_path}'. The destination is an S3 folder, and only virtual folders can be selected as valid copy destinations.") + else: + destination_kind = "Folder" + response = source_client.copy_item( + item=source_item, + destination_id=destination_folder["_id"], + destination_kind=destination_kind + ) + if response.ok: + click.secho("Item copied successfully.", fg="green", bold=True) + else: + raise ValueError(f"Copy failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Copy operation failed. {str(e)}") + + +@datasets.command(name="mkdir") +@click.argument("new_folder_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def mkdir_item(ctx, + new_folder_path, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Create a virtual folder in a CloudOS project. + + NEW_FOLDER_PATH [path]: Full path to the new folder including its name. Must start with 'Data'. + """ + new_folder_path = new_folder_path.strip("/") + if not new_folder_path.startswith("Data"): + raise ValueError("NEW_FOLDER_PATH must start with 'Data'.") + + path_parts = new_folder_path.split("/") + if len(path_parts) < 2: + raise ValueError("NEW_FOLDER_PATH must include at least a parent folder and the new folder name.") + + parent_path = "/".join(path_parts[:-1]) + folder_name = path_parts[-1] + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + # Split parent path to get its parent + name + parent_parts = parent_path.split("/") + parent_name = parent_parts[-1] + parent_of_parent_path = "/".join(parent_parts[:-1]) + + # List the parent of the parent + try: + contents = client.list_folder_content(parent_of_parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_of_parent_path}'. {str(e)}") + + # Find the parent folder in the contents + folder_info = next( + (f for f in contents.get("folders", []) if f.get("name") == parent_name), + None + ) + + if not folder_info: + raise ValueError(f"Could not find folder '{parent_name}' in '{parent_of_parent_path}'.") + + parent_id = folder_info.get("_id") + folder_type = folder_info.get("folderType") + + if folder_type is True: + parent_kind = "Dataset" + elif isinstance(folder_type, str): + parent_kind = "Folder" + else: + raise ValueError(f"Unrecognized folderType for '{parent_path}'.") + + # Create the folder + print(f"Creating folder '{folder_name}' under '{parent_path}' ({parent_kind})...") + try: + response = client.create_virtual_folder(name=folder_name, parent_id=parent_id, parent_kind=parent_kind) + if response.ok: + click.secho(f"Folder '{folder_name}' created under '{parent_path}'", fg="green", bold=True) + else: + raise ValueError(f"Folder creation failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Folder creation failed. {str(e)}") + + +@datasets.command(name="rm") +@click.argument("target_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.option('--force', is_flag=True, help='Force delete files. Required when deleting user uploaded files. This may also delete the file from the cloud provider storage.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def rm_item(ctx, + target_path, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile, + force): + """ + Delete a file or folder in a CloudOS project. + + TARGET_PATH [path]: the full path to the file or folder to delete. Must start with 'Data'. \n + E.g.: 'Data/folderA/file.txt' or 'Data/my_analysis/results/folderB' + """ + if not target_path.strip("/").startswith("Data/"): + raise ValueError("TARGET_PATH must start with 'Data/', pointing to a file or folder.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + parts = target_path.strip("/").split("/") + parent_path = "/".join(parts[:-1]) + item_name = parts[-1] + + try: + contents = client.list_folder_content(parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") + + found_item = None + for item in contents.get('files', []) + contents.get('folders', []): + if item.get("name") == item_name: + found_item = item + break + + if not found_item: + raise ValueError(f"Item '{item_name}' not found in '{parent_path or '[project root]'}'") + + item_id = found_item.get("_id", '') + kind = "Folder" if "folderType" in found_item else "File" + if item_id == '': + raise ValueError(f"Item '{item_name}' could not be removed as the parent folder is an s3 folder and their content cannot be modified.") + # Check if the item is managed by Lifebit + is_managed_by_lifebit = found_item.get("isManagedByLifebit", False) + if is_managed_by_lifebit and not force: + raise ValueError("By removing this file, it will be permanently deleted. If you want to go forward, please use the --force flag.") + print(f"Removing {kind} '{item_name}' from '{parent_path or '[root]'}'...") + try: + response = client.delete_item(item_id=item_id, kind=kind) + if response.ok: + if is_managed_by_lifebit: + click.secho( + f"{kind} '{item_name}' was permanently deleted from '{parent_path or '[root]'}'.", + fg="green", bold=True + ) + else: + click.secho( + f"{kind} '{item_name}' was removed from '{parent_path or '[root]'}'.", + fg="green", bold=True + ) + click.secho("This item will still be available on your Cloud Provider.", fg="yellow") + else: + raise ValueError(f"Removal failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Remove operation failed. {str(e)}") + + +@datasets.command(name="link") +@click.argument("path", required=True) +@click.option('-k', '--apikey', help='Your CloudOS API key', required=True) +@click.option('-c', '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=False) +@click.option('--workspace-id', help='The specific CloudOS workspace id.', required=True) +@click.option('--session-id', help='The specific CloudOS interactive session id.', required=True) +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default='default') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) +def link(ctx, + path, + apikey, + cloudos_url, + project_name, + workspace_id, + session_id, + disable_ssl_verification, + ssl_cert, + profile): + """ + Link a folder (S3 or File Explorer) to an active interactive analysis. + + PATH [path]: the full path to the S3 folder to link or relative to File Explorer. + E.g.: 's3://bucket-name/folder/subfolder', 'Data/Downloads' or 'Data'. + """ + if not path.startswith("s3://") and project_name is None: + # for non-s3 paths we need the project, for S3 we don't + raise click.UsageError("When using File Explorer paths '--project-name' needs to be defined") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + link_p = Link( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + cromwell_token=None, + project_name=project_name, + verify=verify_ssl + ) + + # Minimal folder validation and improved error messages + is_s3 = path.startswith("s3://") + is_folder = True + if is_s3: + # S3 path validation - use heuristics to determine if it's likely a folder + try: + # If path ends with '/', it's likely a folder + if path.endswith('/'): + is_folder = True + else: + # Check the last part of the path + path_parts = path.rstrip("/").split("/") + if path_parts: + last_part = path_parts[-1] + # If the last part has no dot, it's likely a folder + if '.' not in last_part: + is_folder = True + else: + # If it has a dot, it might be a file - set to None for warning + is_folder = None + else: + # Empty path parts, set to None for uncertainty + is_folder = None + except Exception: + # If we can't parse the S3 path, set to None for uncertainty + is_folder = None + else: + # File Explorer path validation (existing logic) + try: + datasets = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + parts = path.strip("/").split("/") + parent_path = "/".join(parts[:-1]) if len(parts) > 1 else "" + item_name = parts[-1] + contents = datasets.list_folder_content(parent_path) + found = None + for item in contents.get("folders", []): + if item.get("name") == item_name: + found = item + break + if not found: + for item in contents.get("files", []): + if item.get("name") == item_name: + found = item + break + if found and ("folderType" not in found): + is_folder = False + except Exception: + is_folder = None + + if is_folder is False: + if is_s3: + raise ValueError("The S3 path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.") + else: + raise ValueError("Linking files or virtual folders is not supported. Link the S3 parent folder instead.", err=True) + return + elif is_folder is None and is_s3: + click.secho("Unable to verify whether the S3 path is a folder. Proceeding with linking; " + + "however, if the operation fails, please confirm that you are linking a folder rather than a file.", fg='yellow', bold=True) + + try: + link_p.link_folder(path, session_id) + except Exception as e: + if is_s3: + print("If you are linking an S3 path, please ensure it is a folder.") + raise ValueError(f"Could not link folder. {e}") + + +@images.command(name="ls") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--page', help='The response page. Defaults to 1.', required=False, default=1) +@click.option('--limit', help='The page size limit. Defaults to 10', required=False, default=10) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def list_images(ctx, + apikey, + cloudos_url, + procurement_id, + disable_ssl_verification, + ssl_cert, + profile, + page, + limit): + """List images associated with organisations of a given procurement.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None, + page=page, + limit=limit + ) + + try: + result = procurement_images.list_procurement_images() + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") + + +@images.command(name="set") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) +@click.option('--image-type', help='The CloudOS resource image type.', required=True, + type=click.Choice([ + 'RegularInteractiveSessions', + 'SparkInteractiveSessions', + 'RStudioInteractiveSessions', + 'JupyterInteractiveSessions', + 'JobDefault', + 'NextflowBatchComputeEnvironment'])) +@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') +@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) +@click.option('--image-id', help='The new image id value.', required=True) +@click.option('--image-name', help='The new image name value.', required=False) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def set_organisation_image(ctx, + apikey, + cloudos_url, + procurement_id, + organisation_id, + image_type, + provider, + region, + image_id, + image_name, + disable_ssl_verification, + ssl_cert, + profile): + """Set a new image id or name to image associated with an organisations of a given procurement.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = procurement_images.set_procurement_organisation_image( + organisation_id, + image_type, + provider, + region, + image_id, + image_name + ) + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") + + +@images.command(name="reset") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) +@click.option('--image-type', help='The CloudOS resource image type.', required=True, + type=click.Choice([ + 'RegularInteractiveSessions', + 'SparkInteractiveSessions', + 'RStudioInteractiveSessions', + 'JupyterInteractiveSessions', + 'JobDefault', + 'NextflowBatchComputeEnvironment'])) +@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') +@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def reset_organisation_image(ctx, + apikey, + cloudos_url, + procurement_id, + organisation_id, + image_type, + provider, + region, + disable_ssl_verification, + ssl_cert, + profile): + """Reset image associated with an organisations of a given procurement to CloudOS defaults.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = procurement_images.reset_procurement_organisation_image( + organisation_id, + image_type, + provider, + region + ) + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") + +@run_cloudos_cli.command('link') +@click.argument('path', required=False) +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS. When provided, links results, workdir and logs by default.', + required=False) +@click.option('--project-name', + help='The name of a CloudOS project. Required for File Explorer paths.', + required=False) +@click.option('--results', + help='Link only results folder (only works with --job-id).', + is_flag=True) +@click.option('--workdir', + help='Link only working directory (only works with --job-id).', + is_flag=True) +@click.option('--logs', + help='Link only logs folder (only works with --job-id).', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) +def link_command(ctx, + path, + apikey, + cloudos_url, + workspace_id, + session_id, + job_id, + project_name, + results, + workdir, + logs, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """ + Link folders to an interactive analysis session. + + This command is used to link folders + to an active interactive analysis session for direct access to data. + + PATH: Optional path to link (S3). + Required if --job-id is not provided. + + Two modes of operation: + + 1. Job-based linking (--job-id): Links job-related folders. + By default, links results, workdir, and logs folders. + Use --results, --workdir, or --logs flags to link only specific folders. + + 2. Direct path linking (PATH argument): Links a specific S3 path. + + Examples: + + # Link all job folders (results, workdir, logs) + cloudos link --job-id 12345 --session-id abc123 + + # Link only results from a job + cloudos link --job-id 12345 --session-id abc123 --results + + # Link a specific S3 path + cloudos link s3://bucket/folder --session-id abc123 + + """ + print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Validate input parameters + if not job_id and not path: + raise click.UsageError("Either --job-id or PATH argument must be provided.") + + if job_id and path: + raise click.UsageError("Cannot use both --job-id and PATH argument. Please provide only one.") + + # Validate folder-specific flags only work with --job-id + if (results or workdir or logs) and not job_id: + raise click.UsageError("--results, --workdir, and --logs flags can only be used with --job-id.") + + # If no specific folders are selected with job-id, link all by default + if job_id and not (results or workdir or logs): + results = True + workdir = True + logs = True + + if verbose: + print('Using the following parameters:') + print(f'\tCloudOS url: {cloudos_url}') + print(f'\tWorkspace ID: {workspace_id}') + print(f'\tSession ID: {session_id}') + if job_id: + print(f'\tJob ID: {job_id}') + print(f'\tLink results: {results}') + print(f'\tLink workdir: {workdir}') + print(f'\tLink logs: {logs}') + else: + print(f'\tPath: {path}') + + # Initialize Link client + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl + ) + + try: + if job_id: + # Job-based linking + print(f'Linking folders from job {job_id} to interactive session {session_id}...\n') + + # Link results + if results: + link_client.link_job_results(job_id, workspace_id, session_id, verify_ssl, verbose) + + # Link workdir + if workdir: + link_client.link_job_workdir(job_id, workspace_id, session_id, verify_ssl, verbose) + + # Link logs + if logs: + link_client.link_job_logs(job_id, workspace_id, session_id, verify_ssl, verbose) + + + else: + # Direct path linking + print(f'Linking path to interactive session {session_id}...\n') + + # Link path with validation + link_client.link_path_with_validation(path, session_id, verify_ssl, project_name, verbose) + + print('\nLinking operation completed.') + + except BadRequestException as e: + raise ValueError(f"Request failed: {str(e)}") + except Exception as e: + raise ValueError(f"Failed to link folder(s): {str(e)}") + +if __name__ == "__main__": + # Setup logging + debug_mode = '--debug' in sys.argv + setup_logging(debug_mode) + logger = logging.getLogger("CloudOS") + # Check if debug flag was passed (fallback for cases where Click doesn't handle it) + try: + run_cloudos_cli() + except Exception as e: + if debug_mode: + logger.error(e, exc_info=True) + traceback.print_exc() + else: + logger.error(e) + click.echo(click.style(f"Error: {e}", fg='red'), err=True) + sys.exit(1) \ No newline at end of file diff --git a/cloudos_cli/bash/__init__.py b/cloudos_cli/bash/__init__.py new file mode 100644 index 00000000..454c6767 --- /dev/null +++ b/cloudos_cli/bash/__init__.py @@ -0,0 +1 @@ +"""Bash job-related CLI commands.""" diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py new file mode 100644 index 00000000..e4150375 --- /dev/null +++ b/cloudos_cli/bash/cli.py @@ -0,0 +1,564 @@ +"""CLI commands for CloudOS bash job management.""" + +import rich_click as click +import cloudos_cli.jobs.job as jb +from cloudos_cli.clos import Cloudos +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.utils.array_job import generate_datasets_for_project +import sys + + +@click.group() +def bash(): + """CloudOS bash-specific job functionality.""" + print(bash.__doc__ + '\n') + + +@bash.command('job') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('--command', + help='The command to run in the bash job.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--cpus', + help='The number of CPUs to use for the task\'s master node. Default=1.', + type=int, + default=1) +@click.option('--memory', + help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', + type=int, + default=4) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + default='aws') +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run_bash_job(ctx, + apikey, + command, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + parameter, + job_name, + do_not_save_logs, + job_queue, + instance_type, + instance_disk, + cpus, + memory, + storage_mode, + lustre_size, + wait_completion, + wait_time, + repository_platform, + execution_platform, + cost_limit, + accelerate_saving_results, + request_interval, + disable_ssl_verification, + ssl_cert, + profile): + """Run a bash job in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + + if do_not_save_logs: + save_logs = False + else: + save_logs = True + + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=None, importsfile=None, + repository_platform=repository_platform, verify=verify_ssl, last=last) + + if job_queue is not None: + batch = True + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, + workspace_id=workspace_id, verify=verify_ssl) + # I have to add 'nextflow', other wise the job queue id is not found + job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, + job_queue=job_queue) + else: + job_queue_id = None + batch = False + j_id = j.send_job(job_config=None, + parameter=parameter, + git_commit=None, + git_tag=None, + git_branch=None, + job_name=job_name, + resumable=False, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + workflow_type='docker', + nextflow_profile=None, + nextflow_version=None, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=None, + cost_limit=cost_limit, + accelerate_saving_results=accelerate_saving_results, + verify=verify_ssl, + command={"command": command}, + cpus=cpus, + memory=memory) + + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=False, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') + + +@bash.command('array-job') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('--command', + help='The command to run in the bash job.') +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + + 'times as parameters you want to include. ' + + 'For parameters pointing to a file, the format expected is ' + + 'parameter_name=/Data/parameter_value. The parameter value must be a ' + + 'file located in the `Data` subfolder. If no is specified, it defaults to ' + + 'the project specified by the profile or --project-name parameter. ' + + 'E.g.: -p "--file=Data/file.txt" or "--file=/Data/folder/file.txt"')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--cpus', + help='The number of CPUs to use for the task\'s master node. Default=1.', + type=int, + default=1) +@click.option('--memory', + help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', + type=int, + default=4) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + type=click.Choice(['aws', 'azure', 'hpc']), + default='aws') +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--array-file', + help=('Path to a file containing an array of commands to run in the bash job.'), + default=None, + required=True) +@click.option('--separator', + help=('Separator to use in the array file. Default=",".'), + type=click.Choice([',', ';', 'tab', 'space', '|']), + default=",", + required=True) +@click.option('--list-columns', + help=('List columns present in the array file. ' + + 'This option will not run any job.'), + is_flag=True) +@click.option('--array-file-project', + help=('Name of the project in which the array file is placed, if different from --project-name.'), + default=None) +@click.option('--disable-column-check', + help=('Disable the check for the columns in the array file. ' + + 'This option is only used when --array-file is provided.'), + is_flag=True) +@click.option('-a', '--array-parameter', + multiple=True, + help=('A single parameter to pass to the job call only for specifying array columns. ' + + 'It should be in the following form: parameter_name=array_file_column_name. E.g.: ' + + '-a --test=value or -a -test=value or -a test=value or -a =value (for no prefix). ' + + 'You can use this option as many times as parameters you want to include.')) +@click.option('--custom-script-path', + help=('Path of a custom script to run in the bash array job instead of a command.'), + default=None) +@click.option('--custom-script-project', + help=('Name of the project to use when running the custom command script, if ' + + 'different than --project-name.'), + default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run_bash_array_job(ctx, + apikey, + command, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + parameter, + job_name, + do_not_save_logs, + job_queue, + instance_type, + instance_disk, + cpus, + memory, + storage_mode, + lustre_size, + wait_completion, + wait_time, + repository_platform, + execution_platform, + cost_limit, + accelerate_saving_results, + request_interval, + disable_ssl_verification, + ssl_cert, + profile, + array_file, + separator, + list_columns, + array_file_project, + disable_column_check, + array_parameter, + custom_script_path, + custom_script_project): + """Run a bash array job in CloudOS.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + if not list_columns and not (command or custom_script_path): + raise click.UsageError("Must provide --command or --custom-script-path if --list-columns is not set.") + + # when not set, use the global project name + if array_file_project is None: + array_file_project = project_name + + # this needs to be in another call to datasets, by default it uses the global project name + if custom_script_project is None: + custom_script_project = project_name + + # setup separators for API and array file (the're different) + separators = { + ",": {"api": ",", "file": ","}, + ";": {"api": "%3B", "file": ";"}, + "space": {"api": "+", "file": " "}, + "tab": {"api": "tab", "file": "tab"}, + "|": {"api": "%7C", "file": "|"} + } + + # setup important options for the job + if do_not_save_logs: + save_logs = False + else: + save_logs = True + + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=None, importsfile=None, + repository_platform=repository_platform, verify=verify_ssl, last=last) + + # retrieve columns + r = j.retrieve_cols_from_array_file( + array_file, + generate_datasets_for_project(cloudos_url, apikey, workspace_id, array_file_project, verify_ssl), + separators[separator]['api'], + verify_ssl + ) + + if not disable_column_check: + columns = json.loads(r.content).get("headers", None) + # pass this to the SEND JOB API call + # b'{"headers":[{"index":0,"name":"id"},{"index":1,"name":"title"},{"index":2,"name":"filename"},{"index":3,"name":"file2name"}]}' + if columns is None: + raise ValueError("No columns found in the array file metadata.") + if list_columns: + print("Columns: ") + for col in columns: + print(f"\t- {col['name']}") + return + else: + columns = [] + + # setup parameters for the job + cmd = j.setup_params_array_file( + custom_script_path, + generate_datasets_for_project(cloudos_url, apikey, workspace_id, custom_script_project, verify_ssl), + command, + separators[separator]['file'] + ) + + # check columns in the array file vs parameters added + if not disable_column_check and array_parameter: + print("\nChecking columns in the array file vs parameters added...\n") + for ap in array_parameter: + ap_split = ap.split('=') + ap_value = '='.join(ap_split[1:]) + for col in columns: + if col['name'] == ap_value: + print(f"Found column '{ap_value}' in the array file.") + break + else: + raise ValueError(f"Column '{ap_value}' not found in the array file. " + \ + f"Columns in array-file: {separator.join([col['name'] for col in columns])}") + + if job_queue is not None: + batch = True + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, + workspace_id=workspace_id, verify=verify_ssl) + # I have to add 'nextflow', other wise the job queue id is not found + job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, + job_queue=job_queue) + else: + job_queue_id = None + batch = False + + # send job + j_id = j.send_job(job_config=None, + parameter=parameter, + array_parameter=array_parameter, + array_file_header=columns, + git_commit=None, + git_tag=None, + git_branch=None, + job_name=job_name, + resumable=False, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + workflow_type='docker', + nextflow_profile=None, + nextflow_version=None, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=None, + cost_limit=cost_limit, + accelerate_saving_results=accelerate_saving_results, + verify=verify_ssl, + command=cmd, + cpus=cpus, + memory=memory) + + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=False, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) + else: + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') diff --git a/cloudos_cli/configure/cli.py b/cloudos_cli/configure/cli.py new file mode 100644 index 00000000..3e108417 --- /dev/null +++ b/cloudos_cli/configure/cli.py @@ -0,0 +1,48 @@ +"""CLI commands for CloudOS configuration management.""" + +import rich_click as click +from cloudos_cli.configure.configure import ConfigurationProfile +from cloudos_cli.logging.logger import update_command_context_from_click + + +# Create the configure group +@click.group(invoke_without_command=True) +@click.option('--profile', help='Profile to use from the config file', default='default') +@click.option('--make-default', + is_flag=True, + help='Make the profile the default one.') +@click.pass_context +def configure(ctx, profile, make_default): + """CloudOS configuration.""" + print(configure.__doc__ + '\n') + update_command_context_from_click(ctx) + profile = profile or ctx.obj['profile'] + config_manager = ConfigurationProfile() + + if ctx.invoked_subcommand is None and profile == "default" and not make_default: + config_manager.create_profile_from_input(profile_name="default") + + if profile != "default" and not make_default: + config_manager.create_profile_from_input(profile_name=profile) + if make_default: + config_manager.make_default_profile(profile_name=profile) + + +@configure.command('list-profiles') +def list_profiles(): + """List all available configuration profiles.""" + config_manager = ConfigurationProfile() + config_manager.list_profiles() + + +@configure.command('remove-profile') +@click.option('--profile', + help='Name of the profile. Not using this option will lead to profile named "deafults" being generated', + required=True) +@click.pass_context +def remove_profile(ctx, profile): + """Remove a configuration profile.""" + update_command_context_from_click(ctx) + profile = profile or ctx.obj['profile'] + config_manager = ConfigurationProfile() + config_manager.remove_profile(profile) diff --git a/cloudos_cli/cromwell/__init__.py b/cloudos_cli/cromwell/__init__.py new file mode 100644 index 00000000..be98fb6f --- /dev/null +++ b/cloudos_cli/cromwell/__init__.py @@ -0,0 +1 @@ +"""Cromwell server-related CLI commands.""" diff --git a/cloudos_cli/cromwell/cli.py b/cloudos_cli/cromwell/cli.py new file mode 100644 index 00000000..1207dc8c --- /dev/null +++ b/cloudos_cli/cromwell/cli.py @@ -0,0 +1,218 @@ +"""CLI commands for CloudOS Cromwell server management.""" + +import rich_click as click +import json +import time +import sys +from cloudos_cli.clos import Cloudos +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL + +# Constants +REQUEST_INTERVAL_CROMWELL = 5 + + +@click.group() +def cromwell(): + """CloudOS Cromwell server functionality.""" + print(cromwell.__doc__ + '\n') + + +@cromwell.command('status') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_status(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Check Cromwell server status in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + print('Executing status...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tChecking Cromwell status in {workspace_id} workspace') + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + + +@cromwell.command('start') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to Cromwell restart. ' + + 'Default=300.'), + default=300) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_restart(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + wait_time, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Restart Cromwell server in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + action = 'restart' + print('Starting Cromwell server...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tStarting Cromwell server in {workspace_id} workspace') + cl.cromwell_switch(workspace_id, action, verify_ssl) + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + elapsed = 0 + while elapsed < wait_time and c_status_h != 'Running': + c_status_old = c_status_h + time.sleep(REQUEST_INTERVAL_CROMWELL) + elapsed += REQUEST_INTERVAL_CROMWELL + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + if c_status_h != c_status_old: + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h != 'Running': + print(f'\tYour current Cromwell status is: {c_status_h}. The ' + + f'selected wait-time of {wait_time} was exceeded. Please, ' + + 'consider to set a longer wait-time.') + print('\tTo further check your Cromwell status you can either go to ' + + f'{cloudos_url} or use the following command:\n' + + '\tcloudos cromwell status \\\n' + + f'\t\t--cloudos-url {cloudos_url} \\\n' + + '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + + f'\t\t--workspace-id {workspace_id}') + sys.exit(1) + + +@cromwell.command('stop') +@click.version_option() +@click.option('-k', + '--apikey', + help='Your CloudOS API key.') +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. You can use it instead of ' + + 'the apikey.')) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['cloudos_url', 'workspace_id']) +def cromwell_stop(ctx, + apikey, + cromwell_token, + cloudos_url, + workspace_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Stop Cromwell server in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if apikey is None and cromwell_token is None: + raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + action = 'stop' + print('Stopping Cromwell server...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tStopping Cromwell server in {workspace_id} workspace') + cl.cromwell_switch(workspace_id, action, verify_ssl) + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') diff --git a/cloudos_cli/datasets/cli.py b/cloudos_cli/datasets/cli.py new file mode 100644 index 00000000..7c1409af --- /dev/null +++ b/cloudos_cli/datasets/cli.py @@ -0,0 +1,849 @@ +"""CLI commands for CloudOS datasets management.""" + +import rich_click as click +import csv +import sys +from cloudos_cli.datasets import Datasets +from cloudos_cli.link import Link +from cloudos_cli.utils.resources import ssl_selector, format_bytes +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.logging.logger import update_command_context_from_click +from rich.console import Console +from rich.table import Table +from rich.style import Style + + +@click.group() +@click.pass_context +def datasets(ctx): + """CloudOS datasets functionality.""" + update_command_context_from_click(ctx) + if ctx.args and ctx.args[0] != 'ls': + print(datasets.__doc__ + '\n') + + +@datasets.command(name="ls") +@click.argument("path", required=False, nargs=1) +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--details', + help=('When selected, it prints the details of the listed files. ' + + 'Details contains "Type", "Owner", "Size", "Last Updated", ' + + '"Virtual Name", "Storage Path".'), + is_flag=True) +@click.option('--output-format', + help=('The desired display for the output, either directly in standard output or saved as file. ' + + 'Default=stdout.'), + type=click.Choice(['stdout', 'csv'], case_sensitive=False), + default='stdout') +@click.option('--output-basename', + help=('Output file base name to save jobs details. ' + + 'Default=datasets_ls'), + default='datasets_ls', + required=False) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def list_files(ctx, + apikey, + cloudos_url, + workspace_id, + disable_ssl_verification, + ssl_cert, + project_name, + profile, + path, + details, + output_format, + output_basename): + """List contents of a path within a CloudOS workspace dataset.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + datasets = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = datasets.list_folder_content(path) + contents = result.get("contents") or result.get("datasets", []) + + if not contents: + contents = result.get("files", []) + result.get("folders", []) + + # Process items to extract data + processed_items = [] + for item in contents: + is_folder = "folderType" in item or item.get("isDir", False) + type_ = "folder" if is_folder else "file" + + # Enhanced type information + if is_folder: + folder_type = item.get("folderType") + if folder_type == "VirtualFolder": + type_ = "virtual folder" + elif folder_type == "S3Folder": + type_ = "s3 folder" + elif folder_type == "AzureBlobFolder": + type_ = "azure folder" + else: + type_ = "folder" + else: + # Check if file is managed by Lifebit (user uploaded) + is_managed_by_lifebit = item.get("isManagedByLifebit", False) + if is_managed_by_lifebit: + type_ = "file (user uploaded)" + else: + type_ = "file (virtual copy)" + + user = item.get("user", {}) + if isinstance(user, dict): + name = user.get("name", "").strip() + surname = user.get("surname", "").strip() + else: + name = surname = "" + if name and surname: + owner = f"{name} {surname}" + elif name: + owner = name + elif surname: + owner = surname + else: + owner = "-" + + raw_size = item.get("sizeInBytes", item.get("size")) + size = format_bytes(raw_size) if not is_folder and raw_size is not None else "-" + + updated = item.get("updatedAt") or item.get("lastModified", "-") + filepath = item.get("name", "-") + + if item.get("fileType") == "S3File" or item.get("folderType") == "S3Folder": + bucket = item.get("s3BucketName") + key = item.get("s3ObjectKey") or item.get("s3Prefix") + storage_path = f"s3://{bucket}/{key}" if bucket and key else "-" + elif item.get("fileType") == "AzureBlobFile" or item.get("folderType") == "AzureBlobFolder": + account = item.get("blobStorageAccountName") + container = item.get("blobContainerName") + key = item.get("blobName") if item.get("fileType") == "AzureBlobFile" else item.get("blobPrefix") + storage_path = f"az://{account}.blob.core.windows.net/{container}/{key}" if account and container and key else "-" + else: + storage_path = "-" + + processed_items.append({ + 'type': type_, + 'owner': owner, + 'size': size, + 'raw_size': raw_size, + 'updated': updated, + 'name': filepath, + 'storage_path': storage_path, + 'is_folder': is_folder + }) + + # Output handling + if output_format == 'csv': + import csv + + csv_filename = f'{output_basename}.csv' + + if details: + # CSV with all details + with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: + fieldnames = ['Type', 'Owner', 'Size', 'Size (bytes)', 'Last Updated', 'Virtual Name', 'Storage Path'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + + for item in processed_items: + writer.writerow({ + 'Type': item['type'], + 'Owner': item['owner'], + 'Size': item['size'], + 'Size (bytes)': item['raw_size'] if item['raw_size'] is not None else '', + 'Last Updated': item['updated'], + 'Virtual Name': item['name'], + 'Storage Path': item['storage_path'] + }) + else: + # CSV with just names + with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(['Name', 'Storage Path']) + for item in processed_items: + writer.writerow([item['name'], item['storage_path']]) + + click.secho(f'\nDatasets list saved to: {csv_filename}', fg='green', bold=True) + + else: # stdout + if details: + console = Console(width=None) + table = Table(show_header=True, header_style="bold white") + table.add_column("Type", style="cyan", no_wrap=True) + table.add_column("Owner", style="white") + table.add_column("Size", style="magenta") + table.add_column("Last Updated", style="green") + table.add_column("Virtual Name", style="bold", overflow="fold") + table.add_column("Storage Path", style="dim", no_wrap=False, overflow="fold", ratio=2) + + for item in processed_items: + style = Style(color="blue", underline=True) if item['is_folder'] else None + table.add_row( + item['type'], + item['owner'], + item['size'], + item['updated'], + item['name'], + item['storage_path'], + style=style + ) + + console.print(table) + + else: + console = Console() + for item in processed_items: + if item['is_folder']: + console.print(f"[blue underline]{item['name']}[/]") + else: + console.print(item['name']) + + except Exception as e: + raise ValueError(f"Failed to list files for project '{project_name}'. {str(e)}") + + +@datasets.command(name="mv") +@click.argument("source_path", required=True) +@click.argument("destination_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The source project name.') +@click.option('--destination-project-name', required=False, + help='The destination project name. Defaults to the source project.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def move_files(ctx, source_path, destination_path, apikey, cloudos_url, workspace_id, + project_name, destination_project_name, + disable_ssl_verification, ssl_cert, profile): + """ + Move a file or folder from a source path to a destination path within or across CloudOS projects. + + SOURCE_PATH [path]: the full path to the file or folder to move. It must be a 'Data' folder path. + E.g.: 'Data/folderA/file.txt'\n + DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. + E.g.: 'Data/folderB' + """ + # Validate destination constraint + if not destination_path.strip("/").startswith("Data/") and destination_path.strip("/") != "Data": + raise ValueError("Destination path must begin with 'Data/' or be 'Data'.") + if not source_path.strip("/").startswith("Data/") and source_path.strip("/") != "Data": + raise ValueError("SOURCE_PATH must start with 'Data/' or be 'Data'.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + destination_project_name = destination_project_name or project_name + # Initialize Datasets clients + source_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + dest_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=destination_project_name, + verify=verify_ssl, + cromwell_token=None + ) + print('Checking source path') + # === Resolve Source Item === + source_parts = source_path.strip("/").split("/") + source_parent_path = "/".join(source_parts[:-1]) if len(source_parts) > 1 else None + source_item_name = source_parts[-1] + + try: + source_contents = source_client.list_folder_content(source_parent_path) + except Exception as e: + raise ValueError(f"Could not resolve source path '{source_path}'. {str(e)}") + + found_source = None + for collection in ["files", "folders"]: + for item in source_contents.get(collection, []): + if item.get("name") == source_item_name: + found_source = item + break + if found_source: + break + if not found_source: + raise ValueError(f"Item '{source_item_name}' not found in '{source_parent_path or '[project root]'}'") + + source_id = found_source["_id"] + source_kind = "Folder" if "folderType" in found_source else "File" + print("Checking destination path") + # === Resolve Destination Folder === + dest_parts = destination_path.strip("/").split("/") + dest_folder_name = dest_parts[-1] + dest_parent_path = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else None + + try: + dest_contents = dest_client.list_folder_content(dest_parent_path) + match = next((f for f in dest_contents.get("folders", []) if f.get("name") == dest_folder_name), None) + if not match: + raise ValueError(f"Could not resolve destination folder '{destination_path}'") + + target_id = match["_id"] + folder_type = match.get("folderType") + # Normalize kind: top-level datasets are kind=Dataset, all other folders are kind=Folder + if folder_type in ("VirtualFolder", "Folder"): + target_kind = "Folder" + elif folder_type == "S3Folder": + raise ValueError(f"Unable to move item '{source_item_name}' to '{destination_path}'. " + + "The destination is an S3 folder, and only virtual folders can be selected as valid move destinations.") + elif isinstance(folder_type, bool) and folder_type: # legacy dataset structure + target_kind = "Dataset" + else: + raise ValueError(f"Unrecognized folderType '{folder_type}' for destination '{destination_path}'") + + except Exception as e: + raise ValueError(f"Could not resolve destination path '{destination_path}'. {str(e)}") + print(f"Moving {source_kind} '{source_item_name}' to '{destination_path}' " + + f"in project '{destination_project_name} ...") + # === Perform Move === + try: + response = source_client.move_files_and_folders( + source_id=source_id, + source_kind=source_kind, + target_id=target_id, + target_kind=target_kind + ) + if response.ok: + click.secho(f"{source_kind} '{source_item_name}' moved to '{destination_path}' " + + f"in project '{destination_project_name}'.", fg="green", bold=True) + else: + raise ValueError(f"Move failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Move operation failed. {str(e)}") + + +@datasets.command(name="rename") +@click.argument("source_path", required=True) +@click.argument("new_name", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def renaming_item(ctx, + source_path, + new_name, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Rename a file or folder in a CloudOS project. + + SOURCE_PATH [path]: the full path to the file or folder to rename. It must be a 'Data' folder path. + E.g.: 'Data/folderA/old_name.txt'\n + NEW_NAME [name]: the new name to assign to the file or folder. E.g.: 'new_name.txt' + """ + if not source_path.strip("/").startswith("Data/"): + raise ValueError("SOURCE_PATH must start with 'Data/', pointing to a file or folder in that dataset.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + parts = source_path.strip("/").split("/") + + parent_path = "/".join(parts[:-1]) + target_name = parts[-1] + + try: + contents = client.list_folder_content(parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") + + # Search for file/folder + found_item = None + for category in ["files", "folders"]: + for item in contents.get(category, []): + if item.get("name") == target_name: + found_item = item + break + if found_item: + break + + if not found_item: + raise ValueError(f"Item '{target_name}' not found in '{parent_path or '[project root]'}'") + + item_id = found_item["_id"] + kind = "Folder" if "folderType" in found_item else "File" + + print(f"Renaming {kind} '{target_name}' to '{new_name}'...") + try: + response = client.rename_item(item_id=item_id, new_name=new_name, kind=kind) + if response.ok: + click.secho( + f"{kind} '{target_name}' renamed to '{new_name}' in folder '{parent_path}'.", + fg="green", + bold=True + ) + else: + raise ValueError(f"Rename failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Rename operation failed. {str(e)}") + + +@datasets.command(name="cp") +@click.argument("source_path", required=True) +@click.argument("destination_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The source project name.') +@click.option('--destination-project-name', required=False, help='The destination project name. Defaults to the source project.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def copy_item_cli(ctx, + source_path, + destination_path, + apikey, + cloudos_url, + workspace_id, + project_name, + destination_project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Copy a file or folder (S3 or virtual) from SOURCE_PATH to DESTINATION_PATH. + + SOURCE_PATH [path]: the full path to the file or folder to copy. + E.g.: AnalysesResults/my_analysis/results/my_plot.png\n + DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. + E.g.: Data/plots + """ + destination_project_name = destination_project_name or project_name + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + # Initialize clients + source_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + dest_client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=destination_project_name, + verify=verify_ssl, + cromwell_token=None + ) + # Validate paths + dest_parts = destination_path.strip("/").split("/") + if not dest_parts or dest_parts[0] != "Data": + raise ValueError("DESTINATION_PATH must start with 'Data/'.") + # Parse source and destination + source_parts = source_path.strip("/").split("/") + source_parent = "/".join(source_parts[:-1]) if len(source_parts) > 1 else "" + source_name = source_parts[-1] + dest_folder_name = dest_parts[-1] + dest_parent = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else "" + try: + source_content = source_client.list_folder_content(source_parent) + dest_content = dest_client.list_folder_content(dest_parent) + except Exception as e: + raise ValueError(f"Could not access paths. {str(e)}") + # Find the source item + source_item = None + for item in source_content.get('files', []) + source_content.get('folders', []): + if item.get("name") == source_name: + source_item = item + break + if not source_item: + raise ValueError(f"Item '{source_name}' not found in '{source_parent or '[project root]'}'") + # Find the destination folder + destination_folder = None + for folder in dest_content.get("folders", []): + if folder.get("name") == dest_folder_name: + destination_folder = folder + break + if not destination_folder: + raise ValueError(f"Destination folder '{destination_path}' not found.") + try: + # Determine item type + if "fileType" in source_item: + item_type = "file" + elif source_item.get("folderType") == "VirtualFolder": + item_type = "virtual_folder" + elif "s3BucketName" in source_item and source_item.get("folderType") == "S3Folder": + item_type = "s3_folder" + else: + raise ValueError("Could not determine item type.") + print(f"Copying {item_type.replace('_', ' ')} '{source_name}' to '{destination_path}'...") + if destination_folder.get("folderType") is True and destination_folder.get("kind") in ("Data", "Cohorts", "AnalysesResults"): + destination_kind = "Dataset" + elif destination_folder.get("folderType") == "S3Folder": + raise ValueError(f"Unable to copy item '{source_name}' to '{destination_path}'. The destination is an S3 folder, and only virtual folders can be selected as valid copy destinations.") + else: + destination_kind = "Folder" + response = source_client.copy_item( + item=source_item, + destination_id=destination_folder["_id"], + destination_kind=destination_kind + ) + if response.ok: + click.secho("Item copied successfully.", fg="green", bold=True) + else: + raise ValueError(f"Copy failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Copy operation failed. {str(e)}") + + +@datasets.command(name="mkdir") +@click.argument("new_folder_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def mkdir_item(ctx, + new_folder_path, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile): + """ + Create a virtual folder in a CloudOS project. + + NEW_FOLDER_PATH [path]: Full path to the new folder including its name. Must start with 'Data'. + """ + new_folder_path = new_folder_path.strip("/") + if not new_folder_path.startswith("Data"): + raise ValueError("NEW_FOLDER_PATH must start with 'Data'.") + + path_parts = new_folder_path.split("/") + if len(path_parts) < 2: + raise ValueError("NEW_FOLDER_PATH must include at least a parent folder and the new folder name.") + + parent_path = "/".join(path_parts[:-1]) + folder_name = path_parts[-1] + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + # Split parent path to get its parent + name + parent_parts = parent_path.split("/") + parent_name = parent_parts[-1] + parent_of_parent_path = "/".join(parent_parts[:-1]) + + # List the parent of the parent + try: + contents = client.list_folder_content(parent_of_parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_of_parent_path}'. {str(e)}") + + # Find the parent folder in the contents + folder_info = next( + (f for f in contents.get("folders", []) if f.get("name") == parent_name), + None + ) + + if not folder_info: + raise ValueError(f"Could not find folder '{parent_name}' in '{parent_of_parent_path}'.") + + parent_id = folder_info.get("_id") + folder_type = folder_info.get("folderType") + + if folder_type is True: + parent_kind = "Dataset" + elif isinstance(folder_type, str): + parent_kind = "Folder" + else: + raise ValueError(f"Unrecognized folderType for '{parent_path}'.") + + # Create the folder + print(f"Creating folder '{folder_name}' under '{parent_path}' ({parent_kind})...") + try: + response = client.create_virtual_folder(name=folder_name, parent_id=parent_id, parent_kind=parent_kind) + if response.ok: + click.secho(f"Folder '{folder_name}' created under '{parent_path}'", fg="green", bold=True) + else: + raise ValueError(f"Folder creation failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Folder creation failed. {str(e)}") + + +@datasets.command(name="rm") +@click.argument("target_path", required=True) +@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') +@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') +@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') +@click.option('--project-name', required=True, help='The project name.') +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', default=None, help='Profile to use from the config file.') +@click.option('--force', is_flag=True, help='Force delete files. Required when deleting user uploaded files. This may also delete the file from the cloud provider storage.') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) +def rm_item(ctx, + target_path, + apikey, + cloudos_url, + workspace_id, + project_name, + disable_ssl_verification, + ssl_cert, + profile, + force): + """ + Delete a file or folder in a CloudOS project. + + TARGET_PATH [path]: the full path to the file or folder to delete. Must start with 'Data'. \n + E.g.: 'Data/folderA/file.txt' or 'Data/my_analysis/results/folderB' + """ + if not target_path.strip("/").startswith("Data/"): + raise ValueError("TARGET_PATH must start with 'Data/', pointing to a file or folder.") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + client = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + + parts = target_path.strip("/").split("/") + parent_path = "/".join(parts[:-1]) + item_name = parts[-1] + + try: + contents = client.list_folder_content(parent_path) + except Exception as e: + raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") + + found_item = None + for item in contents.get('files', []) + contents.get('folders', []): + if item.get("name") == item_name: + found_item = item + break + + if not found_item: + raise ValueError(f"Item '{item_name}' not found in '{parent_path or '[project root]'}'") + + item_id = found_item.get("_id", '') + kind = "Folder" if "folderType" in found_item else "File" + if item_id == '': + raise ValueError(f"Item '{item_name}' could not be removed as the parent folder is an s3 folder and their content cannot be modified.") + # Check if the item is managed by Lifebit + is_managed_by_lifebit = found_item.get("isManagedByLifebit", False) + if is_managed_by_lifebit and not force: + raise ValueError("By removing this file, it will be permanently deleted. If you want to go forward, please use the --force flag.") + print(f"Removing {kind} '{item_name}' from '{parent_path or '[root]'}'...") + try: + response = client.delete_item(item_id=item_id, kind=kind) + if response.ok: + if is_managed_by_lifebit: + click.secho( + f"{kind} '{item_name}' was permanently deleted from '{parent_path or '[root]'}'.", + fg="green", bold=True + ) + else: + click.secho( + f"{kind} '{item_name}' was removed from '{parent_path or '[root]'}'.", + fg="green", bold=True + ) + click.secho("This item will still be available on your Cloud Provider.", fg="yellow") + else: + raise ValueError(f"Removal failed. {response.status_code} - {response.text}") + except Exception as e: + raise ValueError(f"Remove operation failed. {str(e)}") + + +@datasets.command(name="link") +@click.argument("path", required=True) +@click.option('-k', '--apikey', help='Your CloudOS API key', required=True) +@click.option('-c', '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=False) +@click.option('--workspace-id', help='The specific CloudOS workspace id.', required=True) +@click.option('--session-id', help='The specific CloudOS interactive session id.', required=True) +@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') +@click.option('--ssl-cert', help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default='default') +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) +def link(ctx, + path, + apikey, + cloudos_url, + project_name, + workspace_id, + session_id, + disable_ssl_verification, + ssl_cert, + profile): + """ + Link a folder (S3 or File Explorer) to an active interactive analysis. + + PATH [path]: the full path to the S3 folder to link or relative to File Explorer. + E.g.: 's3://bucket-name/folder/subfolder', 'Data/Downloads' or 'Data'. + """ + if not path.startswith("s3://") and project_name is None: + # for non-s3 paths we need the project, for S3 we don't + raise click.UsageError("When using File Explorer paths '--project-name' needs to be defined") + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + link_p = Link( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + cromwell_token=None, + project_name=project_name, + verify=verify_ssl + ) + + # Minimal folder validation and improved error messages + is_s3 = path.startswith("s3://") + is_folder = True + if is_s3: + # S3 path validation - use heuristics to determine if it's likely a folder + try: + # If path ends with '/', it's likely a folder + if path.endswith('/'): + is_folder = True + else: + # Check the last part of the path + path_parts = path.rstrip("/").split("/") + if path_parts: + last_part = path_parts[-1] + # If the last part has no dot, it's likely a folder + if '.' not in last_part: + is_folder = True + else: + # If it has a dot, it might be a file - set to None for warning + is_folder = None + else: + # Empty path parts, set to None for uncertainty + is_folder = None + except Exception: + # If we can't parse the S3 path, set to None for uncertainty + is_folder = None + else: + # File Explorer path validation (existing logic) + try: + datasets = Datasets( + cloudos_url=cloudos_url, + apikey=apikey, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl, + cromwell_token=None + ) + parts = path.strip("/").split("/") + parent_path = "/".join(parts[:-1]) if len(parts) > 1 else "" + item_name = parts[-1] + contents = datasets.list_folder_content(parent_path) + found = None + for item in contents.get("folders", []): + if item.get("name") == item_name: + found = item + break + if not found: + for item in contents.get("files", []): + if item.get("name") == item_name: + found = item + break + if found and ("folderType" not in found): + is_folder = False + except Exception: + is_folder = None + + if is_folder is False: + if is_s3: + raise ValueError("The S3 path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.") + else: + raise ValueError("Linking files or virtual folders is not supported. Link the S3 parent folder instead.", err=True) + return + elif is_folder is None and is_s3: + click.secho("Unable to verify whether the S3 path is a folder. Proceeding with linking; " + + "however, if the operation fails, please confirm that you are linking a folder rather than a file.", fg='yellow', bold=True) + + try: + link_p.link_folder(path, session_id) + except Exception as e: + if is_s3: + print("If you are linking an S3 path, please ensure it is a folder.") + raise ValueError(f"Could not link folder. {e}") diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py new file mode 100644 index 00000000..1c6a0366 --- /dev/null +++ b/cloudos_cli/jobs/cli.py @@ -0,0 +1,1753 @@ +"""CLI commands for CloudOS job management.""" + +import rich_click as click +import cloudos_cli.jobs.job as jb +from cloudos_cli.clos import Cloudos +from cloudos_cli.utils.errors import BadRequestException +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.utils.details import create_job_details, create_job_list_table +from cloudos_cli.cost.cost import CostViewer +from cloudos_cli.related_analyses.related_analyses import related_analyses +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.link import Link +import json +import copy +import time + + +# Import global constants from __main__ (will be available when imported) +# These need to be imported for backward compatibility +JOB_COMPLETED = 'completed' +REQUEST_INTERVAL_CROMWELL = 30 +ABORT_JOB_STATES = ['running', 'initializing'] +AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] +AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] +HPC_NEXTFLOW_VERSIONS = ['22.10.8'] +AWS_NEXTFLOW_LATEST = '24.04.4' +AZURE_NEXTFLOW_LATEST = '22.11.1-edge' +HPC_NEXTFLOW_LATEST = '22.10.8' + + +# Create the job group +@click.group() +def job(): + """CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS.""" + print(job.__doc__ + '\n') + + +@job.command('run', cls=click.RichCommand) +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.', + required=True) +@click.option('--workflow-name', + help='The name of a CloudOS workflow or pipeline.', + required=True) +@click.option('--last', + help=('When the workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('--job-config', + help=('A config file similar to a nextflow.config file, ' + + 'but only with the parameters to use with your job.')) +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p input=s3://path_to_my_file. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--nextflow-profile', + help=('A comma separated string indicating the nextflow profile/s ' + + 'to use with your job.')) +@click.option('--nextflow-version', + help=('Nextflow version to use when executing the workflow in CloudOS. ' + + 'Default=22.10.8.'), + type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']), + default='22.10.8') +@click.option('--git-commit', + help=('The git commit hash to run for ' + + 'the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--git-tag', + help=('The tag to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--git-branch', + help=('The branch to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--job-name', + help='The name of the job. Default=new_job.', + default='new_job') +@click.option('--resumable', + help='Whether to make the job able to be resumed or not.', + is_flag=True) +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help='Name of the job queue to use with a batch job.') +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), + default='NONE_SELECTED') +@click.option('--instance-disk', + help='The disk space of the master node instance, in GB. Default=500.', + type=int, + default=500) +@click.option('--storage-mode', + help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + + 'regular or lustre storage. Default=regular.'), + default='regular') +@click.option('--lustre-size', + help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + + 'be 1200 or a multiple of it. Default=1200.'), + type=int, + default=1200) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--wdl-mainfile', + help='For WDL workflows, which mainFile (.wdl) is configured to use.',) +@click.option('--wdl-importsfile', + help='For WDL workflows, which importsFile (.zip) is configured to use.',) +@click.option('-t', + '--cromwell-token', + help=('Specific Cromwell server authentication token. Currently, not necessary ' + + 'as apikey can be used instead, but maintained for backwards compatibility.')) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--execution-platform', + help='Name of the execution platform implemented in your CloudOS. Default=aws.', + type=click.Choice(['aws', 'azure', 'hpc']), + default='aws') +@click.option('--hpc-id', + help=('ID of your HPC, only applicable when --execution-platform=hpc. ' + + 'Default=660fae20f93358ad61e0104b'), + default='660fae20f93358ad61e0104b') +@click.option('--azure-worker-instance-type', + help=('The worker node instance type to be used in azure. ' + + 'Default=Standard_D4as_v4'), + default='Standard_D4as_v4') +@click.option('--azure-worker-instance-disk', + help='The disk size in GB for the worker node to be used in azure. Default=100', + type=int, + default=100) +@click.option('--azure-worker-instance-spot', + help='Whether the azure worker nodes have to be spot instances or not.', + is_flag=True) +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float, + default=30.0) +@click.option('--accelerate-file-staging', + help='Enables AWS S3 mountpoint for quicker file staging.', + is_flag=True) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--use-private-docker-repository', + help=('Allows to use private docker repository for running jobs. The Docker user ' + + 'account has to be already linked to CloudOS.'), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) +def run(ctx, + apikey, + cloudos_url, + workspace_id, + project_name, + workflow_name, + last, + job_config, + parameter, + git_commit, + git_tag, + git_branch, + job_name, + resumable, + do_not_save_logs, + job_queue, + nextflow_profile, + nextflow_version, + instance_type, + instance_disk, + storage_mode, + lustre_size, + wait_completion, + wait_time, + wdl_mainfile, + wdl_importsfile, + cromwell_token, + repository_platform, + execution_platform, + hpc_id, + azure_worker_instance_type, + azure_worker_instance_disk, + azure_worker_instance_spot, + cost_limit, + accelerate_file_staging, + accelerate_saving_results, + use_private_docker_repository, + verbose, + request_interval, + disable_ssl_verification, + ssl_cert, + profile): + """Run a CloudOS workflow.""" + # Import here to avoid circular dependency and get constants + from cloudos_cli.__main__ import ( + AWS_NEXTFLOW_VERSIONS, AZURE_NEXTFLOW_VERSIONS, HPC_NEXTFLOW_VERSIONS, + AWS_NEXTFLOW_LATEST, AZURE_NEXTFLOW_LATEST, HPC_NEXTFLOW_LATEST, + JOB_COMPLETED + ) + + # apikey, cloudos_url, workspace_id, project_name, and workflow_name are now automatically resolved by the decorator + print('Executing run...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + my_job = jb.Job(cloudos_url, apikey, cromwell_token, workspace_id, project_name, + workflow_name, mainfile=wdl_mainfile, importsfile=wdl_importsfile, + repository_platform=repository_platform, verify=verify_ssl) + if verbose: + print('\tThe following Job object was created:') + print('\t' + str(my_job) + '\n') + + # set the nextflow version + if nextflow_version == 'latest': + if execution_platform == 'aws': + nextflow_version = AWS_NEXTFLOW_LATEST + elif execution_platform == 'azure': + nextflow_version = AZURE_NEXTFLOW_LATEST + else: + nextflow_version = HPC_NEXTFLOW_LATEST + else: + # validate nextflow version is allowed in the execution platform + if execution_platform == 'aws' and nextflow_version not in AWS_NEXTFLOW_VERSIONS: + raise ValueError(f'Nextflow version {nextflow_version} is not supported in AWS. ' + + f'Supported versions: {", ".join(AWS_NEXTFLOW_VERSIONS)}') + elif execution_platform == 'azure' and nextflow_version not in AZURE_NEXTFLOW_VERSIONS: + raise ValueError(f'Nextflow version {nextflow_version} is not supported in Azure. ' + + f'Supported versions: {", ".join(AZURE_NEXTFLOW_VERSIONS)}') + elif execution_platform == 'hpc' and nextflow_version not in HPC_NEXTFLOW_VERSIONS: + raise ValueError(f'Nextflow version {nextflow_version} is not supported in HPC. ' + + f'Supported versions: {", ".join(HPC_NEXTFLOW_VERSIONS)}') + + # Set instance type based on platform if not set + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = 'm5.xlarge' + + # check if the user has defined a configuration file for the job + if job_config is not None: + my_job_params = my_job.parse_job_config(job_config) + else: + my_job_params = {} + + # Allows to override the job_config file with parameters from the command line + if len(parameter) > 0: + # pass the single parameter list to the function + input_params = my_job.parse_individual_params(list(parameter)) + my_job_params.update(input_params) + + if verbose: + print('\tJob is going to be run with the following parameters:') + print('\t' + str(my_job_params) + '\n') + if execution_platform == 'aws' or execution_platform == 'hpc': + my_job_id = my_job.send_job(my_job_params, + job_name, + repository_platform, + execution_platform, + nextflow_profile, + nextflow_version, + instance_type, + instance_disk, + job_queue, + cost_limit, + storage_mode, + lustre_size, + resumable, + do_not_save_logs, + cromwell_token, + last, + git_commit, + git_tag, + git_branch, + accelerate_file_staging, + accelerate_saving_results, + use_private_docker_repository, + hpc_id) + elif execution_platform == 'azure': + my_job_id = my_job.send_job(my_job_params, + job_name, + repository_platform, + execution_platform, + nextflow_profile, + nextflow_version, + instance_type, + instance_disk, + job_queue, + cost_limit, + storage_mode, + lustre_size, + resumable, + do_not_save_logs, + cromwell_token, + last, + git_commit, + git_tag, + git_branch, + azure_worker_instance_type=azure_worker_instance_type, + azure_worker_instance_disk=azure_worker_instance_disk, + azure_worker_instance_spot=azure_worker_instance_spot, + accelerate_file_staging=accelerate_file_staging, + accelerate_saving_results=accelerate_saving_results, + use_private_docker_repository=use_private_docker_repository) + else: + raise ValueError(f'Execution platform {execution_platform} is not supported.') + + if verbose: + print('\tYour job was sent and has ID:') + print(f'\t{my_job_id}') + if not wait_completion: + print(f'\tJob {my_job_id} sent.') + else: + print(f'\tJob {my_job_id} sent. Waiting for it to complete...') + start_time = time.time() + end_time = start_time + wait_time + while True: + # get job status and print it out + j_status = my_job.get_job_status(my_job_id) + status = json.loads(j_status.content)['status'] + print(f'\tJob status: {status}') + if status == JOB_COMPLETED or status == 'failed' or status == 'aborted': + print(f'\tJob completed with status: {status}') + break + # check if we have waited too long + if time.time() > end_time: + print(f'\tWait time limit of {wait_time} seconds reached. ' + + f'Job status: {status}') + break + # wait before checking again + time.sleep(request_interval) + + +@job.command('status') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_status(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the status of a CloudOS job.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + print('Executing status...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job {job_id} in the following workspace: ' + + f'{workspace_id}') + j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) + status_data = json.loads(j_status.content) + print(f'\tJob {job_id} status: {status_data["status"]}') + + +@job.command('workdir') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the working directory to an interactive session.', + is_flag=True) +@click.option('--delete', + help='Delete the results directory of a CloudOS job.', + is_flag=True) +@click.option('-y', '--yes', + help='Skip confirmation prompt when deleting results.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--status', + help='Check the deletion status of the working directory.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_workdir(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + delete, + yes, + session_id, + status, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the working directory of a specified job or check deletion status.""" + from rich.console import Console + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Handle --status flag + if status: + console = Console() + + if verbose: + console.print('[bold cyan]Checking deletion status of job working directory...[/bold cyan]') + console.print('\t[dim]...Preparing objects[/dim]') + console.print('\t[bold]Using the following parameters:[/bold]') + console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') + console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') + console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') + + # Use Cloudos object to access the deletion status method + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + console.print('\t[dim]The following Cloudos object was created:[/dim]') + console.print('\t' + str(cl) + '\n') + + try: + deletion_status = cl.get_workdir_deletion_status( + job_id=job_id, + workspace_id=workspace_id, + verify=verify_ssl + ) + + # Convert API status to user-friendly terminology with color + status_config = { + "ready": ("available", "green"), + "deleting": ("deleting", "yellow"), + "scheduledForDeletion": ("scheduled for deletion", "yellow"), + "deleted": ("deleted", "red"), + "failedToDelete": ("failed to delete", "red") + } + + # Get the status of the workdir folder itself and convert it + api_status = deletion_status.get("status", "unknown") + folder_status, status_color = status_config.get(api_status, (api_status, "white")) + folder_info = deletion_status.get("items", {}) + + # Display results in a clear, styled format with human-readable sentence + console.print(f'The working directory of job [cyan]{deletion_status["job_id"]}[/cyan] is in status: [bold {status_color}]{folder_status}[/bold {status_color}]') + + # For non-available statuses, always show update time and user info + if folder_status != "available": + if folder_info.get("updatedAt"): + console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') + + # Show user information - prefer deletedBy over user field + user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) + if user_info: + user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() + user_email = user_info.get('email', '') + if user_name or user_email: + user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) + console.print(f'[blue]User:[/blue] {user_display}') + + # Display detailed information if verbose + if verbose: + console.print(f'\n[bold]Additional information:[/bold]') + console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') + console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') + console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') + + # Show folder metadata if available + if folder_info.get("createdAt"): + console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') + if folder_info.get("updatedAt"): + console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') + if folder_info.get("folderType"): + console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') + + except ValueError as e: + raise click.ClickException(str(e)) + except Exception as e: + raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") + + return + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Finding working directory path...') + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl) + print(f"Working directory for job {job_id}: {workdir}") + + # Link to interactive session if requested + if link: + if verbose: + print(f'\tLinking working directory to interactive session {session_id}...') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + link_client.link_folder(workdir.strip(), session_id) + + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") + + # Delete workdir directory if requested + if delete: + try: + # Ask for confirmation unless --yes flag is provided + if not yes: + confirmation_message = ( + "\n⚠️ Deleting intermediate results is permanent and cannot be undone. " + "All associated data will be permanently removed and cannot be recovered. " + "The current job, as well as any other jobs sharing the same working directory, " + "will no longer be resumable. This action will be logged in the audit trail " + "(if auditing is enabled for your organisation), and you will be recorded as " + "the user who performed the deletion. You can skip this confirmation step by " + "providing -y or --yes flag to cloudos job workdir --delete. Please confirm " + "that you want to delete intermediate results of this analysis? [y/n] " + ) + click.secho(confirmation_message, fg='black', bg='yellow') + user_input = input().strip().lower() + if user_input != 'y': + print('\nDeletion cancelled.') + return + # Proceed with deletion + job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + job.delete_job_results(job_id, "workDirectory", verify=verify_ssl) + click.secho('\nIntermediate results directories deleted successfully.', fg='green', bold=True) + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve intermediate results for job '{job_id}'. {str(e)}") + else: + if yes: + click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) + + +@job.command('logs') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the logs directories to an interactive session.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_logs(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + session_id, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the logs of a specified job.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Executing logs...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + logs = cl.get_job_logs(job_id, workspace_id, verify_ssl) + for name, path in logs.items(): + print(f"{name}: {path}") + + # Link to interactive session if requested + if link: + if logs: + # Extract the parent logs directory from any log file path + # All log files should be in the same logs directory + first_log_path = next(iter(logs.values())) + # Remove the filename to get the logs directory + # e.g., "s3://bucket/path/to/logs/filename.txt" -> "s3://bucket/path/to/logs" + logs_dir = '/'.join(first_log_path.split('/')[:-1]) + + if verbose: + print(f'\tLinking logs directory to interactive session {session_id}...') + print(f'\t\tLogs directory: {logs_dir}') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + link_client.link_folder(logs_dir, session_id) + else: + if verbose: + print('\tNo logs found to link.') + + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve logs for job '{job_id}'. {str(e)}") + + +@job.command('results') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--link', + help='Link the results directories to an interactive session.', + is_flag=True) +@click.option('--delete', + help='Delete the results directory of a CloudOS job.', + is_flag=True) +@click.option('-y', '--yes', + help='Skip confirmation prompt when deleting results.', + is_flag=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id. Required when using --link flag.', + required=False) +@click.option('--status', + help='Check the deletion status of the job results.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_results(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + link, + delete, + yes, + session_id, + status, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Get the path to the results of a specified job or check deletion status.""" + from rich.console import Console + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + # session_id is also resolved if provided in profile + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Handle --status flag + if status: + console = Console() + + if verbose: + console.print('[bold cyan]Checking deletion status of job results...[/bold cyan]') + console.print('\t[dim]...Preparing objects[/dim]') + console.print('\t[bold]Using the following parameters:[/bold]') + console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') + console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') + console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') + + # Use Cloudos object to access the deletion status method + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + console.print('\t[dim]The following Cloudos object was created:[/dim]') + console.print('\t' + str(cl) + '\n') + + try: + deletion_status = cl.get_results_deletion_status( + job_id=job_id, + workspace_id=workspace_id, + verify=verify_ssl + ) + + # Convert API status to user-friendly terminology with color + status_config = { + "ready": ("available", "green"), + "deleting": ("deleting", "yellow"), + "scheduledForDeletion": ("scheduled for deletion", "yellow"), + "deleted": ("deleted", "red"), + "failedToDelete": ("failed to delete", "red") + } + + # Get the status of the results folder itself and convert it + api_status = deletion_status.get("status", "unknown") + folder_status, status_color = status_config.get(api_status, (api_status, "white")) + folder_info = deletion_status.get("items", {}) + + # Display results in a clear, styled format with human-readable sentence + console.print(f'The results of job [cyan]{deletion_status["job_id"]}[/cyan] are in status: [bold {status_color}]{folder_status}[/bold {status_color}]') + + # For non-available statuses, always show update time and user info + if folder_status != "available": + if folder_info.get("updatedAt"): + console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') + + # Show user information - prefer deletedBy over user field + user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) + if user_info: + user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() + user_email = user_info.get('email', '') + if user_name or user_email: + user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) + console.print(f'[blue]User:[/blue] {user_display}') + + # Display detailed information if verbose + if verbose: + console.print(f'\n[bold]Additional information:[/bold]') + console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') + console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') + console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') + + # Show folder metadata if available + if folder_info.get("createdAt"): + console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') + if folder_info.get("updatedAt"): + console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') + if folder_info.get("folderType"): + console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') + + except ValueError as e: + raise click.ClickException(str(e)) + except Exception as e: + raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") + + return + + # Validate link flag requirements AFTER loading profile + if link and not session_id: + raise click.ClickException("--session-id is required when using --link flag") + + print('Executing results...') + if verbose: + print('\t...Preparing objects') + print('\tUsing the following parameters:') + print(f'\t\tCloudOS url: {cloudos_url}') + print(f'\t\tWorkspace ID: {workspace_id}') + print(f'\t\tJob ID: {job_id}') + if link: + print(f'\t\tSession ID: {session_id}') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + try: + results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) + print(f"results: {results_path}") + + # Link to interactive session if requested + if link: + if verbose: + print(f'\tLinking results directory to interactive session {session_id}...') + + # Use Link class to perform the linking + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, # Not needed for linking operations + workspace_id=workspace_id, + project_name=None, # Not needed for S3 paths + verify=verify_ssl + ) + + if verbose: + print(f'\t\tLinking results ({results_path})...') + + link_client.link_folder(results_path, session_id) + + # Delete results directory if requested + if delete: + # Ask for confirmation unless --yes flag is provided + if not yes: + confirmation_message = ( + "\n⚠️ Deleting final analysis results is irreversible. " + "All data and backups will be permanently removed and cannot be recovered. " + "You can skip this confirmation step by providing '-y' or '--yes' flag to " + "'cloudos job results --delete'. " + "Please confirm that you want to delete final results of this analysis? [y/n] " + ) + click.secho(confirmation_message, fg='black', bg='yellow') + user_input = input().strip().lower() + if user_input != 'y': + print('\nDeletion cancelled.') + return + if verbose: + print(f'\nDeleting result directories from CloudOS...') + # Proceed with deletion + job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + job.delete_job_results(job_id, "analysisResults", verify=verify_ssl) + click.secho('\nResults directories deleted successfully.', fg='green', bold=True) + else: + if yes: + click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve results for job '{job_id}'. {str(e)}") + + +@job.command('details') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to search for.', + required=True) +@click.option('--output-format', + help=('The desired display for the output, either directly in standard output or saved as file. ' + + 'Default=stdout.'), + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--output-basename', + help=('Output file base name to save jobs details. ' + + 'Default={job_id}_details'), + required=False) +@click.option('--parameters', + help=('Whether to generate a ".config" file that can be used as input for --job-config parameter. ' + + 'It will have the same basename as defined in "--output-basename". '), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_details(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + output_basename, + parameters, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve job details in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + if ctx.get_parameter_source('output_basename') == click.core.ParameterSource.DEFAULT: + output_basename = f"{job_id}_details" + + print('Executing details...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\tSearching for job id: {job_id}') + + # check if the API gives a 403 error/forbidden error + try: + j_details = cl.get_job_status(job_id, workspace_id, verify_ssl) + except BadRequestException as e: + if '403' in str(e) or 'Forbidden' in str(e): + raise ValueError("API can only show job details of your own jobs, cannot see other user's job details.") + else: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve details for job '{job_id}'. {str(e)}") + create_job_details(json.loads(j_details.content), job_id, output_format, output_basename, parameters, cloudos_url) + + +@job.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save jobs list. ' + + 'Default=joblist'), + default='joblist', + required=False) +@click.option('--output-format', + help='The desired output format. For json option --all-fields will be automatically set to True. Default=stdout.', + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--table-columns', + help=('Comma-separated list of columns to display in the table. Only applicable when --output-format=stdout. ' + + 'Available columns: status,name,project,owner,pipeline,id,submit_time,end_time,run_time,commit,cost,resources,storage_type. ' + + 'Default: responsive (auto-selects columns based on terminal width)'), + default=None) +@click.option('--all-fields', + help=('Whether to collect all available fields from jobs or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv. Automatically enabled for json output.'), + is_flag=True) +@click.option('--last-n-jobs', + help=("The number of last workspace jobs to retrieve. You can use 'all' to " + + "retrieve all workspace jobs. When adding this option, options " + + "'--page' and '--page-size' are ignored.")) +@click.option('--page', + help=('Page number to fetch from the API. Used with --page-size to control jobs ' + + 'per page (e.g. --page=4 --page-size=20). Default=1.'), + type=int, + default=1) +@click.option('--page-size', + help=('Page size to retrieve from API, corresponds to the number of jobs per page. ' + + 'Maximum allowed integer is 100. Default=10.'), + type=int, + default=10) +@click.option('--archived', + help=('When this flag is used, only archived jobs list is collected.'), + is_flag=True) +@click.option('--filter-status', + help='Filter jobs by status (e.g., completed, running, failed, aborted).') +@click.option('--filter-job-name', + help='Filter jobs by job name ( case insensitive ).') +@click.option('--filter-project', + help='Filter jobs by project name.') +@click.option('--filter-workflow', + help='Filter jobs by workflow/pipeline name.') +@click.option('--last', + help=('When workflows are duplicated, use the latest imported workflow (by date).'), + is_flag=True) +@click.option('--filter-job-id', + help='Filter jobs by specific job ID.') +@click.option('--filter-only-mine', + help='Filter to show only jobs belonging to the current user.', + is_flag=True) +@click.option('--filter-queue', + help='Filter jobs by queue name. Only applies to jobs running in batch environment. Non-batch jobs are preserved in results.') +@click.option('--filter-owner', + help='Filter jobs by owner username.') +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + table_columns, + all_fields, + last_n_jobs, + page, + page_size, + archived, + filter_status, + filter_job_name, + filter_project, + filter_workflow, + last, + filter_job_id, + filter_only_mine, + filter_owner, + filter_queue, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect and display workspace jobs from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Pass table_columns directly to create_job_list_table for validation and processing + selected_columns = table_columns + # Only set outfile if not using stdout + if output_format != 'stdout': + outfile = output_basename + '.' + output_format + + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for jobs in the following workspace: ' + + f'{workspace_id}') + # Check if the user provided the --page option + ctx = click.get_current_context() + if not isinstance(page, int) or page < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') + + if not isinstance(page_size, int) or page_size < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page-size parameter') + + # Validate page_size limit - must be done before API call + if page_size > 100: + click.secho('Error: Page size cannot exceed 100. Please use --page-size with a value <= 100', fg='red', err=True) + raise SystemExit(1) + + result = cl.get_job_list(workspace_id, last_n_jobs, page, page_size, archived, verify_ssl, + filter_status=filter_status, + filter_job_name=filter_job_name, + filter_project=filter_project, + filter_workflow=filter_workflow, + filter_job_id=filter_job_id, + filter_only_mine=filter_only_mine, + filter_owner=filter_owner, + filter_queue=filter_queue, + last=last) + + # Extract jobs and pagination metadata from result + my_jobs_r = result['jobs'] + pagination_metadata = result['pagination_metadata'] + + # Validate requested page exists + if pagination_metadata: + total_jobs = pagination_metadata.get('Pagination-Count', 0) + current_page_size = pagination_metadata.get('Pagination-Limit', page_size) + + if total_jobs > 0: + total_pages = (total_jobs + current_page_size - 1) // current_page_size + if page > total_pages: + click.secho(f'Error: Page {page} does not exist. There are only {total_pages} page(s) available with {total_jobs} total job(s). ' + f'Please use --page with a value between 1 and {total_pages}', fg='red', err=True) + raise SystemExit(1) + + if len(my_jobs_r) == 0: + # Check if any filtering options are being used + filters_used = any([ + filter_status, + filter_job_name, + filter_project, + filter_workflow, + filter_job_id, + filter_only_mine, + filter_owner, + filter_queue + ]) + if output_format == 'stdout': + # For stdout, always show a user-friendly message + create_job_list_table([], cloudos_url, pagination_metadata, selected_columns) + else: + if filters_used: + print('A total of 0 jobs collected.') + elif ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + print('A total of 0 jobs collected. This is likely because your workspace ' + + 'has no jobs created yet.') + else: + print('A total of 0 jobs collected. This is likely because the --page you requested ' + + 'does not exist. Please, try a smaller number for --page or collect all the jobs by not ' + + 'using --page parameter.') + elif output_format == 'stdout': + # Display as table + create_job_list_table(my_jobs_r, cloudos_url, pagination_metadata, selected_columns) + elif output_format == 'csv': + my_jobs = cl.process_job_list(my_jobs_r, all_fields) + cl.save_job_list_to_csv(my_jobs, outfile) + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_jobs_r)) + print(f'\tJob list collected with a total of {len(my_jobs_r)} jobs.') + print(f'\tJob list saved to {outfile}') + else: + raise ValueError('Unrecognised output format. Please use one of [stdout|csv|json]') + + +@job.command('abort') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-ids', + help=('One or more job ids to abort. If more than ' + + 'one is provided, they must be provided as ' + + 'a comma separated list of ids. E.g. id1,id2,id3'), + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.option('--force', + help='Force abort the job even if it is not in a running or initializing state.', + is_flag=True) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def abort_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + job_ids, + verbose, + disable_ssl_verification, + ssl_cert, + profile, + force): + """Abort all specified jobs from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + print('Aborting jobs...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for jobs in the following workspace: ' + + f'{workspace_id}') + # check if the user provided an empty job list + jobs = job_ids.replace(' ', '') + if not jobs: + raise ValueError('No job IDs provided. Please specify at least one job ID to abort.') + jobs = jobs.split(',') + + # Issue warning if using --force flag + if force: + click.secho(f"Warning: Using --force to abort jobs. Some data might be lost.", fg='yellow', bold=True) + + for job in jobs: + try: + j_status = cl.get_job_status(job, workspace_id, verify_ssl) + except Exception as e: + click.secho(f"Failed to get status for job {job}, please make sure it exists in the workspace: {e}", fg='yellow', bold=True) + continue + + j_status_content = json.loads(j_status.content) + job_status = j_status_content['status'] + + # Check if job is in a state that normally allows abortion + is_abortable = job_status in ABORT_JOB_STATES + + # Issue warning if job is in initializing state and not using force + if job_status == 'initializing' and not force: + click.secho(f"Warning: Job {job} is in initializing state.", fg='yellow', bold=True) + + # Check if job can be aborted + if not is_abortable: + click.secho(f"Job {job} is not in a state that can be aborted and is ignored. " + + f"Current status: {job_status}", fg='yellow', bold=True) + else: + try: + cl.abort_job(job, workspace_id, verify_ssl, force) + click.secho(f"Job '{job}' aborted successfully.", fg='green', bold=True) + except Exception as e: + click.secho(f"Failed to abort job {job}. Error: {e}", fg='red', bold=True) + + +@job.command('cost') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to get costs for.', + required=True) +@click.option('--output-format', + help='The desired file format (file extension) for the output. For json option --all-fields will be automatically set to True. Default=csv.', + type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), + default='stdout') +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def job_cost(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve job cost information in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + print('Retrieving cost information...') + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if verbose: + print('\t...Preparing objects') + cost_viewer = CostViewer(cloudos_url, apikey) + if verbose: + print(f'\tSearching for cost data for job id: {job_id}') + # Display costs with pagination + cost_viewer.display_costs(job_id, workspace_id, output_format, verify_ssl) + + +@job.command('related') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS to get costs for.', + required=True) +@click.option('--output-format', + help='The desired output format. Default=stdout.', + type=click.Choice(['stdout', 'json'], case_sensitive=False), + default='stdout') +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def related(ctx, + apikey, + cloudos_url, + workspace_id, + job_id, + output_format, + disable_ssl_verification, + ssl_cert, + profile): + """Retrieve related job analyses in CloudOS.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + related_analyses(cloudos_url, apikey, job_id, workspace_id, output_format, verify_ssl) + + +@click.command() +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--job-ids', + help=('One or more job ids to archive/unarchive. If more than ' + + 'one is provided, they must be provided as ' + + 'a comma separated list of ids. E.g. id1,id2,id3'), + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def archive_unarchive_jobs(ctx, + apikey, + cloudos_url, + workspace_id, + job_ids, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Archive or unarchive specified jobs in a CloudOS workspace.""" + # Determine operation based on the command name used + target_archived_state = ctx.info_name == "archive" + action = "archive" if target_archived_state else "unarchive" + action_past = "archived" if target_archived_state else "unarchived" + action_ing = "archiving" if target_archived_state else "unarchiving" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + print(f'{action_ing.capitalize()} jobs...') + + if verbose: + print('\t...Preparing objects') + + cl = Cloudos(cloudos_url, apikey, None) + + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print(f'\t{action_ing.capitalize()} jobs in the following workspace: {workspace_id}') + + # check if the user provided an empty job list + jobs = job_ids.replace(' ', '') + if not jobs: + raise ValueError(f'No job IDs provided. Please specify at least one job ID to {action}.') + jobs_list = [job for job in jobs.split(',') if job] # Filter out empty strings + + # Check for duplicate job IDs + duplicates = [job_id for job_id in set(jobs_list) if jobs_list.count(job_id) > 1] + if duplicates: + dup_str = ', '.join(duplicates) + click.secho(f'Warning: Duplicate job IDs detected and will be processed only once: {dup_str}', fg='yellow', bold=True) + # Remove duplicates while preserving order + jobs_list = list(dict.fromkeys(jobs_list)) + if verbose: + print(f'\tDuplicate job IDs removed. Processing {len(jobs_list)} unique job(s).') + + # Check archive status for all jobs + status_check = cl.check_jobs_archive_status(jobs_list, workspace_id, target_archived_state=target_archived_state, verify=verify_ssl, verbose=verbose) + valid_jobs = status_check['valid_jobs'] + already_processed = status_check['already_processed'] + invalid_jobs = status_check['invalid_jobs'] + + # Report invalid jobs (but continue processing valid ones) + for job_id, error_msg in invalid_jobs.items(): + click.secho(f"Failed to get status for job {job_id}, please make sure it exists in the workspace: {error_msg}", fg='yellow', bold=True) + + if not valid_jobs and not already_processed: + # All jobs were invalid - exit gracefully + click.secho('No valid job IDs found. Please check that the job IDs exist and are accessible.', fg='yellow', bold=True) + return + + if not valid_jobs: + if len(already_processed) == 1: + click.secho(f"Job '{already_processed[0]}' is already {action_past}. No action needed.", fg='cyan', bold=True) + else: + click.secho(f"All {len(already_processed)} jobs are already {action_past}. No action needed.", fg='cyan', bold=True) + return + + try: + # Call the appropriate action method + if target_archived_state: + cl.archive_jobs(valid_jobs, workspace_id, verify_ssl) + else: + cl.unarchive_jobs(valid_jobs, workspace_id, verify_ssl) + + success_msg = [] + if len(valid_jobs) == 1: + success_msg.append(f"Job '{valid_jobs[0]}' {action_past} successfully.") + else: + success_msg.append(f"{len(valid_jobs)} jobs {action_past} successfully: {', '.join(valid_jobs)}") + + if already_processed: + if len(already_processed) == 1: + success_msg.append(f"Job '{already_processed[0]}' was already {action_past}.") + else: + success_msg.append(f"{len(already_processed)} jobs were already {action_past}: {', '.join(already_processed)}") + + click.secho('\n'.join(success_msg), fg='green', bold=True) + except Exception as e: + raise ValueError(f"Failed to {action} jobs: {str(e)}") + + +@click.command(help='Clone or resume a job with modified parameters') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--project-name', + help='The name of a CloudOS project.') +@click.option('-p', + '--parameter', + multiple=True, + help=('A single parameter to pass to the job call. It should be in the ' + + 'following form: parameter_name=parameter_value. E.g.: ' + + '-p input=s3://path_to_my_file. You can use this option as many ' + + 'times as parameters you want to include.')) +@click.option('--nextflow-profile', + help=('A comma separated string indicating the nextflow profile/s ' + + 'to use with your job.')) +@click.option('--nextflow-version', + help=('Nextflow version to use when executing the workflow in CloudOS. ' + + 'Default=22.10.8.'), + type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest'])) +@click.option('--git-branch', + help=('The branch to run for the selected pipeline. ' + + 'If not specified it defaults to the last commit ' + + 'of the default branch.')) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option('--job-name', + help='The name of the job. If not set, will take the name of the cloned job.') +@click.option('--do-not-save-logs', + help=('Avoids process log saving. If you select this option, your job process ' + + 'logs will not be stored.'), + is_flag=True) +@click.option('--job-queue', + help=('Name of the job queue to use with a batch job. ' + + 'In Azure workspaces, this option is ignored.')) +@click.option('--instance-type', + help=('The type of compute instance to use as master node. ' + + 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).')) +@click.option('--cost-limit', + help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', + type=float) +@click.option('--job-id', + help='The CloudOS job id of the job to be cloned.', + required=True) +@click.option('--accelerate-file-staging', + help='Enables AWS S3 mountpoint for quicker file staging.', + is_flag=True) +@click.option('--accelerate-saving-results', + help='Enables saving results directly to cloud storage bypassing the master node.', + is_flag=True) +@click.option('--resumable', + help='Whether to make the job able to be resumed or not.', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', + help='Profile to use from the config file', + default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def clone_resume(ctx, + apikey, + cloudos_url, + workspace_id, + project_name, + parameter, + nextflow_profile, + nextflow_version, + git_branch, + repository_platform, + job_name, + do_not_save_logs, + job_queue, + instance_type, + cost_limit, + job_id, + accelerate_file_staging, + accelerate_saving_results, + resumable, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + if ctx.info_name == "clone": + mode, action = "clone", "cloning" + elif ctx.info_name == "resume": + mode, action = "resume", "resuming" + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + print(f'{action.capitalize()} job...') + if verbose: + print('\t...Preparing objects') + + # Create Job object (set dummy values for project_name and workflow_name, since they come from the cloned job) + job_obj = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", + mainfile=None, importsfile=None, verify=verify_ssl) + + if verbose: + print('\tThe following Job object was created:') + print('\t' + str(job_obj) + '\n') + print(f'\t{action.capitalize()} job {job_id} in workspace: {workspace_id}') + + try: + + # Clone/resume the job with provided overrides + cloned_resumed_job_id = job_obj.clone_or_resume_job( + source_job_id=job_id, + queue_name=job_queue, + cost_limit=cost_limit, + master_instance=instance_type, + job_name=job_name, + nextflow_version=nextflow_version, + branch=git_branch, + repository_platform=repository_platform, + profile=nextflow_profile, + do_not_save_logs=do_not_save_logs, + use_fusion=accelerate_file_staging, + accelerate_saving_results=accelerate_saving_results, + resumable=resumable, + # only when explicitly setting --project-name will be overridden, else using the original project + project_name=project_name if ctx.get_parameter_source("project_name") == click.core.ParameterSource.COMMANDLINE else None, + parameters=list(parameter) if parameter else None, + verify=verify_ssl, + mode=mode + ) + + if verbose: + print(f'\t{mode.capitalize()}d job ID: {cloned_resumed_job_id}') + + print(f"Job successfully {mode}d. New job ID: {cloned_resumed_job_id}") + + except BadRequestException as e: + raise ValueError(f"Failed to {mode} job. Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") + + +# Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) +archive_unarchive_jobs.help = 'Archive specified jobs in a CloudOS workspace.' +job.add_command(archive_unarchive_jobs, "archive") + +# Create a copy with different help text for unarchive +archive_unarchive_jobs_copy = copy.deepcopy(archive_unarchive_jobs) +archive_unarchive_jobs_copy.help = 'Unarchive specified jobs in a CloudOS workspace.' +job.add_command(archive_unarchive_jobs_copy, "unarchive") + + +# Apply the best Click solution: Set specific help text for each command registration +clone_resume.help = 'Clone a job with modified parameters' +job.add_command(clone_resume, "clone") + +# Create a copy with different help text for resume +clone_resume_copy = copy.deepcopy(clone_resume) +clone_resume_copy.help = 'Resume a job with modified parameters' +job.add_command(clone_resume_copy, "resume") diff --git a/cloudos_cli/procurement/cli.py b/cloudos_cli/procurement/cli.py new file mode 100644 index 00000000..84186f2a --- /dev/null +++ b/cloudos_cli/procurement/cli.py @@ -0,0 +1,71 @@ +"""CLI commands for CloudOS procurement management.""" + +import rich_click as click +from cloudos_cli.procurement.images import Images +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from rich.console import Console + + +@click.group() +def procurement(): + """CloudOS procurement functionality.""" + print(procurement.__doc__ + '\n') + + +@procurement.group() +def images(): + """CloudOS procurement images functionality.""" + + +@images.command(name="ls") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--page', help='The response page. Defaults to 1.', required=False, default=1) +@click.option('--limit', help='The page size limit. Defaults to 10', required=False, default=10) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def list_images(ctx, + apikey, + cloudos_url, + procurement_id, + disable_ssl_verification, + ssl_cert, + profile, + page, + limit): + """List images associated with organisations of a given procurement.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None, + page=page, + limit=limit + ) + + try: + result = procurement_images.list_procurement_images() + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") diff --git a/cloudos_cli/projects/__init__.py b/cloudos_cli/projects/__init__.py new file mode 100644 index 00000000..d80ec310 --- /dev/null +++ b/cloudos_cli/projects/__init__.py @@ -0,0 +1 @@ +"""Project-related CLI commands.""" diff --git a/cloudos_cli/projects/cli.py b/cloudos_cli/projects/cli.py new file mode 100644 index 00000000..4ff23ebf --- /dev/null +++ b/cloudos_cli/projects/cli.py @@ -0,0 +1,174 @@ +"""CLI commands for CloudOS project management.""" + +import rich_click as click +import json +import sys +from cloudos_cli.clos import Cloudos +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL + + +@click.group() +def project(): + """CloudOS project functionality.""" + print(project.__doc__ + '\n') + + +@project.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save project list. ' + + 'Default=project_list'), + default='project_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from projects or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--page', + help=('Response page to retrieve. Default=1.'), + type=int, + default=1) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_projects(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + page, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all projects from a CloudOS workspace in CSV format.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for projects in the following workspace: ' + + f'{workspace_id}') + # Check if the user provided the --page option + ctx = click.get_current_context() + if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + get_all = True + else: + get_all = False + if not isinstance(page, int) or page < 1: + raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') + my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all) + if len(my_projects_r) == 0: + if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: + print('A total of 0 projects collected. This is likely because your workspace ' + + 'has no projects created yet.') + else: + print('A total of 0 projects collected. This is likely because the --page you ' + + 'requested does not exist. Please, try a smaller number for --page or collect all the ' + + 'projects by not using --page parameter.') + elif output_format == 'csv': + my_projects = cl.process_project_list(my_projects_r, all_fields) + my_projects.to_csv(outfile, index=False) + print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_projects_r)) + print(f'\tProject list collected with a total of {len(my_projects_r)} projects.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tProject list saved to {outfile}') + + +@project.command('create') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--new-project', + help='The name for the new project.', + required=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def create_project(ctx, + apikey, + cloudos_url, + workspace_id, + new_project, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Create a new project in CloudOS.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + # verify ssl configuration + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Print basic output + if verbose: + print(f'\tUsing CloudOS URL: {cloudos_url}') + print(f'\tUsing workspace: {workspace_id}') + print(f'\tProject name: {new_project}') + + cl = Cloudos(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None) + + try: + project_id = cl.create_project(workspace_id, new_project, verify_ssl) + print(f'\tProject "{new_project}" created successfully with ID: {project_id}') + if verbose: + print(f'\tProject URL: {cloudos_url}/app/projects/{project_id}') + except Exception as e: + print(f'\tError creating project: {str(e)}') + sys.exit(1) diff --git a/cloudos_cli/queue/cli.py b/cloudos_cli/queue/cli.py new file mode 100644 index 00000000..68f84868 --- /dev/null +++ b/cloudos_cli/queue/cli.py @@ -0,0 +1,83 @@ +"""CLI commands for CloudOS job queue management.""" + +import rich_click as click +import json +from cloudos_cli.queue.queue import Queue +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL + + +# Create the queue group +@click.group() +def queue(): + """CloudOS job queue functionality.""" + print(queue.__doc__ + '\n') + + +@queue.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save job queue list. ' + + 'Default=job_queue_list'), + default='job_queue_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from workflows or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_queues(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all available job queues from a CloudOS workspace.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + j_queue = Queue(cloudos_url, apikey, None, workspace_id, verify=verify_ssl) + my_queues = j_queue.get_job_queues() + if len(my_queues) == 0: + raise ValueError('No AWS batch queues found. Please, make sure that your CloudOS supports AWS bath queues') + if output_format == 'csv': + queues_processed = j_queue.process_queue_list(my_queues, all_fields) + queues_processed.to_csv(outfile, index=False) + print(f'\tJob queue list collected with a total of {queues_processed.shape[0]} queues.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_queues)) + print(f'\tJob queue list collected with a total of {len(my_queues)} queues.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tJob queue list saved to {outfile}') diff --git a/cloudos_cli/utils/cli_helpers.py b/cloudos_cli/utils/cli_helpers.py new file mode 100644 index 00000000..a292b3e2 --- /dev/null +++ b/cloudos_cli/utils/cli_helpers.py @@ -0,0 +1,87 @@ +"""CLI helper utilities for debug mode and exception handling.""" + +import rich_click as click +import sys +import logging +from rich.console import Console +from cloudos_cli.logging.logger import setup_logging + +# Global debug state +_global_debug = False + + +def custom_exception_handler(exc_type, exc_value, exc_traceback): + """Custom exception handler that respects debug mode""" + console = Console(stderr=True) + # Initialise logger + debug_mode = '--debug' in sys.argv + setup_logging(debug_mode) + logger = logging.getLogger("CloudOS") + if get_debug_mode(): + logger.error(exc_value, exc_info=exc_value) + console.print("[yellow]Debug mode: showing full traceback[/yellow]") + sys.__excepthook__(exc_type, exc_value, exc_traceback) + else: + # Extract a clean error message + if hasattr(exc_value, 'message'): + error_msg = exc_value.message + elif str(exc_value): + error_msg = str(exc_value) + else: + error_msg = f"{exc_type.__name__}" + logger.error(exc_value) + console.print(f"[bold red]Error: {error_msg}[/bold red]") + + # For network errors, give helpful context + if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): + console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") + + +def pass_debug_to_subcommands(group_cls=click.RichGroup): + """Custom Group class that passes --debug option to all subcommands""" + + class DebugGroup(group_cls): + def add_command(self, cmd, name=None): + # Add debug option to the command if it doesn't already have it + if isinstance(cmd, (click.Command, click.Group)): + has_debug = any(param.name == 'debug' for param in cmd.params) + if not has_debug: + debug_option = click.Option( + ['--debug'], + is_flag=True, + help='Show detailed error information and tracebacks', + is_eager=True, + expose_value=False, + callback=self._debug_callback + ) + cmd.params.insert(-1, debug_option) # Insert at the end for precedence + + super().add_command(cmd, name) + + def _debug_callback(self, ctx, param, value): + """Callback to handle debug flag""" + global _global_debug + if value: + _global_debug = True + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value + + return DebugGroup + + +def get_debug_mode(): + """Get current debug mode state""" + return _global_debug + + +def setup_debug(ctx, param, value): + """Setup debug mode globally and in context""" + global _global_debug + _global_debug = value + if value: + ctx.meta['debug'] = True + else: + ctx.meta['debug'] = False + return value diff --git a/cloudos_cli/workflows/__init__.py b/cloudos_cli/workflows/__init__.py new file mode 100644 index 00000000..7d26061f --- /dev/null +++ b/cloudos_cli/workflows/__init__.py @@ -0,0 +1 @@ +"""Workflow-related CLI commands.""" diff --git a/cloudos_cli/workflows/cli.py b/cloudos_cli/workflows/cli.py new file mode 100644 index 00000000..578af42e --- /dev/null +++ b/cloudos_cli/workflows/cli.py @@ -0,0 +1,153 @@ +"""CLI commands for CloudOS workflow management.""" + +import rich_click as click +import json +from cloudos_cli.clos import Cloudos +from cloudos_cli.import_wf.import_wf import ImportWorflow +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL + + +# Create the workflow group +@click.group() +def workflow(): + """CloudOS workflow functionality: list and import workflows.""" + print(workflow.__doc__ + '\n') + + +@workflow.command('list') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--output-basename', + help=('Output file base name to save workflow list. ' + + 'Default=workflow_list'), + default='workflow_list', + required=False) +@click.option('--output-format', + help='The desired file format (file extension) for the output. Default=csv.', + type=click.Choice(['csv', 'json'], case_sensitive=False), + default='csv') +@click.option('--all-fields', + help=('Whether to collect all available fields from workflows or ' + + 'just the preconfigured selected fields. Only applicable ' + + 'when --output-format=csv'), + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id']) +def list_workflows(ctx, + apikey, + cloudos_url, + workspace_id, + output_basename, + output_format, + all_fields, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """Collect all workflows from a CloudOS workspace in CSV format.""" + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + outfile = output_basename + '.' + output_format + print('Executing list...') + if verbose: + print('\t...Preparing objects') + cl = Cloudos(cloudos_url, apikey, None) + if verbose: + print('\tThe following Cloudos object was created:') + print('\t' + str(cl) + '\n') + print('\tSearching for workflows in the following workspace: ' + + f'{workspace_id}') + my_workflows_r = cl.get_workflow_list(workspace_id, verify=verify_ssl) + if output_format == 'csv': + my_workflows = cl.process_workflow_list(my_workflows_r, all_fields) + my_workflows.to_csv(outfile, index=False) + print(f'\tWorkflow list collected with a total of {my_workflows.shape[0]} workflows.') + elif output_format == 'json': + with open(outfile, 'w') as o: + o.write(json.dumps(my_workflows_r)) + print(f'\tWorkflow list collected with a total of {len(my_workflows_r)} workflows.') + else: + raise ValueError('Unrecognised output format. Please use one of [csv|json]') + print(f'\tWorkflow list saved to {outfile}') + + +@workflow.command('import') +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=('The CloudOS url you are trying to access to. ' + + f'Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), + help='Name of the repository platform of the workflow. Default=github.', + default='github') +@click.option("--workflow-name", help="The name that the workflow will have in CloudOS.", required=True) +@click.option("-w", "--workflow-url", help="URL of the workflow repository.", required=True) +@click.option("-d", "--workflow-docs-link", help="URL to the documentation of the workflow.", default='') +@click.option("--cost-limit", help="Cost limit for the workflow. Default: $30 USD.", default=30) +@click.option("--workflow-description", help="Workflow description", default="") +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name']) +def import_wf(ctx, + apikey, + cloudos_url, + workspace_id, + workflow_name, + workflow_url, + workflow_docs_link, + cost_limit, + workflow_description, + repository_platform, + disable_ssl_verification, + ssl_cert, + profile): + """ + Import workflows from supported repository providers. + """ + # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + repo_import = ImportWorflow( + cloudos_url=cloudos_url, cloudos_apikey=apikey, workspace_id=workspace_id, platform=repository_platform, + workflow_name=workflow_name, workflow_url=workflow_url, workflow_docs_link=workflow_docs_link, + cost_limit=cost_limit, workflow_description=workflow_description, verify=verify_ssl + ) + workflow_id = repo_import.import_workflow() + print(f'\tWorkflow {workflow_name} was imported successfully with the ' + + f'following ID: {workflow_id}') From deb62c38bb2d0a56a079d4e6334e92b4db5b3811 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 27 Jan 2026 18:39:55 +0100 Subject: [PATCH 02/41] test: update imports --- cloudos_cli/jobs/cli.py | 15 +++++++++------ tests/main/test_ssl_selector.py | 2 +- ...test_accelerate_saving_results_clone_resume.py | 6 +++--- .../test_accelerate_saving_results_run.py | 6 +++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 1c6a0366..57c3a48a 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -230,12 +230,15 @@ def run(ctx, ssl_cert, profile): """Run a CloudOS workflow.""" - # Import here to avoid circular dependency and get constants - from cloudos_cli.__main__ import ( - AWS_NEXTFLOW_VERSIONS, AZURE_NEXTFLOW_VERSIONS, HPC_NEXTFLOW_VERSIONS, - AWS_NEXTFLOW_LATEST, AZURE_NEXTFLOW_LATEST, HPC_NEXTFLOW_LATEST, - JOB_COMPLETED - ) + # Import constants from __main__ + from cloudos_cli import __main__ + AWS_NEXTFLOW_VERSIONS = __main__.AWS_NEXTFLOW_VERSIONS + AZURE_NEXTFLOW_VERSIONS = __main__.AZURE_NEXTFLOW_VERSIONS + HPC_NEXTFLOW_VERSIONS = __main__.HPC_NEXTFLOW_VERSIONS + AWS_NEXTFLOW_LATEST = __main__.AWS_NEXTFLOW_LATEST + AZURE_NEXTFLOW_LATEST = __main__.AZURE_NEXTFLOW_LATEST + HPC_NEXTFLOW_LATEST = __main__.HPC_NEXTFLOW_LATEST + JOB_COMPLETED = __main__.JOB_COMPLETED # apikey, cloudos_url, workspace_id, project_name, and workflow_name are now automatically resolved by the decorator print('Executing run...') diff --git a/tests/main/test_ssl_selector.py b/tests/main/test_ssl_selector.py index 86eadf03..5b8295fe 100644 --- a/tests/main/test_ssl_selector.py +++ b/tests/main/test_ssl_selector.py @@ -3,7 +3,7 @@ import sys from io import StringIO import warnings -from cloudos_cli.__main__ import ssl_selector +from cloudos_cli.utils.resources import ssl_selector DUMMY_SSL_CERT_FILE = "tests/test_data/process_job_list_initial_json.json" diff --git a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py index 36710866..482ba98e 100644 --- a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py +++ b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py @@ -5,14 +5,14 @@ """ import pytest from click.testing import CliRunner -from cloudos_cli.__main__ import clone_resume +from cloudos_cli.jobs.cli import clone_resume def test_resume_accelerate_saving_results_flag_is_boolean(): """ Test that --accelerate-saving-results is properly defined as a boolean flag in resume command """ - from cloudos_cli.__main__ import clone_resume + from cloudos_cli.jobs.cli import clone_resume # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None @@ -30,7 +30,7 @@ def test_resume_accelerate_saving_results_flag_definition(): """ Test that the flag has the correct help text definition in resume command """ - from cloudos_cli.__main__ import clone_resume + from cloudos_cli.jobs.cli import clone_resume # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None diff --git a/tests/test_jobs/test_accelerate_saving_results_run.py b/tests/test_jobs/test_accelerate_saving_results_run.py index 58f46755..80ab5d50 100644 --- a/tests/test_jobs/test_accelerate_saving_results_run.py +++ b/tests/test_jobs/test_accelerate_saving_results_run.py @@ -5,14 +5,14 @@ """ import pytest from click.testing import CliRunner -from cloudos_cli.__main__ import run +from cloudos_cli.jobs.cli import run def test_run_accelerate_saving_results_flag_is_boolean(): """ Test that --accelerate-saving-results is properly defined as a boolean flag """ - from cloudos_cli.__main__ import run as run_command + from cloudos_cli.jobs.cli import run as run_command # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None @@ -43,7 +43,7 @@ def test_run_accelerate_saving_results_flag_definition(): """ Test that the flag has the correct help text definition """ - from cloudos_cli.__main__ import run as run_command + from cloudos_cli.jobs.cli import run as run_command # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None From 300868ad4816632318bc8cd5cca4ded8faad8f5f Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 28 Jan 2026 14:33:09 +0100 Subject: [PATCH 03/41] docs: updage changes and version --- CHANGELOG.md | 6 ++++++ cloudos_cli/_version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5d592d1..e4f2415f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## lifebit-ai/cloudos-cli: changelog +## v2.78.1 (2026-01-28) + +### Patch + +- Refactor `__main__.py` and move comamnds to separate files + ## v2.78.0 (2026-01-13) ### Feat diff --git a/cloudos_cli/_version.py b/cloudos_cli/_version.py index f705aee1..d8c9c360 100644 --- a/cloudos_cli/_version.py +++ b/cloudos_cli/_version.py @@ -1 +1 @@ -__version__ = '2.78.0' +__version__ = '2.78.1' From 69d39b7cbbcbef85f312b467b2fde4a3599ab35c Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 28 Jan 2026 15:35:36 +0100 Subject: [PATCH 04/41] refactor: jobs --- cloudos_cli/jobs/cli.py | 349 +++++++++++++++++++++++++--------------- 1 file changed, 223 insertions(+), 126 deletions(-) diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 57c3a48a..157c19a4 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -13,6 +13,9 @@ import json import copy import time +from cloudos_cli.queue.queue import Queue +import sys +from rich.console import Console # Import global constants from __main__ (will be available when imported) @@ -240,19 +243,141 @@ def run(ctx, HPC_NEXTFLOW_LATEST = __main__.HPC_NEXTFLOW_LATEST JOB_COMPLETED = __main__.JOB_COMPLETED - # apikey, cloudos_url, workspace_id, project_name, and workflow_name are now automatically resolved by the decorator - print('Executing run...') verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + if do_not_save_logs: + save_logs = False + else: + save_logs = True + if instance_type == 'NONE_SELECTED': + if execution_platform == 'aws': + instance_type = 'c5.xlarge' + elif execution_platform == 'azure': + instance_type = 'Standard_D4as_v4' + else: + instance_type = None + if execution_platform == 'azure' or execution_platform == 'hpc': + batch = False + else: + batch = True + if execution_platform == 'hpc': + print('\nHPC execution platform selected') + if hpc_id is None: + raise ValueError('Please, specify your HPC ID using --hpc parameter') + print('Please, take into account that HPC execution do not support ' + + 'the following parameters and all of them will be ignored:\n' + + '\t--job-queue\n' + + '\t--resumable | --do-not-save-logs\n' + + '\t--instance-type | --instance-disk | --cost-limit\n' + + '\t--storage-mode | --lustre-size\n' + + '\t--wdl-mainfile | --wdl-importsfile | --cromwell-token\n') + wdl_mainfile = None + wdl_importsfile = None + storage_mode = 'regular' + save_logs = False + if accelerate_file_staging: + if execution_platform != 'aws': + print('You have selected accelerate file staging, but this function is ' + + 'only available when execution platform is AWS. The accelerate file staging ' + + 'will not be applied') + use_mountpoints = False + else: + use_mountpoints = True + print('Enabling AWS S3 mountpoint for accelerated file staging. ' + + 'Please, take into consideration the following:\n' + + '\t- It significantly reduces runtime and compute costs but may increase network costs.\n' + + '\t- Requires extra memory. Adjust process memory or optimise resource usage if necessary.\n' + + '\t- This is still a CloudOS BETA feature.\n') + else: + use_mountpoints = False + if verbose: + print('\t...Detecting workflow type') + cl = Cloudos(cloudos_url, apikey, cromwell_token) + workflow_type = cl.detect_workflow(workflow_name, workspace_id, verify_ssl, last) + is_module = cl.is_module(workflow_name, workspace_id, verify_ssl, last) + if execution_platform == 'hpc' and workflow_type == 'wdl': + raise ValueError(f'The workflow {workflow_name} is a WDL workflow. ' + + 'WDL is not supported on HPC execution platform.') + if workflow_type == 'wdl': + print('WDL workflow detected') + if wdl_mainfile is None: + raise ValueError('Please, specify WDL mainFile using --wdl-mainfile .') + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h == 'Stopped': + print('\tStarting Cromwell server...\n') + cl.cromwell_switch(workspace_id, 'restart', verify_ssl) + elapsed = 0 + while elapsed < 300 and c_status_h != 'Running': + c_status_old = c_status_h + time.sleep(REQUEST_INTERVAL_CROMWELL) + elapsed += REQUEST_INTERVAL_CROMWELL + c_status = cl.get_cromwell_status(workspace_id, verify_ssl) + c_status_h = json.loads(c_status.content)["status"] + if c_status_h != c_status_old: + print(f'\tCurrent Cromwell server status is: {c_status_h}\n') + if c_status_h != 'Running': + raise Exception('Cromwell server did not restarted properly.') + cromwell_id = json.loads(c_status.content)["_id"] + click.secho('\t' + ('*' * 80) + '\n' + + '\tCromwell server is now running. Please, remember to stop it when ' + + 'your\n' + '\tjob finishes. You can use the following command:\n' + + '\tcloudos cromwell stop \\\n' + + '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + + f'\t\t--cloudos-url {cloudos_url} \\\n' + + f'\t\t--workspace-id {workspace_id}\n' + + '\t' + ('*' * 80) + '\n', fg='yellow', bold=True) + else: + cromwell_id = None if verbose: print('\t...Preparing objects') - my_job = jb.Job(cloudos_url, apikey, cromwell_token, workspace_id, project_name, - workflow_name, mainfile=wdl_mainfile, importsfile=wdl_importsfile, - repository_platform=repository_platform, verify=verify_ssl) + j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, + mainfile=wdl_mainfile, importsfile=wdl_importsfile, + repository_platform=repository_platform, verify=verify_ssl, last=last) if verbose: print('\tThe following Job object was created:') - print('\t' + str(my_job) + '\n') - - # set the nextflow version + print('\t' + str(j)) + print('\t...Sending job to CloudOS\n') + if is_module: + if job_queue is not None: + print(f'Ignoring job queue "{job_queue}" for ' + + f'Platform Workflow "{workflow_name}". Platform Workflows ' + + 'use their own predetermined queues.') + job_queue_id = None + if nextflow_version != '22.10.8': + print(f'The selected worflow \'{workflow_name}\' ' + + 'is a CloudOS module. CloudOS modules only work with ' + + 'Nextflow version 22.10.8. Switching to use 22.10.8') + nextflow_version = '22.10.8' + if execution_platform == 'azure': + print(f'The selected worflow \'{workflow_name}\' ' + + 'is a CloudOS module. For these workflows, worker nodes ' + + 'are managed internally. For this reason, the options ' + + 'azure-worker-instance-type, azure-worker-instance-disk and ' + + 'azure-worker-instance-spot are not taking effect.') + else: + queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=cromwell_token, + workspace_id=workspace_id, verify=verify_ssl) + job_queue_id = queue.fetch_job_queue_id(workflow_type=workflow_type, batch=batch, + job_queue=job_queue) + if use_private_docker_repository: + if is_module: + print(f'Workflow "{workflow_name}" is a CloudOS module. ' + + 'Option --use-private-docker-repository will be ignored.') + docker_login = False + else: + me = j.get_user_info(verify=verify_ssl)['dockerRegistriesCredentials'] + if len(me) == 0: + raise Exception('User private Docker repository has been selected but your user ' + + 'credentials have not been configured yet. Please, link your ' + + 'Docker account to CloudOS before using ' + + '--use-private-docker-repository option.') + print('Use private Docker repository has been selected. A custom job ' + + 'queue to support private Docker containers and/or Lustre FSx will be created for ' + + 'your job. The selected job queue will serve as a template.') + docker_login = True + else: + docker_login = False if nextflow_version == 'latest': if execution_platform == 'aws': nextflow_version = AWS_NEXTFLOW_LATEST @@ -260,119 +385,86 @@ def run(ctx, nextflow_version = AZURE_NEXTFLOW_LATEST else: nextflow_version = HPC_NEXTFLOW_LATEST - else: - # validate nextflow version is allowed in the execution platform - if execution_platform == 'aws' and nextflow_version not in AWS_NEXTFLOW_VERSIONS: - raise ValueError(f'Nextflow version {nextflow_version} is not supported in AWS. ' + - f'Supported versions: {", ".join(AWS_NEXTFLOW_VERSIONS)}') - elif execution_platform == 'azure' and nextflow_version not in AZURE_NEXTFLOW_VERSIONS: - raise ValueError(f'Nextflow version {nextflow_version} is not supported in Azure. ' + - f'Supported versions: {", ".join(AZURE_NEXTFLOW_VERSIONS)}') - elif execution_platform == 'hpc' and nextflow_version not in HPC_NEXTFLOW_VERSIONS: - raise ValueError(f'Nextflow version {nextflow_version} is not supported in HPC. ' + - f'Supported versions: {", ".join(HPC_NEXTFLOW_VERSIONS)}') - - # Set instance type based on platform if not set - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' + print('You have specified Nextflow version \'latest\' for execution platform ' + + f'\'{execution_platform}\'. The workflow will use the ' + + f'latest version available on CloudOS: {nextflow_version}.') + if execution_platform == 'aws': + if nextflow_version not in AWS_NEXTFLOW_VERSIONS: + print('For execution platform \'aws\', the workflow will use the default ' + + '\'22.10.8\' version on CloudOS.') + nextflow_version = '22.10.8' + if execution_platform == 'azure': + if nextflow_version not in AZURE_NEXTFLOW_VERSIONS: + print('For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' ' + + 'version on CloudOS.') + nextflow_version = '22.11.1-edge' + if execution_platform == 'hpc': + if nextflow_version not in HPC_NEXTFLOW_VERSIONS: + print('For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.') + nextflow_version = '22.10.8' + if nextflow_version != '22.10.8' and nextflow_version != '22.11.1-edge': + click.secho(f'You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' + + 'to be written in DSL2 and does not support DSL1.', fg='yellow', bold=True) + print('\nExecuting run...') + if workflow_type == 'nextflow': + print(f'\tNextflow version: {nextflow_version}') + j_id = j.send_job(job_config=job_config, + parameter=parameter, + is_module=is_module, + git_commit=git_commit, + git_tag=git_tag, + git_branch=git_branch, + job_name=job_name, + resumable=resumable, + save_logs=save_logs, + batch=batch, + job_queue_id=job_queue_id, + nextflow_profile=nextflow_profile, + nextflow_version=nextflow_version, + instance_type=instance_type, + instance_disk=instance_disk, + storage_mode=storage_mode, + lustre_size=lustre_size, + execution_platform=execution_platform, + hpc_id=hpc_id, + workflow_type=workflow_type, + cromwell_id=cromwell_id, + azure_worker_instance_type=azure_worker_instance_type, + azure_worker_instance_disk=azure_worker_instance_disk, + azure_worker_instance_spot=azure_worker_instance_spot, + cost_limit=cost_limit, + use_mountpoints=use_mountpoints, + accelerate_saving_results=accelerate_saving_results, + docker_login=docker_login, + verify=verify_ssl) + print(f'\tYour assigned job id is: {j_id}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = j.wait_job_completion(job_id=j_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=verbose, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(0) else: - instance_type = 'm5.xlarge' - - # check if the user has defined a configuration file for the job - if job_config is not None: - my_job_params = my_job.parse_job_config(job_config) - else: - my_job_params = {} - - # Allows to override the job_config file with parameters from the command line - if len(parameter) > 0: - # pass the single parameter list to the function - input_params = my_job.parse_individual_params(list(parameter)) - my_job_params.update(input_params) - - if verbose: - print('\tJob is going to be run with the following parameters:') - print('\t' + str(my_job_params) + '\n') - if execution_platform == 'aws' or execution_platform == 'hpc': - my_job_id = my_job.send_job(my_job_params, - job_name, - repository_platform, - execution_platform, - nextflow_profile, - nextflow_version, - instance_type, - instance_disk, - job_queue, - cost_limit, - storage_mode, - lustre_size, - resumable, - do_not_save_logs, - cromwell_token, - last, - git_commit, - git_tag, - git_branch, - accelerate_file_staging, - accelerate_saving_results, - use_private_docker_repository, - hpc_id) - elif execution_platform == 'azure': - my_job_id = my_job.send_job(my_job_params, - job_name, - repository_platform, - execution_platform, - nextflow_profile, - nextflow_version, - instance_type, - instance_disk, - job_queue, - cost_limit, - storage_mode, - lustre_size, - resumable, - do_not_save_logs, - cromwell_token, - last, - git_commit, - git_tag, - git_branch, - azure_worker_instance_type=azure_worker_instance_type, - azure_worker_instance_disk=azure_worker_instance_disk, - azure_worker_instance_spot=azure_worker_instance_spot, - accelerate_file_staging=accelerate_file_staging, - accelerate_saving_results=accelerate_saving_results, - use_private_docker_repository=use_private_docker_repository) - else: - raise ValueError(f'Execution platform {execution_platform} is not supported.') - - if verbose: - print('\tYour job was sent and has ID:') - print(f'\t{my_job_id}') - if not wait_completion: - print(f'\tJob {my_job_id} sent.') + print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') + sys.exit(1) else: - print(f'\tJob {my_job_id} sent. Waiting for it to complete...') - start_time = time.time() - end_time = start_time + wait_time - while True: - # get job status and print it out - j_status = my_job.get_job_status(my_job_id) - status = json.loads(j_status.content)['status'] - print(f'\tJob status: {status}') - if status == JOB_COMPLETED or status == 'failed' or status == 'aborted': - print(f'\tJob completed with status: {status}') - break - # check if we have waited too long - if time.time() > end_time: - print(f'\tWait time limit of {wait_time} seconds reached. ' + - f'Job status: {status}') - break - # wait before checking again - time.sleep(request_interval) + j_status = j.get_job_status(j_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {j_id}\n') @job.command('status') @@ -423,11 +515,18 @@ def job_status(ctx, if verbose: print('\tThe following Cloudos object was created:') print('\t' + str(cl) + '\n') - print(f'\tSearching for job {job_id} in the following workspace: ' + - f'{workspace_id}') - j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) - status_data = json.loads(j_status.content) - print(f'\tJob {job_id} status: {status_data["status"]}') + print(f'\tSearching for job id: {job_id}') + try: + j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}\n') + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' + print(f'\tTo further check your job status you can either go to {j_url} ' + + 'or repeat the command you just used.') + except BadRequestException as e: + raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") + except Exception as e: + raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") @job.command('workdir') @@ -488,7 +587,6 @@ def job_workdir(ctx, ssl_cert, profile): """Get the path to the working directory of a specified job or check deletion status.""" - from rich.console import Console # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator # session_id is also resolved if provided in profile @@ -817,7 +915,6 @@ def job_results(ctx, ssl_cert, profile): """Get the path to the results of a specified job or check deletion status.""" - from rich.console import Console # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator # session_id is also resolved if provided in profile From 44517f6c0fda13a115e66f2c342b29c5605c79cc Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 28 Jan 2026 16:21:21 +0100 Subject: [PATCH 05/41] refactor: update missing code --- cloudos_cli/bash/cli.py | 10 ++- cloudos_cli/procurement/cli.py | 140 +++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py index e4150375..7cbd99b0 100644 --- a/cloudos_cli/bash/cli.py +++ b/cloudos_cli/bash/cli.py @@ -2,11 +2,12 @@ import rich_click as click import cloudos_cli.jobs.job as jb -from cloudos_cli.clos import Cloudos from cloudos_cli.utils.resources import ssl_selector from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL from cloudos_cli.utils.array_job import generate_datasets_for_project +from cloudos_cli.queue.queue import Queue import sys +import json @click.group() @@ -148,6 +149,9 @@ def run_bash_job(ctx, """Run a bash job in CloudOS.""" # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator + from cloudos_cli import __main__ + JOB_COMPLETED = __main__.JOB_COMPLETED + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) if instance_type == 'NONE_SELECTED': @@ -410,6 +414,10 @@ def run_bash_array_job(ctx, custom_script_path, custom_script_project): """Run a bash array job in CloudOS.""" + + from cloudos_cli import __main__ + JOB_COMPLETED = __main__.JOB_COMPLETED + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) if not list_columns and not (command or custom_script_path): diff --git a/cloudos_cli/procurement/cli.py b/cloudos_cli/procurement/cli.py index 84186f2a..a563ce57 100644 --- a/cloudos_cli/procurement/cli.py +++ b/cloudos_cli/procurement/cli.py @@ -69,3 +69,143 @@ def list_images(ctx, except Exception as e: raise ValueError(f"{str(e)}") + + +@images.command(name="set") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) +@click.option('--image-type', help='The CloudOS resource image type.', required=True, + type=click.Choice([ + 'RegularInteractiveSessions', + 'SparkInteractiveSessions', + 'RStudioInteractiveSessions', + 'JupyterInteractiveSessions', + 'JobDefault', + 'NextflowBatchComputeEnvironment'])) +@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') +@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) +@click.option('--image-id', help='The new image id value.', required=True) +@click.option('--image-name', help='The new image name value.', required=False) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def set_organisation_image(ctx, + apikey, + cloudos_url, + procurement_id, + organisation_id, + image_type, + provider, + region, + image_id, + image_name, + disable_ssl_verification, + ssl_cert, + profile): + """Set a new image id or name to image associated with an organisations of a given procurement.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = procurement_images.set_procurement_organisation_image( + organisation_id, + image_type, + provider, + region, + image_id, + image_name + ) + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") + + +@images.command(name="reset") +@click.option('-k', + '--apikey', + help='Your CloudOS API key.', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) +@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) +@click.option('--image-type', help='The CloudOS resource image type.', required=True, + type=click.Choice([ + 'RegularInteractiveSessions', + 'SparkInteractiveSessions', + 'RStudioInteractiveSessions', + 'JupyterInteractiveSessions', + 'JobDefault', + 'NextflowBatchComputeEnvironment'])) +@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') +@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'procurement_id']) +def reset_organisation_image(ctx, + apikey, + cloudos_url, + procurement_id, + organisation_id, + image_type, + provider, + region, + disable_ssl_verification, + ssl_cert, + profile): + """Reset image associated with an organisations of a given procurement to CloudOS defaults.""" + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + procurement_images = Images( + cloudos_url=cloudos_url, + apikey=apikey, + procurement_id=procurement_id, + verify=verify_ssl, + cromwell_token=None + ) + + try: + result = procurement_images.reset_procurement_organisation_image( + organisation_id, + image_type, + provider, + region + ) + console = Console() + console.print(result) + + except Exception as e: + raise ValueError(f"{str(e)}") From 20d6c294a595bd059deec13baf404a0556a054cf Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 28 Jan 2026 16:45:04 +0100 Subject: [PATCH 06/41] refactor: link --- cloudos_cli/__main__.py | 3 +- cloudos_cli/link/cli.py | 170 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 cloudos_cli/link/cli.py diff --git a/cloudos_cli/__main__.py b/cloudos_cli/__main__.py index 676bc4aa..e07f64ed 100644 --- a/cloudos_cli/__main__.py +++ b/cloudos_cli/__main__.py @@ -24,6 +24,7 @@ from cloudos_cli.procurement.cli import procurement from cloudos_cli.datasets.cli import datasets from cloudos_cli.configure.cli import configure +from cloudos_cli.link.cli import link # GLOBAL CONSTANTS - Keep these for backward compatibility @@ -72,7 +73,7 @@ def run_cloudos_cli(ctx): run_cloudos_cli.add_command(procurement) run_cloudos_cli.add_command(datasets) run_cloudos_cli.add_command(configure) - +run_cloudos_cli.add_command(link) if __name__ == '__main__': run_cloudos_cli() diff --git a/cloudos_cli/link/cli.py b/cloudos_cli/link/cli.py new file mode 100644 index 00000000..ae9dd832 --- /dev/null +++ b/cloudos_cli/link/cli.py @@ -0,0 +1,170 @@ +import rich_click as click +from cloudos_cli.link.link import Link +from cloudos_cli.utils.resources import ssl_selector +from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.utils.errors import BadRequestException + + +@click.command() +@click.argument('path', required=False) +@click.option('-k', + '--apikey', + help='Your CloudOS API key', + required=True) +@click.option('-c', + '--cloudos-url', + help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), + default=CLOUDOS_URL, + required=True) +@click.option('--workspace-id', + help='The specific CloudOS workspace id.', + required=True) +@click.option('--session-id', + help='The specific CloudOS interactive session id.', + required=True) +@click.option('--job-id', + help='The job id in CloudOS. When provided, links results, workdir and logs by default.', + required=False) +@click.option('--project-name', + help='The name of a CloudOS project. Required for File Explorer paths.', + required=False) +@click.option('--results', + help='Link only results folder (only works with --job-id).', + is_flag=True) +@click.option('--workdir', + help='Link only working directory (only works with --job-id).', + is_flag=True) +@click.option('--logs', + help='Link only logs folder (only works with --job-id).', + is_flag=True) +@click.option('--verbose', + help='Whether to print information messages or not.', + is_flag=True) +@click.option('--disable-ssl-verification', + help=('Disable SSL certificate verification. Please, remember that this option is ' + + 'not generally recommended for security reasons.'), + is_flag=True) +@click.option('--ssl-cert', + help='Path to your SSL certificate file.') +@click.option('--profile', help='Profile to use from the config file', default=None) +@click.pass_context +@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) +def link(ctx, + path, + apikey, + cloudos_url, + workspace_id, + session_id, + job_id, + project_name, + results, + workdir, + logs, + verbose, + disable_ssl_verification, + ssl_cert, + profile): + """ + Link folders to an interactive analysis session. + + This command is used to link folders + to an active interactive analysis session for direct access to data. + + PATH: Optional path to link (S3). + Required if --job-id is not provided. + + Two modes of operation: + + 1. Job-based linking (--job-id): Links job-related folders. + By default, links results, workdir, and logs folders. + Use --results, --workdir, or --logs flags to link only specific folders. + + 2. Direct path linking (PATH argument): Links a specific S3 path. + + Examples: + + # Link all job folders (results, workdir, logs) + cloudos link --job-id 12345 --session-id abc123 + + # Link only results from a job + cloudos link --job-id 12345 --session-id abc123 --results + + # Link a specific S3 path + cloudos link s3://bucket/folder --session-id abc123 + + """ + print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') + + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) + + # Validate input parameters + if not job_id and not path: + raise click.UsageError("Either --job-id or PATH argument must be provided.") + + if job_id and path: + raise click.UsageError("Cannot use both --job-id and PATH argument. Please provide only one.") + + # Validate folder-specific flags only work with --job-id + if (results or workdir or logs) and not job_id: + raise click.UsageError("--results, --workdir, and --logs flags can only be used with --job-id.") + + # If no specific folders are selected with job-id, link all by default + if job_id and not (results or workdir or logs): + results = True + workdir = True + logs = True + + if verbose: + print('Using the following parameters:') + print(f'\tCloudOS url: {cloudos_url}') + print(f'\tWorkspace ID: {workspace_id}') + print(f'\tSession ID: {session_id}') + if job_id: + print(f'\tJob ID: {job_id}') + print(f'\tLink results: {results}') + print(f'\tLink workdir: {workdir}') + print(f'\tLink logs: {logs}') + else: + print(f'\tPath: {path}') + + # Initialize Link client + link_client = Link( + cloudos_url=cloudos_url, + apikey=apikey, + cromwell_token=None, + workspace_id=workspace_id, + project_name=project_name, + verify=verify_ssl + ) + + try: + if job_id: + # Job-based linking + print(f'Linking folders from job {job_id} to interactive session {session_id}...\n') + + # Link results + if results: + link_client.link_job_results(job_id, workspace_id, session_id, verify_ssl, verbose) + + # Link workdir + if workdir: + link_client.link_job_workdir(job_id, workspace_id, session_id, verify_ssl, verbose) + + # Link logs + if logs: + link_client.link_job_logs(job_id, workspace_id, session_id, verify_ssl, verbose) + + + else: + # Direct path linking + print(f'Linking path to interactive session {session_id}...\n') + + # Link path with validation + link_client.link_path_with_validation(path, session_id, verify_ssl, project_name, verbose) + + print('\nLinking operation completed.') + + except BadRequestException as e: + raise ValueError(f"Request failed: {str(e)}") + except Exception as e: + raise ValueError(f"Failed to link folder(s): {str(e)}") From 8b5c571b1c60a1e820203bf919fc26582d21dd75 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 14:44:58 +0100 Subject: [PATCH 07/41] ci: add run and abort --- .github/workflows/ci.yml | 73 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 233951c0..697630d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -645,6 +645,72 @@ jobs: run: | cloudos job archive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids 694545c801722f7aa3c626c4 cloudos job unarchive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids 694545c801722f7aa3c626c4 + job_run_and_abort: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: setup.py + - name: Install dependencies + run: | + pip install -e . + - name: Run job, wait for running status, then abort + env: + CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} + CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} + CLOUDOS_URL: "https://cloudos.lifebit.ai" + PROJECT_NAME: "cloudos-cli-tests" + WORKFLOW: "GH-rnatoy" + JOB_CONFIG: "cloudos_cli/examples/rnatoy.config" + JOB_NAME_BASE: "cloudos-cli-CI-test-abort" + COMMIT_HASH: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.number }} + INSTANCE_TYPE: "m4.xlarge" + run: | + # Step 1: Run job (without --wait-completion so it doesn't wait until completion) + JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER" + cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt + JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev) + echo "Job ID: $JOB_ID" + + # Step 2: Poll status until it reaches "running" + MAX_ATTEMPTS=60 + SLEEP_TIME=10 + attempt=0 + + while [ $attempt -lt $MAX_ATTEMPTS ]; do + echo "Checking job status (attempt $((attempt+1))/$MAX_ATTEMPTS)..." + STATUS=$(cloudos job status --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID 2>&1 | grep -i "Your current job status is" | head -1 || echo "unknown") + echo "Current status: $STATUS" + + if echo "$STATUS" | grep -qi "running"; then + echo "Job is now running!" + break + elif echo "$STATUS" | grep -qi "completed\|failed\|aborted"; then + echo "Job finished before we could abort it (status: $STATUS)" + exit 0 + fi + + attempt=$((attempt+1)) + sleep $SLEEP_TIME + done + + if [ $attempt -eq $MAX_ATTEMPTS ]; then + echo "Job did not reach running status within timeout" + exit 1 + fi + + # Step 3: Abort the running job + echo "Aborting job $JOB_ID..." + cloudos job abort --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID + echo "Job abort command executed successfully!" # delete_results: # needs: job_run_and_status # runs-on: ubuntu-latest @@ -668,4 +734,9 @@ jobs: # CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} # CLOUDOS_URL: "https://cloudos.lifebit.ai" # run: | - # cloudos job results --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} \ No newline at end of file + # cloudos job results --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} +# - job abort +# - job resume +# - link +# - configure +# - workdir --delete \ No newline at end of file From e05a81e31e76361a981ea0ee8dbbbba45f9ad9c9 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 14:56:52 +0100 Subject: [PATCH 08/41] ci: adds job resume --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 697630d0..f7d806e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,10 +241,39 @@ jobs: INSTANCE_TYPE: "m4.xlarge" run: | JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER" - cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --wait-completion --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt + cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --wait-completion --resumable --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev) echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT cloudos job status --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID + job_resume: + needs: job_run_and_status + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: setup.py + - name: Install dependencies + run: | + pip install -e . + - name: Resume completed job + env: + CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} + CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} + CLOUDOS_URL: "https://cloudos.lifebit.ai" + PROJECT_NAME: "cloudos-cli-tests" + INSTANCE_TYPE: "m4.xlarge" + run: | + JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" + echo "Resuming job $JOB_ID..." + cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-id $JOB_ID --instance-type $INSTANCE_TYPE --wait-completion + echo "Job resumed successfully!" logs_results_aws: needs: job_run_and_status runs-on: ubuntu-latest @@ -735,7 +764,7 @@ jobs: # CLOUDOS_URL: "https://cloudos.lifebit.ai" # run: | # cloudos job results --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} -# - job abort +# The following tests are not yet implemented: # - job resume # - link # - configure From 0030ddf0003a8553987a2649d321c4528c0b31bd Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 15:05:03 +0100 Subject: [PATCH 09/41] ci: adds configure tests --- .github/workflows/ci.yml | 61 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7d806e4..f9bca21b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -674,6 +674,64 @@ jobs: run: | cloudos job archive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids 694545c801722f7aa3c626c4 cloudos job unarchive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids 694545c801722f7aa3c626c4 + configure_profile: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: setup.py + - name: Install dependencies + run: | + pip install -e . + - name: Test configure profile workflow + env: + CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} + CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} + CLOUDOS_URL: "https://cloudos.lifebit.ai" + PROJECT_NAME: "cloudos-cli-tests" + run: | + # Step 1: Create a test profile by providing inputs via stdin + echo "Creating profile 'ci-test-profile'..." + echo -e "$CLOUDOS_TOKEN\n$CLOUDOS_URL\n$CLOUDOS_WORKSPACE_ID\n\n$PROJECT_NAME\n\n\n\n\n" | cloudos configure --profile ci-test-profile + echo "Profile created successfully!" + + # Step 2: List all profiles to verify creation + echo "Listing all profiles..." + cloudos configure list-profiles | tee profiles_output.txt + + # Verify that our test profile exists in the list + if grep -q "ci-test-profile" profiles_output.txt; then + echo "✅ Profile 'ci-test-profile' found in the list" + else + echo "❌ Profile 'ci-test-profile' not found in the list" + exit 1 + fi + + # Step 3: Remove the test profile + echo "Removing profile 'ci-test-profile'..." + cloudos configure remove-profile --profile ci-test-profile + echo "Profile removed successfully!" + + # Step 4: List profiles again to verify removal + echo "Listing profiles after removal..." + cloudos configure list-profiles | tee profiles_after_removal.txt + + # Verify that our test profile no longer exists + if grep -q "ci-test-profile" profiles_after_removal.txt; then + echo "❌ Profile 'ci-test-profile' still exists after removal" + exit 1 + else + echo "✅ Profile 'ci-test-profile' successfully removed" + fi + + echo "Configure profile workflow test completed successfully!" job_run_and_abort: runs-on: ubuntu-latest strategy: @@ -765,7 +823,4 @@ jobs: # run: | # cloudos job results --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} # The following tests are not yet implemented: -# - job resume -# - link -# - configure # - workdir --delete \ No newline at end of file From 4fa5d8704e52af6208493d732332e69c66cf6b07 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 15:09:00 +0100 Subject: [PATCH 10/41] ci: adds configure tests + workdir --delete --- .github/workflows/ci.yml | 51 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9bca21b..e743ab49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -798,29 +798,28 @@ jobs: echo "Aborting job $JOB_ID..." cloudos job abort --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID echo "Job abort command executed successfully!" - # delete_results: - # needs: job_run_and_status - # runs-on: ubuntu-latest - # strategy: - # matrix: - # python-version: [ "3.9" ] - # steps: - # - uses: actions/checkout@v3 - # - name: Set up Python ${{ matrix.python-version }} - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ matrix.python-version }} - # cache: pip - # cache-dependency-path: setup.py - # - name: Install dependencies - # run: | - # pip install -e . - # - name: Run tests - # env: - # CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} - # CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} - # CLOUDOS_URL: "https://cloudos.lifebit.ai" - # run: | - # cloudos job results --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} -# The following tests are not yet implemented: -# - workdir --delete \ No newline at end of file + delete_workdir: + needs: job_run_and_status + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.9" ] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: setup.py + - name: Install dependencies + run: | + pip install -e . + - name: Run tests + env: + CLOUDOS_URL: "https://cloudos.lifebit.ai" + CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} + CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} + CLOUDOS_URL: "https://cloudos.lifebit.ai" + run: | + cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} From e24b504879a36d3841fef3a56e7aa77e60c6109f Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 15:28:33 +0100 Subject: [PATCH 11/41] ci: remove typo duplicate --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e743ab49..af371f4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -817,7 +817,6 @@ jobs: pip install -e . - name: Run tests env: - CLOUDOS_URL: "https://cloudos.lifebit.ai" CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" From 3b8d3a661e7905bf3129e13a6a94966040f6d7a1 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 15:39:26 +0100 Subject: [PATCH 12/41] ci: update resume option --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af371f4c..c897b5df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,7 +272,7 @@ jobs: run: | JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" echo "Resuming job $JOB_ID..." - cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-id $JOB_ID --instance-type $INSTANCE_TYPE --wait-completion + cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-id $JOB_ID --instance-type $INSTANCE_TYPE echo "Job resumed successfully!" logs_results_aws: needs: job_run_and_status From 76324f7c58723eaad26074e4ee6653ca0b295f55 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 15:43:49 +0100 Subject: [PATCH 13/41] ci: refactor to not have hardcoded jobs --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c897b5df..aaf935ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: run: | cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID job_list_filtering: + needs: job_run_and_status runs-on: ubuntu-latest strategy: matrix: @@ -78,10 +79,11 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | + JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" # Test filtering by status, project and workflow name cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-status completed --filter-project cloudos-cli-tests --filter-workflow GH-rnatoy --last --last-n-jobs 10 # Test filtering job id - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-id 68af3e26628f9dd83dae1c05 + cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-id $JOB_ID # Test filtering job name cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-name "cloudos-cli-CI-test" --last-n-jobs 10 # Test filtering by only mine @@ -89,6 +91,7 @@ jobs: # Test filtering by queue cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "job_queue_nextflow" --last-n-jobs 10 job_details: + needs: job_run_and_status runs-on: ubuntu-latest strategy: matrix: @@ -110,9 +113,10 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | - JOB_ID="68b9aaab76724fd335457ed3" + JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" cloudos job details --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID job_workdir: + needs: job_run_and_status runs-on: ubuntu-latest strategy: matrix: @@ -134,9 +138,10 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | - JOB_ID="68e8b0f503db9b2833b5959f" + JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" cloudos job workdir --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id $JOB_ID job_related_analyses: + needs: job_run_and_status runs-on: ubuntu-latest strategy: matrix: @@ -158,7 +163,7 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | - JOB_ID="68b9aaab76724fd335457ed3" + JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" cloudos job related --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id $JOB_ID import_gitlab: runs-on: ubuntu-latest @@ -445,6 +450,8 @@ jobs: cloudos datasets rename --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/new_renamed_folder test_rename bash_job: runs-on: ubuntu-latest + outputs: + job_id: ${{ steps.get-bash-job-id.outputs.job_id }} strategy: matrix: python-version: ["3.9"] @@ -460,7 +467,7 @@ jobs: run: | pip install -e . - name: Run tests - id: run-bash-job + id: get-bash-job-id env: CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} @@ -483,8 +490,11 @@ jobs: --workflow-name plink2 \ --apikey "$CLOUDOS_TOKEN" \ --cloudos-url "$CLOUDOS_URL" \ - --wait-completion + --wait-completion 2>&1 | tee bash_job_out.txt + JOB_ID=$(grep -e "Your assigned job id is:" bash_job_out.txt | rev | cut -f1 -d " " | rev) + echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT bash_job_clone: + needs: bash_job runs-on: ubuntu-latest strategy: matrix: @@ -508,9 +518,9 @@ jobs: CLOUDOS_URL: "https://cloudos.lifebit.ai" PROJECT_NAME: "cloudos-cli-tests" JOB_NAME_BASE: "cloudos_cli_CI_test_bash_job_cloned" - JOB_ID: "68d51f6b5eb22f6f2f445539" INSTANCE_TYPE: "m4.xlarge" run: | + JOB_ID="${{ needs.bash_job.outputs.job_id }}" cloudos job clone \ --job-id "$JOB_ID" \ --job-name "$JOB_NAME_BASE" \ @@ -521,6 +531,8 @@ jobs: --cloudos-url "$CLOUDOS_URL" bash_array_job_run_and_multiple_projects: runs-on: ubuntu-latest + outputs: + job_id: ${{ steps.get-bash-array-job-id.outputs.job_id }} strategy: matrix: python-version: ["3.9"] @@ -536,7 +548,7 @@ jobs: run: | pip install -e . - name: Run tests - id: run-bash-array-job + id: get-bash-array-job-id env: CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} @@ -563,8 +575,11 @@ jobs: --apikey "$CLOUDOS_TOKEN" \ --cloudos-url "$CLOUDOS_URL" \ --parameter --file=ci-testing/Data/bash_array/input.csv \ - --wait-completion + --wait-completion 2>&1 | tee bash_array_job_out.txt + JOB_ID=$(grep -e "Your assigned job id is:" bash_array_job_out.txt | rev | cut -f1 -d " " | rev) + echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT bash_array_clone: + needs: bash_array_job_run_and_multiple_projects runs-on: ubuntu-latest strategy: matrix: @@ -588,9 +603,9 @@ jobs: CLOUDOS_URL: "https://cloudos.lifebit.ai" PROJECT_NAME: "cloudos-cli-tests" JOB_NAME_BASE: "cloudos_cli_CI_test_bash_array_job_cloned" - JOB_ID: "68d51f7b41b37cb06dea2a11" INSTANCE_TYPE: "m4.xlarge" run: | + JOB_ID="${{ needs.bash_array_job_run_and_multiple_projects.outputs.job_id }}" cloudos job clone \ --job-id "$JOB_ID" \ --job-name "$JOB_NAME_BASE" \ @@ -650,6 +665,7 @@ jobs: cloudos datasets mkdir --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/rm_test cloudos datasets rm --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/rm_test archive_unarchive_job: + needs: job_run_and_status runs-on: ubuntu-latest strategy: matrix: @@ -672,8 +688,9 @@ jobs: PROJECT_NAME: "cloudos-cli-tests" CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | - cloudos job archive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids 694545c801722f7aa3c626c4 - cloudos job unarchive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids 694545c801722f7aa3c626c4 + JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" + cloudos job archive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID + cloudos job unarchive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID configure_profile: runs-on: ubuntu-latest strategy: From 0dc66aadd97b6c1f9588fe72ed875e1664ded23a Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 16:03:39 +0100 Subject: [PATCH 14/41] ci: update job_workdir, job_resume to run before delete_workdir --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aaf935ba..7a983b1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -816,7 +816,7 @@ jobs: cloudos job abort --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID echo "Job abort command executed successfully!" delete_workdir: - needs: job_run_and_status + needs: [job_workdir, job_resume] runs-on: ubuntu-latest strategy: matrix: From f95f64b0af888fee2e4cb731f320259aef24398f Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 29 Jan 2026 16:17:05 +0100 Subject: [PATCH 15/41] ci: add job id dependency --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a983b1c..ecaf634e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -816,7 +816,7 @@ jobs: cloudos job abort --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID echo "Job abort command executed successfully!" delete_workdir: - needs: [job_workdir, job_resume] + needs: [job_run_and_status, job_workdir, job_resume] runs-on: ubuntu-latest strategy: matrix: From 960586f9b47f0a719270e763220ed9ebc41f7e22 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 09:22:11 +0100 Subject: [PATCH 16/41] refactor: remove unsed files --- cloudos_cli/__main__.py.backup | 4367 ----------------------- cloudos_cli/__main__.py.before_refactor | 156 - cloudos_cli/__main__.py.old | 4367 ----------------------- 3 files changed, 8890 deletions(-) delete mode 100644 cloudos_cli/__main__.py.backup delete mode 100644 cloudos_cli/__main__.py.before_refactor delete mode 100644 cloudos_cli/__main__.py.old diff --git a/cloudos_cli/__main__.py.backup b/cloudos_cli/__main__.py.backup deleted file mode 100644 index 1eab8bfd..00000000 --- a/cloudos_cli/__main__.py.backup +++ /dev/null @@ -1,4367 +0,0 @@ -#!/usr/bin/env python3 - -import rich_click as click -import cloudos_cli.jobs.job as jb -from cloudos_cli.clos import Cloudos -from cloudos_cli.import_wf.import_wf import ImportWorflow -from cloudos_cli.queue.queue import Queue -from cloudos_cli.utils.errors import BadRequestException -import json -import time -import sys -import traceback -import copy -from ._version import __version__ -from cloudos_cli.configure.configure import ConfigurationProfile -from rich.console import Console -from rich.table import Table -from cloudos_cli.datasets import Datasets -from cloudos_cli.procurement import Images -from cloudos_cli.utils.resources import ssl_selector, format_bytes -from rich.style import Style -from cloudos_cli.utils.array_job import generate_datasets_for_project -from cloudos_cli.utils.details import create_job_details, create_job_list_table -from cloudos_cli.link import Link -from cloudos_cli.cost.cost import CostViewer -from cloudos_cli.logging.logger import setup_logging, update_command_context_from_click -import logging -from cloudos_cli.configure.configure import ( - with_profile_config, - build_default_map_for_group, - get_shared_config, - CLOUDOS_URL -) -from cloudos_cli.related_analyses.related_analyses import related_analyses - - -# GLOBAL VARS -JOB_COMPLETED = 'completed' -REQUEST_INTERVAL_CROMWELL = 30 -AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] -AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] -HPC_NEXTFLOW_VERSIONS = ['22.10.8'] -AWS_NEXTFLOW_LATEST = '24.04.4' -AZURE_NEXTFLOW_LATEST = '22.11.1-edge' -HPC_NEXTFLOW_LATEST = '22.10.8' -ABORT_JOB_STATES = ['running', 'initializing'] - - -def custom_exception_handler(exc_type, exc_value, exc_traceback): - """Custom exception handler that respects debug mode""" - console = Console(stderr=True) - # Initialise logger - debug_mode = '--debug' in sys.argv - setup_logging(debug_mode) - logger = logging.getLogger("CloudOS") - if get_debug_mode(): - logger.error(exc_value, exc_info=exc_value) - console.print("[yellow]Debug mode: showing full traceback[/yellow]") - sys.__excepthook__(exc_type, exc_value, exc_traceback) - else: - # Extract a clean error message - if hasattr(exc_value, 'message'): - error_msg = exc_value.message - elif str(exc_value): - error_msg = str(exc_value) - else: - error_msg = f"{exc_type.__name__}" - logger.error(exc_value) - console.print(f"[bold red]Error: {error_msg}[/bold red]") - - # For network errors, give helpful context - if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): - console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") - -# Install the custom exception handler -sys.excepthook = custom_exception_handler - - -def pass_debug_to_subcommands(group_cls=click.RichGroup): - """Custom Group class that passes --debug option to all subcommands""" - - class DebugGroup(group_cls): - def add_command(self, cmd, name=None): - # Add debug option to the command if it doesn't already have it - if isinstance(cmd, (click.Command, click.Group)): - has_debug = any(param.name == 'debug' for param in cmd.params) - if not has_debug: - debug_option = click.Option( - ['--debug'], - is_flag=True, - help='Show detailed error information and tracebacks', - is_eager=True, - expose_value=False, - callback=self._debug_callback - ) - cmd.params.insert(-1, debug_option) # Insert at the end for precedence - - super().add_command(cmd, name) - - def _debug_callback(self, ctx, param, value): - """Callback to handle debug flag""" - global _global_debug - if value: - _global_debug = True - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - return DebugGroup - - -def get_debug_mode(): - """Get current debug mode state""" - return _global_debug - - -# Helper function for debug setup -def _setup_debug(ctx, param, value): - """Setup debug mode globally and in context""" - global _global_debug - _global_debug = value - if value: - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - -@click.group(cls=pass_debug_to_subcommands()) -@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', - is_eager=True, expose_value=False, callback=_setup_debug) -@click.version_option(__version__) -@click.pass_context -def run_cloudos_cli(ctx): - """CloudOS python package: a package for interacting with CloudOS.""" - update_command_context_from_click(ctx) - ctx.ensure_object(dict) - - if ctx.invoked_subcommand not in ['datasets']: - print(run_cloudos_cli.__doc__ + '\n') - print('Version: ' + __version__ + '\n') - - # Load shared configuration (handles missing profiles and fields gracefully) - shared_config = get_shared_config() - - # Automatically build default_map from registered commands - ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def job(): - """CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS.""" - print(job.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def workflow(): - """CloudOS workflow functionality: list and import workflows.""" - print(workflow.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def project(): - """CloudOS project functionality: list and create projects in CloudOS.""" - print(project.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def cromwell(): - """Cromwell server functionality: check status, start and stop.""" - print(cromwell.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def queue(): - """CloudOS job queue functionality.""" - print(queue.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def bash(): - """CloudOS bash functionality.""" - print(bash.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def procurement(): - """CloudOS procurement functionality.""" - print(procurement.__doc__ + '\n') - - -@procurement.group(cls=pass_debug_to_subcommands()) -def images(): - """CloudOS procurement images functionality.""" - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -@click.pass_context -def datasets(ctx): - """CloudOS datasets functionality.""" - update_command_context_from_click(ctx) - if ctx.args and ctx.args[0] != 'ls': - print(datasets.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands(), invoke_without_command=True) -@click.option('--profile', help='Profile to use from the config file', default='default') -@click.option('--make-default', - is_flag=True, - help='Make the profile the default one.') -@click.pass_context -def configure(ctx, profile, make_default): - """CloudOS configuration.""" - print(configure.__doc__ + '\n') - update_command_context_from_click(ctx) - profile = profile or ctx.obj['profile'] - config_manager = ConfigurationProfile() - - if ctx.invoked_subcommand is None and profile == "default" and not make_default: - config_manager.create_profile_from_input(profile_name="default") - - if profile != "default" and not make_default: - config_manager.create_profile_from_input(profile_name=profile) - if make_default: - config_manager.make_default_profile(profile_name=profile) - - -@job.command('run', cls=click.RichCommand) -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('--job-config', - help=('A config file similar to a nextflow.config file, ' + - 'but only with the parameters to use with your job.')) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p input=s3://path_to_my_file. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--nextflow-profile', - help=('A comma separated string indicating the nextflow profile/s ' + - 'to use with your job.')) -@click.option('--nextflow-version', - help=('Nextflow version to use when executing the workflow in CloudOS. ' + - 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']), - default='22.10.8') -@click.option('--git-commit', - help=('The git commit hash to run for ' + - 'the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--git-tag', - help=('The tag to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--git-branch', - help=('The branch to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--resumable', - help='Whether to make the job able to be resumed or not.', - is_flag=True) -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--wdl-mainfile', - help='For WDL workflows, which mainFile (.wdl) is configured to use.',) -@click.option('--wdl-importsfile', - help='For WDL workflows, which importsFile (.zip) is configured to use.',) -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. Currently, not necessary ' + - 'as apikey can be used instead, but maintained for backwards compatibility.')) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - type=click.Choice(['aws', 'azure', 'hpc']), - default='aws') -@click.option('--hpc-id', - help=('ID of your HPC, only applicable when --execution-platform=hpc. ' + - 'Default=660fae20f93358ad61e0104b'), - default='660fae20f93358ad61e0104b') -@click.option('--azure-worker-instance-type', - help=('The worker node instance type to be used in azure. ' + - 'Default=Standard_D4as_v4'), - default='Standard_D4as_v4') -@click.option('--azure-worker-instance-disk', - help='The disk size in GB for the worker node to be used in azure. Default=100', - type=int, - default=100) -@click.option('--azure-worker-instance-spot', - help='Whether the azure worker nodes have to be spot instances or not.', - is_flag=True) -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-file-staging', - help='Enables AWS S3 mountpoint for quicker file staging.', - is_flag=True) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--use-private-docker-repository', - help=('Allows to use private docker repository for running jobs. The Docker user ' + - 'account has to be already linked to CloudOS.'), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run(ctx, - apikey, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - job_config, - parameter, - git_commit, - git_tag, - git_branch, - job_name, - resumable, - do_not_save_logs, - job_queue, - nextflow_profile, - nextflow_version, - instance_type, - instance_disk, - storage_mode, - lustre_size, - wait_completion, - wait_time, - wdl_mainfile, - wdl_importsfile, - cromwell_token, - repository_platform, - execution_platform, - hpc_id, - azure_worker_instance_type, - azure_worker_instance_disk, - azure_worker_instance_spot, - cost_limit, - accelerate_file_staging, - accelerate_saving_results, - use_private_docker_repository, - verbose, - request_interval, - disable_ssl_verification, - ssl_cert, - profile): - """Submit a job to CloudOS.""" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if do_not_save_logs: - save_logs = False - else: - save_logs = True - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - if execution_platform == 'azure' or execution_platform == 'hpc': - batch = False - else: - batch = True - if execution_platform == 'hpc': - print('\nHPC execution platform selected') - if hpc_id is None: - raise ValueError('Please, specify your HPC ID using --hpc parameter') - print('Please, take into account that HPC execution do not support ' + - 'the following parameters and all of them will be ignored:\n' + - '\t--job-queue\n' + - '\t--resumable | --do-not-save-logs\n' + - '\t--instance-type | --instance-disk | --cost-limit\n' + - '\t--storage-mode | --lustre-size\n' + - '\t--wdl-mainfile | --wdl-importsfile | --cromwell-token\n') - wdl_mainfile = None - wdl_importsfile = None - storage_mode = 'regular' - save_logs = False - if accelerate_file_staging: - if execution_platform != 'aws': - print('You have selected accelerate file staging, but this function is ' + - 'only available when execution platform is AWS. The accelerate file staging ' + - 'will not be applied') - use_mountpoints = False - else: - use_mountpoints = True - print('Enabling AWS S3 mountpoint for accelerated file staging. ' + - 'Please, take into consideration the following:\n' + - '\t- It significantly reduces runtime and compute costs but may increase network costs.\n' + - '\t- Requires extra memory. Adjust process memory or optimise resource usage if necessary.\n' + - '\t- This is still a CloudOS BETA feature.\n') - else: - use_mountpoints = False - if verbose: - print('\t...Detecting workflow type') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - workflow_type = cl.detect_workflow(workflow_name, workspace_id, verify_ssl, last) - is_module = cl.is_module(workflow_name, workspace_id, verify_ssl, last) - if execution_platform == 'hpc' and workflow_type == 'wdl': - raise ValueError(f'The workflow {workflow_name} is a WDL workflow. ' + - 'WDL is not supported on HPC execution platform.') - if workflow_type == 'wdl': - print('WDL workflow detected') - if wdl_mainfile is None: - raise ValueError('Please, specify WDL mainFile using --wdl-mainfile .') - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h == 'Stopped': - print('\tStarting Cromwell server...\n') - cl.cromwell_switch(workspace_id, 'restart', verify_ssl) - elapsed = 0 - while elapsed < 300 and c_status_h != 'Running': - c_status_old = c_status_h - time.sleep(REQUEST_INTERVAL_CROMWELL) - elapsed += REQUEST_INTERVAL_CROMWELL - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - if c_status_h != c_status_old: - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h != 'Running': - raise Exception('Cromwell server did not restarted properly.') - cromwell_id = json.loads(c_status.content)["_id"] - click.secho('\t' + ('*' * 80) + '\n' + - '\tCromwell server is now running. Please, remember to stop it when ' + - 'your\n' + '\tjob finishes. You can use the following command:\n' + - '\tcloudos cromwell stop \\\n' + - '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + - f'\t\t--cloudos-url {cloudos_url} \\\n' + - f'\t\t--workspace-id {workspace_id}\n' + - '\t' + ('*' * 80) + '\n', fg='yellow', bold=True) - else: - cromwell_id = None - if verbose: - print('\t...Preparing objects') - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=wdl_mainfile, importsfile=wdl_importsfile, - repository_platform=repository_platform, verify=verify_ssl, last=last) - if verbose: - print('\tThe following Job object was created:') - print('\t' + str(j)) - print('\t...Sending job to CloudOS\n') - if is_module: - if job_queue is not None: - print(f'Ignoring job queue "{job_queue}" for ' + - f'Platform Workflow "{workflow_name}". Platform Workflows ' + - 'use their own predetermined queues.') - job_queue_id = None - if nextflow_version != '22.10.8': - print(f'The selected worflow \'{workflow_name}\' ' + - 'is a CloudOS module. CloudOS modules only work with ' + - 'Nextflow version 22.10.8. Switching to use 22.10.8') - nextflow_version = '22.10.8' - if execution_platform == 'azure': - print(f'The selected worflow \'{workflow_name}\' ' + - 'is a CloudOS module. For these workflows, worker nodes ' + - 'are managed internally. For this reason, the options ' + - 'azure-worker-instance-type, azure-worker-instance-disk and ' + - 'azure-worker-instance-spot are not taking effect.') - else: - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=cromwell_token, - workspace_id=workspace_id, verify=verify_ssl) - job_queue_id = queue.fetch_job_queue_id(workflow_type=workflow_type, batch=batch, - job_queue=job_queue) - if use_private_docker_repository: - if is_module: - print(f'Workflow "{workflow_name}" is a CloudOS module. ' + - 'Option --use-private-docker-repository will be ignored.') - docker_login = False - else: - me = j.get_user_info(verify=verify_ssl)['dockerRegistriesCredentials'] - if len(me) == 0: - raise Exception('User private Docker repository has been selected but your user ' + - 'credentials have not been configured yet. Please, link your ' + - 'Docker account to CloudOS before using ' + - '--use-private-docker-repository option.') - print('Use private Docker repository has been selected. A custom job ' + - 'queue to support private Docker containers and/or Lustre FSx will be created for ' + - 'your job. The selected job queue will serve as a template.') - docker_login = True - else: - docker_login = False - if nextflow_version == 'latest': - if execution_platform == 'aws': - nextflow_version = AWS_NEXTFLOW_LATEST - elif execution_platform == 'azure': - nextflow_version = AZURE_NEXTFLOW_LATEST - else: - nextflow_version = HPC_NEXTFLOW_LATEST - print('You have specified Nextflow version \'latest\' for execution platform ' + - f'\'{execution_platform}\'. The workflow will use the ' + - f'latest version available on CloudOS: {nextflow_version}.') - if execution_platform == 'aws': - if nextflow_version not in AWS_NEXTFLOW_VERSIONS: - print('For execution platform \'aws\', the workflow will use the default ' + - '\'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' - if execution_platform == 'azure': - if nextflow_version not in AZURE_NEXTFLOW_VERSIONS: - print('For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' ' + - 'version on CloudOS.') - nextflow_version = '22.11.1-edge' - if execution_platform == 'hpc': - if nextflow_version not in HPC_NEXTFLOW_VERSIONS: - print('For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' - if nextflow_version != '22.10.8' and nextflow_version != '22.11.1-edge': - click.secho(f'You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' + - 'to be written in DSL2 and does not support DSL1.', fg='yellow', bold=True) - print('\nExecuting run...') - if workflow_type == 'nextflow': - print(f'\tNextflow version: {nextflow_version}') - j_id = j.send_job(job_config=job_config, - parameter=parameter, - is_module=is_module, - git_commit=git_commit, - git_tag=git_tag, - git_branch=git_branch, - job_name=job_name, - resumable=resumable, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - nextflow_profile=nextflow_profile, - nextflow_version=nextflow_version, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=hpc_id, - workflow_type=workflow_type, - cromwell_id=cromwell_id, - azure_worker_instance_type=azure_worker_instance_type, - azure_worker_instance_disk=azure_worker_instance_disk, - azure_worker_instance_spot=azure_worker_instance_spot, - cost_limit=cost_limit, - use_mountpoints=use_mountpoints, - accelerate_saving_results=accelerate_saving_results, - docker_login=docker_login, - verify=verify_ssl) - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=verbose, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@job.command('status') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_status(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Check job status in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - print('Executing status...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' - print(f'\tTo further check your job status you can either go to {j_url} ' + - 'or repeat the command you just used.') - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") - - -@job.command('workdir') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the working directory to an interactive session.', - is_flag=True) -@click.option('--delete', - help='Delete the results directory of a CloudOS job.', - is_flag=True) -@click.option('-y', '--yes', - help='Skip confirmation prompt when deleting results.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--status', - help='Check the deletion status of the working directory.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_workdir(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - delete, - yes, - session_id, - status, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the working directory of a specified job or check deletion status.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Handle --status flag - if status: - console = Console() - - if verbose: - console.print('[bold cyan]Checking deletion status of job working directory...[/bold cyan]') - console.print('\t[dim]...Preparing objects[/dim]') - console.print('\t[bold]Using the following parameters:[/bold]') - console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') - console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') - console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - - # Use Cloudos object to access the deletion status method - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - console.print('\t[dim]The following Cloudos object was created:[/dim]') - console.print('\t' + str(cl) + '\n') - - try: - deletion_status = cl.get_workdir_deletion_status( - job_id=job_id, - workspace_id=workspace_id, - verify=verify_ssl - ) - - # Convert API status to user-friendly terminology with color - status_config = { - "ready": ("available", "green"), - "deleting": ("deleting", "yellow"), - "scheduledForDeletion": ("scheduled for deletion", "yellow"), - "deleted": ("deleted", "red"), - "failedToDelete": ("failed to delete", "red") - } - - # Get the status of the workdir folder itself and convert it - api_status = deletion_status.get("status", "unknown") - folder_status, status_color = status_config.get(api_status, (api_status, "white")) - folder_info = deletion_status.get("items", {}) - - # Display results in a clear, styled format with human-readable sentence - console.print(f'The working directory of job [cyan]{deletion_status["job_id"]}[/cyan] is in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - - # For non-available statuses, always show update time and user info - if folder_status != "available": - if folder_info.get("updatedAt"): - console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - - # Show user information - prefer deletedBy over user field - user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) - if user_info: - user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() - user_email = user_info.get('email', '') - if user_name or user_email: - user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) - console.print(f'[blue]User:[/blue] {user_display}') - - # Display detailed information if verbose - if verbose: - console.print(f'\n[bold]Additional information:[/bold]') - console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') - console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') - console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') - - # Show folder metadata if available - if folder_info.get("createdAt"): - console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') - if folder_info.get("updatedAt"): - console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') - if folder_info.get("folderType"): - console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - - except ValueError as e: - raise click.ClickException(str(e)) - except Exception as e: - raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - - return - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Finding working directory path...') - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl) - print(f"Working directory for job {job_id}: {workdir}") - - # Link to interactive session if requested - if link: - if verbose: - print(f'\tLinking working directory to interactive session {session_id}...') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - link_client.link_folder(workdir.strip(), session_id) - - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") - - # Delete workdir directory if requested - if delete: - try: - # Ask for confirmation unless --yes flag is provided - if not yes: - confirmation_message = ( - "\n⚠️ Deleting intermediate results is permanent and cannot be undone. " - "All associated data will be permanently removed and cannot be recovered. " - "The current job, as well as any other jobs sharing the same working directory, " - "will no longer be resumable. This action will be logged in the audit trail " - "(if auditing is enabled for your organisation), and you will be recorded as " - "the user who performed the deletion. You can skip this confirmation step by " - "providing -y or --yes flag to cloudos job workdir --delete. Please confirm " - "that you want to delete intermediate results of this analysis? [y/n] " - ) - click.secho(confirmation_message, fg='black', bg='yellow') - user_input = input().strip().lower() - if user_input != 'y': - print('\nDeletion cancelled.') - return - # Proceed with deletion - job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - job.delete_job_results(job_id, "workDirectory", verify=verify_ssl) - click.secho('\nIntermediate results directories deleted successfully.', fg='green', bold=True) - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve intermediate results for job '{job_id}'. {str(e)}") - else: - if yes: - click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) - - -@job.command('logs') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the logs directories to an interactive session.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_logs(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - session_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the logs of a specified job.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Executing logs...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - logs = cl.get_job_logs(job_id, workspace_id, verify_ssl) - for name, path in logs.items(): - print(f"{name}: {path}") - - # Link to interactive session if requested - if link: - if logs: - # Extract the parent logs directory from any log file path - # All log files should be in the same logs directory - first_log_path = next(iter(logs.values())) - # Remove the filename to get the logs directory - # e.g., "s3://bucket/path/to/logs/filename.txt" -> "s3://bucket/path/to/logs" - logs_dir = '/'.join(first_log_path.split('/')[:-1]) - - if verbose: - print(f'\tLinking logs directory to interactive session {session_id}...') - print(f'\t\tLogs directory: {logs_dir}') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - link_client.link_folder(logs_dir, session_id) - else: - if verbose: - print('\tNo logs found to link.') - - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve logs for job '{job_id}'. {str(e)}") - - -@job.command('results') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the results directories to an interactive session.', - is_flag=True) -@click.option('--delete', - help='Delete the results directory of a CloudOS job.', - is_flag=True) -@click.option('-y', '--yes', - help='Skip confirmation prompt when deleting results.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--status', - help='Check the deletion status of the job results.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_results(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - delete, - yes, - session_id, - status, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the results of a specified job or check deletion status.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Handle --status flag - if status: - console = Console() - - if verbose: - console.print('[bold cyan]Checking deletion status of job results...[/bold cyan]') - console.print('\t[dim]...Preparing objects[/dim]') - console.print('\t[bold]Using the following parameters:[/bold]') - console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') - console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') - console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - - # Use Cloudos object to access the deletion status method - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - console.print('\t[dim]The following Cloudos object was created:[/dim]') - console.print('\t' + str(cl) + '\n') - - try: - deletion_status = cl.get_results_deletion_status( - job_id=job_id, - workspace_id=workspace_id, - verify=verify_ssl - ) - - # Convert API status to user-friendly terminology with color - status_config = { - "ready": ("available", "green"), - "deleting": ("deleting", "yellow"), - "scheduledForDeletion": ("scheduled for deletion", "yellow"), - "deleted": ("deleted", "red"), - "failedToDelete": ("failed to delete", "red") - } - - # Get the status of the results folder itself and convert it - api_status = deletion_status.get("status", "unknown") - folder_status, status_color = status_config.get(api_status, (api_status, "white")) - folder_info = deletion_status.get("items", {}) - - # Display results in a clear, styled format with human-readable sentence - console.print(f'The results of job [cyan]{deletion_status["job_id"]}[/cyan] are in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - - # For non-available statuses, always show update time and user info - if folder_status != "available": - if folder_info.get("updatedAt"): - console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - - # Show user information - prefer deletedBy over user field - user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) - if user_info: - user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() - user_email = user_info.get('email', '') - if user_name or user_email: - user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) - console.print(f'[blue]User:[/blue] {user_display}') - - # Display detailed information if verbose - if verbose: - console.print(f'\n[bold]Additional information:[/bold]') - console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') - console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') - console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') - - # Show folder metadata if available - if folder_info.get("createdAt"): - console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') - if folder_info.get("updatedAt"): - console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') - if folder_info.get("folderType"): - console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - - except ValueError as e: - raise click.ClickException(str(e)) - except Exception as e: - raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - - return - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Executing results...') - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) - print(f"results: {results_path}") - - # Link to interactive session if requested - if link: - if verbose: - print(f'\tLinking results directory to interactive session {session_id}...') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - if verbose: - print(f'\t\tLinking results ({results_path})...') - - link_client.link_folder(results_path, session_id) - - # Delete results directory if requested - if delete: - # Ask for confirmation unless --yes flag is provided - if not yes: - confirmation_message = ( - "\n⚠️ Deleting final analysis results is irreversible. " - "All data and backups will be permanently removed and cannot be recovered. " - "You can skip this confirmation step by providing '-y' or '--yes' flag to " - "'cloudos job results --delete'. " - "Please confirm that you want to delete final results of this analysis? [y/n] " - ) - click.secho(confirmation_message, fg='black', bg='yellow') - user_input = input().strip().lower() - if user_input != 'y': - print('\nDeletion cancelled.') - return - if verbose: - print(f'\nDeleting result directories from CloudOS...') - # Proceed with deletion - job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - job.delete_job_results(job_id, "analysisResults", verify=verify_ssl) - click.secho('\nResults directories deleted successfully.', fg='green', bold=True) - else: - if yes: - click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve results for job '{job_id}'. {str(e)}") - - -@job.command('details') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--output-format', - help=('The desired display for the output, either directly in standard output or saved as file. ' + - 'Default=stdout.'), - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--output-basename', - help=('Output file base name to save jobs details. ' + - 'Default={job_id}_details'), - required=False) -@click.option('--parameters', - help=('Whether to generate a ".config" file that can be used as input for --job-config parameter. ' + - 'It will have the same basename as defined in "--output-basename". '), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_details(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - output_basename, - parameters, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve job details in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if ctx.get_parameter_source('output_basename') == click.core.ParameterSource.DEFAULT: - output_basename = f"{job_id}_details" - - print('Executing details...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - - # check if the API gives a 403 error/forbidden error - try: - j_details = cl.get_job_status(job_id, workspace_id, verify_ssl) - except BadRequestException as e: - if '403' in str(e) or 'Forbidden' in str(e): - raise ValueError("API can only show job details of your own jobs, cannot see other user's job details.") - else: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve details for job '{job_id}'. {str(e)}") - create_job_details(json.loads(j_details.content), job_id, output_format, output_basename, parameters, cloudos_url) - - -@job.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save jobs list. ' + - 'Default=joblist'), - default='joblist', - required=False) -@click.option('--output-format', - help='The desired output format. For json option --all-fields will be automatically set to True. Default=stdout.', - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--table-columns', - help=('Comma-separated list of columns to display in the table. Only applicable when --output-format=stdout. ' + - 'Available columns: status,name,project,owner,pipeline,id,submit_time,end_time,run_time,commit,cost,resources,storage_type. ' + - 'Default: responsive (auto-selects columns based on terminal width)'), - default=None) -@click.option('--all-fields', - help=('Whether to collect all available fields from jobs or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv. Automatically enabled for json output.'), - is_flag=True) -@click.option('--last-n-jobs', - help=("The number of last workspace jobs to retrieve. You can use 'all' to " + - "retrieve all workspace jobs. When adding this option, options " + - "'--page' and '--page-size' are ignored.")) -@click.option('--page', - help=('Page number to fetch from the API. Used with --page-size to control jobs ' + - 'per page (e.g. --page=4 --page-size=20). Default=1.'), - type=int, - default=1) -@click.option('--page-size', - help=('Page size to retrieve from API, corresponds to the number of jobs per page. ' + - 'Maximum allowed integer is 100. Default=10.'), - type=int, - default=10) -@click.option('--archived', - help=('When this flag is used, only archived jobs list is collected.'), - is_flag=True) -@click.option('--filter-status', - help='Filter jobs by status (e.g., completed, running, failed, aborted).') -@click.option('--filter-job-name', - help='Filter jobs by job name ( case insensitive ).') -@click.option('--filter-project', - help='Filter jobs by project name.') -@click.option('--filter-workflow', - help='Filter jobs by workflow/pipeline name.') -@click.option('--last', - help=('When workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('--filter-job-id', - help='Filter jobs by specific job ID.') -@click.option('--filter-only-mine', - help='Filter to show only jobs belonging to the current user.', - is_flag=True) -@click.option('--filter-queue', - help='Filter jobs by queue name. Only applies to jobs running in batch environment. Non-batch jobs are preserved in results.') -@click.option('--filter-owner', - help='Filter jobs by owner username.') -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - table_columns, - all_fields, - last_n_jobs, - page, - page_size, - archived, - filter_status, - filter_job_name, - filter_project, - filter_workflow, - last, - filter_job_id, - filter_only_mine, - filter_owner, - filter_queue, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect and display workspace jobs from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Pass table_columns directly to create_job_list_table for validation and processing - selected_columns = table_columns - # Only set outfile if not using stdout - if output_format != 'stdout': - outfile = output_basename + '.' + output_format - - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for jobs in the following workspace: ' + - f'{workspace_id}') - # Check if the user provided the --page option - ctx = click.get_current_context() - if not isinstance(page, int) or page < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') - - if not isinstance(page_size, int) or page_size < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page-size parameter') - - # Validate page_size limit - must be done before API call - if page_size > 100: - click.secho('Error: Page size cannot exceed 100. Please use --page-size with a value <= 100', fg='red', err=True) - raise SystemExit(1) - - result = cl.get_job_list(workspace_id, last_n_jobs, page, page_size, archived, verify_ssl, - filter_status=filter_status, - filter_job_name=filter_job_name, - filter_project=filter_project, - filter_workflow=filter_workflow, - filter_job_id=filter_job_id, - filter_only_mine=filter_only_mine, - filter_owner=filter_owner, - filter_queue=filter_queue, - last=last) - - # Extract jobs and pagination metadata from result - my_jobs_r = result['jobs'] - pagination_metadata = result['pagination_metadata'] - - # Validate requested page exists - if pagination_metadata: - total_jobs = pagination_metadata.get('Pagination-Count', 0) - current_page_size = pagination_metadata.get('Pagination-Limit', page_size) - - if total_jobs > 0: - total_pages = (total_jobs + current_page_size - 1) // current_page_size - if page > total_pages: - click.secho(f'Error: Page {page} does not exist. There are only {total_pages} page(s) available with {total_jobs} total job(s). ' - f'Please use --page with a value between 1 and {total_pages}', fg='red', err=True) - raise SystemExit(1) - - if len(my_jobs_r) == 0: - # Check if any filtering options are being used - filters_used = any([ - filter_status, - filter_job_name, - filter_project, - filter_workflow, - filter_job_id, - filter_only_mine, - filter_owner, - filter_queue - ]) - if output_format == 'stdout': - # For stdout, always show a user-friendly message - create_job_list_table([], cloudos_url, pagination_metadata, selected_columns) - else: - if filters_used: - print('A total of 0 jobs collected.') - elif ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - print('A total of 0 jobs collected. This is likely because your workspace ' + - 'has no jobs created yet.') - else: - print('A total of 0 jobs collected. This is likely because the --page you requested ' + - 'does not exist. Please, try a smaller number for --page or collect all the jobs by not ' + - 'using --page parameter.') - elif output_format == 'stdout': - # Display as table - create_job_list_table(my_jobs_r, cloudos_url, pagination_metadata, selected_columns) - elif output_format == 'csv': - my_jobs = cl.process_job_list(my_jobs_r, all_fields) - cl.save_job_list_to_csv(my_jobs, outfile) - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_jobs_r)) - print(f'\tJob list collected with a total of {len(my_jobs_r)} jobs.') - print(f'\tJob list saved to {outfile}') - else: - raise ValueError('Unrecognised output format. Please use one of [stdout|csv|json]') - - -@job.command('abort') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-ids', - help=('One or more job ids to abort. If more than ' + - 'one is provided, they must be provided as ' + - 'a comma separated list of ids. E.g. id1,id2,id3'), - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--force', - help='Force abort the job even if it is not in a running or initializing state.', - is_flag=True) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def abort_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - job_ids, - verbose, - disable_ssl_verification, - ssl_cert, - profile, - force): - """Abort all specified jobs from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - print('Aborting jobs...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for jobs in the following workspace: ' + - f'{workspace_id}') - # check if the user provided an empty job list - jobs = job_ids.replace(' ', '') - if not jobs: - raise ValueError('No job IDs provided. Please specify at least one job ID to abort.') - jobs = jobs.split(',') - - # Issue warning if using --force flag - if force: - click.secho(f"Warning: Using --force to abort jobs. Some data might be lost.", fg='yellow', bold=True) - - for job in jobs: - try: - j_status = cl.get_job_status(job, workspace_id, verify_ssl) - except Exception as e: - click.secho(f"Failed to get status for job {job}, please make sure it exists in the workspace: {e}", fg='yellow', bold=True) - continue - - j_status_content = json.loads(j_status.content) - job_status = j_status_content['status'] - - # Check if job is in a state that normally allows abortion - is_abortable = job_status in ABORT_JOB_STATES - - # Issue warning if job is in initializing state and not using force - if job_status == 'initializing' and not force: - click.secho(f"Warning: Job {job} is in initializing state.", fg='yellow', bold=True) - - # Check if job can be aborted - if not is_abortable: - click.secho(f"Job {job} is not in a state that can be aborted and is ignored. " + - f"Current status: {job_status}", fg='yellow', bold=True) - else: - try: - cl.abort_job(job, workspace_id, verify_ssl, force) - click.secho(f"Job '{job}' aborted successfully.", fg='green', bold=True) - except Exception as e: - click.secho(f"Failed to abort job {job}. Error: {e}", fg='red', bold=True) - - -@job.command('cost') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to get costs for.', - required=True) -@click.option('--output-format', - help='The desired file format (file extension) for the output. For json option --all-fields will be automatically set to True. Default=csv.', - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_cost(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve job cost information in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - print('Retrieving cost information...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cost_viewer = CostViewer(cloudos_url, apikey) - if verbose: - print(f'\tSearching for cost data for job id: {job_id}') - # Display costs with pagination - cost_viewer.display_costs(job_id, workspace_id, output_format, verify_ssl) - - -@job.command('related') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to get costs for.', - required=True) -@click.option('--output-format', - help='The desired output format. Default=stdout.', - type=click.Choice(['stdout', 'json'], case_sensitive=False), - default='stdout') -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def related(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve related job analyses in CloudOS.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - related_analyses(cloudos_url, apikey, job_id, workspace_id, output_format, verify_ssl) - - -@click.command() -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-ids', - help=('One or more job ids to archive/unarchive. If more than ' + - 'one is provided, they must be provided as ' + - 'a comma separated list of ids. E.g. id1,id2,id3'), - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def archive_unarchive_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - job_ids, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Archive or unarchive specified jobs in a CloudOS workspace.""" - # Determine operation based on the command name used - target_archived_state = ctx.info_name == "archive" - action = "archive" if target_archived_state else "unarchive" - action_past = "archived" if target_archived_state else "unarchived" - action_ing = "archiving" if target_archived_state else "unarchiving" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - print(f'{action_ing.capitalize()} jobs...') - - if verbose: - print('\t...Preparing objects') - - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\t{action_ing.capitalize()} jobs in the following workspace: {workspace_id}') - - # check if the user provided an empty job list - jobs = job_ids.replace(' ', '') - if not jobs: - raise ValueError(f'No job IDs provided. Please specify at least one job ID to {action}.') - jobs_list = [job for job in jobs.split(',') if job] # Filter out empty strings - - # Check for duplicate job IDs - duplicates = [job_id for job_id in set(jobs_list) if jobs_list.count(job_id) > 1] - if duplicates: - dup_str = ', '.join(duplicates) - click.secho(f'Warning: Duplicate job IDs detected and will be processed only once: {dup_str}', fg='yellow', bold=True) - # Remove duplicates while preserving order - jobs_list = list(dict.fromkeys(jobs_list)) - if verbose: - print(f'\tDuplicate job IDs removed. Processing {len(jobs_list)} unique job(s).') - - # Check archive status for all jobs - status_check = cl.check_jobs_archive_status(jobs_list, workspace_id, target_archived_state=target_archived_state, verify=verify_ssl, verbose=verbose) - valid_jobs = status_check['valid_jobs'] - already_processed = status_check['already_processed'] - invalid_jobs = status_check['invalid_jobs'] - - # Report invalid jobs (but continue processing valid ones) - for job_id, error_msg in invalid_jobs.items(): - click.secho(f"Failed to get status for job {job_id}, please make sure it exists in the workspace: {error_msg}", fg='yellow', bold=True) - - if not valid_jobs and not already_processed: - # All jobs were invalid - exit gracefully - click.secho('No valid job IDs found. Please check that the job IDs exist and are accessible.', fg='yellow', bold=True) - return - - if not valid_jobs: - if len(already_processed) == 1: - click.secho(f"Job '{already_processed[0]}' is already {action_past}. No action needed.", fg='cyan', bold=True) - else: - click.secho(f"All {len(already_processed)} jobs are already {action_past}. No action needed.", fg='cyan', bold=True) - return - - try: - # Call the appropriate action method - if target_archived_state: - cl.archive_jobs(valid_jobs, workspace_id, verify_ssl) - else: - cl.unarchive_jobs(valid_jobs, workspace_id, verify_ssl) - - success_msg = [] - if len(valid_jobs) == 1: - success_msg.append(f"Job '{valid_jobs[0]}' {action_past} successfully.") - else: - success_msg.append(f"{len(valid_jobs)} jobs {action_past} successfully: {', '.join(valid_jobs)}") - - if already_processed: - if len(already_processed) == 1: - success_msg.append(f"Job '{already_processed[0]}' was already {action_past}.") - else: - success_msg.append(f"{len(already_processed)} jobs were already {action_past}: {', '.join(already_processed)}") - - click.secho('\n'.join(success_msg), fg='green', bold=True) - except Exception as e: - raise ValueError(f"Failed to {action} jobs: {str(e)}") - - -@click.command(help='Clone or resume a job with modified parameters') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.') -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p input=s3://path_to_my_file. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--nextflow-profile', - help=('A comma separated string indicating the nextflow profile/s ' + - 'to use with your job.')) -@click.option('--nextflow-version', - help=('Nextflow version to use when executing the workflow in CloudOS. ' + - 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest'])) -@click.option('--git-branch', - help=('The branch to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--job-name', - help='The name of the job. If not set, will take the name of the cloned job.') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help=('Name of the job queue to use with a batch job. ' + - 'In Azure workspaces, this option is ignored.')) -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).')) -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float) -@click.option('--job-id', - help='The CloudOS job id of the job to be cloned.', - required=True) -@click.option('--accelerate-file-staging', - help='Enables AWS S3 mountpoint for quicker file staging.', - is_flag=True) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--resumable', - help='Whether to make the job able to be resumed or not.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', - help='Profile to use from the config file', - default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def clone_resume(ctx, - apikey, - cloudos_url, - workspace_id, - project_name, - parameter, - nextflow_profile, - nextflow_version, - git_branch, - repository_platform, - job_name, - do_not_save_logs, - job_queue, - instance_type, - cost_limit, - job_id, - accelerate_file_staging, - accelerate_saving_results, - resumable, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - if ctx.info_name == "clone": - mode, action = "clone", "cloning" - elif ctx.info_name == "resume": - mode, action = "resume", "resuming" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - print(f'{action.capitalize()} job...') - if verbose: - print('\t...Preparing objects') - - # Create Job object (set dummy values for project_name and workflow_name, since they come from the cloned job) - job_obj = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - - if verbose: - print('\tThe following Job object was created:') - print('\t' + str(job_obj) + '\n') - print(f'\t{action.capitalize()} job {job_id} in workspace: {workspace_id}') - - try: - - # Clone/resume the job with provided overrides - cloned_resumed_job_id = job_obj.clone_or_resume_job( - source_job_id=job_id, - queue_name=job_queue, - cost_limit=cost_limit, - master_instance=instance_type, - job_name=job_name, - nextflow_version=nextflow_version, - branch=git_branch, - repository_platform=repository_platform, - profile=nextflow_profile, - do_not_save_logs=do_not_save_logs, - use_fusion=accelerate_file_staging, - accelerate_saving_results=accelerate_saving_results, - resumable=resumable, - # only when explicitly setting --project-name will be overridden, else using the original project - project_name=project_name if ctx.get_parameter_source("project_name") == click.core.ParameterSource.COMMANDLINE else None, - parameters=list(parameter) if parameter else None, - verify=verify_ssl, - mode=mode - ) - - if verbose: - print(f'\t{mode.capitalize()}d job ID: {cloned_resumed_job_id}') - - print(f"Job successfully {mode}d. New job ID: {cloned_resumed_job_id}") - - except BadRequestException as e: - raise ValueError(f"Failed to {mode} job. Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") - - -# Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) -archive_unarchive_jobs.help = 'Archive specified jobs in a CloudOS workspace.' -job.add_command(archive_unarchive_jobs, "archive") - -# Create a copy with different help text for unarchive -archive_unarchive_jobs_copy = copy.deepcopy(archive_unarchive_jobs) -archive_unarchive_jobs_copy.help = 'Unarchive specified jobs in a CloudOS workspace.' -job.add_command(archive_unarchive_jobs_copy, "unarchive") - - -# Apply the best Click solution: Set specific help text for each command registration -clone_resume.help = 'Clone a job with modified parameters' -job.add_command(clone_resume, "clone") - -# Create a copy with different help text for resume -clone_resume_copy = copy.deepcopy(clone_resume) -clone_resume_copy.help = 'Resume a job with modified parameters' -job.add_command(clone_resume_copy, "resume") - - -@workflow.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save workflow list. ' + - 'Default=workflow_list'), - default='workflow_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from workflows or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_workflows(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all workflows from a CloudOS workspace in CSV format.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for workflows in the following workspace: ' + - f'{workspace_id}') - my_workflows_r = cl.get_workflow_list(workspace_id, verify=verify_ssl) - if output_format == 'csv': - my_workflows = cl.process_workflow_list(my_workflows_r, all_fields) - my_workflows.to_csv(outfile, index=False) - print(f'\tWorkflow list collected with a total of {my_workflows.shape[0]} workflows.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_workflows_r)) - print(f'\tWorkflow list collected with a total of {len(my_workflows_r)} workflows.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tWorkflow list saved to {outfile}') - - -@workflow.command('import') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=('The CloudOS url you are trying to access to. ' + - f'Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option("--workflow-name", help="The name that the workflow will have in CloudOS.", required=True) -@click.option("-w", "--workflow-url", help="URL of the workflow repository.", required=True) -@click.option("-d", "--workflow-docs-link", help="URL to the documentation of the workflow.", default='') -@click.option("--cost-limit", help="Cost limit for the workflow. Default: $30 USD.", default=30) -@click.option("--workflow-description", help="Workflow description", default="") -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name']) -def import_wf(ctx, - apikey, - cloudos_url, - workspace_id, - workflow_name, - workflow_url, - workflow_docs_link, - cost_limit, - workflow_description, - repository_platform, - disable_ssl_verification, - ssl_cert, - profile): - """ - Import workflows from supported repository providers. - """ - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - repo_import = ImportWorflow( - cloudos_url=cloudos_url, cloudos_apikey=apikey, workspace_id=workspace_id, platform=repository_platform, - workflow_name=workflow_name, workflow_url=workflow_url, workflow_docs_link=workflow_docs_link, - cost_limit=cost_limit, workflow_description=workflow_description, verify=verify_ssl - ) - workflow_id = repo_import.import_workflow() - print(f'\tWorkflow {workflow_name} was imported successfully with the ' + - f'following ID: {workflow_id}') - - -@project.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save project list. ' + - 'Default=project_list'), - default='project_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from projects or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--page', - help=('Response page to retrieve. Default=1.'), - type=int, - default=1) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_projects(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - page, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all projects from a CloudOS workspace in CSV format.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for projects in the following workspace: ' + - f'{workspace_id}') - # Check if the user provided the --page option - ctx = click.get_current_context() - if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - get_all = True - else: - get_all = False - if not isinstance(page, int) or page < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') - my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all) - if len(my_projects_r) == 0: - if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - print('A total of 0 projects collected. This is likely because your workspace ' + - 'has no projects created yet.') - else: - print('A total of 0 projects collected. This is likely because the --page you ' + - 'requested does not exist. Please, try a smaller number for --page or collect all the ' + - 'projects by not using --page parameter.') - elif output_format == 'csv': - my_projects = cl.process_project_list(my_projects_r, all_fields) - my_projects.to_csv(outfile, index=False) - print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_projects_r)) - print(f'\tProject list collected with a total of {len(my_projects_r)} projects.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tProject list saved to {outfile}') - - -@project.command('create') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--new-project', - help='The name for the new project.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def create_project(ctx, - apikey, - cloudos_url, - workspace_id, - new_project, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Create a new project in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - # verify ssl configuration - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Print basic output - if verbose: - print(f'\tUsing CloudOS URL: {cloudos_url}') - print(f'\tUsing workspace: {workspace_id}') - print(f'\tProject name: {new_project}') - - cl = Cloudos(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None) - - try: - project_id = cl.create_project(workspace_id, new_project, verify_ssl) - print(f'\tProject "{new_project}" created successfully with ID: {project_id}') - if verbose: - print(f'\tProject URL: {cloudos_url}/app/projects/{project_id}') - except Exception as e: - print(f'\tError creating project: {str(e)}') - sys.exit(1) - - -@cromwell.command('status') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_status(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Check Cromwell server status in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - print('Executing status...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tChecking Cromwell status in {workspace_id} workspace') - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - - -@cromwell.command('start') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to Cromwell restart. ' + - 'Default=300.'), - default=300) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_restart(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - wait_time, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Restart Cromwell server in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - action = 'restart' - print('Starting Cromwell server...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tStarting Cromwell server in {workspace_id} workspace') - cl.cromwell_switch(workspace_id, action, verify_ssl) - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - elapsed = 0 - while elapsed < wait_time and c_status_h != 'Running': - c_status_old = c_status_h - time.sleep(REQUEST_INTERVAL_CROMWELL) - elapsed += REQUEST_INTERVAL_CROMWELL - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - if c_status_h != c_status_old: - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h != 'Running': - print(f'\tYour current Cromwell status is: {c_status_h}. The ' + - f'selected wait-time of {wait_time} was exceeded. Please, ' + - 'consider to set a longer wait-time.') - print('\tTo further check your Cromwell status you can either go to ' + - f'{cloudos_url} or use the following command:\n' + - '\tcloudos cromwell status \\\n' + - f'\t\t--cloudos-url {cloudos_url} \\\n' + - '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + - f'\t\t--workspace-id {workspace_id}') - sys.exit(1) - - -@cromwell.command('stop') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_stop(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Stop Cromwell server in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - action = 'stop' - print('Stopping Cromwell server...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tStopping Cromwell server in {workspace_id} workspace') - cl.cromwell_switch(workspace_id, action, verify_ssl) - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - - -@queue.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save job queue list. ' + - 'Default=job_queue_list'), - default='job_queue_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from workflows or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_queues(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all available job queues from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - j_queue = Queue(cloudos_url, apikey, None, workspace_id, verify=verify_ssl) - my_queues = j_queue.get_job_queues() - if len(my_queues) == 0: - raise ValueError('No AWS batch queues found. Please, make sure that your CloudOS supports AWS bath queues') - if output_format == 'csv': - queues_processed = j_queue.process_queue_list(my_queues, all_fields) - queues_processed.to_csv(outfile, index=False) - print(f'\tJob queue list collected with a total of {queues_processed.shape[0]} queues.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_queues)) - print(f'\tJob queue list collected with a total of {len(my_queues)} queues.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tJob queue list saved to {outfile}') - - -@configure.command('list-profiles') -def list_profiles(): - config_manager = ConfigurationProfile() - config_manager.list_profiles() - - -@configure.command('remove-profile') -@click.option('--profile', - help='Name of the profile. Not using this option will lead to profile named "deafults" being generated', - required=True) -@click.pass_context -def remove_profile(ctx, profile): - update_command_context_from_click(ctx) - profile = profile or ctx.obj['profile'] - config_manager = ConfigurationProfile() - config_manager.remove_profile(profile) - - -@bash.command('job') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('--command', - help='The command to run in the bash job.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--cpus', - help='The number of CPUs to use for the task\'s master node. Default=1.', - type=int, - default=1) -@click.option('--memory', - help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', - type=int, - default=4) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - default='aws') -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run_bash_job(ctx, - apikey, - command, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - parameter, - job_name, - do_not_save_logs, - job_queue, - instance_type, - instance_disk, - cpus, - memory, - storage_mode, - lustre_size, - wait_completion, - wait_time, - repository_platform, - execution_platform, - cost_limit, - accelerate_saving_results, - request_interval, - disable_ssl_verification, - ssl_cert, - profile): - """Run a bash job in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - - if do_not_save_logs: - save_logs = False - else: - save_logs = True - - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=None, importsfile=None, - repository_platform=repository_platform, verify=verify_ssl, last=last) - - if job_queue is not None: - batch = True - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, - workspace_id=workspace_id, verify=verify_ssl) - # I have to add 'nextflow', other wise the job queue id is not found - job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, - job_queue=job_queue) - else: - job_queue_id = None - batch = False - j_id = j.send_job(job_config=None, - parameter=parameter, - git_commit=None, - git_tag=None, - git_branch=None, - job_name=job_name, - resumable=False, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - workflow_type='docker', - nextflow_profile=None, - nextflow_version=None, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=None, - cost_limit=cost_limit, - accelerate_saving_results=accelerate_saving_results, - verify=verify_ssl, - command={"command": command}, - cpus=cpus, - memory=memory) - - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=False, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@bash.command('array-job') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('--command', - help='The command to run in the bash job.') -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + - 'times as parameters you want to include. ' + - 'For parameters pointing to a file, the format expected is ' + - 'parameter_name=/Data/parameter_value. The parameter value must be a ' + - 'file located in the `Data` subfolder. If no is specified, it defaults to ' + - 'the project specified by the profile or --project-name parameter. ' + - 'E.g.: -p "--file=Data/file.txt" or "--file=/Data/folder/file.txt"')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--cpus', - help='The number of CPUs to use for the task\'s master node. Default=1.', - type=int, - default=1) -@click.option('--memory', - help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', - type=int, - default=4) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - type=click.Choice(['aws', 'azure', 'hpc']), - default='aws') -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--array-file', - help=('Path to a file containing an array of commands to run in the bash job.'), - default=None, - required=True) -@click.option('--separator', - help=('Separator to use in the array file. Default=",".'), - type=click.Choice([',', ';', 'tab', 'space', '|']), - default=",", - required=True) -@click.option('--list-columns', - help=('List columns present in the array file. ' + - 'This option will not run any job.'), - is_flag=True) -@click.option('--array-file-project', - help=('Name of the project in which the array file is placed, if different from --project-name.'), - default=None) -@click.option('--disable-column-check', - help=('Disable the check for the columns in the array file. ' + - 'This option is only used when --array-file is provided.'), - is_flag=True) -@click.option('-a', '--array-parameter', - multiple=True, - help=('A single parameter to pass to the job call only for specifying array columns. ' + - 'It should be in the following form: parameter_name=array_file_column_name. E.g.: ' + - '-a --test=value or -a -test=value or -a test=value or -a =value (for no prefix). ' + - 'You can use this option as many times as parameters you want to include.')) -@click.option('--custom-script-path', - help=('Path of a custom script to run in the bash array job instead of a command.'), - default=None) -@click.option('--custom-script-project', - help=('Name of the project to use when running the custom command script, if ' + - 'different than --project-name.'), - default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run_bash_array_job(ctx, - apikey, - command, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - parameter, - job_name, - do_not_save_logs, - job_queue, - instance_type, - instance_disk, - cpus, - memory, - storage_mode, - lustre_size, - wait_completion, - wait_time, - repository_platform, - execution_platform, - cost_limit, - accelerate_saving_results, - request_interval, - disable_ssl_verification, - ssl_cert, - profile, - array_file, - separator, - list_columns, - array_file_project, - disable_column_check, - array_parameter, - custom_script_path, - custom_script_project): - """Run a bash array job in CloudOS.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - if not list_columns and not (command or custom_script_path): - raise click.UsageError("Must provide --command or --custom-script-path if --list-columns is not set.") - - # when not set, use the global project name - if array_file_project is None: - array_file_project = project_name - - # this needs to be in another call to datasets, by default it uses the global project name - if custom_script_project is None: - custom_script_project = project_name - - # setup separators for API and array file (the're different) - separators = { - ",": {"api": ",", "file": ","}, - ";": {"api": "%3B", "file": ";"}, - "space": {"api": "+", "file": " "}, - "tab": {"api": "tab", "file": "tab"}, - "|": {"api": "%7C", "file": "|"} - } - - # setup important options for the job - if do_not_save_logs: - save_logs = False - else: - save_logs = True - - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=None, importsfile=None, - repository_platform=repository_platform, verify=verify_ssl, last=last) - - # retrieve columns - r = j.retrieve_cols_from_array_file( - array_file, - generate_datasets_for_project(cloudos_url, apikey, workspace_id, array_file_project, verify_ssl), - separators[separator]['api'], - verify_ssl - ) - - if not disable_column_check: - columns = json.loads(r.content).get("headers", None) - # pass this to the SEND JOB API call - # b'{"headers":[{"index":0,"name":"id"},{"index":1,"name":"title"},{"index":2,"name":"filename"},{"index":3,"name":"file2name"}]}' - if columns is None: - raise ValueError("No columns found in the array file metadata.") - if list_columns: - print("Columns: ") - for col in columns: - print(f"\t- {col['name']}") - return - else: - columns = [] - - # setup parameters for the job - cmd = j.setup_params_array_file( - custom_script_path, - generate_datasets_for_project(cloudos_url, apikey, workspace_id, custom_script_project, verify_ssl), - command, - separators[separator]['file'] - ) - - # check columns in the array file vs parameters added - if not disable_column_check and array_parameter: - print("\nChecking columns in the array file vs parameters added...\n") - for ap in array_parameter: - ap_split = ap.split('=') - ap_value = '='.join(ap_split[1:]) - for col in columns: - if col['name'] == ap_value: - print(f"Found column '{ap_value}' in the array file.") - break - else: - raise ValueError(f"Column '{ap_value}' not found in the array file. " + \ - f"Columns in array-file: {separator.join([col['name'] for col in columns])}") - - if job_queue is not None: - batch = True - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, - workspace_id=workspace_id, verify=verify_ssl) - # I have to add 'nextflow', other wise the job queue id is not found - job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, - job_queue=job_queue) - else: - job_queue_id = None - batch = False - - # send job - j_id = j.send_job(job_config=None, - parameter=parameter, - array_parameter=array_parameter, - array_file_header=columns, - git_commit=None, - git_tag=None, - git_branch=None, - job_name=job_name, - resumable=False, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - workflow_type='docker', - nextflow_profile=None, - nextflow_version=None, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=None, - cost_limit=cost_limit, - accelerate_saving_results=accelerate_saving_results, - verify=verify_ssl, - command=cmd, - cpus=cpus, - memory=memory) - - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=False, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@datasets.command(name="ls") -@click.argument("path", required=False, nargs=1) -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--details', - help=('When selected, it prints the details of the listed files. ' + - 'Details contains "Type", "Owner", "Size", "Last Updated", ' + - '"Virtual Name", "Storage Path".'), - is_flag=True) -@click.option('--output-format', - help=('The desired display for the output, either directly in standard output or saved as file. ' + - 'Default=stdout.'), - type=click.Choice(['stdout', 'csv'], case_sensitive=False), - default='stdout') -@click.option('--output-basename', - help=('Output file base name to save jobs details. ' + - 'Default=datasets_ls'), - default='datasets_ls', - required=False) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def list_files(ctx, - apikey, - cloudos_url, - workspace_id, - disable_ssl_verification, - ssl_cert, - project_name, - profile, - path, - details, - output_format, - output_basename): - """List contents of a path within a CloudOS workspace dataset.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - datasets = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = datasets.list_folder_content(path) - contents = result.get("contents") or result.get("datasets", []) - - if not contents: - contents = result.get("files", []) + result.get("folders", []) - - # Process items to extract data - processed_items = [] - for item in contents: - is_folder = "folderType" in item or item.get("isDir", False) - type_ = "folder" if is_folder else "file" - - # Enhanced type information - if is_folder: - folder_type = item.get("folderType") - if folder_type == "VirtualFolder": - type_ = "virtual folder" - elif folder_type == "S3Folder": - type_ = "s3 folder" - elif folder_type == "AzureBlobFolder": - type_ = "azure folder" - else: - type_ = "folder" - else: - # Check if file is managed by Lifebit (user uploaded) - is_managed_by_lifebit = item.get("isManagedByLifebit", False) - if is_managed_by_lifebit: - type_ = "file (user uploaded)" - else: - type_ = "file (virtual copy)" - - user = item.get("user", {}) - if isinstance(user, dict): - name = user.get("name", "").strip() - surname = user.get("surname", "").strip() - else: - name = surname = "" - if name and surname: - owner = f"{name} {surname}" - elif name: - owner = name - elif surname: - owner = surname - else: - owner = "-" - - raw_size = item.get("sizeInBytes", item.get("size")) - size = format_bytes(raw_size) if not is_folder and raw_size is not None else "-" - - updated = item.get("updatedAt") or item.get("lastModified", "-") - filepath = item.get("name", "-") - - if item.get("fileType") == "S3File" or item.get("folderType") == "S3Folder": - bucket = item.get("s3BucketName") - key = item.get("s3ObjectKey") or item.get("s3Prefix") - storage_path = f"s3://{bucket}/{key}" if bucket and key else "-" - elif item.get("fileType") == "AzureBlobFile" or item.get("folderType") == "AzureBlobFolder": - account = item.get("blobStorageAccountName") - container = item.get("blobContainerName") - key = item.get("blobName") if item.get("fileType") == "AzureBlobFile" else item.get("blobPrefix") - storage_path = f"az://{account}.blob.core.windows.net/{container}/{key}" if account and container and key else "-" - else: - storage_path = "-" - - processed_items.append({ - 'type': type_, - 'owner': owner, - 'size': size, - 'raw_size': raw_size, - 'updated': updated, - 'name': filepath, - 'storage_path': storage_path, - 'is_folder': is_folder - }) - - # Output handling - if output_format == 'csv': - import csv - - csv_filename = f'{output_basename}.csv' - - if details: - # CSV with all details - with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: - fieldnames = ['Type', 'Owner', 'Size', 'Size (bytes)', 'Last Updated', 'Virtual Name', 'Storage Path'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - - for item in processed_items: - writer.writerow({ - 'Type': item['type'], - 'Owner': item['owner'], - 'Size': item['size'], - 'Size (bytes)': item['raw_size'] if item['raw_size'] is not None else '', - 'Last Updated': item['updated'], - 'Virtual Name': item['name'], - 'Storage Path': item['storage_path'] - }) - else: - # CSV with just names - with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: - writer = csv.writer(csvfile) - writer.writerow(['Name', 'Storage Path']) - for item in processed_items: - writer.writerow([item['name'], item['storage_path']]) - - click.secho(f'\nDatasets list saved to: {csv_filename}', fg='green', bold=True) - - else: # stdout - if details: - console = Console(width=None) - table = Table(show_header=True, header_style="bold white") - table.add_column("Type", style="cyan", no_wrap=True) - table.add_column("Owner", style="white") - table.add_column("Size", style="magenta") - table.add_column("Last Updated", style="green") - table.add_column("Virtual Name", style="bold", overflow="fold") - table.add_column("Storage Path", style="dim", no_wrap=False, overflow="fold", ratio=2) - - for item in processed_items: - style = Style(color="blue", underline=True) if item['is_folder'] else None - table.add_row( - item['type'], - item['owner'], - item['size'], - item['updated'], - item['name'], - item['storage_path'], - style=style - ) - - console.print(table) - - else: - console = Console() - for item in processed_items: - if item['is_folder']: - console.print(f"[blue underline]{item['name']}[/]") - else: - console.print(item['name']) - - except Exception as e: - raise ValueError(f"Failed to list files for project '{project_name}'. {str(e)}") - - -@datasets.command(name="mv") -@click.argument("source_path", required=True) -@click.argument("destination_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The source project name.') -@click.option('--destination-project-name', required=False, - help='The destination project name. Defaults to the source project.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def move_files(ctx, source_path, destination_path, apikey, cloudos_url, workspace_id, - project_name, destination_project_name, - disable_ssl_verification, ssl_cert, profile): - """ - Move a file or folder from a source path to a destination path within or across CloudOS projects. - - SOURCE_PATH [path]: the full path to the file or folder to move. It must be a 'Data' folder path. - E.g.: 'Data/folderA/file.txt'\n - DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. - E.g.: 'Data/folderB' - """ - # Validate destination constraint - if not destination_path.strip("/").startswith("Data/") and destination_path.strip("/") != "Data": - raise ValueError("Destination path must begin with 'Data/' or be 'Data'.") - if not source_path.strip("/").startswith("Data/") and source_path.strip("/") != "Data": - raise ValueError("SOURCE_PATH must start with 'Data/' or be 'Data'.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - destination_project_name = destination_project_name or project_name - # Initialize Datasets clients - source_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - dest_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=destination_project_name, - verify=verify_ssl, - cromwell_token=None - ) - print('Checking source path') - # === Resolve Source Item === - source_parts = source_path.strip("/").split("/") - source_parent_path = "/".join(source_parts[:-1]) if len(source_parts) > 1 else None - source_item_name = source_parts[-1] - - try: - source_contents = source_client.list_folder_content(source_parent_path) - except Exception as e: - raise ValueError(f"Could not resolve source path '{source_path}'. {str(e)}") - - found_source = None - for collection in ["files", "folders"]: - for item in source_contents.get(collection, []): - if item.get("name") == source_item_name: - found_source = item - break - if found_source: - break - if not found_source: - raise ValueError(f"Item '{source_item_name}' not found in '{source_parent_path or '[project root]'}'") - - source_id = found_source["_id"] - source_kind = "Folder" if "folderType" in found_source else "File" - print("Checking destination path") - # === Resolve Destination Folder === - dest_parts = destination_path.strip("/").split("/") - dest_folder_name = dest_parts[-1] - dest_parent_path = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else None - - try: - dest_contents = dest_client.list_folder_content(dest_parent_path) - match = next((f for f in dest_contents.get("folders", []) if f.get("name") == dest_folder_name), None) - if not match: - raise ValueError(f"Could not resolve destination folder '{destination_path}'") - - target_id = match["_id"] - folder_type = match.get("folderType") - # Normalize kind: top-level datasets are kind=Dataset, all other folders are kind=Folder - if folder_type in ("VirtualFolder", "Folder"): - target_kind = "Folder" - elif folder_type == "S3Folder": - raise ValueError(f"Unable to move item '{source_item_name}' to '{destination_path}'. " + - "The destination is an S3 folder, and only virtual folders can be selected as valid move destinations.") - elif isinstance(folder_type, bool) and folder_type: # legacy dataset structure - target_kind = "Dataset" - else: - raise ValueError(f"Unrecognized folderType '{folder_type}' for destination '{destination_path}'") - - except Exception as e: - raise ValueError(f"Could not resolve destination path '{destination_path}'. {str(e)}") - print(f"Moving {source_kind} '{source_item_name}' to '{destination_path}' " + - f"in project '{destination_project_name} ...") - # === Perform Move === - try: - response = source_client.move_files_and_folders( - source_id=source_id, - source_kind=source_kind, - target_id=target_id, - target_kind=target_kind - ) - if response.ok: - click.secho(f"{source_kind} '{source_item_name}' moved to '{destination_path}' " + - f"in project '{destination_project_name}'.", fg="green", bold=True) - else: - raise ValueError(f"Move failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Move operation failed. {str(e)}") - - -@datasets.command(name="rename") -@click.argument("source_path", required=True) -@click.argument("new_name", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def renaming_item(ctx, - source_path, - new_name, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Rename a file or folder in a CloudOS project. - - SOURCE_PATH [path]: the full path to the file or folder to rename. It must be a 'Data' folder path. - E.g.: 'Data/folderA/old_name.txt'\n - NEW_NAME [name]: the new name to assign to the file or folder. E.g.: 'new_name.txt' - """ - if not source_path.strip("/").startswith("Data/"): - raise ValueError("SOURCE_PATH must start with 'Data/', pointing to a file or folder in that dataset.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - parts = source_path.strip("/").split("/") - - parent_path = "/".join(parts[:-1]) - target_name = parts[-1] - - try: - contents = client.list_folder_content(parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") - - # Search for file/folder - found_item = None - for category in ["files", "folders"]: - for item in contents.get(category, []): - if item.get("name") == target_name: - found_item = item - break - if found_item: - break - - if not found_item: - raise ValueError(f"Item '{target_name}' not found in '{parent_path or '[project root]'}'") - - item_id = found_item["_id"] - kind = "Folder" if "folderType" in found_item else "File" - - print(f"Renaming {kind} '{target_name}' to '{new_name}'...") - try: - response = client.rename_item(item_id=item_id, new_name=new_name, kind=kind) - if response.ok: - click.secho( - f"{kind} '{target_name}' renamed to '{new_name}' in folder '{parent_path}'.", - fg="green", - bold=True - ) - else: - raise ValueError(f"Rename failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Rename operation failed. {str(e)}") - - -@datasets.command(name="cp") -@click.argument("source_path", required=True) -@click.argument("destination_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The source project name.') -@click.option('--destination-project-name', required=False, help='The destination project name. Defaults to the source project.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def copy_item_cli(ctx, - source_path, - destination_path, - apikey, - cloudos_url, - workspace_id, - project_name, - destination_project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Copy a file or folder (S3 or virtual) from SOURCE_PATH to DESTINATION_PATH. - - SOURCE_PATH [path]: the full path to the file or folder to copy. - E.g.: AnalysesResults/my_analysis/results/my_plot.png\n - DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. - E.g.: Data/plots - """ - destination_project_name = destination_project_name or project_name - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - # Initialize clients - source_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - dest_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=destination_project_name, - verify=verify_ssl, - cromwell_token=None - ) - # Validate paths - dest_parts = destination_path.strip("/").split("/") - if not dest_parts or dest_parts[0] != "Data": - raise ValueError("DESTINATION_PATH must start with 'Data/'.") - # Parse source and destination - source_parts = source_path.strip("/").split("/") - source_parent = "/".join(source_parts[:-1]) if len(source_parts) > 1 else "" - source_name = source_parts[-1] - dest_folder_name = dest_parts[-1] - dest_parent = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else "" - try: - source_content = source_client.list_folder_content(source_parent) - dest_content = dest_client.list_folder_content(dest_parent) - except Exception as e: - raise ValueError(f"Could not access paths. {str(e)}") - # Find the source item - source_item = None - for item in source_content.get('files', []) + source_content.get('folders', []): - if item.get("name") == source_name: - source_item = item - break - if not source_item: - raise ValueError(f"Item '{source_name}' not found in '{source_parent or '[project root]'}'") - # Find the destination folder - destination_folder = None - for folder in dest_content.get("folders", []): - if folder.get("name") == dest_folder_name: - destination_folder = folder - break - if not destination_folder: - raise ValueError(f"Destination folder '{destination_path}' not found.") - try: - # Determine item type - if "fileType" in source_item: - item_type = "file" - elif source_item.get("folderType") == "VirtualFolder": - item_type = "virtual_folder" - elif "s3BucketName" in source_item and source_item.get("folderType") == "S3Folder": - item_type = "s3_folder" - else: - raise ValueError("Could not determine item type.") - print(f"Copying {item_type.replace('_', ' ')} '{source_name}' to '{destination_path}'...") - if destination_folder.get("folderType") is True and destination_folder.get("kind") in ("Data", "Cohorts", "AnalysesResults"): - destination_kind = "Dataset" - elif destination_folder.get("folderType") == "S3Folder": - raise ValueError(f"Unable to copy item '{source_name}' to '{destination_path}'. The destination is an S3 folder, and only virtual folders can be selected as valid copy destinations.") - else: - destination_kind = "Folder" - response = source_client.copy_item( - item=source_item, - destination_id=destination_folder["_id"], - destination_kind=destination_kind - ) - if response.ok: - click.secho("Item copied successfully.", fg="green", bold=True) - else: - raise ValueError(f"Copy failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Copy operation failed. {str(e)}") - - -@datasets.command(name="mkdir") -@click.argument("new_folder_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def mkdir_item(ctx, - new_folder_path, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Create a virtual folder in a CloudOS project. - - NEW_FOLDER_PATH [path]: Full path to the new folder including its name. Must start with 'Data'. - """ - new_folder_path = new_folder_path.strip("/") - if not new_folder_path.startswith("Data"): - raise ValueError("NEW_FOLDER_PATH must start with 'Data'.") - - path_parts = new_folder_path.split("/") - if len(path_parts) < 2: - raise ValueError("NEW_FOLDER_PATH must include at least a parent folder and the new folder name.") - - parent_path = "/".join(path_parts[:-1]) - folder_name = path_parts[-1] - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - # Split parent path to get its parent + name - parent_parts = parent_path.split("/") - parent_name = parent_parts[-1] - parent_of_parent_path = "/".join(parent_parts[:-1]) - - # List the parent of the parent - try: - contents = client.list_folder_content(parent_of_parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_of_parent_path}'. {str(e)}") - - # Find the parent folder in the contents - folder_info = next( - (f for f in contents.get("folders", []) if f.get("name") == parent_name), - None - ) - - if not folder_info: - raise ValueError(f"Could not find folder '{parent_name}' in '{parent_of_parent_path}'.") - - parent_id = folder_info.get("_id") - folder_type = folder_info.get("folderType") - - if folder_type is True: - parent_kind = "Dataset" - elif isinstance(folder_type, str): - parent_kind = "Folder" - else: - raise ValueError(f"Unrecognized folderType for '{parent_path}'.") - - # Create the folder - print(f"Creating folder '{folder_name}' under '{parent_path}' ({parent_kind})...") - try: - response = client.create_virtual_folder(name=folder_name, parent_id=parent_id, parent_kind=parent_kind) - if response.ok: - click.secho(f"Folder '{folder_name}' created under '{parent_path}'", fg="green", bold=True) - else: - raise ValueError(f"Folder creation failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Folder creation failed. {str(e)}") - - -@datasets.command(name="rm") -@click.argument("target_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.option('--force', is_flag=True, help='Force delete files. Required when deleting user uploaded files. This may also delete the file from the cloud provider storage.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def rm_item(ctx, - target_path, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile, - force): - """ - Delete a file or folder in a CloudOS project. - - TARGET_PATH [path]: the full path to the file or folder to delete. Must start with 'Data'. \n - E.g.: 'Data/folderA/file.txt' or 'Data/my_analysis/results/folderB' - """ - if not target_path.strip("/").startswith("Data/"): - raise ValueError("TARGET_PATH must start with 'Data/', pointing to a file or folder.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - parts = target_path.strip("/").split("/") - parent_path = "/".join(parts[:-1]) - item_name = parts[-1] - - try: - contents = client.list_folder_content(parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") - - found_item = None - for item in contents.get('files', []) + contents.get('folders', []): - if item.get("name") == item_name: - found_item = item - break - - if not found_item: - raise ValueError(f"Item '{item_name}' not found in '{parent_path or '[project root]'}'") - - item_id = found_item.get("_id", '') - kind = "Folder" if "folderType" in found_item else "File" - if item_id == '': - raise ValueError(f"Item '{item_name}' could not be removed as the parent folder is an s3 folder and their content cannot be modified.") - # Check if the item is managed by Lifebit - is_managed_by_lifebit = found_item.get("isManagedByLifebit", False) - if is_managed_by_lifebit and not force: - raise ValueError("By removing this file, it will be permanently deleted. If you want to go forward, please use the --force flag.") - print(f"Removing {kind} '{item_name}' from '{parent_path or '[root]'}'...") - try: - response = client.delete_item(item_id=item_id, kind=kind) - if response.ok: - if is_managed_by_lifebit: - click.secho( - f"{kind} '{item_name}' was permanently deleted from '{parent_path or '[root]'}'.", - fg="green", bold=True - ) - else: - click.secho( - f"{kind} '{item_name}' was removed from '{parent_path or '[root]'}'.", - fg="green", bold=True - ) - click.secho("This item will still be available on your Cloud Provider.", fg="yellow") - else: - raise ValueError(f"Removal failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Remove operation failed. {str(e)}") - - -@datasets.command(name="link") -@click.argument("path", required=True) -@click.option('-k', '--apikey', help='Your CloudOS API key', required=True) -@click.option('-c', '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=False) -@click.option('--workspace-id', help='The specific CloudOS workspace id.', required=True) -@click.option('--session-id', help='The specific CloudOS interactive session id.', required=True) -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default='default') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) -def link(ctx, - path, - apikey, - cloudos_url, - project_name, - workspace_id, - session_id, - disable_ssl_verification, - ssl_cert, - profile): - """ - Link a folder (S3 or File Explorer) to an active interactive analysis. - - PATH [path]: the full path to the S3 folder to link or relative to File Explorer. - E.g.: 's3://bucket-name/folder/subfolder', 'Data/Downloads' or 'Data'. - """ - if not path.startswith("s3://") and project_name is None: - # for non-s3 paths we need the project, for S3 we don't - raise click.UsageError("When using File Explorer paths '--project-name' needs to be defined") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - link_p = Link( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - cromwell_token=None, - project_name=project_name, - verify=verify_ssl - ) - - # Minimal folder validation and improved error messages - is_s3 = path.startswith("s3://") - is_folder = True - if is_s3: - # S3 path validation - use heuristics to determine if it's likely a folder - try: - # If path ends with '/', it's likely a folder - if path.endswith('/'): - is_folder = True - else: - # Check the last part of the path - path_parts = path.rstrip("/").split("/") - if path_parts: - last_part = path_parts[-1] - # If the last part has no dot, it's likely a folder - if '.' not in last_part: - is_folder = True - else: - # If it has a dot, it might be a file - set to None for warning - is_folder = None - else: - # Empty path parts, set to None for uncertainty - is_folder = None - except Exception: - # If we can't parse the S3 path, set to None for uncertainty - is_folder = None - else: - # File Explorer path validation (existing logic) - try: - datasets = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - parts = path.strip("/").split("/") - parent_path = "/".join(parts[:-1]) if len(parts) > 1 else "" - item_name = parts[-1] - contents = datasets.list_folder_content(parent_path) - found = None - for item in contents.get("folders", []): - if item.get("name") == item_name: - found = item - break - if not found: - for item in contents.get("files", []): - if item.get("name") == item_name: - found = item - break - if found and ("folderType" not in found): - is_folder = False - except Exception: - is_folder = None - - if is_folder is False: - if is_s3: - raise ValueError("The S3 path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.") - else: - raise ValueError("Linking files or virtual folders is not supported. Link the S3 parent folder instead.", err=True) - return - elif is_folder is None and is_s3: - click.secho("Unable to verify whether the S3 path is a folder. Proceeding with linking; " + - "however, if the operation fails, please confirm that you are linking a folder rather than a file.", fg='yellow', bold=True) - - try: - link_p.link_folder(path, session_id) - except Exception as e: - if is_s3: - print("If you are linking an S3 path, please ensure it is a folder.") - raise ValueError(f"Could not link folder. {e}") - - -@images.command(name="ls") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--page', help='The response page. Defaults to 1.', required=False, default=1) -@click.option('--limit', help='The page size limit. Defaults to 10', required=False, default=10) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def list_images(ctx, - apikey, - cloudos_url, - procurement_id, - disable_ssl_verification, - ssl_cert, - profile, - page, - limit): - """List images associated with organisations of a given procurement.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None, - page=page, - limit=limit - ) - - try: - result = procurement_images.list_procurement_images() - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - - -@images.command(name="set") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) -@click.option('--image-type', help='The CloudOS resource image type.', required=True, - type=click.Choice([ - 'RegularInteractiveSessions', - 'SparkInteractiveSessions', - 'RStudioInteractiveSessions', - 'JupyterInteractiveSessions', - 'JobDefault', - 'NextflowBatchComputeEnvironment'])) -@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') -@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) -@click.option('--image-id', help='The new image id value.', required=True) -@click.option('--image-name', help='The new image name value.', required=False) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def set_organisation_image(ctx, - apikey, - cloudos_url, - procurement_id, - organisation_id, - image_type, - provider, - region, - image_id, - image_name, - disable_ssl_verification, - ssl_cert, - profile): - """Set a new image id or name to image associated with an organisations of a given procurement.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = procurement_images.set_procurement_organisation_image( - organisation_id, - image_type, - provider, - region, - image_id, - image_name - ) - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - - -@images.command(name="reset") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) -@click.option('--image-type', help='The CloudOS resource image type.', required=True, - type=click.Choice([ - 'RegularInteractiveSessions', - 'SparkInteractiveSessions', - 'RStudioInteractiveSessions', - 'JupyterInteractiveSessions', - 'JobDefault', - 'NextflowBatchComputeEnvironment'])) -@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') -@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def reset_organisation_image(ctx, - apikey, - cloudos_url, - procurement_id, - organisation_id, - image_type, - provider, - region, - disable_ssl_verification, - ssl_cert, - profile): - """Reset image associated with an organisations of a given procurement to CloudOS defaults.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = procurement_images.reset_procurement_organisation_image( - organisation_id, - image_type, - provider, - region - ) - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - -@run_cloudos_cli.command('link') -@click.argument('path', required=False) -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS. When provided, links results, workdir and logs by default.', - required=False) -@click.option('--project-name', - help='The name of a CloudOS project. Required for File Explorer paths.', - required=False) -@click.option('--results', - help='Link only results folder (only works with --job-id).', - is_flag=True) -@click.option('--workdir', - help='Link only working directory (only works with --job-id).', - is_flag=True) -@click.option('--logs', - help='Link only logs folder (only works with --job-id).', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) -def link_command(ctx, - path, - apikey, - cloudos_url, - workspace_id, - session_id, - job_id, - project_name, - results, - workdir, - logs, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """ - Link folders to an interactive analysis session. - - This command is used to link folders - to an active interactive analysis session for direct access to data. - - PATH: Optional path to link (S3). - Required if --job-id is not provided. - - Two modes of operation: - - 1. Job-based linking (--job-id): Links job-related folders. - By default, links results, workdir, and logs folders. - Use --results, --workdir, or --logs flags to link only specific folders. - - 2. Direct path linking (PATH argument): Links a specific S3 path. - - Examples: - - # Link all job folders (results, workdir, logs) - cloudos link --job-id 12345 --session-id abc123 - - # Link only results from a job - cloudos link --job-id 12345 --session-id abc123 --results - - # Link a specific S3 path - cloudos link s3://bucket/folder --session-id abc123 - - """ - print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Validate input parameters - if not job_id and not path: - raise click.UsageError("Either --job-id or PATH argument must be provided.") - - if job_id and path: - raise click.UsageError("Cannot use both --job-id and PATH argument. Please provide only one.") - - # Validate folder-specific flags only work with --job-id - if (results or workdir or logs) and not job_id: - raise click.UsageError("--results, --workdir, and --logs flags can only be used with --job-id.") - - # If no specific folders are selected with job-id, link all by default - if job_id and not (results or workdir or logs): - results = True - workdir = True - logs = True - - if verbose: - print('Using the following parameters:') - print(f'\tCloudOS url: {cloudos_url}') - print(f'\tWorkspace ID: {workspace_id}') - print(f'\tSession ID: {session_id}') - if job_id: - print(f'\tJob ID: {job_id}') - print(f'\tLink results: {results}') - print(f'\tLink workdir: {workdir}') - print(f'\tLink logs: {logs}') - else: - print(f'\tPath: {path}') - - # Initialize Link client - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl - ) - - try: - if job_id: - # Job-based linking - print(f'Linking folders from job {job_id} to interactive session {session_id}...\n') - - # Link results - if results: - link_client.link_job_results(job_id, workspace_id, session_id, verify_ssl, verbose) - - # Link workdir - if workdir: - link_client.link_job_workdir(job_id, workspace_id, session_id, verify_ssl, verbose) - - # Link logs - if logs: - link_client.link_job_logs(job_id, workspace_id, session_id, verify_ssl, verbose) - - - else: - # Direct path linking - print(f'Linking path to interactive session {session_id}...\n') - - # Link path with validation - link_client.link_path_with_validation(path, session_id, verify_ssl, project_name, verbose) - - print('\nLinking operation completed.') - - except BadRequestException as e: - raise ValueError(f"Request failed: {str(e)}") - except Exception as e: - raise ValueError(f"Failed to link folder(s): {str(e)}") - -if __name__ == "__main__": - # Setup logging - debug_mode = '--debug' in sys.argv - setup_logging(debug_mode) - logger = logging.getLogger("CloudOS") - # Check if debug flag was passed (fallback for cases where Click doesn't handle it) - try: - run_cloudos_cli() - except Exception as e: - if debug_mode: - logger.error(e, exc_info=True) - traceback.print_exc() - else: - logger.error(e) - click.echo(click.style(f"Error: {e}", fg='red'), err=True) - sys.exit(1) \ No newline at end of file diff --git a/cloudos_cli/__main__.py.before_refactor b/cloudos_cli/__main__.py.before_refactor deleted file mode 100644 index c9804d3e..00000000 --- a/cloudos_cli/__main__.py.before_refactor +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 - -import rich_click as click -import sys -import logging -from ._version import __version__ -from rich.console import Console -from cloudos_cli.logging.logger import setup_logging, update_command_context_from_click -from cloudos_cli.configure.configure import ( - build_default_map_for_group, - get_shared_config -) - -# Import all command groups from their cli modules -from cloudos_cli.jobs.cli import job -from cloudos_cli.workflows.cli import workflow -from cloudos_cli.projects.cli import project -from cloudos_cli.cromwell.cli import cromwell -from cloudos_cli.queue.cli import queue -from cloudos_cli.bash.cli import bash -from cloudos_cli.procurement.cli import procurement -from cloudos_cli.datasets.cli import datasets -from cloudos_cli.configure.cli import configure - - -# GLOBAL VARS - Keep these for backward compatibility -JOB_COMPLETED = 'completed' -REQUEST_INTERVAL_CROMWELL = 30 -AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] -AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] -HPC_NEXTFLOW_VERSIONS = ['22.10.8'] -AWS_NEXTFLOW_LATEST = '24.04.4' -AZURE_NEXTFLOW_LATEST = '22.11.1-edge' -HPC_NEXTFLOW_LATEST = '22.10.8' -ABORT_JOB_STATES = ['running', 'initializing'] - -# Global debug state -_global_debug = False - - -def custom_exception_handler(exc_type, exc_value, exc_traceback): - """Custom exception handler that respects debug mode""" - console = Console(stderr=True) - # Initialise logger - debug_mode = '--debug' in sys.argv - setup_logging(debug_mode) - logger = logging.getLogger("CloudOS") - if get_debug_mode(): - logger.error(exc_value, exc_info=exc_value) - console.print("[yellow]Debug mode: showing full traceback[/yellow]") - sys.__excepthook__(exc_type, exc_value, exc_traceback) - else: - # Extract a clean error message - if hasattr(exc_value, 'message'): - error_msg = exc_value.message - elif str(exc_value): - error_msg = str(exc_value) - else: - error_msg = f"{exc_type.__name__}" - logger.error(exc_value) - console.print(f"[bold red]Error: {error_msg}[/bold red]") - - # For network errors, give helpful context - if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): - console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") - -# Install the custom exception handler -sys.excepthook = custom_exception_handler - - -def pass_debug_to_subcommands(group_cls=click.RichGroup): - """Custom Group class that passes --debug option to all subcommands""" - - class DebugGroup(group_cls): - def add_command(self, cmd, name=None): - # Add debug option to the command if it doesn't already have it - if isinstance(cmd, (click.Command, click.Group)): - has_debug = any(param.name == 'debug' for param in cmd.params) - if not has_debug: - debug_option = click.Option( - ['--debug'], - is_flag=True, - help='Show detailed error information and tracebacks', - is_eager=True, - expose_value=False, - callback=self._debug_callback - ) - cmd.params.insert(-1, debug_option) # Insert at the end for precedence - - super().add_command(cmd, name) - - def _debug_callback(self, ctx, param, value): - """Callback to handle debug flag""" - global _global_debug - if value: - _global_debug = True - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - return DebugGroup - - -def get_debug_mode(): - """Get current debug mode state""" - return _global_debug - - -# Helper function for debug setup -def _setup_debug(ctx, param, value): - """Setup debug mode globally and in context""" - global _global_debug - _global_debug = value - if value: - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - -@click.group(cls=pass_debug_to_subcommands()) -@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', - is_eager=True, expose_value=False, callback=_setup_debug) -@click.version_option(__version__) -@click.pass_context -def run_cloudos_cli(ctx): - """CloudOS python package: a package for interacting with CloudOS.""" - update_command_context_from_click(ctx) - ctx.ensure_object(dict) - - if ctx.invoked_subcommand not in ['datasets']: - print(run_cloudos_cli.__doc__ + '\n') - print('Version: ' + __version__ + '\n') - - # Load shared configuration (handles missing profiles and fields gracefully) - shared_config = get_shared_config() - - # Automatically build default_map from registered commands - ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) - - -# Register all command groups -run_cloudos_cli.add_command(job) -run_cloudos_cli.add_command(workflow) -run_cloudos_cli.add_command(project) -run_cloudos_cli.add_command(cromwell) -run_cloudos_cli.add_command(queue) -run_cloudos_cli.add_command(bash) -run_cloudos_cli.add_command(procurement) -run_cloudos_cli.add_command(datasets) -run_cloudos_cli.add_command(configure) - - -if __name__ == '__main__': - run_cloudos_cli() diff --git a/cloudos_cli/__main__.py.old b/cloudos_cli/__main__.py.old deleted file mode 100644 index 1eab8bfd..00000000 --- a/cloudos_cli/__main__.py.old +++ /dev/null @@ -1,4367 +0,0 @@ -#!/usr/bin/env python3 - -import rich_click as click -import cloudos_cli.jobs.job as jb -from cloudos_cli.clos import Cloudos -from cloudos_cli.import_wf.import_wf import ImportWorflow -from cloudos_cli.queue.queue import Queue -from cloudos_cli.utils.errors import BadRequestException -import json -import time -import sys -import traceback -import copy -from ._version import __version__ -from cloudos_cli.configure.configure import ConfigurationProfile -from rich.console import Console -from rich.table import Table -from cloudos_cli.datasets import Datasets -from cloudos_cli.procurement import Images -from cloudos_cli.utils.resources import ssl_selector, format_bytes -from rich.style import Style -from cloudos_cli.utils.array_job import generate_datasets_for_project -from cloudos_cli.utils.details import create_job_details, create_job_list_table -from cloudos_cli.link import Link -from cloudos_cli.cost.cost import CostViewer -from cloudos_cli.logging.logger import setup_logging, update_command_context_from_click -import logging -from cloudos_cli.configure.configure import ( - with_profile_config, - build_default_map_for_group, - get_shared_config, - CLOUDOS_URL -) -from cloudos_cli.related_analyses.related_analyses import related_analyses - - -# GLOBAL VARS -JOB_COMPLETED = 'completed' -REQUEST_INTERVAL_CROMWELL = 30 -AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] -AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] -HPC_NEXTFLOW_VERSIONS = ['22.10.8'] -AWS_NEXTFLOW_LATEST = '24.04.4' -AZURE_NEXTFLOW_LATEST = '22.11.1-edge' -HPC_NEXTFLOW_LATEST = '22.10.8' -ABORT_JOB_STATES = ['running', 'initializing'] - - -def custom_exception_handler(exc_type, exc_value, exc_traceback): - """Custom exception handler that respects debug mode""" - console = Console(stderr=True) - # Initialise logger - debug_mode = '--debug' in sys.argv - setup_logging(debug_mode) - logger = logging.getLogger("CloudOS") - if get_debug_mode(): - logger.error(exc_value, exc_info=exc_value) - console.print("[yellow]Debug mode: showing full traceback[/yellow]") - sys.__excepthook__(exc_type, exc_value, exc_traceback) - else: - # Extract a clean error message - if hasattr(exc_value, 'message'): - error_msg = exc_value.message - elif str(exc_value): - error_msg = str(exc_value) - else: - error_msg = f"{exc_type.__name__}" - logger.error(exc_value) - console.print(f"[bold red]Error: {error_msg}[/bold red]") - - # For network errors, give helpful context - if 'HTTPSConnectionPool' in str(exc_value) or 'Max retries exceeded' in str(exc_value): - console.print("[yellow]Tip: This appears to be a network connectivity issue. Please check your internet connection and try again.[/yellow]") - -# Install the custom exception handler -sys.excepthook = custom_exception_handler - - -def pass_debug_to_subcommands(group_cls=click.RichGroup): - """Custom Group class that passes --debug option to all subcommands""" - - class DebugGroup(group_cls): - def add_command(self, cmd, name=None): - # Add debug option to the command if it doesn't already have it - if isinstance(cmd, (click.Command, click.Group)): - has_debug = any(param.name == 'debug' for param in cmd.params) - if not has_debug: - debug_option = click.Option( - ['--debug'], - is_flag=True, - help='Show detailed error information and tracebacks', - is_eager=True, - expose_value=False, - callback=self._debug_callback - ) - cmd.params.insert(-1, debug_option) # Insert at the end for precedence - - super().add_command(cmd, name) - - def _debug_callback(self, ctx, param, value): - """Callback to handle debug flag""" - global _global_debug - if value: - _global_debug = True - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - return DebugGroup - - -def get_debug_mode(): - """Get current debug mode state""" - return _global_debug - - -# Helper function for debug setup -def _setup_debug(ctx, param, value): - """Setup debug mode globally and in context""" - global _global_debug - _global_debug = value - if value: - ctx.meta['debug'] = True - else: - ctx.meta['debug'] = False - return value - - -@click.group(cls=pass_debug_to_subcommands()) -@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', - is_eager=True, expose_value=False, callback=_setup_debug) -@click.version_option(__version__) -@click.pass_context -def run_cloudos_cli(ctx): - """CloudOS python package: a package for interacting with CloudOS.""" - update_command_context_from_click(ctx) - ctx.ensure_object(dict) - - if ctx.invoked_subcommand not in ['datasets']: - print(run_cloudos_cli.__doc__ + '\n') - print('Version: ' + __version__ + '\n') - - # Load shared configuration (handles missing profiles and fields gracefully) - shared_config = get_shared_config() - - # Automatically build default_map from registered commands - ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def job(): - """CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS.""" - print(job.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def workflow(): - """CloudOS workflow functionality: list and import workflows.""" - print(workflow.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def project(): - """CloudOS project functionality: list and create projects in CloudOS.""" - print(project.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def cromwell(): - """Cromwell server functionality: check status, start and stop.""" - print(cromwell.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def queue(): - """CloudOS job queue functionality.""" - print(queue.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def bash(): - """CloudOS bash functionality.""" - print(bash.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -def procurement(): - """CloudOS procurement functionality.""" - print(procurement.__doc__ + '\n') - - -@procurement.group(cls=pass_debug_to_subcommands()) -def images(): - """CloudOS procurement images functionality.""" - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands()) -@click.pass_context -def datasets(ctx): - """CloudOS datasets functionality.""" - update_command_context_from_click(ctx) - if ctx.args and ctx.args[0] != 'ls': - print(datasets.__doc__ + '\n') - - -@run_cloudos_cli.group(cls=pass_debug_to_subcommands(), invoke_without_command=True) -@click.option('--profile', help='Profile to use from the config file', default='default') -@click.option('--make-default', - is_flag=True, - help='Make the profile the default one.') -@click.pass_context -def configure(ctx, profile, make_default): - """CloudOS configuration.""" - print(configure.__doc__ + '\n') - update_command_context_from_click(ctx) - profile = profile or ctx.obj['profile'] - config_manager = ConfigurationProfile() - - if ctx.invoked_subcommand is None and profile == "default" and not make_default: - config_manager.create_profile_from_input(profile_name="default") - - if profile != "default" and not make_default: - config_manager.create_profile_from_input(profile_name=profile) - if make_default: - config_manager.make_default_profile(profile_name=profile) - - -@job.command('run', cls=click.RichCommand) -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('--job-config', - help=('A config file similar to a nextflow.config file, ' + - 'but only with the parameters to use with your job.')) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p input=s3://path_to_my_file. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--nextflow-profile', - help=('A comma separated string indicating the nextflow profile/s ' + - 'to use with your job.')) -@click.option('--nextflow-version', - help=('Nextflow version to use when executing the workflow in CloudOS. ' + - 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']), - default='22.10.8') -@click.option('--git-commit', - help=('The git commit hash to run for ' + - 'the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--git-tag', - help=('The tag to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--git-branch', - help=('The branch to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--resumable', - help='Whether to make the job able to be resumed or not.', - is_flag=True) -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--wdl-mainfile', - help='For WDL workflows, which mainFile (.wdl) is configured to use.',) -@click.option('--wdl-importsfile', - help='For WDL workflows, which importsFile (.zip) is configured to use.',) -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. Currently, not necessary ' + - 'as apikey can be used instead, but maintained for backwards compatibility.')) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - type=click.Choice(['aws', 'azure', 'hpc']), - default='aws') -@click.option('--hpc-id', - help=('ID of your HPC, only applicable when --execution-platform=hpc. ' + - 'Default=660fae20f93358ad61e0104b'), - default='660fae20f93358ad61e0104b') -@click.option('--azure-worker-instance-type', - help=('The worker node instance type to be used in azure. ' + - 'Default=Standard_D4as_v4'), - default='Standard_D4as_v4') -@click.option('--azure-worker-instance-disk', - help='The disk size in GB for the worker node to be used in azure. Default=100', - type=int, - default=100) -@click.option('--azure-worker-instance-spot', - help='Whether the azure worker nodes have to be spot instances or not.', - is_flag=True) -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-file-staging', - help='Enables AWS S3 mountpoint for quicker file staging.', - is_flag=True) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--use-private-docker-repository', - help=('Allows to use private docker repository for running jobs. The Docker user ' + - 'account has to be already linked to CloudOS.'), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run(ctx, - apikey, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - job_config, - parameter, - git_commit, - git_tag, - git_branch, - job_name, - resumable, - do_not_save_logs, - job_queue, - nextflow_profile, - nextflow_version, - instance_type, - instance_disk, - storage_mode, - lustre_size, - wait_completion, - wait_time, - wdl_mainfile, - wdl_importsfile, - cromwell_token, - repository_platform, - execution_platform, - hpc_id, - azure_worker_instance_type, - azure_worker_instance_disk, - azure_worker_instance_spot, - cost_limit, - accelerate_file_staging, - accelerate_saving_results, - use_private_docker_repository, - verbose, - request_interval, - disable_ssl_verification, - ssl_cert, - profile): - """Submit a job to CloudOS.""" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if do_not_save_logs: - save_logs = False - else: - save_logs = True - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - if execution_platform == 'azure' or execution_platform == 'hpc': - batch = False - else: - batch = True - if execution_platform == 'hpc': - print('\nHPC execution platform selected') - if hpc_id is None: - raise ValueError('Please, specify your HPC ID using --hpc parameter') - print('Please, take into account that HPC execution do not support ' + - 'the following parameters and all of them will be ignored:\n' + - '\t--job-queue\n' + - '\t--resumable | --do-not-save-logs\n' + - '\t--instance-type | --instance-disk | --cost-limit\n' + - '\t--storage-mode | --lustre-size\n' + - '\t--wdl-mainfile | --wdl-importsfile | --cromwell-token\n') - wdl_mainfile = None - wdl_importsfile = None - storage_mode = 'regular' - save_logs = False - if accelerate_file_staging: - if execution_platform != 'aws': - print('You have selected accelerate file staging, but this function is ' + - 'only available when execution platform is AWS. The accelerate file staging ' + - 'will not be applied') - use_mountpoints = False - else: - use_mountpoints = True - print('Enabling AWS S3 mountpoint for accelerated file staging. ' + - 'Please, take into consideration the following:\n' + - '\t- It significantly reduces runtime and compute costs but may increase network costs.\n' + - '\t- Requires extra memory. Adjust process memory or optimise resource usage if necessary.\n' + - '\t- This is still a CloudOS BETA feature.\n') - else: - use_mountpoints = False - if verbose: - print('\t...Detecting workflow type') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - workflow_type = cl.detect_workflow(workflow_name, workspace_id, verify_ssl, last) - is_module = cl.is_module(workflow_name, workspace_id, verify_ssl, last) - if execution_platform == 'hpc' and workflow_type == 'wdl': - raise ValueError(f'The workflow {workflow_name} is a WDL workflow. ' + - 'WDL is not supported on HPC execution platform.') - if workflow_type == 'wdl': - print('WDL workflow detected') - if wdl_mainfile is None: - raise ValueError('Please, specify WDL mainFile using --wdl-mainfile .') - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h == 'Stopped': - print('\tStarting Cromwell server...\n') - cl.cromwell_switch(workspace_id, 'restart', verify_ssl) - elapsed = 0 - while elapsed < 300 and c_status_h != 'Running': - c_status_old = c_status_h - time.sleep(REQUEST_INTERVAL_CROMWELL) - elapsed += REQUEST_INTERVAL_CROMWELL - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - if c_status_h != c_status_old: - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h != 'Running': - raise Exception('Cromwell server did not restarted properly.') - cromwell_id = json.loads(c_status.content)["_id"] - click.secho('\t' + ('*' * 80) + '\n' + - '\tCromwell server is now running. Please, remember to stop it when ' + - 'your\n' + '\tjob finishes. You can use the following command:\n' + - '\tcloudos cromwell stop \\\n' + - '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + - f'\t\t--cloudos-url {cloudos_url} \\\n' + - f'\t\t--workspace-id {workspace_id}\n' + - '\t' + ('*' * 80) + '\n', fg='yellow', bold=True) - else: - cromwell_id = None - if verbose: - print('\t...Preparing objects') - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=wdl_mainfile, importsfile=wdl_importsfile, - repository_platform=repository_platform, verify=verify_ssl, last=last) - if verbose: - print('\tThe following Job object was created:') - print('\t' + str(j)) - print('\t...Sending job to CloudOS\n') - if is_module: - if job_queue is not None: - print(f'Ignoring job queue "{job_queue}" for ' + - f'Platform Workflow "{workflow_name}". Platform Workflows ' + - 'use their own predetermined queues.') - job_queue_id = None - if nextflow_version != '22.10.8': - print(f'The selected worflow \'{workflow_name}\' ' + - 'is a CloudOS module. CloudOS modules only work with ' + - 'Nextflow version 22.10.8. Switching to use 22.10.8') - nextflow_version = '22.10.8' - if execution_platform == 'azure': - print(f'The selected worflow \'{workflow_name}\' ' + - 'is a CloudOS module. For these workflows, worker nodes ' + - 'are managed internally. For this reason, the options ' + - 'azure-worker-instance-type, azure-worker-instance-disk and ' + - 'azure-worker-instance-spot are not taking effect.') - else: - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=cromwell_token, - workspace_id=workspace_id, verify=verify_ssl) - job_queue_id = queue.fetch_job_queue_id(workflow_type=workflow_type, batch=batch, - job_queue=job_queue) - if use_private_docker_repository: - if is_module: - print(f'Workflow "{workflow_name}" is a CloudOS module. ' + - 'Option --use-private-docker-repository will be ignored.') - docker_login = False - else: - me = j.get_user_info(verify=verify_ssl)['dockerRegistriesCredentials'] - if len(me) == 0: - raise Exception('User private Docker repository has been selected but your user ' + - 'credentials have not been configured yet. Please, link your ' + - 'Docker account to CloudOS before using ' + - '--use-private-docker-repository option.') - print('Use private Docker repository has been selected. A custom job ' + - 'queue to support private Docker containers and/or Lustre FSx will be created for ' + - 'your job. The selected job queue will serve as a template.') - docker_login = True - else: - docker_login = False - if nextflow_version == 'latest': - if execution_platform == 'aws': - nextflow_version = AWS_NEXTFLOW_LATEST - elif execution_platform == 'azure': - nextflow_version = AZURE_NEXTFLOW_LATEST - else: - nextflow_version = HPC_NEXTFLOW_LATEST - print('You have specified Nextflow version \'latest\' for execution platform ' + - f'\'{execution_platform}\'. The workflow will use the ' + - f'latest version available on CloudOS: {nextflow_version}.') - if execution_platform == 'aws': - if nextflow_version not in AWS_NEXTFLOW_VERSIONS: - print('For execution platform \'aws\', the workflow will use the default ' + - '\'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' - if execution_platform == 'azure': - if nextflow_version not in AZURE_NEXTFLOW_VERSIONS: - print('For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' ' + - 'version on CloudOS.') - nextflow_version = '22.11.1-edge' - if execution_platform == 'hpc': - if nextflow_version not in HPC_NEXTFLOW_VERSIONS: - print('For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' - if nextflow_version != '22.10.8' and nextflow_version != '22.11.1-edge': - click.secho(f'You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' + - 'to be written in DSL2 and does not support DSL1.', fg='yellow', bold=True) - print('\nExecuting run...') - if workflow_type == 'nextflow': - print(f'\tNextflow version: {nextflow_version}') - j_id = j.send_job(job_config=job_config, - parameter=parameter, - is_module=is_module, - git_commit=git_commit, - git_tag=git_tag, - git_branch=git_branch, - job_name=job_name, - resumable=resumable, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - nextflow_profile=nextflow_profile, - nextflow_version=nextflow_version, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=hpc_id, - workflow_type=workflow_type, - cromwell_id=cromwell_id, - azure_worker_instance_type=azure_worker_instance_type, - azure_worker_instance_disk=azure_worker_instance_disk, - azure_worker_instance_spot=azure_worker_instance_spot, - cost_limit=cost_limit, - use_mountpoints=use_mountpoints, - accelerate_saving_results=accelerate_saving_results, - docker_login=docker_login, - verify=verify_ssl) - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=verbose, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@job.command('status') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_status(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Check job status in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - print('Executing status...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - j_status = cl.get_job_status(job_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' - print(f'\tTo further check your job status you can either go to {j_url} ' + - 'or repeat the command you just used.') - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") - - -@job.command('workdir') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the working directory to an interactive session.', - is_flag=True) -@click.option('--delete', - help='Delete the results directory of a CloudOS job.', - is_flag=True) -@click.option('-y', '--yes', - help='Skip confirmation prompt when deleting results.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--status', - help='Check the deletion status of the working directory.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_workdir(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - delete, - yes, - session_id, - status, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the working directory of a specified job or check deletion status.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Handle --status flag - if status: - console = Console() - - if verbose: - console.print('[bold cyan]Checking deletion status of job working directory...[/bold cyan]') - console.print('\t[dim]...Preparing objects[/dim]') - console.print('\t[bold]Using the following parameters:[/bold]') - console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') - console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') - console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - - # Use Cloudos object to access the deletion status method - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - console.print('\t[dim]The following Cloudos object was created:[/dim]') - console.print('\t' + str(cl) + '\n') - - try: - deletion_status = cl.get_workdir_deletion_status( - job_id=job_id, - workspace_id=workspace_id, - verify=verify_ssl - ) - - # Convert API status to user-friendly terminology with color - status_config = { - "ready": ("available", "green"), - "deleting": ("deleting", "yellow"), - "scheduledForDeletion": ("scheduled for deletion", "yellow"), - "deleted": ("deleted", "red"), - "failedToDelete": ("failed to delete", "red") - } - - # Get the status of the workdir folder itself and convert it - api_status = deletion_status.get("status", "unknown") - folder_status, status_color = status_config.get(api_status, (api_status, "white")) - folder_info = deletion_status.get("items", {}) - - # Display results in a clear, styled format with human-readable sentence - console.print(f'The working directory of job [cyan]{deletion_status["job_id"]}[/cyan] is in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - - # For non-available statuses, always show update time and user info - if folder_status != "available": - if folder_info.get("updatedAt"): - console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - - # Show user information - prefer deletedBy over user field - user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) - if user_info: - user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() - user_email = user_info.get('email', '') - if user_name or user_email: - user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) - console.print(f'[blue]User:[/blue] {user_display}') - - # Display detailed information if verbose - if verbose: - console.print(f'\n[bold]Additional information:[/bold]') - console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') - console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') - console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') - - # Show folder metadata if available - if folder_info.get("createdAt"): - console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') - if folder_info.get("updatedAt"): - console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') - if folder_info.get("folderType"): - console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - - except ValueError as e: - raise click.ClickException(str(e)) - except Exception as e: - raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - - return - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Finding working directory path...') - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl) - print(f"Working directory for job {job_id}: {workdir}") - - # Link to interactive session if requested - if link: - if verbose: - print(f'\tLinking working directory to interactive session {session_id}...') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - link_client.link_folder(workdir.strip(), session_id) - - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve working directory for job '{job_id}'. {str(e)}") - - # Delete workdir directory if requested - if delete: - try: - # Ask for confirmation unless --yes flag is provided - if not yes: - confirmation_message = ( - "\n⚠️ Deleting intermediate results is permanent and cannot be undone. " - "All associated data will be permanently removed and cannot be recovered. " - "The current job, as well as any other jobs sharing the same working directory, " - "will no longer be resumable. This action will be logged in the audit trail " - "(if auditing is enabled for your organisation), and you will be recorded as " - "the user who performed the deletion. You can skip this confirmation step by " - "providing -y or --yes flag to cloudos job workdir --delete. Please confirm " - "that you want to delete intermediate results of this analysis? [y/n] " - ) - click.secho(confirmation_message, fg='black', bg='yellow') - user_input = input().strip().lower() - if user_input != 'y': - print('\nDeletion cancelled.') - return - # Proceed with deletion - job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - job.delete_job_results(job_id, "workDirectory", verify=verify_ssl) - click.secho('\nIntermediate results directories deleted successfully.', fg='green', bold=True) - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve intermediate results for job '{job_id}'. {str(e)}") - else: - if yes: - click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) - - -@job.command('logs') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the logs directories to an interactive session.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_logs(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - session_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the logs of a specified job.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Executing logs...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - logs = cl.get_job_logs(job_id, workspace_id, verify_ssl) - for name, path in logs.items(): - print(f"{name}: {path}") - - # Link to interactive session if requested - if link: - if logs: - # Extract the parent logs directory from any log file path - # All log files should be in the same logs directory - first_log_path = next(iter(logs.values())) - # Remove the filename to get the logs directory - # e.g., "s3://bucket/path/to/logs/filename.txt" -> "s3://bucket/path/to/logs" - logs_dir = '/'.join(first_log_path.split('/')[:-1]) - - if verbose: - print(f'\tLinking logs directory to interactive session {session_id}...') - print(f'\t\tLogs directory: {logs_dir}') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - link_client.link_folder(logs_dir, session_id) - else: - if verbose: - print('\tNo logs found to link.') - - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve logs for job '{job_id}'. {str(e)}") - - -@job.command('results') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--link', - help='Link the results directories to an interactive session.', - is_flag=True) -@click.option('--delete', - help='Delete the results directory of a CloudOS job.', - is_flag=True) -@click.option('-y', '--yes', - help='Skip confirmation prompt when deleting results.', - is_flag=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id. Required when using --link flag.', - required=False) -@click.option('--status', - help='Check the deletion status of the job results.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_results(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - link, - delete, - yes, - session_id, - status, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Get the path to the results of a specified job or check deletion status.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - # session_id is also resolved if provided in profile - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Handle --status flag - if status: - console = Console() - - if verbose: - console.print('[bold cyan]Checking deletion status of job results...[/bold cyan]') - console.print('\t[dim]...Preparing objects[/dim]') - console.print('\t[bold]Using the following parameters:[/bold]') - console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') - console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') - console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - - # Use Cloudos object to access the deletion status method - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - console.print('\t[dim]The following Cloudos object was created:[/dim]') - console.print('\t' + str(cl) + '\n') - - try: - deletion_status = cl.get_results_deletion_status( - job_id=job_id, - workspace_id=workspace_id, - verify=verify_ssl - ) - - # Convert API status to user-friendly terminology with color - status_config = { - "ready": ("available", "green"), - "deleting": ("deleting", "yellow"), - "scheduledForDeletion": ("scheduled for deletion", "yellow"), - "deleted": ("deleted", "red"), - "failedToDelete": ("failed to delete", "red") - } - - # Get the status of the results folder itself and convert it - api_status = deletion_status.get("status", "unknown") - folder_status, status_color = status_config.get(api_status, (api_status, "white")) - folder_info = deletion_status.get("items", {}) - - # Display results in a clear, styled format with human-readable sentence - console.print(f'The results of job [cyan]{deletion_status["job_id"]}[/cyan] are in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - - # For non-available statuses, always show update time and user info - if folder_status != "available": - if folder_info.get("updatedAt"): - console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - - # Show user information - prefer deletedBy over user field - user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) - if user_info: - user_name = f"{user_info.get('name', '')} {user_info.get('surname', '')}".strip() - user_email = user_info.get('email', '') - if user_name or user_email: - user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) - console.print(f'[blue]User:[/blue] {user_display}') - - # Display detailed information if verbose - if verbose: - console.print(f'\n[bold]Additional information:[/bold]') - console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') - console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') - console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') - - # Show folder metadata if available - if folder_info.get("createdAt"): - console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') - if folder_info.get("updatedAt"): - console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') - if folder_info.get("folderType"): - console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - - except ValueError as e: - raise click.ClickException(str(e)) - except Exception as e: - raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - - return - - # Validate link flag requirements AFTER loading profile - if link and not session_id: - raise click.ClickException("--session-id is required when using --link flag") - - print('Executing results...') - if verbose: - print('\t...Preparing objects') - print('\tUsing the following parameters:') - print(f'\t\tCloudOS url: {cloudos_url}') - print(f'\t\tWorkspace ID: {workspace_id}') - print(f'\t\tJob ID: {job_id}') - if link: - print(f'\t\tSession ID: {session_id}') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - try: - results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) - print(f"results: {results_path}") - - # Link to interactive session if requested - if link: - if verbose: - print(f'\tLinking results directory to interactive session {session_id}...') - - # Use Link class to perform the linking - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, # Not needed for linking operations - workspace_id=workspace_id, - project_name=None, # Not needed for S3 paths - verify=verify_ssl - ) - - if verbose: - print(f'\t\tLinking results ({results_path})...') - - link_client.link_folder(results_path, session_id) - - # Delete results directory if requested - if delete: - # Ask for confirmation unless --yes flag is provided - if not yes: - confirmation_message = ( - "\n⚠️ Deleting final analysis results is irreversible. " - "All data and backups will be permanently removed and cannot be recovered. " - "You can skip this confirmation step by providing '-y' or '--yes' flag to " - "'cloudos job results --delete'. " - "Please confirm that you want to delete final results of this analysis? [y/n] " - ) - click.secho(confirmation_message, fg='black', bg='yellow') - user_input = input().strip().lower() - if user_input != 'y': - print('\nDeletion cancelled.') - return - if verbose: - print(f'\nDeleting result directories from CloudOS...') - # Proceed with deletion - job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - job.delete_job_results(job_id, "analysisResults", verify=verify_ssl) - click.secho('\nResults directories deleted successfully.', fg='green', bold=True) - else: - if yes: - click.secho("\n'--yes' flag is ignored when '--delete' is not specified.", fg='yellow', bold=True) - except BadRequestException as e: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve results for job '{job_id}'. {str(e)}") - - -@job.command('details') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to search for.', - required=True) -@click.option('--output-format', - help=('The desired display for the output, either directly in standard output or saved as file. ' + - 'Default=stdout.'), - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--output-basename', - help=('Output file base name to save jobs details. ' + - 'Default={job_id}_details'), - required=False) -@click.option('--parameters', - help=('Whether to generate a ".config" file that can be used as input for --job-config parameter. ' + - 'It will have the same basename as defined in "--output-basename". '), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_details(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - output_basename, - parameters, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve job details in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if ctx.get_parameter_source('output_basename') == click.core.ParameterSource.DEFAULT: - output_basename = f"{job_id}_details" - - print('Executing details...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tSearching for job id: {job_id}') - - # check if the API gives a 403 error/forbidden error - try: - j_details = cl.get_job_status(job_id, workspace_id, verify_ssl) - except BadRequestException as e: - if '403' in str(e) or 'Forbidden' in str(e): - raise ValueError("API can only show job details of your own jobs, cannot see other user's job details.") - else: - raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to retrieve details for job '{job_id}'. {str(e)}") - create_job_details(json.loads(j_details.content), job_id, output_format, output_basename, parameters, cloudos_url) - - -@job.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save jobs list. ' + - 'Default=joblist'), - default='joblist', - required=False) -@click.option('--output-format', - help='The desired output format. For json option --all-fields will be automatically set to True. Default=stdout.', - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--table-columns', - help=('Comma-separated list of columns to display in the table. Only applicable when --output-format=stdout. ' + - 'Available columns: status,name,project,owner,pipeline,id,submit_time,end_time,run_time,commit,cost,resources,storage_type. ' + - 'Default: responsive (auto-selects columns based on terminal width)'), - default=None) -@click.option('--all-fields', - help=('Whether to collect all available fields from jobs or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv. Automatically enabled for json output.'), - is_flag=True) -@click.option('--last-n-jobs', - help=("The number of last workspace jobs to retrieve. You can use 'all' to " + - "retrieve all workspace jobs. When adding this option, options " + - "'--page' and '--page-size' are ignored.")) -@click.option('--page', - help=('Page number to fetch from the API. Used with --page-size to control jobs ' + - 'per page (e.g. --page=4 --page-size=20). Default=1.'), - type=int, - default=1) -@click.option('--page-size', - help=('Page size to retrieve from API, corresponds to the number of jobs per page. ' + - 'Maximum allowed integer is 100. Default=10.'), - type=int, - default=10) -@click.option('--archived', - help=('When this flag is used, only archived jobs list is collected.'), - is_flag=True) -@click.option('--filter-status', - help='Filter jobs by status (e.g., completed, running, failed, aborted).') -@click.option('--filter-job-name', - help='Filter jobs by job name ( case insensitive ).') -@click.option('--filter-project', - help='Filter jobs by project name.') -@click.option('--filter-workflow', - help='Filter jobs by workflow/pipeline name.') -@click.option('--last', - help=('When workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('--filter-job-id', - help='Filter jobs by specific job ID.') -@click.option('--filter-only-mine', - help='Filter to show only jobs belonging to the current user.', - is_flag=True) -@click.option('--filter-queue', - help='Filter jobs by queue name. Only applies to jobs running in batch environment. Non-batch jobs are preserved in results.') -@click.option('--filter-owner', - help='Filter jobs by owner username.') -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - table_columns, - all_fields, - last_n_jobs, - page, - page_size, - archived, - filter_status, - filter_job_name, - filter_project, - filter_workflow, - last, - filter_job_id, - filter_only_mine, - filter_owner, - filter_queue, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect and display workspace jobs from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Pass table_columns directly to create_job_list_table for validation and processing - selected_columns = table_columns - # Only set outfile if not using stdout - if output_format != 'stdout': - outfile = output_basename + '.' + output_format - - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for jobs in the following workspace: ' + - f'{workspace_id}') - # Check if the user provided the --page option - ctx = click.get_current_context() - if not isinstance(page, int) or page < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') - - if not isinstance(page_size, int) or page_size < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page-size parameter') - - # Validate page_size limit - must be done before API call - if page_size > 100: - click.secho('Error: Page size cannot exceed 100. Please use --page-size with a value <= 100', fg='red', err=True) - raise SystemExit(1) - - result = cl.get_job_list(workspace_id, last_n_jobs, page, page_size, archived, verify_ssl, - filter_status=filter_status, - filter_job_name=filter_job_name, - filter_project=filter_project, - filter_workflow=filter_workflow, - filter_job_id=filter_job_id, - filter_only_mine=filter_only_mine, - filter_owner=filter_owner, - filter_queue=filter_queue, - last=last) - - # Extract jobs and pagination metadata from result - my_jobs_r = result['jobs'] - pagination_metadata = result['pagination_metadata'] - - # Validate requested page exists - if pagination_metadata: - total_jobs = pagination_metadata.get('Pagination-Count', 0) - current_page_size = pagination_metadata.get('Pagination-Limit', page_size) - - if total_jobs > 0: - total_pages = (total_jobs + current_page_size - 1) // current_page_size - if page > total_pages: - click.secho(f'Error: Page {page} does not exist. There are only {total_pages} page(s) available with {total_jobs} total job(s). ' - f'Please use --page with a value between 1 and {total_pages}', fg='red', err=True) - raise SystemExit(1) - - if len(my_jobs_r) == 0: - # Check if any filtering options are being used - filters_used = any([ - filter_status, - filter_job_name, - filter_project, - filter_workflow, - filter_job_id, - filter_only_mine, - filter_owner, - filter_queue - ]) - if output_format == 'stdout': - # For stdout, always show a user-friendly message - create_job_list_table([], cloudos_url, pagination_metadata, selected_columns) - else: - if filters_used: - print('A total of 0 jobs collected.') - elif ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - print('A total of 0 jobs collected. This is likely because your workspace ' + - 'has no jobs created yet.') - else: - print('A total of 0 jobs collected. This is likely because the --page you requested ' + - 'does not exist. Please, try a smaller number for --page or collect all the jobs by not ' + - 'using --page parameter.') - elif output_format == 'stdout': - # Display as table - create_job_list_table(my_jobs_r, cloudos_url, pagination_metadata, selected_columns) - elif output_format == 'csv': - my_jobs = cl.process_job_list(my_jobs_r, all_fields) - cl.save_job_list_to_csv(my_jobs, outfile) - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_jobs_r)) - print(f'\tJob list collected with a total of {len(my_jobs_r)} jobs.') - print(f'\tJob list saved to {outfile}') - else: - raise ValueError('Unrecognised output format. Please use one of [stdout|csv|json]') - - -@job.command('abort') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-ids', - help=('One or more job ids to abort. If more than ' + - 'one is provided, they must be provided as ' + - 'a comma separated list of ids. E.g. id1,id2,id3'), - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--force', - help='Force abort the job even if it is not in a running or initializing state.', - is_flag=True) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def abort_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - job_ids, - verbose, - disable_ssl_verification, - ssl_cert, - profile, - force): - """Abort all specified jobs from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - print('Aborting jobs...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for jobs in the following workspace: ' + - f'{workspace_id}') - # check if the user provided an empty job list - jobs = job_ids.replace(' ', '') - if not jobs: - raise ValueError('No job IDs provided. Please specify at least one job ID to abort.') - jobs = jobs.split(',') - - # Issue warning if using --force flag - if force: - click.secho(f"Warning: Using --force to abort jobs. Some data might be lost.", fg='yellow', bold=True) - - for job in jobs: - try: - j_status = cl.get_job_status(job, workspace_id, verify_ssl) - except Exception as e: - click.secho(f"Failed to get status for job {job}, please make sure it exists in the workspace: {e}", fg='yellow', bold=True) - continue - - j_status_content = json.loads(j_status.content) - job_status = j_status_content['status'] - - # Check if job is in a state that normally allows abortion - is_abortable = job_status in ABORT_JOB_STATES - - # Issue warning if job is in initializing state and not using force - if job_status == 'initializing' and not force: - click.secho(f"Warning: Job {job} is in initializing state.", fg='yellow', bold=True) - - # Check if job can be aborted - if not is_abortable: - click.secho(f"Job {job} is not in a state that can be aborted and is ignored. " + - f"Current status: {job_status}", fg='yellow', bold=True) - else: - try: - cl.abort_job(job, workspace_id, verify_ssl, force) - click.secho(f"Job '{job}' aborted successfully.", fg='green', bold=True) - except Exception as e: - click.secho(f"Failed to abort job {job}. Error: {e}", fg='red', bold=True) - - -@job.command('cost') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to get costs for.', - required=True) -@click.option('--output-format', - help='The desired file format (file extension) for the output. For json option --all-fields will be automatically set to True. Default=csv.', - type=click.Choice(['stdout', 'csv', 'json'], case_sensitive=False), - default='stdout') -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def job_cost(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve job cost information in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - print('Retrieving cost information...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cost_viewer = CostViewer(cloudos_url, apikey) - if verbose: - print(f'\tSearching for cost data for job id: {job_id}') - # Display costs with pagination - cost_viewer.display_costs(job_id, workspace_id, output_format, verify_ssl) - - -@job.command('related') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS to get costs for.', - required=True) -@click.option('--output-format', - help='The desired output format. Default=stdout.', - type=click.Choice(['stdout', 'json'], case_sensitive=False), - default='stdout') -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def related(ctx, - apikey, - cloudos_url, - workspace_id, - job_id, - output_format, - disable_ssl_verification, - ssl_cert, - profile): - """Retrieve related job analyses in CloudOS.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - related_analyses(cloudos_url, apikey, job_id, workspace_id, output_format, verify_ssl) - - -@click.command() -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--job-ids', - help=('One or more job ids to archive/unarchive. If more than ' + - 'one is provided, they must be provided as ' + - 'a comma separated list of ids. E.g. id1,id2,id3'), - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def archive_unarchive_jobs(ctx, - apikey, - cloudos_url, - workspace_id, - job_ids, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Archive or unarchive specified jobs in a CloudOS workspace.""" - # Determine operation based on the command name used - target_archived_state = ctx.info_name == "archive" - action = "archive" if target_archived_state else "unarchive" - action_past = "archived" if target_archived_state else "unarchived" - action_ing = "archiving" if target_archived_state else "unarchiving" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - print(f'{action_ing.capitalize()} jobs...') - - if verbose: - print('\t...Preparing objects') - - cl = Cloudos(cloudos_url, apikey, None) - - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\t{action_ing.capitalize()} jobs in the following workspace: {workspace_id}') - - # check if the user provided an empty job list - jobs = job_ids.replace(' ', '') - if not jobs: - raise ValueError(f'No job IDs provided. Please specify at least one job ID to {action}.') - jobs_list = [job for job in jobs.split(',') if job] # Filter out empty strings - - # Check for duplicate job IDs - duplicates = [job_id for job_id in set(jobs_list) if jobs_list.count(job_id) > 1] - if duplicates: - dup_str = ', '.join(duplicates) - click.secho(f'Warning: Duplicate job IDs detected and will be processed only once: {dup_str}', fg='yellow', bold=True) - # Remove duplicates while preserving order - jobs_list = list(dict.fromkeys(jobs_list)) - if verbose: - print(f'\tDuplicate job IDs removed. Processing {len(jobs_list)} unique job(s).') - - # Check archive status for all jobs - status_check = cl.check_jobs_archive_status(jobs_list, workspace_id, target_archived_state=target_archived_state, verify=verify_ssl, verbose=verbose) - valid_jobs = status_check['valid_jobs'] - already_processed = status_check['already_processed'] - invalid_jobs = status_check['invalid_jobs'] - - # Report invalid jobs (but continue processing valid ones) - for job_id, error_msg in invalid_jobs.items(): - click.secho(f"Failed to get status for job {job_id}, please make sure it exists in the workspace: {error_msg}", fg='yellow', bold=True) - - if not valid_jobs and not already_processed: - # All jobs were invalid - exit gracefully - click.secho('No valid job IDs found. Please check that the job IDs exist and are accessible.', fg='yellow', bold=True) - return - - if not valid_jobs: - if len(already_processed) == 1: - click.secho(f"Job '{already_processed[0]}' is already {action_past}. No action needed.", fg='cyan', bold=True) - else: - click.secho(f"All {len(already_processed)} jobs are already {action_past}. No action needed.", fg='cyan', bold=True) - return - - try: - # Call the appropriate action method - if target_archived_state: - cl.archive_jobs(valid_jobs, workspace_id, verify_ssl) - else: - cl.unarchive_jobs(valid_jobs, workspace_id, verify_ssl) - - success_msg = [] - if len(valid_jobs) == 1: - success_msg.append(f"Job '{valid_jobs[0]}' {action_past} successfully.") - else: - success_msg.append(f"{len(valid_jobs)} jobs {action_past} successfully: {', '.join(valid_jobs)}") - - if already_processed: - if len(already_processed) == 1: - success_msg.append(f"Job '{already_processed[0]}' was already {action_past}.") - else: - success_msg.append(f"{len(already_processed)} jobs were already {action_past}: {', '.join(already_processed)}") - - click.secho('\n'.join(success_msg), fg='green', bold=True) - except Exception as e: - raise ValueError(f"Failed to {action} jobs: {str(e)}") - - -@click.command(help='Clone or resume a job with modified parameters') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.') -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p input=s3://path_to_my_file. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--nextflow-profile', - help=('A comma separated string indicating the nextflow profile/s ' + - 'to use with your job.')) -@click.option('--nextflow-version', - help=('Nextflow version to use when executing the workflow in CloudOS. ' + - 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest'])) -@click.option('--git-branch', - help=('The branch to run for the selected pipeline. ' + - 'If not specified it defaults to the last commit ' + - 'of the default branch.')) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--job-name', - help='The name of the job. If not set, will take the name of the cloned job.') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help=('Name of the job queue to use with a batch job. ' + - 'In Azure workspaces, this option is ignored.')) -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).')) -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float) -@click.option('--job-id', - help='The CloudOS job id of the job to be cloned.', - required=True) -@click.option('--accelerate-file-staging', - help='Enables AWS S3 mountpoint for quicker file staging.', - is_flag=True) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--resumable', - help='Whether to make the job able to be resumed or not.', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', - help='Profile to use from the config file', - default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def clone_resume(ctx, - apikey, - cloudos_url, - workspace_id, - project_name, - parameter, - nextflow_profile, - nextflow_version, - git_branch, - repository_platform, - job_name, - do_not_save_logs, - job_queue, - instance_type, - cost_limit, - job_id, - accelerate_file_staging, - accelerate_saving_results, - resumable, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - if ctx.info_name == "clone": - mode, action = "clone", "cloning" - elif ctx.info_name == "resume": - mode, action = "resume", "resuming" - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - print(f'{action.capitalize()} job...') - if verbose: - print('\t...Preparing objects') - - # Create Job object (set dummy values for project_name and workflow_name, since they come from the cloned job) - job_obj = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", - mainfile=None, importsfile=None, verify=verify_ssl) - - if verbose: - print('\tThe following Job object was created:') - print('\t' + str(job_obj) + '\n') - print(f'\t{action.capitalize()} job {job_id} in workspace: {workspace_id}') - - try: - - # Clone/resume the job with provided overrides - cloned_resumed_job_id = job_obj.clone_or_resume_job( - source_job_id=job_id, - queue_name=job_queue, - cost_limit=cost_limit, - master_instance=instance_type, - job_name=job_name, - nextflow_version=nextflow_version, - branch=git_branch, - repository_platform=repository_platform, - profile=nextflow_profile, - do_not_save_logs=do_not_save_logs, - use_fusion=accelerate_file_staging, - accelerate_saving_results=accelerate_saving_results, - resumable=resumable, - # only when explicitly setting --project-name will be overridden, else using the original project - project_name=project_name if ctx.get_parameter_source("project_name") == click.core.ParameterSource.COMMANDLINE else None, - parameters=list(parameter) if parameter else None, - verify=verify_ssl, - mode=mode - ) - - if verbose: - print(f'\t{mode.capitalize()}d job ID: {cloned_resumed_job_id}') - - print(f"Job successfully {mode}d. New job ID: {cloned_resumed_job_id}") - - except BadRequestException as e: - raise ValueError(f"Failed to {mode} job. Job '{job_id}' not found or not accessible. {str(e)}") - except Exception as e: - raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") - - -# Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) -archive_unarchive_jobs.help = 'Archive specified jobs in a CloudOS workspace.' -job.add_command(archive_unarchive_jobs, "archive") - -# Create a copy with different help text for unarchive -archive_unarchive_jobs_copy = copy.deepcopy(archive_unarchive_jobs) -archive_unarchive_jobs_copy.help = 'Unarchive specified jobs in a CloudOS workspace.' -job.add_command(archive_unarchive_jobs_copy, "unarchive") - - -# Apply the best Click solution: Set specific help text for each command registration -clone_resume.help = 'Clone a job with modified parameters' -job.add_command(clone_resume, "clone") - -# Create a copy with different help text for resume -clone_resume_copy = copy.deepcopy(clone_resume) -clone_resume_copy.help = 'Resume a job with modified parameters' -job.add_command(clone_resume_copy, "resume") - - -@workflow.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save workflow list. ' + - 'Default=workflow_list'), - default='workflow_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from workflows or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_workflows(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all workflows from a CloudOS workspace in CSV format.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for workflows in the following workspace: ' + - f'{workspace_id}') - my_workflows_r = cl.get_workflow_list(workspace_id, verify=verify_ssl) - if output_format == 'csv': - my_workflows = cl.process_workflow_list(my_workflows_r, all_fields) - my_workflows.to_csv(outfile, index=False) - print(f'\tWorkflow list collected with a total of {my_workflows.shape[0]} workflows.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_workflows_r)) - print(f'\tWorkflow list collected with a total of {len(my_workflows_r)} workflows.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tWorkflow list saved to {outfile}') - - -@workflow.command('import') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=('The CloudOS url you are trying to access to. ' + - f'Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option("--workflow-name", help="The name that the workflow will have in CloudOS.", required=True) -@click.option("-w", "--workflow-url", help="URL of the workflow repository.", required=True) -@click.option("-d", "--workflow-docs-link", help="URL to the documentation of the workflow.", default='') -@click.option("--cost-limit", help="Cost limit for the workflow. Default: $30 USD.", default=30) -@click.option("--workflow-description", help="Workflow description", default="") -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name']) -def import_wf(ctx, - apikey, - cloudos_url, - workspace_id, - workflow_name, - workflow_url, - workflow_docs_link, - cost_limit, - workflow_description, - repository_platform, - disable_ssl_verification, - ssl_cert, - profile): - """ - Import workflows from supported repository providers. - """ - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - repo_import = ImportWorflow( - cloudos_url=cloudos_url, cloudos_apikey=apikey, workspace_id=workspace_id, platform=repository_platform, - workflow_name=workflow_name, workflow_url=workflow_url, workflow_docs_link=workflow_docs_link, - cost_limit=cost_limit, workflow_description=workflow_description, verify=verify_ssl - ) - workflow_id = repo_import.import_workflow() - print(f'\tWorkflow {workflow_name} was imported successfully with the ' + - f'following ID: {workflow_id}') - - -@project.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save project list. ' + - 'Default=project_list'), - default='project_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from projects or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--page', - help=('Response page to retrieve. Default=1.'), - type=int, - default=1) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_projects(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - page, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all projects from a CloudOS workspace in CSV format.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, None) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print('\tSearching for projects in the following workspace: ' + - f'{workspace_id}') - # Check if the user provided the --page option - ctx = click.get_current_context() - if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - get_all = True - else: - get_all = False - if not isinstance(page, int) or page < 1: - raise ValueError('Please, use a positive integer (>= 1) for the --page parameter') - my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all) - if len(my_projects_r) == 0: - if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT: - print('A total of 0 projects collected. This is likely because your workspace ' + - 'has no projects created yet.') - else: - print('A total of 0 projects collected. This is likely because the --page you ' + - 'requested does not exist. Please, try a smaller number for --page or collect all the ' + - 'projects by not using --page parameter.') - elif output_format == 'csv': - my_projects = cl.process_project_list(my_projects_r, all_fields) - my_projects.to_csv(outfile, index=False) - print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_projects_r)) - print(f'\tProject list collected with a total of {len(my_projects_r)} projects.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tProject list saved to {outfile}') - - -@project.command('create') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--new-project', - help='The name for the new project.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def create_project(ctx, - apikey, - cloudos_url, - workspace_id, - new_project, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Create a new project in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - # verify ssl configuration - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Print basic output - if verbose: - print(f'\tUsing CloudOS URL: {cloudos_url}') - print(f'\tUsing workspace: {workspace_id}') - print(f'\tProject name: {new_project}') - - cl = Cloudos(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None) - - try: - project_id = cl.create_project(workspace_id, new_project, verify_ssl) - print(f'\tProject "{new_project}" created successfully with ID: {project_id}') - if verbose: - print(f'\tProject URL: {cloudos_url}/app/projects/{project_id}') - except Exception as e: - print(f'\tError creating project: {str(e)}') - sys.exit(1) - - -@cromwell.command('status') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_status(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Check Cromwell server status in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - print('Executing status...') - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tChecking Cromwell status in {workspace_id} workspace') - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - - -@cromwell.command('start') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to Cromwell restart. ' + - 'Default=300.'), - default=300) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_restart(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - wait_time, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Restart Cromwell server in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - action = 'restart' - print('Starting Cromwell server...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tStarting Cromwell server in {workspace_id} workspace') - cl.cromwell_switch(workspace_id, action, verify_ssl) - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - elapsed = 0 - while elapsed < wait_time and c_status_h != 'Running': - c_status_old = c_status_h - time.sleep(REQUEST_INTERVAL_CROMWELL) - elapsed += REQUEST_INTERVAL_CROMWELL - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - if c_status_h != c_status_old: - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - if c_status_h != 'Running': - print(f'\tYour current Cromwell status is: {c_status_h}. The ' + - f'selected wait-time of {wait_time} was exceeded. Please, ' + - 'consider to set a longer wait-time.') - print('\tTo further check your Cromwell status you can either go to ' + - f'{cloudos_url} or use the following command:\n' + - '\tcloudos cromwell status \\\n' + - f'\t\t--cloudos-url {cloudos_url} \\\n' + - '\t\t--cromwell-token $CROMWELL_TOKEN \\\n' + - f'\t\t--workspace-id {workspace_id}') - sys.exit(1) - - -@cromwell.command('stop') -@click.version_option() -@click.option('-k', - '--apikey', - help='Your CloudOS API key.') -@click.option('-t', - '--cromwell-token', - help=('Specific Cromwell server authentication token. You can use it instead of ' + - 'the apikey.')) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['cloudos_url', 'workspace_id']) -def cromwell_stop(ctx, - apikey, - cromwell_token, - cloudos_url, - workspace_id, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """Stop Cromwell server in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - if apikey is None and cromwell_token is None: - raise ValueError("Please, use one of the following tokens: '--apikey', '--cromwell_token'") - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - action = 'stop' - print('Stopping Cromwell server...') - if verbose: - print('\t...Preparing objects') - cl = Cloudos(cloudos_url, apikey, cromwell_token) - if verbose: - print('\tThe following Cloudos object was created:') - print('\t' + str(cl) + '\n') - print(f'\tStopping Cromwell server in {workspace_id} workspace') - cl.cromwell_switch(workspace_id, action, verify_ssl) - c_status = cl.get_cromwell_status(workspace_id, verify_ssl) - c_status_h = json.loads(c_status.content)["status"] - print(f'\tCurrent Cromwell server status is: {c_status_h}\n') - - -@queue.command('list') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--output-basename', - help=('Output file base name to save job queue list. ' + - 'Default=job_queue_list'), - default='job_queue_list', - required=False) -@click.option('--output-format', - help='The desired file format (file extension) for the output. Default=csv.', - type=click.Choice(['csv', 'json'], case_sensitive=False), - default='csv') -@click.option('--all-fields', - help=('Whether to collect all available fields from workflows or ' + - 'just the preconfigured selected fields. Only applicable ' + - 'when --output-format=csv'), - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id']) -def list_queues(ctx, - apikey, - cloudos_url, - workspace_id, - output_basename, - output_format, - all_fields, - disable_ssl_verification, - ssl_cert, - profile): - """Collect all available job queues from a CloudOS workspace.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - outfile = output_basename + '.' + output_format - print('Executing list...') - j_queue = Queue(cloudos_url, apikey, None, workspace_id, verify=verify_ssl) - my_queues = j_queue.get_job_queues() - if len(my_queues) == 0: - raise ValueError('No AWS batch queues found. Please, make sure that your CloudOS supports AWS bath queues') - if output_format == 'csv': - queues_processed = j_queue.process_queue_list(my_queues, all_fields) - queues_processed.to_csv(outfile, index=False) - print(f'\tJob queue list collected with a total of {queues_processed.shape[0]} queues.') - elif output_format == 'json': - with open(outfile, 'w') as o: - o.write(json.dumps(my_queues)) - print(f'\tJob queue list collected with a total of {len(my_queues)} queues.') - else: - raise ValueError('Unrecognised output format. Please use one of [csv|json]') - print(f'\tJob queue list saved to {outfile}') - - -@configure.command('list-profiles') -def list_profiles(): - config_manager = ConfigurationProfile() - config_manager.list_profiles() - - -@configure.command('remove-profile') -@click.option('--profile', - help='Name of the profile. Not using this option will lead to profile named "deafults" being generated', - required=True) -@click.pass_context -def remove_profile(ctx, profile): - update_command_context_from_click(ctx) - profile = profile or ctx.obj['profile'] - config_manager = ConfigurationProfile() - config_manager.remove_profile(profile) - - -@bash.command('job') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('--command', - help='The command to run in the bash job.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + - 'times as parameters you want to include.')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--cpus', - help='The number of CPUs to use for the task\'s master node. Default=1.', - type=int, - default=1) -@click.option('--memory', - help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', - type=int, - default=4) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - default='aws') -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run_bash_job(ctx, - apikey, - command, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - parameter, - job_name, - do_not_save_logs, - job_queue, - instance_type, - instance_disk, - cpus, - memory, - storage_mode, - lustre_size, - wait_completion, - wait_time, - repository_platform, - execution_platform, - cost_limit, - accelerate_saving_results, - request_interval, - disable_ssl_verification, - ssl_cert, - profile): - """Run a bash job in CloudOS.""" - # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - - if do_not_save_logs: - save_logs = False - else: - save_logs = True - - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=None, importsfile=None, - repository_platform=repository_platform, verify=verify_ssl, last=last) - - if job_queue is not None: - batch = True - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, - workspace_id=workspace_id, verify=verify_ssl) - # I have to add 'nextflow', other wise the job queue id is not found - job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, - job_queue=job_queue) - else: - job_queue_id = None - batch = False - j_id = j.send_job(job_config=None, - parameter=parameter, - git_commit=None, - git_tag=None, - git_branch=None, - job_name=job_name, - resumable=False, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - workflow_type='docker', - nextflow_profile=None, - nextflow_version=None, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=None, - cost_limit=cost_limit, - accelerate_saving_results=accelerate_saving_results, - verify=verify_ssl, - command={"command": command}, - cpus=cpus, - memory=memory) - - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=False, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@bash.command('array-job') -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('--command', - help='The command to run in the bash job.') -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--workflow-name', - help='The name of a CloudOS workflow or pipeline.', - required=True) -@click.option('--last', - help=('When the workflows are duplicated, use the latest imported workflow (by date).'), - is_flag=True) -@click.option('-p', - '--parameter', - multiple=True, - help=('A single parameter to pass to the job call. It should be in the ' + - 'following form: parameter_name=parameter_value. E.g.: ' + - '-p --test=value or -p -test=value or -p test=value. You can use this option as many ' + - 'times as parameters you want to include. ' + - 'For parameters pointing to a file, the format expected is ' + - 'parameter_name=/Data/parameter_value. The parameter value must be a ' + - 'file located in the `Data` subfolder. If no is specified, it defaults to ' + - 'the project specified by the profile or --project-name parameter. ' + - 'E.g.: -p "--file=Data/file.txt" or "--file=/Data/folder/file.txt"')) -@click.option('--job-name', - help='The name of the job. Default=new_job.', - default='new_job') -@click.option('--do-not-save-logs', - help=('Avoids process log saving. If you select this option, your job process ' + - 'logs will not be stored.'), - is_flag=True) -@click.option('--job-queue', - help='Name of the job queue to use with a batch job.') -@click.option('--instance-type', - help=('The type of compute instance to use as master node. ' + - 'Default=c5.xlarge(aws)|Standard_D4as_v4(azure).'), - default='NONE_SELECTED') -@click.option('--instance-disk', - help='The disk space of the master node instance, in GB. Default=500.', - type=int, - default=500) -@click.option('--cpus', - help='The number of CPUs to use for the task\'s master node. Default=1.', - type=int, - default=1) -@click.option('--memory', - help='The amount of memory, in GB, to use for the task\'s master node. Default=4.', - type=int, - default=4) -@click.option('--storage-mode', - help=('Either \'lustre\' or \'regular\'. Indicates if the user wants to select ' + - 'regular or lustre storage. Default=regular.'), - default='regular') -@click.option('--lustre-size', - help=('The lustre storage to be used when --storage-mode=lustre, in GB. It should ' + - 'be 1200 or a multiple of it. Default=1200.'), - type=int, - default=1200) -@click.option('--wait-completion', - help=('Whether to wait to job completion and report final ' + - 'job status.'), - is_flag=True) -@click.option('--wait-time', - help=('Max time to wait (in seconds) to job completion. ' + - 'Default=3600.'), - default=3600) -@click.option('--repository-platform', type=click.Choice(["github", "gitlab", "bitbucketServer"]), - help='Name of the repository platform of the workflow. Default=github.', - default='github') -@click.option('--execution-platform', - help='Name of the execution platform implemented in your CloudOS. Default=aws.', - type=click.Choice(['aws', 'azure', 'hpc']), - default='aws') -@click.option('--cost-limit', - help='Add a cost limit to your job. Default=30.0 (For no cost limit please use -1).', - type=float, - default=30.0) -@click.option('--accelerate-saving-results', - help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) -@click.option('--request-interval', - help=('Time interval to request (in seconds) the job status. ' + - 'For large jobs is important to use a high number to ' + - 'make fewer requests so that is not considered spamming by the API. ' + - 'Default=30.'), - default=30) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--array-file', - help=('Path to a file containing an array of commands to run in the bash job.'), - default=None, - required=True) -@click.option('--separator', - help=('Separator to use in the array file. Default=",".'), - type=click.Choice([',', ';', 'tab', 'space', '|']), - default=",", - required=True) -@click.option('--list-columns', - help=('List columns present in the array file. ' + - 'This option will not run any job.'), - is_flag=True) -@click.option('--array-file-project', - help=('Name of the project in which the array file is placed, if different from --project-name.'), - default=None) -@click.option('--disable-column-check', - help=('Disable the check for the columns in the array file. ' + - 'This option is only used when --array-file is provided.'), - is_flag=True) -@click.option('-a', '--array-parameter', - multiple=True, - help=('A single parameter to pass to the job call only for specifying array columns. ' + - 'It should be in the following form: parameter_name=array_file_column_name. E.g.: ' + - '-a --test=value or -a -test=value or -a test=value or -a =value (for no prefix). ' + - 'You can use this option as many times as parameters you want to include.')) -@click.option('--custom-script-path', - help=('Path of a custom script to run in the bash array job instead of a command.'), - default=None) -@click.option('--custom-script-project', - help=('Name of the project to use when running the custom command script, if ' + - 'different than --project-name.'), - default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'workflow_name', 'project_name']) -def run_bash_array_job(ctx, - apikey, - command, - cloudos_url, - workspace_id, - project_name, - workflow_name, - last, - parameter, - job_name, - do_not_save_logs, - job_queue, - instance_type, - instance_disk, - cpus, - memory, - storage_mode, - lustre_size, - wait_completion, - wait_time, - repository_platform, - execution_platform, - cost_limit, - accelerate_saving_results, - request_interval, - disable_ssl_verification, - ssl_cert, - profile, - array_file, - separator, - list_columns, - array_file_project, - disable_column_check, - array_parameter, - custom_script_path, - custom_script_project): - """Run a bash array job in CloudOS.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - if not list_columns and not (command or custom_script_path): - raise click.UsageError("Must provide --command or --custom-script-path if --list-columns is not set.") - - # when not set, use the global project name - if array_file_project is None: - array_file_project = project_name - - # this needs to be in another call to datasets, by default it uses the global project name - if custom_script_project is None: - custom_script_project = project_name - - # setup separators for API and array file (the're different) - separators = { - ",": {"api": ",", "file": ","}, - ";": {"api": "%3B", "file": ";"}, - "space": {"api": "+", "file": " "}, - "tab": {"api": "tab", "file": "tab"}, - "|": {"api": "%7C", "file": "|"} - } - - # setup important options for the job - if do_not_save_logs: - save_logs = False - else: - save_logs = True - - if instance_type == 'NONE_SELECTED': - if execution_platform == 'aws': - instance_type = 'c5.xlarge' - elif execution_platform == 'azure': - instance_type = 'Standard_D4as_v4' - else: - instance_type = None - - j = jb.Job(cloudos_url, apikey, None, workspace_id, project_name, workflow_name, - mainfile=None, importsfile=None, - repository_platform=repository_platform, verify=verify_ssl, last=last) - - # retrieve columns - r = j.retrieve_cols_from_array_file( - array_file, - generate_datasets_for_project(cloudos_url, apikey, workspace_id, array_file_project, verify_ssl), - separators[separator]['api'], - verify_ssl - ) - - if not disable_column_check: - columns = json.loads(r.content).get("headers", None) - # pass this to the SEND JOB API call - # b'{"headers":[{"index":0,"name":"id"},{"index":1,"name":"title"},{"index":2,"name":"filename"},{"index":3,"name":"file2name"}]}' - if columns is None: - raise ValueError("No columns found in the array file metadata.") - if list_columns: - print("Columns: ") - for col in columns: - print(f"\t- {col['name']}") - return - else: - columns = [] - - # setup parameters for the job - cmd = j.setup_params_array_file( - custom_script_path, - generate_datasets_for_project(cloudos_url, apikey, workspace_id, custom_script_project, verify_ssl), - command, - separators[separator]['file'] - ) - - # check columns in the array file vs parameters added - if not disable_column_check and array_parameter: - print("\nChecking columns in the array file vs parameters added...\n") - for ap in array_parameter: - ap_split = ap.split('=') - ap_value = '='.join(ap_split[1:]) - for col in columns: - if col['name'] == ap_value: - print(f"Found column '{ap_value}' in the array file.") - break - else: - raise ValueError(f"Column '{ap_value}' not found in the array file. " + \ - f"Columns in array-file: {separator.join([col['name'] for col in columns])}") - - if job_queue is not None: - batch = True - queue = Queue(cloudos_url=cloudos_url, apikey=apikey, cromwell_token=None, - workspace_id=workspace_id, verify=verify_ssl) - # I have to add 'nextflow', other wise the job queue id is not found - job_queue_id = queue.fetch_job_queue_id(workflow_type='nextflow', batch=batch, - job_queue=job_queue) - else: - job_queue_id = None - batch = False - - # send job - j_id = j.send_job(job_config=None, - parameter=parameter, - array_parameter=array_parameter, - array_file_header=columns, - git_commit=None, - git_tag=None, - git_branch=None, - job_name=job_name, - resumable=False, - save_logs=save_logs, - batch=batch, - job_queue_id=job_queue_id, - workflow_type='docker', - nextflow_profile=None, - nextflow_version=None, - instance_type=instance_type, - instance_disk=instance_disk, - storage_mode=storage_mode, - lustre_size=lustre_size, - execution_platform=execution_platform, - hpc_id=None, - cost_limit=cost_limit, - accelerate_saving_results=accelerate_saving_results, - verify=verify_ssl, - command=cmd, - cpus=cpus, - memory=memory) - - print(f'\tYour assigned job id is: {j_id}\n') - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{j_id}' - if wait_completion: - print('\tPlease, wait until job completion (max wait time of ' + - f'{wait_time} seconds).\n') - j_status = j.wait_job_completion(job_id=j_id, - workspace_id=workspace_id, - wait_time=wait_time, - request_interval=request_interval, - verbose=False, - verify=verify_ssl) - j_name = j_status['name'] - j_final_s = j_status['status'] - if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(0) - else: - print(f'\nJob status for job "{j_name}" (ID: {j_id}): {j_final_s}') - sys.exit(1) - else: - j_status = j.get_job_status(j_id, workspace_id, verify_ssl) - j_status_h = json.loads(j_status.content)["status"] - print(f'\tYour current job status is: {j_status_h}') - print('\tTo further check your job status you can either go to ' + - f'{j_url} or use the following command:\n' + - '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {j_id}\n') - - -@datasets.command(name="ls") -@click.argument("path", required=False, nargs=1) -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--project-name', - help='The name of a CloudOS project.', - required=True) -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.option('--details', - help=('When selected, it prints the details of the listed files. ' + - 'Details contains "Type", "Owner", "Size", "Last Updated", ' + - '"Virtual Name", "Storage Path".'), - is_flag=True) -@click.option('--output-format', - help=('The desired display for the output, either directly in standard output or saved as file. ' + - 'Default=stdout.'), - type=click.Choice(['stdout', 'csv'], case_sensitive=False), - default='stdout') -@click.option('--output-basename', - help=('Output file base name to save jobs details. ' + - 'Default=datasets_ls'), - default='datasets_ls', - required=False) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def list_files(ctx, - apikey, - cloudos_url, - workspace_id, - disable_ssl_verification, - ssl_cert, - project_name, - profile, - path, - details, - output_format, - output_basename): - """List contents of a path within a CloudOS workspace dataset.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - datasets = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = datasets.list_folder_content(path) - contents = result.get("contents") or result.get("datasets", []) - - if not contents: - contents = result.get("files", []) + result.get("folders", []) - - # Process items to extract data - processed_items = [] - for item in contents: - is_folder = "folderType" in item or item.get("isDir", False) - type_ = "folder" if is_folder else "file" - - # Enhanced type information - if is_folder: - folder_type = item.get("folderType") - if folder_type == "VirtualFolder": - type_ = "virtual folder" - elif folder_type == "S3Folder": - type_ = "s3 folder" - elif folder_type == "AzureBlobFolder": - type_ = "azure folder" - else: - type_ = "folder" - else: - # Check if file is managed by Lifebit (user uploaded) - is_managed_by_lifebit = item.get("isManagedByLifebit", False) - if is_managed_by_lifebit: - type_ = "file (user uploaded)" - else: - type_ = "file (virtual copy)" - - user = item.get("user", {}) - if isinstance(user, dict): - name = user.get("name", "").strip() - surname = user.get("surname", "").strip() - else: - name = surname = "" - if name and surname: - owner = f"{name} {surname}" - elif name: - owner = name - elif surname: - owner = surname - else: - owner = "-" - - raw_size = item.get("sizeInBytes", item.get("size")) - size = format_bytes(raw_size) if not is_folder and raw_size is not None else "-" - - updated = item.get("updatedAt") or item.get("lastModified", "-") - filepath = item.get("name", "-") - - if item.get("fileType") == "S3File" or item.get("folderType") == "S3Folder": - bucket = item.get("s3BucketName") - key = item.get("s3ObjectKey") or item.get("s3Prefix") - storage_path = f"s3://{bucket}/{key}" if bucket and key else "-" - elif item.get("fileType") == "AzureBlobFile" or item.get("folderType") == "AzureBlobFolder": - account = item.get("blobStorageAccountName") - container = item.get("blobContainerName") - key = item.get("blobName") if item.get("fileType") == "AzureBlobFile" else item.get("blobPrefix") - storage_path = f"az://{account}.blob.core.windows.net/{container}/{key}" if account and container and key else "-" - else: - storage_path = "-" - - processed_items.append({ - 'type': type_, - 'owner': owner, - 'size': size, - 'raw_size': raw_size, - 'updated': updated, - 'name': filepath, - 'storage_path': storage_path, - 'is_folder': is_folder - }) - - # Output handling - if output_format == 'csv': - import csv - - csv_filename = f'{output_basename}.csv' - - if details: - # CSV with all details - with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: - fieldnames = ['Type', 'Owner', 'Size', 'Size (bytes)', 'Last Updated', 'Virtual Name', 'Storage Path'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - - for item in processed_items: - writer.writerow({ - 'Type': item['type'], - 'Owner': item['owner'], - 'Size': item['size'], - 'Size (bytes)': item['raw_size'] if item['raw_size'] is not None else '', - 'Last Updated': item['updated'], - 'Virtual Name': item['name'], - 'Storage Path': item['storage_path'] - }) - else: - # CSV with just names - with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile: - writer = csv.writer(csvfile) - writer.writerow(['Name', 'Storage Path']) - for item in processed_items: - writer.writerow([item['name'], item['storage_path']]) - - click.secho(f'\nDatasets list saved to: {csv_filename}', fg='green', bold=True) - - else: # stdout - if details: - console = Console(width=None) - table = Table(show_header=True, header_style="bold white") - table.add_column("Type", style="cyan", no_wrap=True) - table.add_column("Owner", style="white") - table.add_column("Size", style="magenta") - table.add_column("Last Updated", style="green") - table.add_column("Virtual Name", style="bold", overflow="fold") - table.add_column("Storage Path", style="dim", no_wrap=False, overflow="fold", ratio=2) - - for item in processed_items: - style = Style(color="blue", underline=True) if item['is_folder'] else None - table.add_row( - item['type'], - item['owner'], - item['size'], - item['updated'], - item['name'], - item['storage_path'], - style=style - ) - - console.print(table) - - else: - console = Console() - for item in processed_items: - if item['is_folder']: - console.print(f"[blue underline]{item['name']}[/]") - else: - console.print(item['name']) - - except Exception as e: - raise ValueError(f"Failed to list files for project '{project_name}'. {str(e)}") - - -@datasets.command(name="mv") -@click.argument("source_path", required=True) -@click.argument("destination_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The source project name.') -@click.option('--destination-project-name', required=False, - help='The destination project name. Defaults to the source project.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def move_files(ctx, source_path, destination_path, apikey, cloudos_url, workspace_id, - project_name, destination_project_name, - disable_ssl_verification, ssl_cert, profile): - """ - Move a file or folder from a source path to a destination path within or across CloudOS projects. - - SOURCE_PATH [path]: the full path to the file or folder to move. It must be a 'Data' folder path. - E.g.: 'Data/folderA/file.txt'\n - DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. - E.g.: 'Data/folderB' - """ - # Validate destination constraint - if not destination_path.strip("/").startswith("Data/") and destination_path.strip("/") != "Data": - raise ValueError("Destination path must begin with 'Data/' or be 'Data'.") - if not source_path.strip("/").startswith("Data/") and source_path.strip("/") != "Data": - raise ValueError("SOURCE_PATH must start with 'Data/' or be 'Data'.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - destination_project_name = destination_project_name or project_name - # Initialize Datasets clients - source_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - dest_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=destination_project_name, - verify=verify_ssl, - cromwell_token=None - ) - print('Checking source path') - # === Resolve Source Item === - source_parts = source_path.strip("/").split("/") - source_parent_path = "/".join(source_parts[:-1]) if len(source_parts) > 1 else None - source_item_name = source_parts[-1] - - try: - source_contents = source_client.list_folder_content(source_parent_path) - except Exception as e: - raise ValueError(f"Could not resolve source path '{source_path}'. {str(e)}") - - found_source = None - for collection in ["files", "folders"]: - for item in source_contents.get(collection, []): - if item.get("name") == source_item_name: - found_source = item - break - if found_source: - break - if not found_source: - raise ValueError(f"Item '{source_item_name}' not found in '{source_parent_path or '[project root]'}'") - - source_id = found_source["_id"] - source_kind = "Folder" if "folderType" in found_source else "File" - print("Checking destination path") - # === Resolve Destination Folder === - dest_parts = destination_path.strip("/").split("/") - dest_folder_name = dest_parts[-1] - dest_parent_path = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else None - - try: - dest_contents = dest_client.list_folder_content(dest_parent_path) - match = next((f for f in dest_contents.get("folders", []) if f.get("name") == dest_folder_name), None) - if not match: - raise ValueError(f"Could not resolve destination folder '{destination_path}'") - - target_id = match["_id"] - folder_type = match.get("folderType") - # Normalize kind: top-level datasets are kind=Dataset, all other folders are kind=Folder - if folder_type in ("VirtualFolder", "Folder"): - target_kind = "Folder" - elif folder_type == "S3Folder": - raise ValueError(f"Unable to move item '{source_item_name}' to '{destination_path}'. " + - "The destination is an S3 folder, and only virtual folders can be selected as valid move destinations.") - elif isinstance(folder_type, bool) and folder_type: # legacy dataset structure - target_kind = "Dataset" - else: - raise ValueError(f"Unrecognized folderType '{folder_type}' for destination '{destination_path}'") - - except Exception as e: - raise ValueError(f"Could not resolve destination path '{destination_path}'. {str(e)}") - print(f"Moving {source_kind} '{source_item_name}' to '{destination_path}' " + - f"in project '{destination_project_name} ...") - # === Perform Move === - try: - response = source_client.move_files_and_folders( - source_id=source_id, - source_kind=source_kind, - target_id=target_id, - target_kind=target_kind - ) - if response.ok: - click.secho(f"{source_kind} '{source_item_name}' moved to '{destination_path}' " + - f"in project '{destination_project_name}'.", fg="green", bold=True) - else: - raise ValueError(f"Move failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Move operation failed. {str(e)}") - - -@datasets.command(name="rename") -@click.argument("source_path", required=True) -@click.argument("new_name", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def renaming_item(ctx, - source_path, - new_name, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Rename a file or folder in a CloudOS project. - - SOURCE_PATH [path]: the full path to the file or folder to rename. It must be a 'Data' folder path. - E.g.: 'Data/folderA/old_name.txt'\n - NEW_NAME [name]: the new name to assign to the file or folder. E.g.: 'new_name.txt' - """ - if not source_path.strip("/").startswith("Data/"): - raise ValueError("SOURCE_PATH must start with 'Data/', pointing to a file or folder in that dataset.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - parts = source_path.strip("/").split("/") - - parent_path = "/".join(parts[:-1]) - target_name = parts[-1] - - try: - contents = client.list_folder_content(parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") - - # Search for file/folder - found_item = None - for category in ["files", "folders"]: - for item in contents.get(category, []): - if item.get("name") == target_name: - found_item = item - break - if found_item: - break - - if not found_item: - raise ValueError(f"Item '{target_name}' not found in '{parent_path or '[project root]'}'") - - item_id = found_item["_id"] - kind = "Folder" if "folderType" in found_item else "File" - - print(f"Renaming {kind} '{target_name}' to '{new_name}'...") - try: - response = client.rename_item(item_id=item_id, new_name=new_name, kind=kind) - if response.ok: - click.secho( - f"{kind} '{target_name}' renamed to '{new_name}' in folder '{parent_path}'.", - fg="green", - bold=True - ) - else: - raise ValueError(f"Rename failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Rename operation failed. {str(e)}") - - -@datasets.command(name="cp") -@click.argument("source_path", required=True) -@click.argument("destination_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The source project name.') -@click.option('--destination-project-name', required=False, help='The destination project name. Defaults to the source project.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def copy_item_cli(ctx, - source_path, - destination_path, - apikey, - cloudos_url, - workspace_id, - project_name, - destination_project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Copy a file or folder (S3 or virtual) from SOURCE_PATH to DESTINATION_PATH. - - SOURCE_PATH [path]: the full path to the file or folder to copy. - E.g.: AnalysesResults/my_analysis/results/my_plot.png\n - DESTINATION_PATH [path]: the full path to the destination folder. It must be a 'Data' folder path. - E.g.: Data/plots - """ - destination_project_name = destination_project_name or project_name - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - # Initialize clients - source_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - dest_client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=destination_project_name, - verify=verify_ssl, - cromwell_token=None - ) - # Validate paths - dest_parts = destination_path.strip("/").split("/") - if not dest_parts or dest_parts[0] != "Data": - raise ValueError("DESTINATION_PATH must start with 'Data/'.") - # Parse source and destination - source_parts = source_path.strip("/").split("/") - source_parent = "/".join(source_parts[:-1]) if len(source_parts) > 1 else "" - source_name = source_parts[-1] - dest_folder_name = dest_parts[-1] - dest_parent = "/".join(dest_parts[:-1]) if len(dest_parts) > 1 else "" - try: - source_content = source_client.list_folder_content(source_parent) - dest_content = dest_client.list_folder_content(dest_parent) - except Exception as e: - raise ValueError(f"Could not access paths. {str(e)}") - # Find the source item - source_item = None - for item in source_content.get('files', []) + source_content.get('folders', []): - if item.get("name") == source_name: - source_item = item - break - if not source_item: - raise ValueError(f"Item '{source_name}' not found in '{source_parent or '[project root]'}'") - # Find the destination folder - destination_folder = None - for folder in dest_content.get("folders", []): - if folder.get("name") == dest_folder_name: - destination_folder = folder - break - if not destination_folder: - raise ValueError(f"Destination folder '{destination_path}' not found.") - try: - # Determine item type - if "fileType" in source_item: - item_type = "file" - elif source_item.get("folderType") == "VirtualFolder": - item_type = "virtual_folder" - elif "s3BucketName" in source_item and source_item.get("folderType") == "S3Folder": - item_type = "s3_folder" - else: - raise ValueError("Could not determine item type.") - print(f"Copying {item_type.replace('_', ' ')} '{source_name}' to '{destination_path}'...") - if destination_folder.get("folderType") is True and destination_folder.get("kind") in ("Data", "Cohorts", "AnalysesResults"): - destination_kind = "Dataset" - elif destination_folder.get("folderType") == "S3Folder": - raise ValueError(f"Unable to copy item '{source_name}' to '{destination_path}'. The destination is an S3 folder, and only virtual folders can be selected as valid copy destinations.") - else: - destination_kind = "Folder" - response = source_client.copy_item( - item=source_item, - destination_id=destination_folder["_id"], - destination_kind=destination_kind - ) - if response.ok: - click.secho("Item copied successfully.", fg="green", bold=True) - else: - raise ValueError(f"Copy failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Copy operation failed. {str(e)}") - - -@datasets.command(name="mkdir") -@click.argument("new_folder_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def mkdir_item(ctx, - new_folder_path, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile): - """ - Create a virtual folder in a CloudOS project. - - NEW_FOLDER_PATH [path]: Full path to the new folder including its name. Must start with 'Data'. - """ - new_folder_path = new_folder_path.strip("/") - if not new_folder_path.startswith("Data"): - raise ValueError("NEW_FOLDER_PATH must start with 'Data'.") - - path_parts = new_folder_path.split("/") - if len(path_parts) < 2: - raise ValueError("NEW_FOLDER_PATH must include at least a parent folder and the new folder name.") - - parent_path = "/".join(path_parts[:-1]) - folder_name = path_parts[-1] - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - # Split parent path to get its parent + name - parent_parts = parent_path.split("/") - parent_name = parent_parts[-1] - parent_of_parent_path = "/".join(parent_parts[:-1]) - - # List the parent of the parent - try: - contents = client.list_folder_content(parent_of_parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_of_parent_path}'. {str(e)}") - - # Find the parent folder in the contents - folder_info = next( - (f for f in contents.get("folders", []) if f.get("name") == parent_name), - None - ) - - if not folder_info: - raise ValueError(f"Could not find folder '{parent_name}' in '{parent_of_parent_path}'.") - - parent_id = folder_info.get("_id") - folder_type = folder_info.get("folderType") - - if folder_type is True: - parent_kind = "Dataset" - elif isinstance(folder_type, str): - parent_kind = "Folder" - else: - raise ValueError(f"Unrecognized folderType for '{parent_path}'.") - - # Create the folder - print(f"Creating folder '{folder_name}' under '{parent_path}' ({parent_kind})...") - try: - response = client.create_virtual_folder(name=folder_name, parent_id=parent_id, parent_kind=parent_kind) - if response.ok: - click.secho(f"Folder '{folder_name}' created under '{parent_path}'", fg="green", bold=True) - else: - raise ValueError(f"Folder creation failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Folder creation failed. {str(e)}") - - -@datasets.command(name="rm") -@click.argument("target_path", required=True) -@click.option('-k', '--apikey', required=True, help='Your CloudOS API key.') -@click.option('-c', '--cloudos-url', default=CLOUDOS_URL, required=True, help='The CloudOS URL.') -@click.option('--workspace-id', required=True, help='The CloudOS workspace ID.') -@click.option('--project-name', required=True, help='The project name.') -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', default=None, help='Profile to use from the config file.') -@click.option('--force', is_flag=True, help='Force delete files. Required when deleting user uploaded files. This may also delete the file from the cloud provider storage.') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'project_name']) -def rm_item(ctx, - target_path, - apikey, - cloudos_url, - workspace_id, - project_name, - disable_ssl_verification, - ssl_cert, - profile, - force): - """ - Delete a file or folder in a CloudOS project. - - TARGET_PATH [path]: the full path to the file or folder to delete. Must start with 'Data'. \n - E.g.: 'Data/folderA/file.txt' or 'Data/my_analysis/results/folderB' - """ - if not target_path.strip("/").startswith("Data/"): - raise ValueError("TARGET_PATH must start with 'Data/', pointing to a file or folder.") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - client = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - - parts = target_path.strip("/").split("/") - parent_path = "/".join(parts[:-1]) - item_name = parts[-1] - - try: - contents = client.list_folder_content(parent_path) - except Exception as e: - raise ValueError(f"Could not list contents at '{parent_path or '[project root]'}'. {str(e)}") - - found_item = None - for item in contents.get('files', []) + contents.get('folders', []): - if item.get("name") == item_name: - found_item = item - break - - if not found_item: - raise ValueError(f"Item '{item_name}' not found in '{parent_path or '[project root]'}'") - - item_id = found_item.get("_id", '') - kind = "Folder" if "folderType" in found_item else "File" - if item_id == '': - raise ValueError(f"Item '{item_name}' could not be removed as the parent folder is an s3 folder and their content cannot be modified.") - # Check if the item is managed by Lifebit - is_managed_by_lifebit = found_item.get("isManagedByLifebit", False) - if is_managed_by_lifebit and not force: - raise ValueError("By removing this file, it will be permanently deleted. If you want to go forward, please use the --force flag.") - print(f"Removing {kind} '{item_name}' from '{parent_path or '[root]'}'...") - try: - response = client.delete_item(item_id=item_id, kind=kind) - if response.ok: - if is_managed_by_lifebit: - click.secho( - f"{kind} '{item_name}' was permanently deleted from '{parent_path or '[root]'}'.", - fg="green", bold=True - ) - else: - click.secho( - f"{kind} '{item_name}' was removed from '{parent_path or '[root]'}'.", - fg="green", bold=True - ) - click.secho("This item will still be available on your Cloud Provider.", fg="yellow") - else: - raise ValueError(f"Removal failed. {response.status_code} - {response.text}") - except Exception as e: - raise ValueError(f"Remove operation failed. {str(e)}") - - -@datasets.command(name="link") -@click.argument("path", required=True) -@click.option('-k', '--apikey', help='Your CloudOS API key', required=True) -@click.option('-c', '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL) -@click.option('--project-name', - help='The name of a CloudOS project.', - required=False) -@click.option('--workspace-id', help='The specific CloudOS workspace id.', required=True) -@click.option('--session-id', help='The specific CloudOS interactive session id.', required=True) -@click.option('--disable-ssl-verification', is_flag=True, help='Disable SSL certificate verification.') -@click.option('--ssl-cert', help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default='default') -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) -def link(ctx, - path, - apikey, - cloudos_url, - project_name, - workspace_id, - session_id, - disable_ssl_verification, - ssl_cert, - profile): - """ - Link a folder (S3 or File Explorer) to an active interactive analysis. - - PATH [path]: the full path to the S3 folder to link or relative to File Explorer. - E.g.: 's3://bucket-name/folder/subfolder', 'Data/Downloads' or 'Data'. - """ - if not path.startswith("s3://") and project_name is None: - # for non-s3 paths we need the project, for S3 we don't - raise click.UsageError("When using File Explorer paths '--project-name' needs to be defined") - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - link_p = Link( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - cromwell_token=None, - project_name=project_name, - verify=verify_ssl - ) - - # Minimal folder validation and improved error messages - is_s3 = path.startswith("s3://") - is_folder = True - if is_s3: - # S3 path validation - use heuristics to determine if it's likely a folder - try: - # If path ends with '/', it's likely a folder - if path.endswith('/'): - is_folder = True - else: - # Check the last part of the path - path_parts = path.rstrip("/").split("/") - if path_parts: - last_part = path_parts[-1] - # If the last part has no dot, it's likely a folder - if '.' not in last_part: - is_folder = True - else: - # If it has a dot, it might be a file - set to None for warning - is_folder = None - else: - # Empty path parts, set to None for uncertainty - is_folder = None - except Exception: - # If we can't parse the S3 path, set to None for uncertainty - is_folder = None - else: - # File Explorer path validation (existing logic) - try: - datasets = Datasets( - cloudos_url=cloudos_url, - apikey=apikey, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl, - cromwell_token=None - ) - parts = path.strip("/").split("/") - parent_path = "/".join(parts[:-1]) if len(parts) > 1 else "" - item_name = parts[-1] - contents = datasets.list_folder_content(parent_path) - found = None - for item in contents.get("folders", []): - if item.get("name") == item_name: - found = item - break - if not found: - for item in contents.get("files", []): - if item.get("name") == item_name: - found = item - break - if found and ("folderType" not in found): - is_folder = False - except Exception: - is_folder = None - - if is_folder is False: - if is_s3: - raise ValueError("The S3 path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.") - else: - raise ValueError("Linking files or virtual folders is not supported. Link the S3 parent folder instead.", err=True) - return - elif is_folder is None and is_s3: - click.secho("Unable to verify whether the S3 path is a folder. Proceeding with linking; " + - "however, if the operation fails, please confirm that you are linking a folder rather than a file.", fg='yellow', bold=True) - - try: - link_p.link_folder(path, session_id) - except Exception as e: - if is_s3: - print("If you are linking an S3 path, please ensure it is a folder.") - raise ValueError(f"Could not link folder. {e}") - - -@images.command(name="ls") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--page', help='The response page. Defaults to 1.', required=False, default=1) -@click.option('--limit', help='The page size limit. Defaults to 10', required=False, default=10) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def list_images(ctx, - apikey, - cloudos_url, - procurement_id, - disable_ssl_verification, - ssl_cert, - profile, - page, - limit): - """List images associated with organisations of a given procurement.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None, - page=page, - limit=limit - ) - - try: - result = procurement_images.list_procurement_images() - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - - -@images.command(name="set") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) -@click.option('--image-type', help='The CloudOS resource image type.', required=True, - type=click.Choice([ - 'RegularInteractiveSessions', - 'SparkInteractiveSessions', - 'RStudioInteractiveSessions', - 'JupyterInteractiveSessions', - 'JobDefault', - 'NextflowBatchComputeEnvironment'])) -@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') -@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) -@click.option('--image-id', help='The new image id value.', required=True) -@click.option('--image-name', help='The new image name value.', required=False) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def set_organisation_image(ctx, - apikey, - cloudos_url, - procurement_id, - organisation_id, - image_type, - provider, - region, - image_id, - image_name, - disable_ssl_verification, - ssl_cert, - profile): - """Set a new image id or name to image associated with an organisations of a given procurement.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = procurement_images.set_procurement_organisation_image( - organisation_id, - image_type, - provider, - region, - image_id, - image_name - ) - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - - -@images.command(name="reset") -@click.option('-k', - '--apikey', - help='Your CloudOS API key.', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--procurement-id', help='The specific CloudOS procurement id.', required=True) -@click.option('--organisation-id', help='The Organisation Id where the change is going to be applied.', required=True) -@click.option('--image-type', help='The CloudOS resource image type.', required=True, - type=click.Choice([ - 'RegularInteractiveSessions', - 'SparkInteractiveSessions', - 'RStudioInteractiveSessions', - 'JupyterInteractiveSessions', - 'JobDefault', - 'NextflowBatchComputeEnvironment'])) -@click.option('--provider', help='The cloud provider. Only aws is supported.', required=True, type=click.Choice(['aws']), default='aws') -@click.option('--region', help='The cloud region. Only aws regions are supported.', required=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'procurement_id']) -def reset_organisation_image(ctx, - apikey, - cloudos_url, - procurement_id, - organisation_id, - image_type, - provider, - region, - disable_ssl_verification, - ssl_cert, - profile): - """Reset image associated with an organisations of a given procurement to CloudOS defaults.""" - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - procurement_images = Images( - cloudos_url=cloudos_url, - apikey=apikey, - procurement_id=procurement_id, - verify=verify_ssl, - cromwell_token=None - ) - - try: - result = procurement_images.reset_procurement_organisation_image( - organisation_id, - image_type, - provider, - region - ) - console = Console() - console.print(result) - - except Exception as e: - raise ValueError(f"{str(e)}") - -@run_cloudos_cli.command('link') -@click.argument('path', required=False) -@click.option('-k', - '--apikey', - help='Your CloudOS API key', - required=True) -@click.option('-c', - '--cloudos-url', - help=(f'The CloudOS url you are trying to access to. Default={CLOUDOS_URL}.'), - default=CLOUDOS_URL, - required=True) -@click.option('--workspace-id', - help='The specific CloudOS workspace id.', - required=True) -@click.option('--session-id', - help='The specific CloudOS interactive session id.', - required=True) -@click.option('--job-id', - help='The job id in CloudOS. When provided, links results, workdir and logs by default.', - required=False) -@click.option('--project-name', - help='The name of a CloudOS project. Required for File Explorer paths.', - required=False) -@click.option('--results', - help='Link only results folder (only works with --job-id).', - is_flag=True) -@click.option('--workdir', - help='Link only working directory (only works with --job-id).', - is_flag=True) -@click.option('--logs', - help='Link only logs folder (only works with --job-id).', - is_flag=True) -@click.option('--verbose', - help='Whether to print information messages or not.', - is_flag=True) -@click.option('--disable-ssl-verification', - help=('Disable SSL certificate verification. Please, remember that this option is ' + - 'not generally recommended for security reasons.'), - is_flag=True) -@click.option('--ssl-cert', - help='Path to your SSL certificate file.') -@click.option('--profile', help='Profile to use from the config file', default=None) -@click.pass_context -@with_profile_config(required_params=['apikey', 'workspace_id', 'session_id']) -def link_command(ctx, - path, - apikey, - cloudos_url, - workspace_id, - session_id, - job_id, - project_name, - results, - workdir, - logs, - verbose, - disable_ssl_verification, - ssl_cert, - profile): - """ - Link folders to an interactive analysis session. - - This command is used to link folders - to an active interactive analysis session for direct access to data. - - PATH: Optional path to link (S3). - Required if --job-id is not provided. - - Two modes of operation: - - 1. Job-based linking (--job-id): Links job-related folders. - By default, links results, workdir, and logs folders. - Use --results, --workdir, or --logs flags to link only specific folders. - - 2. Direct path linking (PATH argument): Links a specific S3 path. - - Examples: - - # Link all job folders (results, workdir, logs) - cloudos link --job-id 12345 --session-id abc123 - - # Link only results from a job - cloudos link --job-id 12345 --session-id abc123 --results - - # Link a specific S3 path - cloudos link s3://bucket/folder --session-id abc123 - - """ - print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') - - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - - # Validate input parameters - if not job_id and not path: - raise click.UsageError("Either --job-id or PATH argument must be provided.") - - if job_id and path: - raise click.UsageError("Cannot use both --job-id and PATH argument. Please provide only one.") - - # Validate folder-specific flags only work with --job-id - if (results or workdir or logs) and not job_id: - raise click.UsageError("--results, --workdir, and --logs flags can only be used with --job-id.") - - # If no specific folders are selected with job-id, link all by default - if job_id and not (results or workdir or logs): - results = True - workdir = True - logs = True - - if verbose: - print('Using the following parameters:') - print(f'\tCloudOS url: {cloudos_url}') - print(f'\tWorkspace ID: {workspace_id}') - print(f'\tSession ID: {session_id}') - if job_id: - print(f'\tJob ID: {job_id}') - print(f'\tLink results: {results}') - print(f'\tLink workdir: {workdir}') - print(f'\tLink logs: {logs}') - else: - print(f'\tPath: {path}') - - # Initialize Link client - link_client = Link( - cloudos_url=cloudos_url, - apikey=apikey, - cromwell_token=None, - workspace_id=workspace_id, - project_name=project_name, - verify=verify_ssl - ) - - try: - if job_id: - # Job-based linking - print(f'Linking folders from job {job_id} to interactive session {session_id}...\n') - - # Link results - if results: - link_client.link_job_results(job_id, workspace_id, session_id, verify_ssl, verbose) - - # Link workdir - if workdir: - link_client.link_job_workdir(job_id, workspace_id, session_id, verify_ssl, verbose) - - # Link logs - if logs: - link_client.link_job_logs(job_id, workspace_id, session_id, verify_ssl, verbose) - - - else: - # Direct path linking - print(f'Linking path to interactive session {session_id}...\n') - - # Link path with validation - link_client.link_path_with_validation(path, session_id, verify_ssl, project_name, verbose) - - print('\nLinking operation completed.') - - except BadRequestException as e: - raise ValueError(f"Request failed: {str(e)}") - except Exception as e: - raise ValueError(f"Failed to link folder(s): {str(e)}") - -if __name__ == "__main__": - # Setup logging - debug_mode = '--debug' in sys.argv - setup_logging(debug_mode) - logger = logging.getLogger("CloudOS") - # Check if debug flag was passed (fallback for cases where Click doesn't handle it) - try: - run_cloudos_cli() - except Exception as e: - if debug_mode: - logger.error(e, exc_info=True) - traceback.print_exc() - else: - logger.error(e) - click.echo(click.style(f"Error: {e}", fg='red'), err=True) - sys.exit(1) \ No newline at end of file From 332430493bb7b32758492e5a8814116a9675c10e Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 10:05:31 +0100 Subject: [PATCH 17/41] ci: change dependency for workdir --delete --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecaf634e..170f8f96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -816,7 +816,7 @@ jobs: cloudos job abort --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID echo "Job abort command executed successfully!" delete_workdir: - needs: [job_run_and_status, job_workdir, job_resume] + needs: [job_workdir, job_resume] runs-on: ubuntu-latest strategy: matrix: @@ -838,4 +838,4 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | - cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} + cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_resume.outputs.job_id }} From bc6b34aaf6340e4590e1c1d94afc92a60ff7c821 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 10:54:04 +0100 Subject: [PATCH 18/41] ci: update workdir --delete --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 170f8f96..e6979c85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,6 +252,8 @@ jobs: cloudos job status --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID job_resume: needs: job_run_and_status + outputs: + job_id: ${{ steps.get-resumed-job-id.outputs.job_id }} runs-on: ubuntu-latest strategy: matrix: @@ -274,10 +276,11 @@ jobs: CLOUDOS_URL: "https://cloudos.lifebit.ai" PROJECT_NAME: "cloudos-cli-tests" INSTANCE_TYPE: "m4.xlarge" + JOB_NAME_BASE: "cloudos-cli-CI-test-abort" run: | JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" echo "Resuming job $JOB_ID..." - cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-id $JOB_ID --instance-type $INSTANCE_TYPE + cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-name "$JOB_NAME_BASE" --job-id $JOB_ID --instance-type $INSTANCE_TYPE echo "Job resumed successfully!" logs_results_aws: needs: job_run_and_status From a5f9e7799deabc787eeaa0d1f3a612ddaae004b2 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 11:40:56 +0100 Subject: [PATCH 19/41] ci: update resume --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6979c85..61a026a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -270,13 +270,14 @@ jobs: run: | pip install -e . - name: Resume completed job + id: get-resumed-job-id env: CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN }} CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" PROJECT_NAME: "cloudos-cli-tests" INSTANCE_TYPE: "m4.xlarge" - JOB_NAME_BASE: "cloudos-cli-CI-test-abort" + JOB_NAME_BASE: "cloudos-cli-CI-test-resume" run: | JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" echo "Resuming job $JOB_ID..." From daf0f16e4b92f8dedfaeb420ea6fba23c3bd69ec Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 11:58:19 +0100 Subject: [PATCH 20/41] ci: update resume --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61a026a7..fda6e57a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -820,7 +820,7 @@ jobs: cloudos job abort --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID echo "Job abort command executed successfully!" delete_workdir: - needs: [job_workdir, job_resume] + needs: [job_run_and_status, job_workdir, job_resume] runs-on: ubuntu-latest strategy: matrix: @@ -842,4 +842,4 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | - cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_resume.outputs.job_id }} + cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} From 64810031c45881e100a1bfd7487e523776871b86 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 12:31:15 +0100 Subject: [PATCH 21/41] update wait completion --- .github/workflows/ci.yml | 9 ++++---- cloudos_cli/jobs/cli.py | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fda6e57a..511530c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -279,9 +279,10 @@ jobs: INSTANCE_TYPE: "m4.xlarge" JOB_NAME_BASE: "cloudos-cli-CI-test-resume" run: | - JOB_ID="${{ needs.job_run_and_status.outputs.job_id }}" - echo "Resuming job $JOB_ID..." - cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-name "$JOB_NAME_BASE" --job-id $JOB_ID --instance-type $INSTANCE_TYPE + JOB_ID_TO_RESUME="${{ needs.job_run_and_status.outputs.job_id }}" + echo "Resuming job $JOB_ID_TO_RESUME..." + cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-name "$JOB_NAME_BASE" --job-id $JOB_ID_TO_RESUME --wait-completion --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt + JOB_ID=$(grep -e "Job successfully resumed. New job ID:" out.txt | rev | cut -f1 -d " " | rev) echo "Job resumed successfully!" logs_results_aws: needs: job_run_and_status @@ -842,4 +843,4 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID }} CLOUDOS_URL: "https://cloudos.lifebit.ai" run: | - cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status.outputs.job_id }} + cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_resume.outputs.job_id }} diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 157c19a4..4738e947 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -1739,6 +1739,20 @@ def archive_unarchive_jobs(ctx, @click.option('--resumable', help='Whether to make the job able to be resumed or not.', is_flag=True) +@click.option('--wait-completion', + help=('Whether to wait to job completion and report final ' + + 'job status.'), + is_flag=True) +@click.option('--wait-time', + help=('Max time to wait (in seconds) to job completion. ' + + 'Default=3600.'), + default=3600) +@click.option('--request-interval', + help=('Time interval to request (in seconds) the job status. ' + + 'For large jobs is important to use a high number to ' + + 'make fewer requests so that is not considered spamming by the API. ' + + 'Default=30.'), + default=30) @click.option('--verbose', help='Whether to print information messages or not.', is_flag=True) @@ -1772,6 +1786,9 @@ def clone_resume(ctx, accelerate_file_staging, accelerate_saving_results, resumable, + wait_completion, + wait_time, + request_interval, verbose, disable_ssl_verification, ssl_cert, @@ -1832,6 +1849,34 @@ def clone_resume(ctx, except Exception as e: raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' + if wait_completion: + print('\tPlease, wait until job completion (max wait time of ' + + f'{wait_time} seconds).\n') + j_status = job_obj.wait_job_completion(job_id=job_id, + workspace_id=workspace_id, + wait_time=wait_time, + request_interval=request_interval, + verbose=verbose, + verify=verify_ssl) + j_name = j_status['name'] + j_final_s = j_status['status'] + if j_final_s == JOB_COMPLETED: + print(f'\nJob status for job "{j_name}" (ID: {job_id}): {j_final_s}') + sys.exit(0) + else: + print(f'\nJob status for job "{j_name}" (ID: {job_id}): {j_final_s}') + sys.exit(1) + else: + j_status = job_obj.get_job_status(job_id, workspace_id, verify_ssl) + j_status_h = json.loads(j_status.content)["status"] + print(f'\tYour current job status is: {j_status_h}') + print('\tTo further check your job status you can either go to ' + + f'{j_url} or use the following command:\n' + + '\tcloudos job status \\\n' + + f'\t\t--profile my_profile \\\n' + + f'\t\t--job-id {job_id}\n') + # Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) archive_unarchive_jobs.help = 'Archive specified jobs in a CloudOS workspace.' From 65ad1736f6d113c28e78ff81618b92d9f9bd4ae1 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 12:45:27 +0100 Subject: [PATCH 22/41] update resume job id --- cloudos_cli/jobs/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 4738e947..6ba617c5 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -1849,11 +1849,11 @@ def clone_resume(ctx, except Exception as e: raise ValueError(f"Failed to {mode} job. Failed to {action} job '{job_id}'. {str(e)}") - j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{job_id}' + j_url = f'{cloudos_url}/app/advanced-analytics/analyses/{cloned_resumed_job_id}' if wait_completion: print('\tPlease, wait until job completion (max wait time of ' + f'{wait_time} seconds).\n') - j_status = job_obj.wait_job_completion(job_id=job_id, + j_status = job_obj.wait_job_completion(job_id=cloned_resumed_job_id, workspace_id=workspace_id, wait_time=wait_time, request_interval=request_interval, @@ -1862,20 +1862,20 @@ def clone_resume(ctx, j_name = j_status['name'] j_final_s = j_status['status'] if j_final_s == JOB_COMPLETED: - print(f'\nJob status for job "{j_name}" (ID: {job_id}): {j_final_s}') + print(f'\nJob status for job "{j_name}" (ID: {cloned_resumed_job_id}): {j_final_s}') sys.exit(0) else: - print(f'\nJob status for job "{j_name}" (ID: {job_id}): {j_final_s}') + print(f'\nJob status for job "{j_name}" (ID: {cloned_resumed_job_id}): {j_final_s}') sys.exit(1) else: - j_status = job_obj.get_job_status(job_id, workspace_id, verify_ssl) + j_status = job_obj.get_job_status(cloned_resumed_job_id, workspace_id, verify_ssl) j_status_h = json.loads(j_status.content)["status"] print(f'\tYour current job status is: {j_status_h}') print('\tTo further check your job status you can either go to ' + f'{j_url} or use the following command:\n' + '\tcloudos job status \\\n' + f'\t\t--profile my_profile \\\n' + - f'\t\t--job-id {job_id}\n') + f'\t\t--job-id {cloned_resumed_job_id}\n') # Register archive_unarchive_jobs with both command names using aliases (same pattern as clone/resume) From 14c5e9da1053e75d77850c5e9aae5ba6d1cc21e9 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Tue, 3 Feb 2026 13:17:29 +0100 Subject: [PATCH 23/41] ci: update resume output --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 511530c7..74941d06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -283,6 +283,7 @@ jobs: echo "Resuming job $JOB_ID_TO_RESUME..." cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-name "$JOB_NAME_BASE" --job-id $JOB_ID_TO_RESUME --wait-completion --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt JOB_ID=$(grep -e "Job successfully resumed. New job ID:" out.txt | rev | cut -f1 -d " " | rev) + echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT echo "Job resumed successfully!" logs_results_aws: needs: job_run_and_status From 7d5745f6d0e59ba499dbeaa3ed9a7b4f00fe80ee Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 10:29:23 +0100 Subject: [PATCH 24/41] review: add debug to all commands --- cloudos_cli/bash/cli.py | 3 ++- cloudos_cli/configure/cli.py | 3 ++- cloudos_cli/cromwell/cli.py | 4 +++- cloudos_cli/datasets/cli.py | 3 ++- cloudos_cli/jobs/cli.py | 3 ++- cloudos_cli/link/cli.py | 1 + cloudos_cli/projects/cli.py | 3 ++- cloudos_cli/queue/cli.py | 3 ++- cloudos_cli/workflows/cli.py | 3 ++- 9 files changed, 18 insertions(+), 8 deletions(-) diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py index 7cbd99b0..90979208 100644 --- a/cloudos_cli/bash/cli.py +++ b/cloudos_cli/bash/cli.py @@ -8,9 +8,10 @@ from cloudos_cli.queue.queue import Queue import sys import json +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands -@click.group() +@click.group(cls=pass_debug_to_subcommands()) def bash(): """CloudOS bash-specific job functionality.""" print(bash.__doc__ + '\n') diff --git a/cloudos_cli/configure/cli.py b/cloudos_cli/configure/cli.py index 3e108417..67409f15 100644 --- a/cloudos_cli/configure/cli.py +++ b/cloudos_cli/configure/cli.py @@ -3,10 +3,11 @@ import rich_click as click from cloudos_cli.configure.configure import ConfigurationProfile from cloudos_cli.logging.logger import update_command_context_from_click +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands # Create the configure group -@click.group(invoke_without_command=True) +@click.group(cls=pass_debug_to_subcommands(), invoke_without_command=True) @click.option('--profile', help='Profile to use from the config file', default='default') @click.option('--make-default', is_flag=True, diff --git a/cloudos_cli/cromwell/cli.py b/cloudos_cli/cromwell/cli.py index 1207dc8c..cae421ab 100644 --- a/cloudos_cli/cromwell/cli.py +++ b/cloudos_cli/cromwell/cli.py @@ -7,12 +7,14 @@ from cloudos_cli.clos import Cloudos from cloudos_cli.utils.resources import ssl_selector from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands + # Constants REQUEST_INTERVAL_CROMWELL = 5 -@click.group() +@click.group(cls=pass_debug_to_subcommands()) def cromwell(): """CloudOS Cromwell server functionality.""" print(cromwell.__doc__ + '\n') diff --git a/cloudos_cli/datasets/cli.py b/cloudos_cli/datasets/cli.py index 7c1409af..8ba9e78b 100644 --- a/cloudos_cli/datasets/cli.py +++ b/cloudos_cli/datasets/cli.py @@ -11,9 +11,10 @@ from rich.console import Console from rich.table import Table from rich.style import Style +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands -@click.group() +@click.group(cls=pass_debug_to_subcommands()) @click.pass_context def datasets(ctx): """CloudOS datasets functionality.""" diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 6ba617c5..4daada40 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -16,6 +16,7 @@ from cloudos_cli.queue.queue import Queue import sys from rich.console import Console +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands # Import global constants from __main__ (will be available when imported) @@ -32,7 +33,7 @@ # Create the job group -@click.group() +@click.group(cls=pass_debug_to_subcommands()) def job(): """CloudOS job functionality: run, clone, resume, check and abort jobs in CloudOS.""" print(job.__doc__ + '\n') diff --git a/cloudos_cli/link/cli.py b/cloudos_cli/link/cli.py index ae9dd832..01f1fce4 100644 --- a/cloudos_cli/link/cli.py +++ b/cloudos_cli/link/cli.py @@ -3,6 +3,7 @@ from cloudos_cli.utils.resources import ssl_selector from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL from cloudos_cli.utils.errors import BadRequestException +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands @click.command() diff --git a/cloudos_cli/projects/cli.py b/cloudos_cli/projects/cli.py index 4ff23ebf..bf8b794a 100644 --- a/cloudos_cli/projects/cli.py +++ b/cloudos_cli/projects/cli.py @@ -6,9 +6,10 @@ from cloudos_cli.clos import Cloudos from cloudos_cli.utils.resources import ssl_selector from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands -@click.group() +@click.group(cls=pass_debug_to_subcommands()) def project(): """CloudOS project functionality.""" print(project.__doc__ + '\n') diff --git a/cloudos_cli/queue/cli.py b/cloudos_cli/queue/cli.py index 68f84868..45897a20 100644 --- a/cloudos_cli/queue/cli.py +++ b/cloudos_cli/queue/cli.py @@ -5,10 +5,11 @@ from cloudos_cli.queue.queue import Queue from cloudos_cli.utils.resources import ssl_selector from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands # Create the queue group -@click.group() +@click.group(cls=pass_debug_to_subcommands()) def queue(): """CloudOS job queue functionality.""" print(queue.__doc__ + '\n') diff --git a/cloudos_cli/workflows/cli.py b/cloudos_cli/workflows/cli.py index 578af42e..7c47b476 100644 --- a/cloudos_cli/workflows/cli.py +++ b/cloudos_cli/workflows/cli.py @@ -6,10 +6,11 @@ from cloudos_cli.import_wf.import_wf import ImportWorflow from cloudos_cli.utils.resources import ssl_selector from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL +from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands # Create the workflow group -@click.group() +@click.group(cls=pass_debug_to_subcommands()) def workflow(): """CloudOS workflow functionality: list and import workflows.""" print(workflow.__doc__ + '\n') From 14ce4936e0d8c8d3d5e394da794c14106de87738 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 13:59:08 +0100 Subject: [PATCH 25/41] review: resolve Leila's conversations --- CHANGELOG.md | 3 ++- .../test_accelerate_saving_results_clone_resume.py | 4 ---- tests/test_jobs/test_accelerate_saving_results_run.py | 7 ++----- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4f2415f..b4ca8e9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ### Patch -- Refactor `__main__.py` and move comamnds to separate files +- Refactor `__main__.py` to move commands into separate files for better organization +- Improve CI/CD pipeline to test additional functionalities and use dynamically generated job IDs from tests instead of predefined values ## v2.78.0 (2026-01-13) diff --git a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py index 482ba98e..20822951 100644 --- a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py +++ b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py @@ -3,8 +3,6 @@ This test file provides testing for the --accelerate-saving-results flag functionality in the job resume command of CloudOS CLI. """ -import pytest -from click.testing import CliRunner from cloudos_cli.jobs.cli import clone_resume @@ -12,7 +10,6 @@ def test_resume_accelerate_saving_results_flag_is_boolean(): """ Test that --accelerate-saving-results is properly defined as a boolean flag in resume command """ - from cloudos_cli.jobs.cli import clone_resume # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None @@ -30,7 +27,6 @@ def test_resume_accelerate_saving_results_flag_definition(): """ Test that the flag has the correct help text definition in resume command """ - from cloudos_cli.jobs.cli import clone_resume # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None diff --git a/tests/test_jobs/test_accelerate_saving_results_run.py b/tests/test_jobs/test_accelerate_saving_results_run.py index 80ab5d50..1d43337a 100644 --- a/tests/test_jobs/test_accelerate_saving_results_run.py +++ b/tests/test_jobs/test_accelerate_saving_results_run.py @@ -3,7 +3,6 @@ This test file provides testing for the --accelerate-saving-results flag functionality in the job run command of CloudOS CLI. """ -import pytest from click.testing import CliRunner from cloudos_cli.jobs.cli import run @@ -12,11 +11,10 @@ def test_run_accelerate_saving_results_flag_is_boolean(): """ Test that --accelerate-saving-results is properly defined as a boolean flag """ - from cloudos_cli.jobs.cli import run as run_command # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None - for param in run_command.params: + for param in run.params: if hasattr(param, 'name') and param.name == 'accelerate_saving_results': accelerate_saving_results_option = param break @@ -43,11 +41,10 @@ def test_run_accelerate_saving_results_flag_definition(): """ Test that the flag has the correct help text definition """ - from cloudos_cli.jobs.cli import run as run_command # Get the accelerate-saving-results option from the command accelerate_saving_results_option = None - for param in run_command.params: + for param in run.params: if hasattr(param, 'name') and param.name == 'accelerate_saving_results': accelerate_saving_results_option = param break From c24d2bb05f5f42eed4fe5649119e93c20ce493b4 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 14:05:17 +0100 Subject: [PATCH 26/41] review: add python versions 3.9, 3.11, 3.15 for all the tests --- .github/workflows/ci.yml | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74941d06..277085cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -95,7 +95,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -120,7 +120,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -145,7 +145,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -169,7 +169,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9" ] + python-version: [ "3.9", "3.11", "3.15" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -193,7 +193,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9" ] + python-version: [ "3.9", "3.11", "3.15" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -219,7 +219,7 @@ jobs: job_id: ${{ steps.get-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -257,7 +257,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -290,7 +290,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9" ] + python-version: [ "3.9", "3.11", "3.15" ] feature: ["logs", "results"] steps: - uses: actions/checkout@v3 @@ -315,7 +315,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -338,7 +338,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -361,7 +361,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -384,7 +384,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -408,7 +408,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -460,7 +460,7 @@ jobs: job_id: ${{ steps.get-bash-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -504,7 +504,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -541,7 +541,7 @@ jobs: job_id: ${{ steps.get-bash-array-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -589,7 +589,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -624,7 +624,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -649,7 +649,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -675,7 +675,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -701,7 +701,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -759,7 +759,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.15"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -826,7 +826,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9" ] + python-version: [ "3.9", "3.11", "3.15" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 70b2a2fd989144b528fa910f4c93d7c63825da52 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 14:10:57 +0100 Subject: [PATCH 27/41] review: change latest supported python versions 3.13 for all the tests --- .github/workflows/ci.yml | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 277085cb..c4db6245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -95,7 +95,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -120,7 +120,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -145,7 +145,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -169,7 +169,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.15" ] + python-version: [ "3.9", "3.11", "3.13" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -193,7 +193,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.15" ] + python-version: [ "3.9", "3.11", "3.13" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -219,7 +219,7 @@ jobs: job_id: ${{ steps.get-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -257,7 +257,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -290,7 +290,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.15" ] + python-version: [ "3.9", "3.11", "3.13" ] feature: ["logs", "results"] steps: - uses: actions/checkout@v3 @@ -315,7 +315,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -338,7 +338,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -361,7 +361,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -384,7 +384,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -408,7 +408,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -460,7 +460,7 @@ jobs: job_id: ${{ steps.get-bash-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -504,7 +504,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -541,7 +541,7 @@ jobs: job_id: ${{ steps.get-bash-array-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -589,7 +589,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -624,7 +624,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -649,7 +649,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -675,7 +675,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -701,7 +701,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -759,7 +759,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.15"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -826,7 +826,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.15" ] + python-version: [ "3.9", "3.11", "3.13" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From f5d2363cd67fe4ef0dae65891f331312347b93e0 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 15:14:54 +0100 Subject: [PATCH 28/41] review: use exit 1 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4db6245..be3ab007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -805,7 +805,7 @@ jobs: break elif echo "$STATUS" | grep -qi "completed\|failed\|aborted"; then echo "Job finished before we could abort it (status: $STATUS)" - exit 0 + exit 1 fi attempt=$((attempt+1)) From 379bcd75f4edbb4d89be492b79a99e293e29b7cc Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 16:20:22 +0100 Subject: [PATCH 29/41] review: moved imports at the top --- cloudos_cli/bash/cli.py | 11 +++-------- cloudos_cli/jobs/cli.py | 34 +++++++++++----------------------- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py index 90979208..b2a5784d 100644 --- a/cloudos_cli/bash/cli.py +++ b/cloudos_cli/bash/cli.py @@ -9,8 +9,11 @@ import sys import json from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands +from cloudos_cli import __main__ +JOB_COMPLETED = __main__.JOB_COMPLETED + @click.group(cls=pass_debug_to_subcommands()) def bash(): """CloudOS bash-specific job functionality.""" @@ -149,10 +152,6 @@ def run_bash_job(ctx, profile): """Run a bash job in CloudOS.""" # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - - from cloudos_cli import __main__ - JOB_COMPLETED = __main__.JOB_COMPLETED - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) if instance_type == 'NONE_SELECTED': @@ -415,10 +414,6 @@ def run_bash_array_job(ctx, custom_script_path, custom_script_project): """Run a bash array job in CloudOS.""" - - from cloudos_cli import __main__ - JOB_COMPLETED = __main__.JOB_COMPLETED - verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) if not list_columns and not (command or custom_script_path): diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 4daada40..f3af20c7 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -17,20 +17,18 @@ import sys from rich.console import Console from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands +from cloudos_cli import __main__ -# Import global constants from __main__ (will be available when imported) -# These need to be imported for backward compatibility -JOB_COMPLETED = 'completed' -REQUEST_INTERVAL_CROMWELL = 30 -ABORT_JOB_STATES = ['running', 'initializing'] -AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] -AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] -HPC_NEXTFLOW_VERSIONS = ['22.10.8'] -AWS_NEXTFLOW_LATEST = '24.04.4' -AZURE_NEXTFLOW_LATEST = '22.11.1-edge' -HPC_NEXTFLOW_LATEST = '22.10.8' - +# Import constants from __main__ +JOB_COMPLETED = __main__.JOB_COMPLETED +REQUEST_INTERVAL_CROMWELL = __main__.REQUEST_INTERVAL_CROMWELL +AWS_NEXTFLOW_VERSIONS = __main__.AWS_NEXTFLOW_VERSIONS +AZURE_NEXTFLOW_VERSIONS = __main__.AZURE_NEXTFLOW_VERSIONS +HPC_NEXTFLOW_VERSIONS = __main__.HPC_NEXTFLOW_VERSIONS +AWS_NEXTFLOW_LATEST = __main__.AWS_NEXTFLOW_LATEST +AZURE_NEXTFLOW_LATEST = __main__.AZURE_NEXTFLOW_LATEST +HPC_NEXTFLOW_LATEST = __main__.HPC_NEXTFLOW_LATEST # Create the job group @click.group(cls=pass_debug_to_subcommands()) @@ -233,17 +231,7 @@ def run(ctx, disable_ssl_verification, ssl_cert, profile): - """Run a CloudOS workflow.""" - # Import constants from __main__ - from cloudos_cli import __main__ - AWS_NEXTFLOW_VERSIONS = __main__.AWS_NEXTFLOW_VERSIONS - AZURE_NEXTFLOW_VERSIONS = __main__.AZURE_NEXTFLOW_VERSIONS - HPC_NEXTFLOW_VERSIONS = __main__.HPC_NEXTFLOW_VERSIONS - AWS_NEXTFLOW_LATEST = __main__.AWS_NEXTFLOW_LATEST - AZURE_NEXTFLOW_LATEST = __main__.AZURE_NEXTFLOW_LATEST - HPC_NEXTFLOW_LATEST = __main__.HPC_NEXTFLOW_LATEST - JOB_COMPLETED = __main__.JOB_COMPLETED - + """Run a CloudOS workflow.""" verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) if do_not_save_logs: save_logs = False From d85f68b176a5ec96123246d6fc598ccf5f2f7e78 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 16:21:17 +0100 Subject: [PATCH 30/41] Update CHANGELOG.md Co-authored-by: dapineyro <45285897+dapineyro@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4ca8e9d..36c234e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## lifebit-ai/cloudos-cli: changelog -## v2.78.1 (2026-01-28) +## v2.79.0 (2026-01-28) ### Patch From dbe038960344849554e1bb79877c53863ccbf76c Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 16:22:02 +0100 Subject: [PATCH 31/41] Update cloudos_cli/_version.py Co-authored-by: dapineyro <45285897+dapineyro@users.noreply.github.com> --- cloudos_cli/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudos_cli/_version.py b/cloudos_cli/_version.py index d8c9c360..3126b4f0 100644 --- a/cloudos_cli/_version.py +++ b/cloudos_cli/_version.py @@ -1 +1 @@ -__version__ = '2.78.1' +__version__ = '2.79.0' From 150fd44af405f94802878916fdb8f732587823df Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 16:27:00 +0100 Subject: [PATCH 32/41] revert: use python 3.9 only for now in platform tests --- .github/workflows/ci.yml | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be3ab007..8ddb654a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -61,7 +61,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -95,7 +95,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -120,7 +120,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -145,7 +145,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -169,7 +169,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.13" ] + python-version: [ "3.9" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -193,7 +193,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.13" ] + python-version: [ "3.9" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -219,7 +219,7 @@ jobs: job_id: ${{ steps.get-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -257,7 +257,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -290,7 +290,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.13" ] + python-version: [ "3.9" ] feature: ["logs", "results"] steps: - uses: actions/checkout@v3 @@ -315,7 +315,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -338,7 +338,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -361,7 +361,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -384,7 +384,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -408,7 +408,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -433,7 +433,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -460,7 +460,7 @@ jobs: job_id: ${{ steps.get-bash-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -504,7 +504,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -541,7 +541,7 @@ jobs: job_id: ${{ steps.get-bash-array-job-id.outputs.job_id }} strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -589,7 +589,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -624,7 +624,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -649,7 +649,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -675,7 +675,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -701,7 +701,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -759,7 +759,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -826,7 +826,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.11", "3.13" ] + python-version: [ "3.9" ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From c6c4ad82881e594e722b2b94b539635bb0911d0e Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Wed, 11 Feb 2026 16:28:56 +0100 Subject: [PATCH 33/41] refactor: add missing variable --- cloudos_cli/jobs/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index f3af20c7..6a7a1037 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -29,6 +29,7 @@ AWS_NEXTFLOW_LATEST = __main__.AWS_NEXTFLOW_LATEST AZURE_NEXTFLOW_LATEST = __main__.AZURE_NEXTFLOW_LATEST HPC_NEXTFLOW_LATEST = __main__.HPC_NEXTFLOW_LATEST +ABORT_JOB_STATES = __main__.ABORT_JOB_STATES # Create the job group @click.group(cls=pass_debug_to_subcommands()) From 7869693d580e1f777526411ae8176f0709a2c81e Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 12:42:40 +0100 Subject: [PATCH 34/41] ci: update versions --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ddb654a..ae172883 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11"] + python-version: ["3.9", "3.11", "3.13"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -145,7 +145,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From dac4d2c89ec9ec2c6ede4fe00acbb6cf6bbf1e3a Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 12:43:30 +0100 Subject: [PATCH 35/41] refactor: add constants in own file --- cloudos_cli/__main__.py | 11 ----------- cloudos_cli/bash/cli.py | 4 +--- cloudos_cli/clos.py | 8 +------- cloudos_cli/configure/configure.py | 6 +----- cloudos_cli/constants.py | 24 ++++++++++++++++++++++++ cloudos_cli/cromwell/cli.py | 5 +---- cloudos_cli/jobs/cli.py | 27 ++++++++++++++------------- 7 files changed, 42 insertions(+), 43 deletions(-) create mode 100644 cloudos_cli/constants.py diff --git a/cloudos_cli/__main__.py b/cloudos_cli/__main__.py index e07f64ed..ce104c9c 100644 --- a/cloudos_cli/__main__.py +++ b/cloudos_cli/__main__.py @@ -27,17 +27,6 @@ from cloudos_cli.link.cli import link -# GLOBAL CONSTANTS - Keep these for backward compatibility -JOB_COMPLETED = 'completed' -REQUEST_INTERVAL_CROMWELL = 30 -AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] -AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] -HPC_NEXTFLOW_VERSIONS = ['22.10.8'] -AWS_NEXTFLOW_LATEST = '24.04.4' -AZURE_NEXTFLOW_LATEST = '22.11.1-edge' -HPC_NEXTFLOW_LATEST = '22.10.8' -ABORT_JOB_STATES = ['running', 'initializing'] - # Install the custom exception handler sys.excepthook = custom_exception_handler diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py index b2a5784d..53133a95 100644 --- a/cloudos_cli/bash/cli.py +++ b/cloudos_cli/bash/cli.py @@ -6,14 +6,12 @@ from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL from cloudos_cli.utils.array_job import generate_datasets_for_project from cloudos_cli.queue.queue import Queue +from cloudos_cli.constants import JOB_COMPLETED import sys import json from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands -from cloudos_cli import __main__ -JOB_COMPLETED = __main__.JOB_COMPLETED - @click.group(cls=pass_debug_to_subcommands()) def bash(): """CloudOS bash-specific job functionality.""" diff --git a/cloudos_cli/clos.py b/cloudos_cli/clos.py index 6e84e010..7f886aa4 100644 --- a/cloudos_cli/clos.py +++ b/cloudos_cli/clos.py @@ -12,13 +12,7 @@ import pandas as pd from cloudos_cli.utils.last_wf import youngest_workflow_id_by_name from datetime import datetime, timezone - - -# GLOBAL VARS -JOB_COMPLETED = 'completed' -JOB_FAILED = 'failed' -JOB_ABORTED = 'aborted' - +from cloudos_cli.constants import JOB_COMPLETED, JOB_FAILED, JOB_ABORTED @dataclass class Cloudos: diff --git a/cloudos_cli/configure/configure.py b/cloudos_cli/configure/configure.py index dbe1375c..6d331628 100644 --- a/cloudos_cli/configure/configure.py +++ b/cloudos_cli/configure/configure.py @@ -3,6 +3,7 @@ import configparser import click from cloudos_cli.logging.logger import update_command_context_from_click +from cloudos_cli.constants import CLOUDOS_URL, INIT_PROFILE class ConfigurationProfile: @@ -592,11 +593,6 @@ def load_profile_and_validate_data(self, ctx, init_profile, cloudos_url_default, return resolved_params -# Not part of the class, but related to configuration -# Global constants for CloudOS CLI -CLOUDOS_URL = 'https://cloudos.lifebit.ai' -INIT_PROFILE = 'initialisingProfile' - # Define all standard configuration keys with their default empty values # This is the single source of truth for configuration fields STANDARD_CONFIG_KEYS = { diff --git a/cloudos_cli/constants.py b/cloudos_cli/constants.py new file mode 100644 index 00000000..e3a53a80 --- /dev/null +++ b/cloudos_cli/constants.py @@ -0,0 +1,24 @@ +"""Global constants for CloudOS CLI.""" + +# Job status constants +JOB_COMPLETED = 'completed' +JOB_FAILED = 'failed' +JOB_ABORTED = 'aborted' + +# Nextflow version constants +AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] +AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] +HPC_NEXTFLOW_VERSIONS = ['22.10.8'] +AWS_NEXTFLOW_LATEST = '24.04.4' +AZURE_NEXTFLOW_LATEST = '22.11.1-edge' +HPC_NEXTFLOW_LATEST = '22.10.8' + +# Job abort states +ABORT_JOB_STATES = ['running', 'initializing'] + +# Request interval for Cromwell +REQUEST_INTERVAL_CROMWELL = 30 + +# Global constants for CloudOS CLI +CLOUDOS_URL = 'https://cloudos.lifebit.ai' +INIT_PROFILE = 'initialisingProfile' diff --git a/cloudos_cli/cromwell/cli.py b/cloudos_cli/cromwell/cli.py index cae421ab..af0f9ab4 100644 --- a/cloudos_cli/cromwell/cli.py +++ b/cloudos_cli/cromwell/cli.py @@ -8,10 +8,7 @@ from cloudos_cli.utils.resources import ssl_selector from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands - - -# Constants -REQUEST_INTERVAL_CROMWELL = 5 +from cloudos_cli.constants import REQUEST_INTERVAL_CROMWELL @click.group(cls=pass_debug_to_subcommands()) diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 6a7a1037..0404e12c 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -10,6 +10,17 @@ from cloudos_cli.related_analyses.related_analyses import related_analyses from cloudos_cli.configure.configure import with_profile_config, CLOUDOS_URL from cloudos_cli.link import Link +from cloudos_cli.constants import ( + JOB_COMPLETED, + REQUEST_INTERVAL_CROMWELL, + AWS_NEXTFLOW_VERSIONS, + AZURE_NEXTFLOW_VERSIONS, + HPC_NEXTFLOW_VERSIONS, + AWS_NEXTFLOW_LATEST, + AZURE_NEXTFLOW_LATEST, + HPC_NEXTFLOW_LATEST, + ABORT_JOB_STATES +) import json import copy import time @@ -17,19 +28,9 @@ import sys from rich.console import Console from cloudos_cli.utils.cli_helpers import pass_debug_to_subcommands -from cloudos_cli import __main__ - - -# Import constants from __main__ -JOB_COMPLETED = __main__.JOB_COMPLETED -REQUEST_INTERVAL_CROMWELL = __main__.REQUEST_INTERVAL_CROMWELL -AWS_NEXTFLOW_VERSIONS = __main__.AWS_NEXTFLOW_VERSIONS -AZURE_NEXTFLOW_VERSIONS = __main__.AZURE_NEXTFLOW_VERSIONS -HPC_NEXTFLOW_VERSIONS = __main__.HPC_NEXTFLOW_VERSIONS -AWS_NEXTFLOW_LATEST = __main__.AWS_NEXTFLOW_LATEST -AZURE_NEXTFLOW_LATEST = __main__.AZURE_NEXTFLOW_LATEST -HPC_NEXTFLOW_LATEST = __main__.HPC_NEXTFLOW_LATEST -ABORT_JOB_STATES = __main__.ABORT_JOB_STATES + + + # Create the job group @click.group(cls=pass_debug_to_subcommands()) From f099f1a2230c799691a543da65cac7ae7f7aeb9b Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 14:08:08 +0100 Subject: [PATCH 36/41] ci: filter by default queue --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae172883..f2ab1486 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: # Test filtering by only mine cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-only-mine --last-n-jobs 10 # Test filtering by queue - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "job_queue_nextflow" --last-n-jobs 10 + cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "cost_saving_standard_nextflow" --last-n-jobs 10 job_details: needs: job_run_and_status runs-on: ubuntu-latest From 7b92ab66fee71996bd5f2d9d83325a4dcc001ca1 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 14:08:28 +0100 Subject: [PATCH 37/41] ci: fix intermittent failure --- tests/test_cost/test_job_cost.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_cost/test_job_cost.py b/tests/test_cost/test_job_cost.py index 866b9d89..68d85448 100644 --- a/tests/test_cost/test_job_cost.py +++ b/tests/test_cost/test_job_cost.py @@ -287,11 +287,15 @@ def test_error_handling_401(self, mock_get): @mock.patch('cloudos_cli.cost.cost.retry_requests_get') def test_error_handling_400(self, mock_get): """Test error handling for 400 Bad Request""" - # Mock a 400 response + # Mock a 400 response that behaves like the real BadRequestException mock_response = MagicMock() mock_response.status_code = 400 - mock_get.return_value = mock_response - mock_get.side_effect = BadRequestException(mock_response) + mock_response.reason = "Bad Request" + mock_response.json.return_value = {"error": "Bad Request"} + + # Create a BadRequestException that matches what the real code would produce + exception = BadRequestException(mock_response) + mock_get.side_effect = exception with pytest.raises(ValueError) as excinfo: self.cost_viewer.display_costs(JOB_ID, WORKSPACE_ID, "stdout") From b7ac09684c36f20ba50e5a300487b37e2c061cf8 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 14:22:19 +0100 Subject: [PATCH 38/41] refactor: remove unused interpolations --- cloudos_cli/bash/cli.py | 4 ++-- cloudos_cli/jobs/cli.py | 10 +++++----- cloudos_cli/link/link.py | 8 ++++---- tests/test_error_messages.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py index 53133a95..50ac944d 100644 --- a/cloudos_cli/bash/cli.py +++ b/cloudos_cli/bash/cli.py @@ -231,7 +231,7 @@ def run_bash_job(ctx, print('\tTo further check your job status you can either go to ' + f'{j_url} or use the following command:\n' + '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + + '\t\t--profile my_profile \\\n' + f'\t\t--job-id {j_id}\n') @@ -562,5 +562,5 @@ def run_bash_array_job(ctx, print('\tTo further check your job status you can either go to ' + f'{j_url} or use the following command:\n' + '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + + '\t\t--profile my_profile \\\n' + f'\t\t--job-id {j_id}\n') diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 0404e12c..868844c3 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -454,7 +454,7 @@ def run(ctx, print('\tTo further check your job status you can either go to ' + f'{j_url} or use the following command:\n' + '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + + '\t\t--profile my_profile \\\n' + f'\t\t--job-id {j_id}\n') @@ -642,7 +642,7 @@ def job_workdir(ctx, # Display detailed information if verbose if verbose: - console.print(f'\n[bold]Additional information:[/bold]') + console.print('\n[bold]Additional information:[/bold]') console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') @@ -970,7 +970,7 @@ def job_results(ctx, # Display detailed information if verbose if verbose: - console.print(f'\n[bold]Additional information:[/bold]') + console.print('\n[bold]Additional information:[/bold]') console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') @@ -1049,7 +1049,7 @@ def job_results(ctx, print('\nDeletion cancelled.') return if verbose: - print(f'\nDeleting result directories from CloudOS...') + print('\nDeleting result directories from CloudOS...') # Proceed with deletion job = jb.Job(cloudos_url, apikey, None, workspace_id, None, None, workflow_id=1234, project_id="None", mainfile=None, importsfile=None, verify=verify_ssl) @@ -1865,7 +1865,7 @@ def clone_resume(ctx, print('\tTo further check your job status you can either go to ' + f'{j_url} or use the following command:\n' + '\tcloudos job status \\\n' + - f'\t\t--profile my_profile \\\n' + + '\t\t--profile my_profile \\\n' + f'\t\t--job-id {cloned_resumed_job_id}\n') diff --git a/cloudos_cli/link/link.py b/cloudos_cli/link/link.py index 417544aa..1edd3493 100644 --- a/cloudos_cli/link/link.py +++ b/cloudos_cli/link/link.py @@ -323,7 +323,7 @@ def link_job_results(self, job_id: str, workspace_id: str, session_id: str, veri results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) if results_path: - print(f'\tLinking results directory...') + print('\tLinking results directory...') if verbose: print(f'\t\tResults: {results_path}') self.link_folder(results_path, session_id) @@ -370,7 +370,7 @@ def link_job_workdir(self, job_id: str, workspace_id: str, session_id: str, veri workdir_path = cl.get_job_workdir(job_id, workspace_id, verify_ssl) if workdir_path: - print(f'\tLinking working directory...') + print('\tLinking working directory...') if verbose: print(f'\t\tWorkdir: {workdir_path}') self.link_folder(workdir_path.strip(), session_id) @@ -419,7 +419,7 @@ def link_job_logs(self, job_id: str, workspace_id: str, session_id: str, verify_ first_log_path = next(iter(logs_dict.values())) logs_dir = '/'.join(first_log_path.split('/')[:-1]) - print(f'\tLinking logs directory...') + print('\tLinking logs directory...') if verbose: print(f'\t\tLogs directory: {logs_dir}') self.link_folder(logs_dir, session_id) @@ -549,7 +549,7 @@ def link_path_with_validation(self, path: str, session_id: str, verify_ssl, proj fg='yellow', bold=True) if verbose: - print(f'\tLinking {path}...') + print('\tLinking {path}...') self.link_folder(path, session_id) diff --git a/tests/test_error_messages.py b/tests/test_error_messages.py index aa364339..eb6d893b 100644 --- a/tests/test_error_messages.py +++ b/tests/test_error_messages.py @@ -19,7 +19,7 @@ class TestJobErrorMessages(unittest.TestCase): def test_invalid_parameters_error_message(self): """Test that invalid parameter error uses period instead of colon""" with self.assertRaises(ValueError) as context: - raise ValueError(f'The provided parameters "test_param" are not valid. ') + raise ValueError('The provided parameters "test_param" are not valid. ') self.assertIn("not valid.", str(context.exception)) self.assertNotIn("not valid:", str(context.exception)) From 0b310f25d6f72e6612b08fe5deff23a756ec9340 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 14:33:30 +0100 Subject: [PATCH 39/41] style: remove whitespaces --- cloudos_cli/__main__.py | 6 +- cloudos_cli/jobs/cli.py | 36 +++--- cloudos_cli/link/cli.py | 30 ++--- .../related_analyses/related_analyses.py | 4 +- cloudos_cli/utils/details.py | 18 +-- cloudos_cli/utils/last_wf.py | 10 +- tests/test_cli_project_create.py | 14 +-- tests/test_clos/test_archive_integration.py | 6 +- tests/test_clos/test_archive_jobs.py | 8 +- .../test_clos/test_archive_status_checking.py | 10 +- tests/test_clos/test_cli_archive_command.py | 6 +- tests/test_clos/test_cli_unarchive_command.py | 6 +- tests/test_clos/test_create_project.py | 28 ++--- tests/test_clos/test_get_job_workdir.py | 58 +++++----- .../test_get_results_deletion_status.py | 108 +++++++++--------- .../test_get_workdir_deletion_status.py | 72 ++++++------ tests/test_clos/test_unarchive_integration.py | 6 +- tests/test_datasets/test_link.py | 6 +- tests/test_jobs/test_clone_job.py | 2 +- tests/test_jobs/test_delete_job_results.py | 70 ++++++------ tests/test_jobs/test_resume_job.py | 4 +- ...st_reset_procurement_organisation_image.py | 8 +- ...test_set_procurement_organisation_image.py | 6 +- utils/delete_project_jobs.sh | 2 +- 24 files changed, 262 insertions(+), 262 deletions(-) diff --git a/cloudos_cli/__main__.py b/cloudos_cli/__main__.py index ce104c9c..ae89133b 100644 --- a/cloudos_cli/__main__.py +++ b/cloudos_cli/__main__.py @@ -32,7 +32,7 @@ @click.group(cls=pass_debug_to_subcommands()) -@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', +@click.option('--debug', is_flag=True, help='Show detailed error information and tracebacks', is_eager=True, expose_value=False, callback=setup_debug) @click.version_option(__version__) @click.pass_context @@ -44,10 +44,10 @@ def run_cloudos_cli(ctx): if ctx.invoked_subcommand not in ['datasets']: print(run_cloudos_cli.__doc__ + '\n') print('Version: ' + __version__ + '\n') - + # Load shared configuration (handles missing profiles and fields gracefully) shared_config = get_shared_config() - + # Automatically build default_map from registered commands ctx.default_map = build_default_map_for_group(run_cloudos_cli, shared_config) diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 868844c3..d82d78d9 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -497,7 +497,7 @@ def job_status(ctx, profile): """Get the status of a CloudOS job.""" # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator - + print('Executing status...') verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) if verbose: @@ -1256,13 +1256,13 @@ def list_jobs(ctx, # apikey, cloudos_url, and workspace_id are now automatically resolved by the decorator verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - + # Pass table_columns directly to create_job_list_table for validation and processing selected_columns = table_columns # Only set outfile if not using stdout if output_format != 'stdout': outfile = output_basename + '.' + output_format - + print('Executing list...') if verbose: print('\t...Preparing objects') @@ -1279,7 +1279,7 @@ def list_jobs(ctx, if not isinstance(page_size, int) or page_size < 1: raise ValueError('Please, use a positive integer (>= 1) for the --page-size parameter') - + # Validate page_size limit - must be done before API call if page_size > 100: click.secho('Error: Page size cannot exceed 100. Please use --page-size with a value <= 100', fg='red', err=True) @@ -1295,11 +1295,11 @@ def list_jobs(ctx, filter_owner=filter_owner, filter_queue=filter_queue, last=last) - + # Extract jobs and pagination metadata from result my_jobs_r = result['jobs'] pagination_metadata = result['pagination_metadata'] - + # Validate requested page exists if pagination_metadata: total_jobs = pagination_metadata.get('Pagination-Count', 0) @@ -1311,7 +1311,7 @@ def list_jobs(ctx, click.secho(f'Error: Page {page} does not exist. There are only {total_pages} page(s) available with {total_jobs} total job(s). ' f'Please use --page with a value between 1 and {total_pages}', fg='red', err=True) raise SystemExit(1) - + if len(my_jobs_r) == 0: # Check if any filtering options are being used filters_used = any([ @@ -1591,26 +1591,26 @@ def archive_unarchive_jobs(ctx, action = "archive" if target_archived_state else "unarchive" action_past = "archived" if target_archived_state else "unarchived" action_ing = "archiving" if target_archived_state else "unarchiving" - + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) print(f'{action_ing.capitalize()} jobs...') - + if verbose: print('\t...Preparing objects') - + cl = Cloudos(cloudos_url, apikey, None) - + if verbose: print('\tThe following Cloudos object was created:') print('\t' + str(cl) + '\n') print(f'\t{action_ing.capitalize()} jobs in the following workspace: {workspace_id}') - + # check if the user provided an empty job list jobs = job_ids.replace(' ', '') if not jobs: raise ValueError(f'No job IDs provided. Please specify at least one job ID to {action}.') jobs_list = [job for job in jobs.split(',') if job] # Filter out empty strings - + # Check for duplicate job IDs duplicates = [job_id for job_id in set(jobs_list) if jobs_list.count(job_id) > 1] if duplicates: @@ -1620,29 +1620,29 @@ def archive_unarchive_jobs(ctx, jobs_list = list(dict.fromkeys(jobs_list)) if verbose: print(f'\tDuplicate job IDs removed. Processing {len(jobs_list)} unique job(s).') - + # Check archive status for all jobs status_check = cl.check_jobs_archive_status(jobs_list, workspace_id, target_archived_state=target_archived_state, verify=verify_ssl, verbose=verbose) valid_jobs = status_check['valid_jobs'] already_processed = status_check['already_processed'] invalid_jobs = status_check['invalid_jobs'] - + # Report invalid jobs (but continue processing valid ones) for job_id, error_msg in invalid_jobs.items(): click.secho(f"Failed to get status for job {job_id}, please make sure it exists in the workspace: {error_msg}", fg='yellow', bold=True) - + if not valid_jobs and not already_processed: # All jobs were invalid - exit gracefully click.secho('No valid job IDs found. Please check that the job IDs exist and are accessible.', fg='yellow', bold=True) return - + if not valid_jobs: if len(already_processed) == 1: click.secho(f"Job '{already_processed[0]}' is already {action_past}. No action needed.", fg='cyan', bold=True) else: click.secho(f"All {len(already_processed)} jobs are already {action_past}. No action needed.", fg='cyan', bold=True) return - + try: # Call the appropriate action method if target_archived_state: diff --git a/cloudos_cli/link/cli.py b/cloudos_cli/link/cli.py index 01f1fce4..55aaf8af 100644 --- a/cloudos_cli/link/cli.py +++ b/cloudos_cli/link/cli.py @@ -67,23 +67,23 @@ def link(ctx, profile): """ Link folders to an interactive analysis session. - + This command is used to link folders to an active interactive analysis session for direct access to data. - + PATH: Optional path to link (S3). Required if --job-id is not provided. - + Two modes of operation: - + 1. Job-based linking (--job-id): Links job-related folders. By default, links results, workdir, and logs folders. Use --results, --workdir, or --logs flags to link only specific folders. - + 2. Direct path linking (PATH argument): Links a specific S3 path. - + Examples: - + # Link all job folders (results, workdir, logs) cloudos link --job-id 12345 --session-id abc123 @@ -95,26 +95,26 @@ def link(ctx, """ print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') - + verify_ssl = ssl_selector(disable_ssl_verification, ssl_cert) - + # Validate input parameters if not job_id and not path: raise click.UsageError("Either --job-id or PATH argument must be provided.") - + if job_id and path: raise click.UsageError("Cannot use both --job-id and PATH argument. Please provide only one.") - + # Validate folder-specific flags only work with --job-id if (results or workdir or logs) and not job_id: raise click.UsageError("--results, --workdir, and --logs flags can only be used with --job-id.") - + # If no specific folders are selected with job-id, link all by default if job_id and not (results or workdir or logs): results = True workdir = True logs = True - + if verbose: print('Using the following parameters:') print(f'\tCloudOS url: {cloudos_url}') @@ -127,7 +127,7 @@ def link(ctx, print(f'\tLink logs: {logs}') else: print(f'\tPath: {path}') - + # Initialize Link client link_client = Link( cloudos_url=cloudos_url, @@ -137,7 +137,7 @@ def link(ctx, project_name=project_name, verify=verify_ssl ) - + try: if job_id: # Job-based linking diff --git a/cloudos_cli/related_analyses/related_analyses.py b/cloudos_cli/related_analyses/related_analyses.py index 06e5883d..0a18087b 100644 --- a/cloudos_cli/related_analyses/related_analyses.py +++ b/cloudos_cli/related_analyses/related_analyses.py @@ -94,7 +94,7 @@ def save_as_json(data, filename): def save_as_stdout(data, j_workdir_parent, cloudos_url="https://cloudos.lifebit.ai"): """Display related analyses in a formatted table with pagination. - + Parameters ---------- data : dict @@ -178,7 +178,7 @@ def format_cost(cost): # Display with pagination show_error = None # Track error messages to display - + while True: start = current_page * limit end = start + limit diff --git a/cloudos_cli/utils/details.py b/cloudos_cli/utils/details.py index 02a5e832..83501e6c 100644 --- a/cloudos_cli/utils/details.py +++ b/cloudos_cli/utils/details.py @@ -205,7 +205,7 @@ def create_job_details(j_details_h, job_id, output_format, output_basename, para # calculate the run time start_time_raw = j_details_h.get("startTime") end_time_raw = j_details_h.get("endTime") - + if start_time_raw and end_time_raw: try: start_dt = datetime.fromisoformat(str(start_time_raw).replace('Z', '+00:00')) @@ -390,7 +390,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ terminal_width = os.get_terminal_size().columns except OSError: terminal_width = 80 # Default fallback - + # Define column priority groups for small terminals priority_columns = { 'essential': ['status', 'name', 'pipeline', 'id'], # ~40 chars minimum @@ -398,7 +398,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ 'useful': [ 'submit_time', 'end_time', 'commit'], # +50 chars 'extended': [ 'resources', 'storage_type'] # +30 chars } - + # Define all available columns with their configurations all_columns = { 'status': {"header": "Status", "style": "cyan", "no_wrap": True, "min_width": 6, "max_width": 6}, @@ -442,7 +442,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ raise ValueError(f"Invalid column names: {', '.join(invalid_cols)}. " f"Valid columns are: {', '.join(valid_columns)}") columns_to_show = selected_columns # Preserve user-specified order - + if not jobs: console.print("\n[yellow]No jobs found matching the criteria.[/yellow]") # Still show pagination info even when no jobs @@ -456,10 +456,10 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ console.print(f"[cyan]Page:[/cyan] {current_page} of {total_pages}") console.print(f"[cyan]Jobs on this page:[/cyan] {len(jobs)}") return - + # Create table table = Table(title="Job List") - + # Add columns to table for col_key in columns_to_show: col_config = all_columns[col_key] @@ -471,7 +471,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ min_width=col_config.get("min_width"), max_width=col_config.get("max_width") ) - + # Process each job for job in jobs: # Status with colored and bold ANSI symbols @@ -635,9 +635,9 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ # Add row to table with only selected columns row_values = [column_values[col] for col in columns_to_show] table.add_row(*row_values) - + console.print(table) - + # Display pagination info at the bottom if pagination_metadata: total_jobs = pagination_metadata.get('Pagination-Count', 0) diff --git a/cloudos_cli/utils/last_wf.py b/cloudos_cli/utils/last_wf.py index aa52ebab..029831a9 100644 --- a/cloudos_cli/utils/last_wf.py +++ b/cloudos_cli/utils/last_wf.py @@ -19,7 +19,7 @@ def _parse_iso8601_z(dt_str): def youngest_workflow_id_by_name(content, target_name, ignore_case=False, return_workflow=True): """ Find the most recently created workflow whose .name matches `target_name`. - + Parameters ---------- content : dict @@ -30,7 +30,7 @@ def youngest_workflow_id_by_name(content, target_name, ignore_case=False, return If True, case-insensitive comparison. return_workflow : bool, default False If True, return the whole workflow dict; else return its _id. - + Returns ------- str | dict | None @@ -42,16 +42,16 @@ def youngest_workflow_id_by_name(content, target_name, ignore_case=False, return matches = [wf for wf in workflows if wf.get('name', '').lower() == target_cmp] else: matches = [wf for wf in workflows if wf.get('name') == target_name] - + if not matches: return None - + def sort_key(wf): created = _parse_iso8601_z(wf.get('createdAt')) updated = _parse_iso8601_z(wf.get('updatedAt')) # Prefer createdAt; fall back to updatedAt; else epoch 0 return created or updated or datetime.fromtimestamp(0, tz=timezone.utc) - + youngest = max(matches, key=sort_key) # keep structure as dictionary, will return inner dictionary just for the selected workflow youngest_d = {"workflows":[ youngest ]} diff --git a/tests/test_cli_project_create.py b/tests/test_cli_project_create.py index 8f7cc8a8..0f384daa 100644 --- a/tests/test_cli_project_create.py +++ b/tests/test_cli_project_create.py @@ -8,13 +8,13 @@ def test_project_create_command_exists(): Test that the 'project create' command exists and shows proper help """ runner = CliRunner() - + # Test that project create command exists result = runner.invoke(run_cloudos_cli, ['project', 'create', '--help']) - + # Command should exist and not error out assert result.exit_code == 0 - + # Check that the help text contains expected options assert 'Create a new project in CloudOS' in result.output assert '--new-project' in result.output @@ -28,10 +28,10 @@ def test_project_create_command_structure(): Test that the 'project create' command has the correct structure and options """ runner = CliRunner() - + # Test that the command exists and can show help without making API calls result = runner.invoke(run_cloudos_cli, ['project', 'create', '--help']) - + # Command should exist and show help properly assert result.exit_code == 0 assert 'Create a new project in CloudOS' in result.output @@ -44,10 +44,10 @@ def test_project_group_contains_create_command(): Test that the 'project' group contains the 'create' command """ runner = CliRunner() - + # Test that project group shows create command result = runner.invoke(run_cloudos_cli, ['project', '--help']) - + assert result.exit_code == 0 assert 'create' in result.output assert 'Create a new project in CloudOS' in result.output diff --git a/tests/test_clos/test_archive_integration.py b/tests/test_clos/test_archive_integration.py index eca850e1..f65f95b1 100644 --- a/tests/test_clos/test_archive_integration.py +++ b/tests/test_clos/test_archive_integration.py @@ -9,7 +9,7 @@ def test_job_archive_successful_flow(): """Test a successful job archiving flow end-to-end.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock checking if job is archived (should return empty for unarchived job) m.get( @@ -48,7 +48,7 @@ def test_job_archive_successful_flow(): def test_job_archive_multiple_jobs_successful_flow(): """Test successful archiving of multiple jobs.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock job status checks for multiple jobs for job_id in ['job1', 'job2', 'job3']: @@ -89,7 +89,7 @@ def test_job_archive_multiple_jobs_successful_flow(): def test_job_archive_mixed_valid_invalid_jobs(): """Test archiving when some jobs are valid and others are not.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock job status check - valid job (not archived) m.get( diff --git a/tests/test_clos/test_archive_jobs.py b/tests/test_clos/test_archive_jobs.py index 0f3e6fa4..d1ce7c8f 100644 --- a/tests/test_clos/test_archive_jobs.py +++ b/tests/test_clos/test_archive_jobs.py @@ -38,7 +38,7 @@ def test_archive_jobs_correct_response(self): timestamp = request_data["update"]["archived"]["archivalTimestamp"] assert isinstance(timestamp, str) assert timestamp.endswith("Z") - + def test_archive_jobs_bad_request_response(self): """Test archive jobs with bad request response.""" cloudos_url = "https://cloudos.lifebit.ai" @@ -57,7 +57,7 @@ def test_archive_jobs_bad_request_response(self): cl = Cloudos(cloudos_url, apikey, None) with pytest.raises(BadRequestException): cl.archive_jobs(job_ids, workspace_id) - + def test_archive_jobs_with_ssl_verification(self): """Test archive jobs with SSL verification disabled.""" cloudos_url = "https://cloudos.lifebit.ai" @@ -76,7 +76,7 @@ def test_archive_jobs_with_ssl_verification(self): response = cl.archive_jobs(job_ids, workspace_id, verify=False) assert response.status_code == 200 - + def test_archive_jobs_single_job(self): """Test archiving a single job.""" cloudos_url = "https://cloudos.lifebit.ai" @@ -98,7 +98,7 @@ def test_archive_jobs_single_job(self): request_data = json.loads(m.last_request.text) assert len(request_data["jobIds"]) == 1 assert request_data["jobIds"][0] == job_ids[0] - + def test_archive_jobs_multiple_jobs(self): """Test archiving multiple jobs.""" cloudos_url = "https://cloudos.lifebit.ai" diff --git a/tests/test_clos/test_archive_status_checking.py b/tests/test_clos/test_archive_status_checking.py index 3770b5e9..f2adf7e4 100644 --- a/tests/test_clos/test_archive_status_checking.py +++ b/tests/test_clos/test_archive_status_checking.py @@ -9,7 +9,7 @@ def test_archive_already_archived_job(): """Test archiving a job that is already archived.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock checking if job is archived (should return the job since it's archived) m.get( @@ -36,7 +36,7 @@ def test_archive_already_archived_job(): def test_unarchive_already_unarchived_job(): """Test unarchiving a job that is already unarchived.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock checking if job is archived (should return empty for unarchived job) m.get( @@ -70,7 +70,7 @@ def test_unarchive_already_unarchived_job(): def test_archive_mixed_status_jobs(): """Test archiving a mix of archived and unarchived jobs.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock job status - one archived, one not archived m.get( @@ -113,7 +113,7 @@ def test_archive_mixed_status_jobs(): def test_unarchive_mixed_status_jobs(): """Test unarchiving a mix of archived and unarchived jobs.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock job status - one archived, one not archived m.get( @@ -156,7 +156,7 @@ def test_unarchive_mixed_status_jobs(): def test_archive_verbose_already_archived(): """Test archiving with verbose output for already archived job.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock checking if job is archived (should return the job since it's archived) m.get( diff --git a/tests/test_clos/test_cli_archive_command.py b/tests/test_clos/test_cli_archive_command.py index 96aa05ae..c13750fc 100644 --- a/tests/test_clos/test_cli_archive_command.py +++ b/tests/test_clos/test_cli_archive_command.py @@ -34,7 +34,7 @@ def test_job_archive_help(): def test_job_archive_command_structure(job_ids, expected_count): """Test that the archive command has the expected structure and validates job IDs.""" runner = CliRunner() - + # Test that the command fails when missing required parameters result = runner.invoke(run_cloudos_cli, ['job', 'archive']) assert result.exit_code != 0 @@ -44,7 +44,7 @@ def test_job_archive_command_structure(job_ids, expected_count): def test_job_archive_empty_job_ids(): """Test that empty job IDs raise appropriate error.""" runner = CliRunner() - + with requests_mock.Mocker(): result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', @@ -60,7 +60,7 @@ def test_job_archive_empty_job_ids(): def test_job_archive_invalid_job_ids(): """Test archiving with invalid job IDs (jobs that don't exist).""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock the job status check to fail (job doesn't exist in either list) m.get( diff --git a/tests/test_clos/test_cli_unarchive_command.py b/tests/test_clos/test_cli_unarchive_command.py index 65502ef3..9a664413 100644 --- a/tests/test_clos/test_cli_unarchive_command.py +++ b/tests/test_clos/test_cli_unarchive_command.py @@ -34,7 +34,7 @@ def test_job_unarchive_help(): def test_job_unarchive_command_structure(job_ids, expected_count): """Test that the unarchive command has the expected structure and validates job IDs.""" runner = CliRunner() - + # Test that the command fails when missing required parameters result = runner.invoke(run_cloudos_cli, ['job', 'unarchive']) assert result.exit_code != 0 @@ -44,7 +44,7 @@ def test_job_unarchive_command_structure(job_ids, expected_count): def test_job_unarchive_empty_job_ids(): """Test that empty job IDs raise appropriate error.""" runner = CliRunner() - + with requests_mock.Mocker(): result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', @@ -60,7 +60,7 @@ def test_job_unarchive_empty_job_ids(): def test_job_unarchive_invalid_job_ids(): """Test unarchiving with invalid job IDs (jobs that don't exist).""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock the job status check to fail (job doesn't exist in either list) m.get( diff --git a/tests/test_clos/test_create_project.py b/tests/test_clos/test_create_project.py index 8d219ec8..ba046a58 100644 --- a/tests/test_clos/test_create_project.py +++ b/tests/test_clos/test_create_project.py @@ -29,7 +29,7 @@ def test_create_project_correct_response(): "Content-Type": "application/json;charset=UTF-8", "apikey": APIKEY } - + # mock POST method with the .json responses.add( responses.POST, @@ -38,13 +38,13 @@ def test_create_project_correct_response(): headers=header, match=[matchers.json_params_matcher(expected_data)], status=200) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # get mock response project_id = clos.create_project(WORKSPACE_ID, PROJECT_NAME) - + # check the response assert isinstance(project_id, str) assert project_id == '64a7b1c2f8d9e1a2b3c4d5e6' @@ -62,7 +62,7 @@ def test_create_project_bad_request(): error_json = json.dumps(error_message) search_str = f"teamId={WORKSPACE_ID}" expected_data = {"name": PROJECT_NAME} - + # mock POST method with error response responses.add( responses.POST, @@ -70,10 +70,10 @@ def test_create_project_bad_request(): body=error_json, match=[matchers.json_params_matcher(expected_data)], status=400) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # test that BadRequestException is raised with pytest.raises(BadRequestException): clos.create_project(WORKSPACE_ID, PROJECT_NAME) @@ -87,7 +87,7 @@ def test_create_project_unauthorized(): """ search_str = f"teamId={WORKSPACE_ID}" expected_data = {"name": PROJECT_NAME} - + # mock POST method with 401 response responses.add( responses.POST, @@ -95,10 +95,10 @@ def test_create_project_unauthorized(): body="Unauthorized", match=[matchers.json_params_matcher(expected_data)], status=401) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # test that ValueError is raised for unauthorized access with pytest.raises(ValueError, match="It seems your API key is not authorised"): clos.create_project(WORKSPACE_ID, PROJECT_NAME) @@ -113,7 +113,7 @@ def test_create_project_with_ssl_verification(): create_json = load_json_file(OUTPUT) search_str = f"teamId={WORKSPACE_ID}" expected_data = {"name": PROJECT_NAME} - + # mock POST method with the .json responses.add( responses.POST, @@ -121,13 +121,13 @@ def test_create_project_with_ssl_verification(): body=create_json, match=[matchers.json_params_matcher(expected_data)], status=200) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # get mock response with SSL verification disabled project_id = clos.create_project(WORKSPACE_ID, PROJECT_NAME, verify=False) - + # check the response assert isinstance(project_id, str) assert project_id == '64a7b1c2f8d9e1a2b3c4d5e6' diff --git a/tests/test_clos/test_get_job_workdir.py b/tests/test_clos/test_get_job_workdir.py index 0e3711dc..e95674ef 100644 --- a/tests/test_clos/test_get_job_workdir.py +++ b/tests/test_clos/test_get_job_workdir.py @@ -33,12 +33,12 @@ def test_get_job_workdir_aws_correct_response(): "s3Prefix": "jobs/logs/path" } } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET method for job details responses.add( responses.GET, @@ -47,7 +47,7 @@ def test_get_job_workdir_aws_correct_response(): headers=header, status=200 ) - + # Mock cloud detection requests (for AWS, we don't need the azure endpoint) responses.add( responses.GET, @@ -55,13 +55,13 @@ def test_get_job_workdir_aws_correct_response(): json=None, status=404 ) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # get workdir path workdir_path = clos.get_job_workdir(JOB_ID, WORKSPACE_ID) - + # check the response (workdir path should replace /logs with /work) expected_path = "s3://my-bucket/jobs/work/path" assert workdir_path == expected_path @@ -85,12 +85,12 @@ def test_get_job_workdir_azure_correct_response(): "blobPrefix": "jobs/logs/path" } } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET method for job details responses.add( responses.GET, @@ -99,7 +99,7 @@ def test_get_job_workdir_azure_correct_response(): headers=header, status=200 ) - + # Mock Azure cloud detection request responses.add( responses.GET, @@ -107,13 +107,13 @@ def test_get_job_workdir_azure_correct_response(): json={"storage": {"storageAccount": "testaccount"}}, status=200 ) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # get workdir path workdir_path = clos.get_job_workdir(JOB_ID, WORKSPACE_ID) - + # check the response (workdir path should replace /logs with /work) expected_path = f"az://testaccount.blob.core.windows.net/my-container/jobs/work/path" assert workdir_path == expected_path @@ -132,12 +132,12 @@ def test_get_job_workdir_workspace_mismatch(): "resumeWorkDir": WORKDIR_ID, "status": "completed" } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET method for job details responses.add( responses.GET, @@ -146,14 +146,14 @@ def test_get_job_workdir_workspace_mismatch(): headers=header, status=200 ) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # Test that it raises ValueError for workspace mismatch with pytest.raises(ValueError) as error: clos.get_job_workdir(JOB_ID, WORKSPACE_ID) - + assert "Workspace provided or configured is different from workspace where the job was executed" in str(error.value) @@ -166,12 +166,12 @@ def test_get_job_workdir_job_not_found(): # prepare error message error_message = {"statusCode": 404, "code": "NotFound", "message": "Job not found.", "time": "2022-11-23_17:31:07"} - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # mock GET method with error response responses.add( responses.GET, @@ -180,7 +180,7 @@ def test_get_job_workdir_job_not_found(): headers=header, status=404 ) - + # Test that it raises BadRequestException with pytest.raises(BadRequestException): clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) @@ -197,7 +197,7 @@ def test_get_job_workdir_unauthorized(): "Content-type": "application/json", "apikey": APIKEY } - + # mock GET method with 401 response responses.add( responses.GET, @@ -206,7 +206,7 @@ def test_get_job_workdir_unauthorized(): headers=header, status=401 ) - + # Test that it raises NotAuthorisedException with pytest.raises(NotAuthorisedException): clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) @@ -226,18 +226,18 @@ def test_get_job_workdir_unsupported_cloud(): "resumeWorkDir": WORKDIR_ID, "status": "completed" } - + # Mock folder details response with unsupported folderType folder_response = [{ "folderType": "UnsupportedFolder", "someOtherField": "value" }] - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET method for job details responses.add( responses.GET, @@ -246,7 +246,7 @@ def test_get_job_workdir_unsupported_cloud(): headers=header, status=200 ) - + # Mock GET method for folder details responses.add( responses.GET, @@ -255,12 +255,12 @@ def test_get_job_workdir_unsupported_cloud(): headers=header, status=200 ) - + # start cloudOS service clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) - + # Test that it raises ValueError for unsupported cloud provider with pytest.raises(ValueError) as error: clos.get_job_workdir(JOB_ID, WORKSPACE_ID) - + assert "Unsupported cloud provider" in str(error.value) diff --git a/tests/test_clos/test_get_results_deletion_status.py b/tests/test_clos/test_get_results_deletion_status.py index ed19c3b2..4211ab53 100644 --- a/tests/test_clos/test_get_results_deletion_status.py +++ b/tests/test_clos/test_get_results_deletion_status.py @@ -34,7 +34,7 @@ def test_get_results_deletion_status_ready(): "folderId": RESULTS_FOLDER_ID } } - + # Mock project content response (list of folders including Analysis Results) project_content_response = { "folders": [ @@ -49,7 +49,7 @@ def test_get_results_deletion_status_ready(): ], "files": [] } - + # Mock datasets API response with job results folder datasets_response = { "folders": [ @@ -70,12 +70,12 @@ def test_get_results_deletion_status_ready(): ], "files": [] } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET method for job details responses.add( responses.GET, @@ -84,7 +84,7 @@ def test_get_results_deletion_status_ready(): headers=header, status=200 ) - + # Mock GET method for projects API (used by Datasets class) responses.add( responses.GET, @@ -93,7 +93,7 @@ def test_get_results_deletion_status_ready(): headers=header, status=200 ) - + # Mock GET method for project content responses.add( responses.GET, @@ -102,7 +102,7 @@ def test_get_results_deletion_status_ready(): headers=header, status=200 ) - + # Mock GET method for datasets items (Analysis Results folder contents) responses.add( responses.GET, @@ -111,11 +111,11 @@ def test_get_results_deletion_status_ready(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_results_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["job_name"] == "test_job_results" @@ -150,7 +150,7 @@ def test_get_results_deletion_status_scheduled_for_deletion(): } } } - + # Mock project content response project_content_response = { "folders": [ @@ -161,7 +161,7 @@ def test_get_results_deletion_status_scheduled_for_deletion(): ], "files": [] } - + # Mock datasets API response with scheduledForDeletion status datasets_response = { "folders": [ @@ -182,12 +182,12 @@ def test_get_results_deletion_status_scheduled_for_deletion(): ], "files": [] } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -196,7 +196,7 @@ def test_get_results_deletion_status_scheduled_for_deletion(): headers=header, status=200 ) - + # Mock GET method for projects API (used by Datasets class) responses.add( responses.GET, @@ -205,7 +205,7 @@ def test_get_results_deletion_status_scheduled_for_deletion(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v2/datasets?projectId={PROJECT_ID}&teamId={WORKSPACE_ID}", @@ -213,7 +213,7 @@ def test_get_results_deletion_status_scheduled_for_deletion(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/datasets/{ANALYSIS_RESULTS_FOLDER_ID}/items", @@ -221,11 +221,11 @@ def test_get_results_deletion_status_scheduled_for_deletion(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_results_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["job_name"] == "test_job_scheduled" @@ -261,7 +261,7 @@ def test_get_results_deletion_status_deleting(): } } } - + # Mock project content response project_content_response = { "folders": [ @@ -272,7 +272,7 @@ def test_get_results_deletion_status_deleting(): ], "files": [] } - + # Mock datasets API response with deleting status datasets_response = { "folders": [ @@ -287,12 +287,12 @@ def test_get_results_deletion_status_deleting(): ], "files": [] } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -301,7 +301,7 @@ def test_get_results_deletion_status_deleting(): headers=header, status=200 ) - + # Mock GET method for projects API (used by Datasets class) responses.add( responses.GET, @@ -310,7 +310,7 @@ def test_get_results_deletion_status_deleting(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v2/datasets?projectId={PROJECT_ID}&teamId={WORKSPACE_ID}", @@ -318,7 +318,7 @@ def test_get_results_deletion_status_deleting(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/datasets/{ANALYSIS_RESULTS_FOLDER_ID}/items", @@ -326,11 +326,11 @@ def test_get_results_deletion_status_deleting(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_results_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["status"] == "deleting" @@ -362,7 +362,7 @@ def test_get_results_deletion_status_deleted(): } } } - + # Mock project content response project_content_response = { "folders": [ @@ -373,7 +373,7 @@ def test_get_results_deletion_status_deleted(): ], "files": [] } - + # Mock datasets API response with deleted status datasets_response = { "folders": [ @@ -388,12 +388,12 @@ def test_get_results_deletion_status_deleted(): ], "files": [] } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -402,7 +402,7 @@ def test_get_results_deletion_status_deleted(): headers=header, status=200 ) - + # Mock GET method for projects API (used by Datasets class) responses.add( responses.GET, @@ -411,7 +411,7 @@ def test_get_results_deletion_status_deleted(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v2/datasets?projectId={PROJECT_ID}&teamId={WORKSPACE_ID}", @@ -419,7 +419,7 @@ def test_get_results_deletion_status_deleted(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/datasets/{ANALYSIS_RESULTS_FOLDER_ID}/items", @@ -427,11 +427,11 @@ def test_get_results_deletion_status_deleted(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_results_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["status"] == "deleted" @@ -463,7 +463,7 @@ def test_get_results_deletion_status_failed_to_delete(): } } } - + # Mock project content response project_content_response = { "folders": [ @@ -474,7 +474,7 @@ def test_get_results_deletion_status_failed_to_delete(): ], "files": [] } - + # Mock datasets API response with failedToDelete status datasets_response = { "folders": [ @@ -489,12 +489,12 @@ def test_get_results_deletion_status_failed_to_delete(): ], "files": [] } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -503,7 +503,7 @@ def test_get_results_deletion_status_failed_to_delete(): headers=header, status=200 ) - + # Mock GET method for projects API (used by Datasets class) responses.add( responses.GET, @@ -512,7 +512,7 @@ def test_get_results_deletion_status_failed_to_delete(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v2/datasets?projectId={PROJECT_ID}&teamId={WORKSPACE_ID}", @@ -520,7 +520,7 @@ def test_get_results_deletion_status_failed_to_delete(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/datasets/{ANALYSIS_RESULTS_FOLDER_ID}/items", @@ -528,11 +528,11 @@ def test_get_results_deletion_status_failed_to_delete(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_results_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["status"] == "failedToDelete" @@ -558,7 +558,7 @@ def test_get_results_deletion_status_alternative_folder_name(): "folderId": RESULTS_FOLDER_ID } } - + # Mock project content response with alternative folder name project_content_response = { "folders": [ @@ -569,7 +569,7 @@ def test_get_results_deletion_status_alternative_folder_name(): ], "files": [] } - + # Mock datasets API response datasets_response = { "folders": [ @@ -584,12 +584,12 @@ def test_get_results_deletion_status_alternative_folder_name(): ], "files": [] } - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -598,7 +598,7 @@ def test_get_results_deletion_status_alternative_folder_name(): headers=header, status=200 ) - + # Mock GET method for projects API (used by Datasets class) responses.add( responses.GET, @@ -607,7 +607,7 @@ def test_get_results_deletion_status_alternative_folder_name(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v2/datasets?projectId={PROJECT_ID}&teamId={WORKSPACE_ID}", @@ -615,7 +615,7 @@ def test_get_results_deletion_status_alternative_folder_name(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/datasets/{ANALYSIS_RESULTS_FOLDER_ID}/items", @@ -623,11 +623,11 @@ def test_get_results_deletion_status_alternative_folder_name(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_results_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["job_name"] == "test_job_alt" diff --git a/tests/test_clos/test_get_workdir_deletion_status.py b/tests/test_clos/test_get_workdir_deletion_status.py index 7ac7bec0..b6eb8293 100644 --- a/tests/test_clos/test_get_workdir_deletion_status.py +++ b/tests/test_clos/test_get_workdir_deletion_status.py @@ -28,7 +28,7 @@ def test_get_workdir_deletion_status_ready(): "folderId": WORKDIR_FOLDER_ID, } } - + # Mock folder details response with ready status folder_response = [{ "_id": WORKDIR_FOLDER_ID, @@ -44,12 +44,12 @@ def test_get_workdir_deletion_status_ready(): "email": "test@example.com" } }] - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET method for job details responses.add( responses.GET, @@ -58,7 +58,7 @@ def test_get_workdir_deletion_status_ready(): headers=header, status=200 ) - + # Mock GET method for folder details with status filters responses.add( responses.GET, @@ -67,11 +67,11 @@ def test_get_workdir_deletion_status_ready(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_workdir_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["job_name"] == "test_job" @@ -102,7 +102,7 @@ def test_get_workdir_deletion_status_scheduled_for_deletion(): } } } - + # Mock folder details response with scheduledForDeletion status folder_response = [{ "_id": WORKDIR_FOLDER_ID, @@ -118,12 +118,12 @@ def test_get_workdir_deletion_status_scheduled_for_deletion(): "email": "test@example.com" } }] - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET method for job details responses.add( responses.GET, @@ -132,7 +132,7 @@ def test_get_workdir_deletion_status_scheduled_for_deletion(): headers=header, status=200 ) - + # Mock GET method for folder details responses.add( responses.GET, @@ -141,11 +141,11 @@ def test_get_workdir_deletion_status_scheduled_for_deletion(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_workdir_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["job_name"] == "test_job_scheduled" @@ -177,7 +177,7 @@ def test_get_workdir_deletion_status_deleting(): } } } - + # Mock folder details response with deleting status folder_response = [{ "_id": WORKDIR_FOLDER_ID, @@ -187,12 +187,12 @@ def test_get_workdir_deletion_status_deleting(): "createdAt": "2024-11-10T15:24:20.528Z", "updatedAt": "2024-11-12T14:00:00.000Z" }] - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -201,7 +201,7 @@ def test_get_workdir_deletion_status_deleting(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/folders/", @@ -209,11 +209,11 @@ def test_get_workdir_deletion_status_deleting(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_workdir_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["status"] == "deleting" @@ -241,7 +241,7 @@ def test_get_workdir_deletion_status_deleted(): } } } - + # Mock folder details response with deleted status folder_response = [{ "_id": WORKDIR_FOLDER_ID, @@ -251,12 +251,12 @@ def test_get_workdir_deletion_status_deleted(): "createdAt": "2024-11-10T15:24:20.528Z", "updatedAt": "2024-11-12T15:00:00.000Z" }] - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -265,7 +265,7 @@ def test_get_workdir_deletion_status_deleted(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/folders/", @@ -273,11 +273,11 @@ def test_get_workdir_deletion_status_deleted(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_workdir_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["status"] == "deleted" @@ -305,7 +305,7 @@ def test_get_workdir_deletion_status_failed_to_delete(): } } } - + # Mock folder details response with failedToDelete status folder_response = [{ "_id": WORKDIR_FOLDER_ID, @@ -315,12 +315,12 @@ def test_get_workdir_deletion_status_failed_to_delete(): "createdAt": "2024-11-10T15:24:20.528Z", "updatedAt": "2024-11-12T16:00:00.000Z" }] - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -329,7 +329,7 @@ def test_get_workdir_deletion_status_failed_to_delete(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/folders/", @@ -337,11 +337,11 @@ def test_get_workdir_deletion_status_failed_to_delete(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_workdir_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["status"] == "failedToDelete" @@ -361,7 +361,7 @@ def test_get_workdir_deletion_status_legacy_resume_workdir(): "status": "completed", "resumeWorkDir": WORKDIR_FOLDER_ID } - + # Mock folder details response folder_response = [{ "_id": WORKDIR_FOLDER_ID, @@ -371,12 +371,12 @@ def test_get_workdir_deletion_status_legacy_resume_workdir(): "createdAt": "2024-11-10T15:24:20.528Z", "updatedAt": "2024-11-10T15:24:20.528Z" }] - + header = { "Content-type": "application/json", "apikey": APIKEY } - + # Mock GET methods responses.add( responses.GET, @@ -385,7 +385,7 @@ def test_get_workdir_deletion_status_legacy_resume_workdir(): headers=header, status=200 ) - + responses.add( responses.GET, url=f"{CLOUDOS_URL}/api/v1/folders/", @@ -393,11 +393,11 @@ def test_get_workdir_deletion_status_legacy_resume_workdir(): headers=header, status=200 ) - + # Create Cloudos instance and call method clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL) result = clos.get_workdir_deletion_status(JOB_ID, WORKSPACE_ID) - + # Assertions assert result["job_id"] == JOB_ID assert result["job_name"] == "test_job_legacy" diff --git a/tests/test_clos/test_unarchive_integration.py b/tests/test_clos/test_unarchive_integration.py index 0f3381af..17fbebae 100644 --- a/tests/test_clos/test_unarchive_integration.py +++ b/tests/test_clos/test_unarchive_integration.py @@ -9,7 +9,7 @@ def test_job_unarchive_successful_flow(): """Test a successful job unarchiving flow end-to-end.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock checking if job is archived (should return the job since it's archived) m.get( @@ -41,7 +41,7 @@ def test_job_unarchive_successful_flow(): def test_job_unarchive_multiple_jobs_successful_flow(): """Test successful unarchiving of multiple jobs.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock job status checks for multiple jobs for job_id in ['job1', 'job2', 'job3']: @@ -120,7 +120,7 @@ def test_job_unarchive_mixed_valid_invalid_jobs(): def test_job_unarchive_verbose_output(): """Test unarchiving with verbose output.""" runner = CliRunner() - + with requests_mock.Mocker() as m: # Mock checking if job is archived (should return the job since it's archived) m.get( diff --git a/tests/test_datasets/test_link.py b/tests/test_datasets/test_link.py index 39bf3622..c3add8fd 100644 --- a/tests/test_datasets/test_link.py +++ b/tests/test_datasets/test_link.py @@ -132,7 +132,7 @@ def test_link_folder_204_s3(capsys, link_instance_test_response, monkeypatch): """Test successful S3 folder linking and mounting.""" url = f"https://lifebit.ai/api/v1/interactive-sessions/sessionABC/fuse-filesystem/mount?teamId=team123" responses.add(responses.POST, url, status=204) - + # Mock the GET request for checking fuse filesystem status status_url = f"https://lifebit.ai/api/v1/interactive-sessions/sessionABC/fuse-filesystems?teamId=team123" mock_response = { @@ -176,7 +176,7 @@ def test_link_folder_204_file_explorer(capsys, link_instance_test_response, monk """Test successful File Explorer folder linking and mounting.""" url = f"https://lifebit.ai/api/v1/interactive-sessions/sessionABC/fuse-filesystem/mount?teamId=team123" responses.add(responses.POST, url, status=204) - + # Mock the GET request for checking fuse filesystem status status_url = f"https://lifebit.ai/api/v1/interactive-sessions/sessionABC/fuse-filesystems?teamId=team123" mock_response = { @@ -227,7 +227,7 @@ def test_get_fuse_filesystems_status_success(link_instance_test_response): ] } responses.add(responses.GET, status_url, json=mock_response, status=200) - + result = link_instance_test_response.get_fuse_filesystems_status("sessionABC") assert len(result) == 1 assert result[0]["mountName"] == "test-mount" diff --git a/tests/test_jobs/test_clone_job.py b/tests/test_jobs/test_clone_job.py index 0c4ad183..dd5029b2 100644 --- a/tests/test_jobs/test_clone_job.py +++ b/tests/test_jobs/test_clone_job.py @@ -409,7 +409,7 @@ def test_clone_job_get_payload_error(): # Load test data for initialization projects_data = load_json_file(PROJECTS_INPUT) workflows_data = load_json_file(WORKFLOWS_INPUT) - + headers = { "Content-type": "application/json", "apikey": APIKEY diff --git a/tests/test_jobs/test_delete_job_results.py b/tests/test_jobs/test_delete_job_results.py index 1b8dd3b5..46324203 100644 --- a/tests/test_jobs/test_delete_job_results.py +++ b/tests/test_jobs/test_delete_job_results.py @@ -22,17 +22,17 @@ def test_delete_job_results_success_204(mock_workflow_id, mock_project_id): """ mock_project_id.return_value = "test_project_id" mock_workflow_id.return_value = "test_workflow_id" - + url = f"{CLOUDOS_URL}/api/v1/jobs/{JOB_ID}/data" params = {"properties[]": MODE, "teamId": WORKSPACE_ID} - + responses.add( responses.DELETE, url=url, status=204, match=[matchers.query_param_matcher(params)] ) - + job = Job( apikey=APIKEY, cloudos_url=CLOUDOS_URL, @@ -41,9 +41,9 @@ def test_delete_job_results_success_204(mock_workflow_id, mock_project_id): project_name="test_project", workflow_name="test_workflow" ) - + result = job.delete_job_results(JOB_ID, MODE) - + assert result["message"] == "Results deleted successfully" assert result["status"] == "deleted" @@ -57,10 +57,10 @@ def test_delete_job_results_not_found_404(mock_workflow_id, mock_project_id): """ mock_project_id.return_value = "test_project_id" mock_workflow_id.return_value = "test_workflow_id" - + url = f"{CLOUDOS_URL}/api/v1/jobs/{JOB_ID}/data" params = {"properties[]": MODE, "teamId": WORKSPACE_ID} - + responses.add( responses.DELETE, url=url, @@ -68,7 +68,7 @@ def test_delete_job_results_not_found_404(mock_workflow_id, mock_project_id): json={"message": f"Job with ID '{JOB_ID}' not found"}, match=[matchers.query_param_matcher(params)] ) - + job = Job( apikey=APIKEY, cloudos_url=CLOUDOS_URL, @@ -77,10 +77,10 @@ def test_delete_job_results_not_found_404(mock_workflow_id, mock_project_id): project_name="test_project", workflow_name="test_workflow" ) - + with pytest.raises(ValueError) as exc_info: job.delete_job_results(JOB_ID, MODE) - + assert JOB_ID in str(exc_info.value) assert "not found" in str(exc_info.value).lower() @@ -94,17 +94,17 @@ def test_delete_job_results_unauthorized_401(mock_workflow_id, mock_project_id): """ mock_project_id.return_value = "test_project_id" mock_workflow_id.return_value = "test_workflow_id" - + url = f"{CLOUDOS_URL}/api/v1/jobs/{JOB_ID}/data" params = {"properties[]": MODE, "teamId": WORKSPACE_ID} - + responses.add( responses.DELETE, url=url, status=401, match=[matchers.query_param_matcher(params)] ) - + job = Job( apikey=APIKEY, cloudos_url=CLOUDOS_URL, @@ -113,10 +113,10 @@ def test_delete_job_results_unauthorized_401(mock_workflow_id, mock_project_id): project_name="test_project", workflow_name="test_workflow" ) - + with pytest.raises(ValueError) as exc_info: job.delete_job_results(JOB_ID, MODE) - + assert "Unauthorized" in str(exc_info.value) assert "API key" in str(exc_info.value) @@ -130,17 +130,17 @@ def test_delete_job_results_forbidden_403(mock_workflow_id, mock_project_id): """ mock_project_id.return_value = "test_project_id" mock_workflow_id.return_value = "test_workflow_id" - + url = f"{CLOUDOS_URL}/api/v1/jobs/{JOB_ID}/data" params = {"properties[]": MODE, "teamId": WORKSPACE_ID} - + responses.add( responses.DELETE, url=url, status=403, match=[matchers.query_param_matcher(params)] ) - + job = Job( apikey=APIKEY, cloudos_url=CLOUDOS_URL, @@ -149,10 +149,10 @@ def test_delete_job_results_forbidden_403(mock_workflow_id, mock_project_id): project_name="test_project", workflow_name="test_workflow" ) - + with pytest.raises(ValueError) as exc_info: job.delete_job_results(JOB_ID, MODE) - + assert "Forbidden" in str(exc_info.value) assert "permission" in str(exc_info.value) @@ -166,17 +166,17 @@ def test_delete_job_results_conflict_409(mock_workflow_id, mock_project_id): """ mock_project_id.return_value = "test_project_id" mock_workflow_id.return_value = "test_workflow_id" - + url = f"{CLOUDOS_URL}/api/v1/jobs/{JOB_ID}/data" params = {"properties[]": MODE, "teamId": WORKSPACE_ID} - + responses.add( responses.DELETE, url=url, status=409, match=[matchers.query_param_matcher(params)] ) - + job = Job( apikey=APIKEY, cloudos_url=CLOUDOS_URL, @@ -185,10 +185,10 @@ def test_delete_job_results_conflict_409(mock_workflow_id, mock_project_id): project_name="test_project", workflow_name="test_workflow" ) - + with pytest.raises(ValueError) as exc_info: job.delete_job_results(JOB_ID, MODE) - + assert "Conflict" in str(exc_info.value) @@ -201,17 +201,17 @@ def test_delete_job_results_bad_request_400(mock_workflow_id, mock_project_id): """ mock_project_id.return_value = "test_project_id" mock_workflow_id.return_value = "test_workflow_id" - + url = f"{CLOUDOS_URL}/api/v1/jobs/{JOB_ID}/data" params = {"properties[]": MODE, "teamId": WORKSPACE_ID} - + responses.add( responses.DELETE, url=url, status=400, match=[matchers.query_param_matcher(params)] ) - + job = Job( apikey=APIKEY, cloudos_url=CLOUDOS_URL, @@ -220,10 +220,10 @@ def test_delete_job_results_bad_request_400(mock_workflow_id, mock_project_id): project_name="test_project", workflow_name="test_workflow" ) - + with pytest.raises(ValueError) as exc_info: job.delete_job_results(JOB_ID, MODE) - + assert "Operation not permitted" in str(exc_info.value) assert "Your workspace does not have the option to delete results folders enabled. Please consult with the organisation owner to enable this feature." in str(exc_info.value) @@ -237,17 +237,17 @@ def test_delete_job_results_server_error_500(mock_workflow_id, mock_project_id): """ mock_project_id.return_value = "test_project_id" mock_workflow_id.return_value = "test_workflow_id" - + url = f"{CLOUDOS_URL}/api/v1/jobs/{JOB_ID}/data" params = {"properties[]": MODE, "teamId": WORKSPACE_ID} - + responses.add( responses.DELETE, url=url, status=500, match=[matchers.query_param_matcher(params)] ) - + job = Job( apikey=APIKEY, cloudos_url=CLOUDOS_URL, @@ -256,10 +256,10 @@ def test_delete_job_results_server_error_500(mock_workflow_id, mock_project_id): project_name="test_project", workflow_name="test_workflow" ) - + # The retry mechanism will exhaust retries on 500 errors with pytest.raises(Exception) as exc_info: job.delete_job_results(JOB_ID, MODE) - + # Should raise either RetryError or the underlying ValueError after retries assert "Max retries exceeded" in str(exc_info.value) or "Internal server error" in str(exc_info.value) diff --git a/tests/test_jobs/test_resume_job.py b/tests/test_jobs/test_resume_job.py index 4eaa7657..d4a0f78a 100644 --- a/tests/test_jobs/test_resume_job.py +++ b/tests/test_jobs/test_resume_job.py @@ -444,7 +444,7 @@ def test_resume_job_get_payload_error(): # Load test data for initialization projects_data = load_json_file(PROJECTS_INPUT) workflows_data = load_json_file(WORKFLOWS_INPUT) - + headers = { "Content-type": "application/json", "apikey": APIKEY @@ -661,7 +661,7 @@ def test_get_resume_work_dir_error(): # Load test data for initialization projects_data = load_json_file(PROJECTS_INPUT) workflows_data = load_json_file(WORKFLOWS_INPUT) - + headers = { "Content-type": "application/json", "apikey": APIKEY diff --git a/tests/test_procurement/test_reset_procurement_organisation_image.py b/tests/test_procurement/test_reset_procurement_organisation_image.py index faa9c1a7..36ead71a 100644 --- a/tests/test_procurement/test_reset_procurement_organisation_image.py +++ b/tests/test_procurement/test_reset_procurement_organisation_image.py @@ -75,7 +75,7 @@ def test_reset_procurement_organisation_image(): @responses.activate def test_reset_procurement_organisation_image_different_types(): """Test resetting different image types""" - + image_types = [ "RegularInteractiveSessions", "SparkInteractiveSessions", @@ -83,7 +83,7 @@ def test_reset_procurement_organisation_image_different_types(): "JupyterInteractiveSessions", "NextflowBatchComputeEnvironment" ] - + for image_type in image_types: mock_response = { "id": f"config-{image_type.lower()}", @@ -139,9 +139,9 @@ def test_reset_procurement_organisation_image_different_types(): @responses.activate def test_reset_procurement_organisation_image_different_regions(): """Test resetting image configuration for different AWS regions""" - + aws_regions = ["eu-west-1", "eu-west-2", "us-east-1", "us-west-2"] - + for region in aws_regions: mock_response = { "id": f"config-{region}", diff --git a/tests/test_procurement/test_set_procurement_organisation_image.py b/tests/test_procurement/test_set_procurement_organisation_image.py index ae286f20..7bd78002 100644 --- a/tests/test_procurement/test_set_procurement_organisation_image.py +++ b/tests/test_procurement/test_set_procurement_organisation_image.py @@ -78,7 +78,7 @@ def test_set_procurement_organisation_image(): @responses.activate def test_set_procurement_organisation_image_different_types(): """Test setting different image types""" - + image_types = [ "RegularInteractiveSessions", "SparkInteractiveSessions", @@ -86,7 +86,7 @@ def test_set_procurement_organisation_image_different_types(): "JupyterInteractiveSessions", "NextflowBatchComputeEnvironment" ] - + for image_type in image_types: mock_response = { "id": f"config-{image_type.lower()}", @@ -144,7 +144,7 @@ def test_set_procurement_organisation_image_different_types(): @responses.activate def test_set_procurement_organisation_image_without_image_name(): """Test setting image configuration without providing image_name parameter""" - + mock_response = { "id": "68667809e13a844401d10f6c", "organisationId": ORGANISATION_ID, diff --git a/utils/delete_project_jobs.sh b/utils/delete_project_jobs.sh index f19e3136..3b3355ca 100755 --- a/utils/delete_project_jobs.sh +++ b/utils/delete_project_jobs.sh @@ -117,7 +117,7 @@ if [[ -n "$FILTER_STATUS" ]]; then break fi done - + if [[ "$STATUS_VALID" == false ]]; then echo -e "${RED}Error: Invalid status '$FILTER_STATUS'. Valid statuses are: $(IFS=','; echo "${VALID_STATUSES[*]}")${NC}" exit 1 From 58ff6365bf1f05ace9e10351ab14a8fc2a30c339 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 15:55:28 +0100 Subject: [PATCH 40/41] style: remove whitespaces --- cloudos_cli/clos.py | 142 +++++++++--------- cloudos_cli/configure/configure.py | 26 ++-- cloudos_cli/datasets/cli.py | 2 +- cloudos_cli/datasets/datasets.py | 2 +- cloudos_cli/jobs/cli.py | 88 +++++------ cloudos_cli/jobs/job.py | 10 +- cloudos_cli/link/cli.py | 22 +-- cloudos_cli/link/link.py | 80 +++++----- cloudos_cli/utils/details.py | 32 ++-- cloudos_cli/utils/errors.py | 4 +- tests/test_clos/test_archive_integration.py | 24 +-- tests/test_clos/test_archive_jobs.py | 28 ++-- .../test_clos/test_archive_status_checking.py | 30 ++-- tests/test_clos/test_cli_archive_command.py | 4 +- tests/test_clos/test_cli_unarchive_command.py | 4 +- tests/test_clos/test_unarchive_integration.py | 26 ++-- tests/test_clos/test_unarchive_jobs.py | 6 +- tests/test_cost/test_job_cost.py | 96 ++++++------ .../test_related_analyses.py | 34 ++--- 19 files changed, 330 insertions(+), 330 deletions(-) diff --git a/cloudos_cli/clos.py b/cloudos_cli/clos.py index 7f886aa4..13efc51a 100644 --- a/cloudos_cli/clos.py +++ b/cloudos_cli/clos.py @@ -214,7 +214,7 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): if "resumeWorkDir" not in r_json: raise ValueError("Working directories are not available. This may be because the analysis was run without resumable mode enabled, or because intermediate results have since been removed.") - + # Check if intermediate results have been deleted # When intermediate results are deleted, resumeWorkDir becomes None but workDirectory still exists with folderId resume_workdir_id = r_json.get("resumeWorkDir") @@ -228,7 +228,7 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): folder_id = work_directory["folderId"] folder_response = self.get_folder_deletion_status(folder_id, workspace_id, verify) folder_data = json.loads(folder_response.content) - + # If the API returns the folder, get its status if folder_data and len(folder_data) > 0: api_status = folder_data[0].get("status") @@ -236,12 +236,12 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): # If the folder is not returned, check if deletedBy exists in workDirectory if "deletedBy" in work_directory: api_status = "scheduledForDeletion" # Assume scheduled for deletion - + except Exception: # If we can't get the status, check if deletedBy exists if "deletedBy" in work_directory: api_status = "scheduledForDeletion" # Assume scheduled for deletion - + # Build contextually appropriate error message based on status # Only raise error for non-ready statuses (ready means it's available, so no error) if api_status == "deleting": @@ -263,7 +263,7 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): # If status is "ready", set resume_workdir_id so we can retrieve the workdir path elif api_status == "ready": resume_workdir_id = folder_id - + # If resumeWorkDir exists, use the folders API to get the shared working directory if resume_workdir_id: try: @@ -280,7 +280,7 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): if len(workdir_bucket_o) > 1: raise ValueError(f"Request returned more than one result for folder id {resume_workdir_id}") workdir_bucket_info = workdir_bucket_o[0] - + if workdir_bucket_info["folderType"] == "S3Folder": bucket_name = workdir_bucket_info["s3BucketName"] bucket_path = workdir_bucket_info["s3Prefix"] @@ -292,13 +292,13 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): workdir_path = f"{storage_account}/{container_name}/{blob_prefix}" else: raise ValueError("Unsupported cloud provider") - + return workdir_path except Exception as e: # If folders API fails, fall back to logs-based approach print(f"Warning: Could not get shared workdir from folders API: {e}") pass - + # Check if logs field exists for fallback approach if "logs" in r_json: # Get workdir information from logs object using the same pattern as get_job_logs @@ -308,10 +308,10 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): prefix_name = cloud_storage["prefix"] logs_bucket = logs_obj[container_name] logs_path = logs_obj[prefix_name] - + # Construct workdir path by replacing '/logs' with '/work' in the logs path workdir_path_suffix = logs_path.replace('/logs', '/work') - + if cloud_name == "aws": workdir_path = f"s3://{logs_bucket}/{workdir_path_suffix}" elif cloud_name == "azure": @@ -322,7 +322,7 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): workdir_path = f"{storage_account_prefix}/{logs_bucket}/{workdir_path_suffix}" else: raise ValueError("Unsupported cloud provider") - + return workdir_path else: # Fallback to original folder-based approach for backward compatibility @@ -359,7 +359,7 @@ def get_job_workdir(self, j_id, workspace_id, verify=True): workdir_path = f"{storage_account}/{container_name}/{blob_prefix}" else: raise ValueError("Unsupported cloud provider") - + return workdir_path def _handle_job_access_denied(self, job_id, workspace_id, verify=True): @@ -380,7 +380,7 @@ def _handle_job_access_denied(self, job_id, workspace_id, verify=True): result = self.get_job_list(workspace_id, last_n_jobs='all', verify=verify) jobs = result['jobs'] # Extract jobs list from the dictionary job_owner_name = None - + for job in jobs: if job.get('_id') == job_id: user_info = job.get('user', {}) @@ -388,7 +388,7 @@ def _handle_job_access_denied(self, job_id, workspace_id, verify=True): if not job_owner_name: job_owner_name = user_info.get('email', 'Unknown') break - + raise JobAccessDeniedException(job_id, job_owner_name, current_user_name) except JobAccessDeniedException: # Re-raise the specific exception @@ -409,7 +409,7 @@ def get_job_logs(self, j_id, workspace_id, verify=True): } r = self.get_job_status(j_id, workspace_id, verify) r_json = r.json() - + job_workspace = r_json["team"] if job_workspace != workspace_id: raise ValueError("Workspace provided or configured is different from workspace where the job was executed") @@ -461,20 +461,20 @@ def get_job_results(self, j_id, workspace_id, verify=True): job_workspace = req_obj["team"] if job_workspace != workspace_id: raise ValueError("Workspace provided or configured is different from workspace where the job was executed") - + # Check if analysis results have been deleted or scheduled for deletion # Similar to workdir check - if analysisResults exists with folderId, check its status if "analysisResults" in req_obj and req_obj.get("analysisResults"): analysis_results = req_obj["analysisResults"] results_folder_id = analysis_results.get("folderId") - + if results_folder_id: # Get the actual deletion status from the folders API api_status = None try: folder_response = self.get_folder_deletion_status(results_folder_id, workspace_id, verify) folder_data = json.loads(folder_response.content) - + # If the API returns the folder, get its status if folder_data and len(folder_data) > 0: api_status = folder_data[0].get("status") @@ -482,12 +482,12 @@ def get_job_results(self, j_id, workspace_id, verify=True): # If the folder is not returned, check if deletedBy exists in analysisResults if "deletedBy" in analysis_results: api_status = "scheduledForDeletion" # Assume scheduled for deletion - + except Exception: # If we can't get the status, check if deletedBy exists if "deletedBy" in analysis_results: api_status = "scheduledForDeletion" # Assume scheduled for deletion - + # Build contextually appropriate error message based on status # Only raise error for non-ready statuses (ready means it's available, so no error) if api_status == "deleting": @@ -507,7 +507,7 @@ def get_job_results(self, j_id, workspace_id, verify=True): error_msg = "Analysis results have been removed. The results folder is no longer available." raise ValueError(error_msg) # If status is "ready" or None, don't raise error - let the code continue to retrieve the results path - + cloud_name, meta, cloud_storage = find_cloud(self.cloudos_url, self.apikey, workspace_id, req_obj["logs"]) # cont_name results_obj = req_obj["results"] @@ -523,12 +523,12 @@ def get_job_results(self, j_id, workspace_id, verify=True): for item in contents_obj: if item["isDir"] and item["name"] == "results": return f"{scheme}://{storage_account_prefix}{results_container}/{item['path']}" - + # Fallback: if no "results" directory found, return the first directory for item in contents_obj: if item["isDir"]: return f"{scheme}://{storage_account_prefix}{results_container}/{item['path']}" - + # If no directories found, raise an error raise ValueError("No result directories found for this job") @@ -563,19 +563,19 @@ def get_folder_items_deletion_status(self, folder_id, workspace_id, verify=True) "Content-type": "application/json", "apikey": self.apikey } - + # Query all possible deletion statuses params = { "status": ["ready", "deleted", "deleting", "scheduledForDeletion", "failedToDelete"], "teamId": workspace_id } - + url = f"{self.cloudos_url}/api/v1/datasets/{folder_id}/items" response = retry_requests_get(url, params=params, headers=headers, verify=verify) - + if response.status_code >= 400: raise BadRequestException(response) - + return response def get_results_deletion_status(self, job_id, workspace_id, verify=True): @@ -619,24 +619,24 @@ def get_results_deletion_status(self, job_id, workspace_id, verify=True): job_data = json.loads(job_status.content) job_name = job_data.get("name", job_id) project_info = job_data.get("project") - + # Extract deletedBy info from analysisResults if available analysis_results_deleted_by = None if "analysisResults" in job_data and job_data.get("analysisResults"): analysis_results_deleted_by = job_data["analysisResults"].get("deletedBy") - + if not project_info: raise ValueError(f"Could not find project for job '{job_id}'") - + # Extract project ID and name from the project info dict project_id = project_info.get("_id") project_name = project_info.get("name") - + if not project_name or not project_id: raise ValueError(f"Could not extract project information from job '{job_id}'") - + from cloudos_cli.datasets.datasets import Datasets - + # Create Datasets object to navigate to the Analysis Results folder ds = Datasets( cloudos_url=self.cloudos_url, @@ -646,23 +646,23 @@ def get_results_deletion_status(self, job_id, workspace_id, verify=True): project_name=project_name, verify=verify ) - + # Get project content to find Analysis Results folder try: project_content = ds.list_project_content() except Exception as e: raise ValueError(f"Failed to list project content for project '{project_name}'. {str(e)}") - + # Find the Analysis Results folder ID analysis_results_id = None for folder in project_content.get("folders", []): if folder['name'] in ['Analyses Results', 'AnalysesResults']: analysis_results_id = folder['_id'] break - + if not analysis_results_id: raise ValueError(f"Analyses Results folder not found in project '{project_name}'.") - + # Get items in Analysis Results folder to find the job's specific results folder # The Analysis Results folder contains folders for each job's results try: @@ -670,11 +670,11 @@ def get_results_deletion_status(self, job_id, workspace_id, verify=True): content = json.loads(response.content) except Exception as e: raise ValueError(f"Failed to get items from Analyses Results folder. {str(e)}") - + # The API response contains folders and files arrays # Find the entry matching our job_id job_status_info = None - + # Check if it's a dict with folders/files arrays if isinstance(content, dict): # Check for 'folders' or 'files' keys (common dataset API response format) @@ -683,17 +683,17 @@ def get_results_deletion_status(self, job_id, workspace_id, verify=True): items_to_search.extend(content['folders']) if 'files' in content: items_to_search.extend(content['files']) - + # If no folders/files keys, treat dict values as items if not items_to_search: items_to_search = list(content.values()) - + for item in items_to_search: if not isinstance(item, dict): continue - + item_name = item.get("name", "") - + # Match by exact job ID in the item name (format: workflowname-jobid) # The folder name should contain the exact job ID if job_id in item_name: @@ -703,25 +703,25 @@ def get_results_deletion_status(self, job_id, workspace_id, verify=True): for item in content: if not isinstance(item, dict): continue - + item_name = item.get("name", "") - + # Match by exact job ID in the item name if job_id in item_name: job_status_info = item break - + if not job_status_info: raise ValueError( f"Results folder for job '{job_name}' (ID: {job_id}) not found in Analyses Results.\n" f"This may indicate that the results have been deleted or are scheduled for deletion." ) - + # Merge the deletedBy info from job data with the folder info # The deletedBy from analysisResults is more reliable than folder's user field if analysis_results_deleted_by: job_status_info["deletedBy"] = analysis_results_deleted_by - + return { "job_id": job_id, "job_name": job_name, @@ -762,20 +762,20 @@ def get_folder_deletion_status(self, folder_id, workspace_id, verify=True): "Content-type": "application/json", "apikey": self.apikey } - + # Query with all possible deletion statuses params = { "id": folder_id, "status": ["ready", "deleted", "deleting", "scheduledForDeletion", "failedToDelete"], "teamId": workspace_id } - + url = f"{self.cloudos_url}/api/v1/folders/" response = retry_requests_get(url, params=params, headers=headers, verify=verify) - + if response.status_code >= 400: raise BadRequestException(response) - + return response def get_workdir_deletion_status(self, job_id, workspace_id, verify=True): @@ -819,47 +819,47 @@ def get_workdir_deletion_status(self, job_id, workspace_id, verify=True): job_status = self.get_job_status(job_id, workspace_id, verify) job_data = json.loads(job_status.content) job_name = job_data.get("name", job_id) - + # Try to get the workdir folder ID from workDirectory.folderId first (new format) # If not available, fall back to resumeWorkDir (old format) workdir_folder_id = None workdir_deleted_by = None - + if "workDirectory" in job_data and job_data.get("workDirectory"): workdir_folder_id = job_data["workDirectory"].get("folderId") # Get deletedBy info if available (contains user who scheduled deletion) workdir_deleted_by = job_data["workDirectory"].get("deletedBy") - + if not workdir_folder_id and "resumeWorkDir" in job_data: workdir_folder_id = job_data.get("resumeWorkDir") - + if not workdir_folder_id: raise ValueError( "Working directory is not available for this job. " "This may be because the analysis was run without resumable mode enabled, " "or because intermediate results have been removed." ) - + # Use the folders API to get the working directory status response = self.get_folder_deletion_status(workdir_folder_id, workspace_id, verify) - + # Parse the response content = json.loads(response.content) - + # The API returns an array with the folder info if not content or len(content) == 0: raise ValueError( f"Working directory for job '{job_name}' (ID: {job_id}) not found.\n" f"This may indicate that the working directory has been deleted or is scheduled for deletion." ) - + workdir_info = content[0] # Get the first (and should be only) result - + # Merge the deletedBy info from job data with the folder info # The deletedBy from workDirectory is more reliable than folder's user field if workdir_deleted_by: workdir_info["deletedBy"] = workdir_deleted_by - + return { "job_id": job_id, "job_name": job_name, @@ -931,7 +931,7 @@ def resolve_user_id(self, filter_owner, workspace_id, verify=True): params=search_params, headers=search_headers, verify=verify) if user_search_r.status_code >= 400: raise ValueError(f"Error searching for user '{filter_owner}'") - + user_search_content = user_search_r.json() user_items = user_search_content.get('items', []) if user_items and len(user_items) > 0: @@ -940,7 +940,7 @@ def resolve_user_id(self, filter_owner, workspace_id, verify=True): if user.get("username") == filter_owner or user.get("name") == filter_owner: user_match = user break - + if user_match: return user_match.get("id") else: @@ -1193,7 +1193,7 @@ def get_job_list(self, workspace_id, last_n_jobs=None, page=None, page_size=None content = r.json() page_jobs = content.get('jobs', []) - + # Capture pagination metadata last_pagination_metadata = content.get('paginationMetadata', None) @@ -1895,7 +1895,7 @@ def _update_job_archive_status(self, job_ids, workspace_id, archive_status, veri "Content-type": "application/json", "apikey": apikey } - + # Create the payload with current timestamp in ISO format (UTC) current_time = datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') payload = { @@ -1907,7 +1907,7 @@ def _update_job_archive_status(self, job_ids, workspace_id, archive_status, veri } } } - + r = retry_requests_put( f"{cloudos_url}/api/v1/jobs?teamId={workspace_id}", headers=headers, @@ -1995,20 +1995,20 @@ def check_jobs_archive_status(self, job_ids, workspace_id, target_archived_state valid_jobs = [] already_processed = [] invalid_jobs = {} - + for job_id in job_ids: try: # Check if job exists in archived list archived_jobs = self.get_job_list(workspace_id, archived=True, filter_job_id=job_id, page_size=1, verify=verify) is_archived = len(archived_jobs.get('jobs', [])) > 0 - + if not is_archived: # Check if job exists in unarchived list to verify it's a valid job unarchived_jobs = self.get_job_list(workspace_id, archived=False, filter_job_id=job_id, page_size=1, verify=verify) if len(unarchived_jobs.get('jobs', [])) == 0: # Job doesn't exist in either list raise Exception("Job not found") - + if target_archived_state: # Archiving operation: we want jobs that are NOT archived if is_archived: @@ -2040,7 +2040,7 @@ def check_jobs_archive_status(self, job_ids, workspace_id, target_archived_state except Exception as e: # Job not found or other error - collect and continue processing invalid_jobs[job_id] = str(e) - + return { 'valid_jobs': valid_jobs, 'already_processed': already_processed, diff --git a/cloudos_cli/configure/configure.py b/cloudos_cli/configure/configure.py index 6d331628..1c816d37 100644 --- a/cloudos_cli/configure/configure.py +++ b/cloudos_cli/configure/configure.py @@ -258,7 +258,7 @@ def create_profile_from_input(self, profile_name): config[profile_name]['default'] = str(default_profile) if session_id is not None: config[profile_name]['session_id'] = session_id - + with open(self.config_file, 'w') as conf_file: config.write(conf_file) @@ -362,10 +362,10 @@ def make_default_profile(self, profile_name): def load_profile(self, profile_name): """Load a profile from the config and credentials files dynamically. - + This method now returns ALL parameters from the profile, not just predefined ones. This makes it extensible - you can add new parameters to profiles without modifying this method. - + Parameters: ---------- profile_name : str @@ -375,12 +375,12 @@ def load_profile(self, profile_name): dict A dictionary containing all profile parameters. Returns all keys from both credentials and config files for the specified profile. - + Examples -------- # If you add accelerate_saving_results to your profile config: config[profile_name]['accelerate_saving_results'] = 'true' - + # It will automatically be included in the returned dictionary: profile_data = load_profile('myprofile') # profile_data will contain 'accelerate_saving_results': 'true' @@ -400,19 +400,19 @@ def load_profile(self, profile_name): # Dynamically load all parameters from the profile profile_data = {} - + # Load all items from credentials file if credentials.has_section(profile_name): for key, value in credentials[profile_name].items(): profile_data[key] = value - + # Load all items from config file if config.has_section(profile_name): for key, value in config[profile_name].items(): # Skip the 'default' flag as it's not a user parameter if key != 'default': profile_data[key] = value - + return profile_data def check_if_profile_exists(self, profile_name): @@ -481,7 +481,7 @@ def get_param_value(ctx, param_value, param_name, default_value, required=False, def load_profile_and_validate_data(self, ctx, init_profile, cloudos_url_default, profile, required_dict, **cli_params): """ Load profile data and validate required parameters dynamically. - + This method now accepts any parameters via **cli_params, making it extensible. You can add new parameters to profiles (like accelerate_saving_results) without modifying this method. @@ -501,7 +501,7 @@ def load_profile_and_validate_data(self, ctx, init_profile, cloudos_url_default, **cli_params : dict All CLI parameters passed as keyword arguments. Any parameter can be passed here and will be resolved from the profile if available. - + Examples: apikey, cloudos_url, workspace_id, project_name, workflow_name, execution_platform, repository_platform, session_id, procurement_id, accelerate_saving_results, etc. @@ -510,7 +510,7 @@ def load_profile_and_validate_data(self, ctx, init_profile, cloudos_url_default, ------- dict A dictionary containing all loaded and validated parameters. - + Examples -------- # Add a new parameter to profile without changing this method: @@ -525,7 +525,7 @@ def load_profile_and_validate_data(self, ctx, init_profile, cloudos_url_default, if profile != init_profile: # Load profile data profile_data = self.load_profile(profile_name=profile) - + # Dynamically process all parameters passed in cli_params for param_name, cli_value in cli_params.items(): profile_value = profile_data.get(param_name, "") @@ -540,7 +540,7 @@ def load_profile_and_validate_data(self, ctx, init_profile, cloudos_url_default, required=is_required, missing_required_params=missing ) - + # Convert empty strings to None for optional parameters # This prevents issues with functions that expect None for unset values if resolved_value == "" and not is_required: diff --git a/cloudos_cli/datasets/cli.py b/cloudos_cli/datasets/cli.py index 8ba9e78b..8d365911 100644 --- a/cloudos_cli/datasets/cli.py +++ b/cloudos_cli/datasets/cli.py @@ -119,7 +119,7 @@ def list_files(ctx, type_ = "file (user uploaded)" else: type_ = "file (virtual copy)" - + user = item.get("user", {}) if isinstance(user, dict): name = user.get("name", "").strip() diff --git a/cloudos_cli/datasets/datasets.py b/cloudos_cli/datasets/datasets.py index 6fd87959..623ed7c0 100644 --- a/cloudos_cli/datasets/datasets.py +++ b/cloudos_cli/datasets/datasets.py @@ -362,7 +362,7 @@ def list_folder_content(self, path=None): "files": [file_item], "folders": [] } - + # Also check in contents array (for different API response formats) for item in folder_content.get("contents", []): if item["name"] == job_name and not item.get("isDir", True): diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index d82d78d9..b94bfd48 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -586,7 +586,7 @@ def job_workdir(ctx, # Handle --status flag if status: console = Console() - + if verbose: console.print('[bold cyan]Checking deletion status of job working directory...[/bold cyan]') console.print('\t[dim]...Preparing objects[/dim]') @@ -594,21 +594,21 @@ def job_workdir(ctx, console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - + # Use Cloudos object to access the deletion status method cl = Cloudos(cloudos_url, apikey, None) - + if verbose: console.print('\t[dim]The following Cloudos object was created:[/dim]') console.print('\t' + str(cl) + '\n') - + try: deletion_status = cl.get_workdir_deletion_status( job_id=job_id, workspace_id=workspace_id, verify=verify_ssl ) - + # Convert API status to user-friendly terminology with color status_config = { "ready": ("available", "green"), @@ -617,20 +617,20 @@ def job_workdir(ctx, "deleted": ("deleted", "red"), "failedToDelete": ("failed to delete", "red") } - + # Get the status of the workdir folder itself and convert it api_status = deletion_status.get("status", "unknown") folder_status, status_color = status_config.get(api_status, (api_status, "white")) folder_info = deletion_status.get("items", {}) - + # Display results in a clear, styled format with human-readable sentence console.print(f'The working directory of job [cyan]{deletion_status["job_id"]}[/cyan] is in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - + # For non-available statuses, always show update time and user info if folder_status != "available": if folder_info.get("updatedAt"): console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - + # Show user information - prefer deletedBy over user field user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) if user_info: @@ -639,14 +639,14 @@ def job_workdir(ctx, if user_name or user_email: user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) console.print(f'[blue]User:[/blue] {user_display}') - + # Display detailed information if verbose if verbose: console.print('\n[bold]Additional information:[/bold]') console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') console.print(f' [cyan]Working directory folder name:[/cyan] {deletion_status["workdir_folder_name"]}') console.print(f' [cyan]Working directory folder ID:[/cyan] {deletion_status["workdir_folder_id"]}') - + # Show folder metadata if available if folder_info.get("createdAt"): console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') @@ -654,12 +654,12 @@ def job_workdir(ctx, console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') if folder_info.get("folderType"): console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - + except ValueError as e: raise click.ClickException(str(e)) except Exception as e: raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - + return # Validate link flag requirements AFTER loading profile @@ -683,12 +683,12 @@ def job_workdir(ctx, try: workdir = cl.get_job_workdir(job_id, workspace_id, verify_ssl) print(f"Working directory for job {job_id}: {workdir}") - + # Link to interactive session if requested if link: if verbose: print(f'\tLinking working directory to interactive session {session_id}...') - + # Use Link class to perform the linking link_client = Link( cloudos_url=cloudos_url, @@ -698,9 +698,9 @@ def job_workdir(ctx, project_name=None, # Not needed for S3 paths verify=verify_ssl ) - + link_client.link_folder(workdir.strip(), session_id) - + except BadRequestException as e: raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") except Exception as e: @@ -812,7 +812,7 @@ def job_logs(ctx, logs = cl.get_job_logs(job_id, workspace_id, verify_ssl) for name, path in logs.items(): print(f"{name}: {path}") - + # Link to interactive session if requested if link: if logs: @@ -822,11 +822,11 @@ def job_logs(ctx, # Remove the filename to get the logs directory # e.g., "s3://bucket/path/to/logs/filename.txt" -> "s3://bucket/path/to/logs" logs_dir = '/'.join(first_log_path.split('/')[:-1]) - + if verbose: print(f'\tLinking logs directory to interactive session {session_id}...') print(f'\t\tLogs directory: {logs_dir}') - + # Use Link class to perform the linking link_client = Link( cloudos_url=cloudos_url, @@ -836,12 +836,12 @@ def job_logs(ctx, project_name=None, # Not needed for S3 paths verify=verify_ssl ) - + link_client.link_folder(logs_dir, session_id) else: if verbose: print('\tNo logs found to link.') - + except BadRequestException as e: raise ValueError(f"Job '{job_id}' not found or not accessible. {str(e)}") except Exception as e: @@ -914,7 +914,7 @@ def job_results(ctx, # Handle --status flag if status: console = Console() - + if verbose: console.print('[bold cyan]Checking deletion status of job results...[/bold cyan]') console.print('\t[dim]...Preparing objects[/dim]') @@ -922,21 +922,21 @@ def job_results(ctx, console.print(f'\t\t[cyan]CloudOS url:[/cyan] {cloudos_url}') console.print(f'\t\t[cyan]Workspace ID:[/cyan] {workspace_id}') console.print(f'\t\t[cyan]Job ID:[/cyan] {job_id}') - + # Use Cloudos object to access the deletion status method cl = Cloudos(cloudos_url, apikey, None) - + if verbose: console.print('\t[dim]The following Cloudos object was created:[/dim]') console.print('\t' + str(cl) + '\n') - + try: deletion_status = cl.get_results_deletion_status( job_id=job_id, workspace_id=workspace_id, verify=verify_ssl ) - + # Convert API status to user-friendly terminology with color status_config = { "ready": ("available", "green"), @@ -945,20 +945,20 @@ def job_results(ctx, "deleted": ("deleted", "red"), "failedToDelete": ("failed to delete", "red") } - + # Get the status of the results folder itself and convert it api_status = deletion_status.get("status", "unknown") folder_status, status_color = status_config.get(api_status, (api_status, "white")) folder_info = deletion_status.get("items", {}) - + # Display results in a clear, styled format with human-readable sentence console.print(f'The results of job [cyan]{deletion_status["job_id"]}[/cyan] are in status: [bold {status_color}]{folder_status}[/bold {status_color}]') - + # For non-available statuses, always show update time and user info if folder_status != "available": if folder_info.get("updatedAt"): console.print(f'[magenta]Status changed at:[/magenta] {folder_info.get("updatedAt")}') - + # Show user information - prefer deletedBy over user field user_info = folder_info.get("deletedBy") or folder_info.get("user", {}) if user_info: @@ -967,14 +967,14 @@ def job_results(ctx, if user_name or user_email: user_display = f'{user_name} ({user_email})' if user_name and user_email else (user_name or user_email) console.print(f'[blue]User:[/blue] {user_display}') - + # Display detailed information if verbose if verbose: console.print('\n[bold]Additional information:[/bold]') console.print(f' [cyan]Job name:[/cyan] {deletion_status["job_name"]}') console.print(f' [cyan]Results folder name:[/cyan] {deletion_status["results_folder_name"]}') console.print(f' [cyan]Results folder ID:[/cyan] {deletion_status["results_folder_id"]}') - + # Show folder metadata if available if folder_info.get("createdAt"): console.print(f' [cyan]Created at:[/cyan] {folder_info.get("createdAt")}') @@ -982,12 +982,12 @@ def job_results(ctx, console.print(f' [cyan]Updated at:[/cyan] {folder_info.get("updatedAt")}') if folder_info.get("folderType"): console.print(f' [cyan]Folder type:[/cyan] {folder_info.get("folderType")}') - + except ValueError as e: raise click.ClickException(str(e)) except Exception as e: raise click.ClickException(f"Failed to retrieve deletion status: {str(e)}") - + return # Validate link flag requirements AFTER loading profile @@ -1029,7 +1029,7 @@ def job_results(ctx, if verbose: print(f'\t\tLinking results ({results_path})...') - + link_client.link_folder(results_path, session_id) # Delete results directory if requested @@ -1304,7 +1304,7 @@ def list_jobs(ctx, if pagination_metadata: total_jobs = pagination_metadata.get('Pagination-Count', 0) current_page_size = pagination_metadata.get('Pagination-Limit', page_size) - + if total_jobs > 0: total_pages = (total_jobs + current_page_size - 1) // current_page_size if page > total_pages: @@ -1424,17 +1424,17 @@ def abort_jobs(ctx, except Exception as e: click.secho(f"Failed to get status for job {job}, please make sure it exists in the workspace: {e}", fg='yellow', bold=True) continue - + j_status_content = json.loads(j_status.content) job_status = j_status_content['status'] - + # Check if job is in a state that normally allows abortion is_abortable = job_status in ABORT_JOB_STATES - + # Issue warning if job is in initializing state and not using force if job_status == 'initializing' and not force: click.secho(f"Warning: Job {job} is in initializing state.", fg='yellow', bold=True) - + # Check if job can be aborted if not is_abortable: click.secho(f"Job {job} is not in a state that can be aborted and is ignored. " + @@ -1649,19 +1649,19 @@ def archive_unarchive_jobs(ctx, cl.archive_jobs(valid_jobs, workspace_id, verify_ssl) else: cl.unarchive_jobs(valid_jobs, workspace_id, verify_ssl) - + success_msg = [] if len(valid_jobs) == 1: success_msg.append(f"Job '{valid_jobs[0]}' {action_past} successfully.") else: success_msg.append(f"{len(valid_jobs)} jobs {action_past} successfully: {', '.join(valid_jobs)}") - + if already_processed: if len(already_processed) == 1: success_msg.append(f"Job '{already_processed[0]}' was already {action_past}.") else: success_msg.append(f"{len(already_processed)} jobs were already {action_past}: {', '.join(already_processed)}") - + click.secho('\n'.join(success_msg), fg='green', bold=True) except Exception as e: raise ValueError(f"Failed to {action} jobs: {str(e)}") diff --git a/cloudos_cli/jobs/job.py b/cloudos_cli/jobs/job.py index 24f77911..2ac729be 100644 --- a/cloudos_cli/jobs/job.py +++ b/cloudos_cli/jobs/job.py @@ -924,7 +924,7 @@ def docker_workflow_param_processing(self, param, project_name): def get_job_request_payload(self, job_id, verify=True): """Get the original request payload for a job. - + Parameters ---------- job_id : str @@ -933,7 +933,7 @@ def get_job_request_payload(self, job_id, verify=True): Whether to use SSL verification or not. Alternatively, if a string is passed, it will be interpreted as the path to the SSL certificate file. - + Returns ------- dict @@ -953,7 +953,7 @@ def get_job_request_payload(self, job_id, verify=True): def update_parameter_value(self, parameters, param_name, new_value): """Update a parameter value in the parameters list. - + Parameters ---------- parameters : list @@ -962,7 +962,7 @@ def update_parameter_value(self, parameters, param_name, new_value): Name of the parameter to update. new_value : str New value for the parameter. - + Returns ------- bool @@ -1035,7 +1035,7 @@ def clone_or_resume_job(self, verify=True, mode=None): """Clone or resume an existing job with optional parameter overrides. - + Parameters ---------- source_job_id : str diff --git a/cloudos_cli/link/cli.py b/cloudos_cli/link/cli.py index 55aaf8af..4946e1c9 100644 --- a/cloudos_cli/link/cli.py +++ b/cloudos_cli/link/cli.py @@ -86,13 +86,13 @@ def link(ctx, # Link all job folders (results, workdir, logs) cloudos link --job-id 12345 --session-id abc123 - + # Link only results from a job cloudos link --job-id 12345 --session-id abc123 --results - + # Link a specific S3 path cloudos link s3://bucket/folder --session-id abc123 - + """ print('CloudOS link functionality: link s3 folders to interactive analysis sessions.\n') @@ -142,29 +142,29 @@ def link(ctx, if job_id: # Job-based linking print(f'Linking folders from job {job_id} to interactive session {session_id}...\n') - + # Link results if results: link_client.link_job_results(job_id, workspace_id, session_id, verify_ssl, verbose) - + # Link workdir if workdir: link_client.link_job_workdir(job_id, workspace_id, session_id, verify_ssl, verbose) - + # Link logs if logs: link_client.link_job_logs(job_id, workspace_id, session_id, verify_ssl, verbose) - - + + else: # Direct path linking print(f'Linking path to interactive session {session_id}...\n') - + # Link path with validation link_client.link_path_with_validation(path, session_id, verify_ssl, project_name, verbose) - + print('\nLinking operation completed.') - + except BadRequestException as e: raise ValueError(f"Request failed: {str(e)}") except Exception as e: diff --git a/cloudos_cli/link/link.py b/cloudos_cli/link/link.py index 1edd3493..1239a107 100644 --- a/cloudos_cli/link/link.py +++ b/cloudos_cli/link/link.py @@ -13,7 +13,7 @@ import json import time import rich_click as click - + @dataclass class Link(Cloudos): @@ -64,14 +64,14 @@ def link_folder(self, "Content-type": "application/json", "apikey": self.apikey } - + # Block Azure Blob Storage URLs as they are not supported by the API if folder.startswith('az://'): raise ValueError( "Azure Blob Storage paths (az://) are not supported for linking. " "Azure environments do not support linking folders to Interactive Analysis sessions. " ) - + # determine if is file explorer or s3 if folder.startswith('s3://'): data = self.parse_s3_path(folder) @@ -80,7 +80,7 @@ def link_folder(self, data = self.parse_file_explorer_path(folder) type_folder = "File Explorer" r = retry_requests_post(url, headers=headers, json=data, verify=self.verify) - + if r.status_code == 403: raise ValueError(f"Provided {type_folder} folder already exists with 'mounted' status") elif r.status_code == 401: @@ -103,11 +103,11 @@ def link_folder(self, else: full_path = folder mount_name = data['dataItem']['name'] - + try: # Wait for mount completion and check final status final_status = self.wait_for_mount_completion(session_id, mount_name) - + if final_status["status"] == "mounted": click.secho(f"Successfully mounted {type_folder} folder: {full_path}", fg='green', bold=True) elif final_status["status"] == "failed": @@ -116,7 +116,7 @@ def link_folder(self, click.secho(f" Error: {error_msg}", fg='red') else: click.secho(f"Mount status: {final_status['status']} for {type_folder} folder: {full_path}", fg='yellow', bold=True) - + except ValueError as e: click.secho(f"Warning: Could not verify mount status - {str(e)}", fg='yellow', bold=True) click.secho(f" The linking request was submitted, but verification failed.", fg='yellow') @@ -234,16 +234,16 @@ def get_fuse_filesystems_status(self, session_id: str) -> List[Dict]: "Content-type": "application/json", "apikey": self.apikey } - + r = retry_requests_get(url, headers=headers, verify=self.verify) - + if r.status_code == 401: raise ValueError("Forbidden. Invalid API key or insufficient permissions.") elif r.status_code == 404: raise ValueError(f"Interactive session {session_id} not found") elif r.status_code != 200: raise ValueError(f"Failed to get fuse filesystem status: HTTP {r.status_code}") - + response_data = json.loads(r.content) return response_data.get("fuseFileSystems", []) @@ -273,29 +273,29 @@ def wait_for_mount_completion(self, session_id: str, mount_name: str, If the mount is not found or timeout is reached. """ start_time = time.time() - + while time.time() - start_time < timeout: filesystems = self.get_fuse_filesystems_status(session_id) - + # Find the mount by name target_mount = None for fs in filesystems: if fs.get("mountName") == mount_name: target_mount = fs break - + if target_mount and target_mount.get("status") in ["mounted", "failed"]: return target_mount # If mount not found or still in progress, continue waiting - + time.sleep(check_interval) - + raise ValueError(f"Timeout waiting for mount '{mount_name}' to complete after {timeout} seconds") def link_job_results(self, job_id: str, workspace_id: str, session_id: str, verify_ssl, verbose: bool = False): """ Link job results to an interactive session. - + Parameters ---------- job_id : str @@ -308,7 +308,7 @@ def link_job_results(self, job_id: str, workspace_id: str, session_id: str, veri SSL verification setting verbose : bool Whether to print verbose output - + Returns ------- None @@ -317,11 +317,11 @@ def link_job_results(self, job_id: str, workspace_id: str, session_id: str, veri try: if verbose: print('\tFetching job results...') - + # Create a temporary Cloudos client for API calls cl = Cloudos(self.cloudos_url, self.apikey, None) results_path = cl.get_job_results(job_id, workspace_id, verify_ssl) - + if results_path: print('\tLinking results directory...') if verbose: @@ -329,7 +329,7 @@ def link_job_results(self, job_id: str, workspace_id: str, session_id: str, veri self.link_folder(results_path, session_id) else: click.secho('\tNo results found to link.', fg='yellow') - + except JoBNotCompletedException as e: click.secho(f'\tCannot link results: {str(e)}', fg='red') except Exception as e: @@ -342,7 +342,7 @@ def link_job_results(self, job_id: str, workspace_id: str, session_id: str, veri def link_job_workdir(self, job_id: str, workspace_id: str, session_id: str, verify_ssl, verbose: bool = False): """ Link job working directory to an interactive session. - + Parameters ---------- job_id : str @@ -355,7 +355,7 @@ def link_job_workdir(self, job_id: str, workspace_id: str, session_id: str, veri SSL verification setting verbose : bool Whether to print verbose output - + Returns ------- None @@ -364,11 +364,11 @@ def link_job_workdir(self, job_id: str, workspace_id: str, session_id: str, veri try: if verbose: print('\tFetching job working directory...') - + # Create a temporary Cloudos client for API calls cl = Cloudos(self.cloudos_url, self.apikey, None) workdir_path = cl.get_job_workdir(job_id, workspace_id, verify_ssl) - + if workdir_path: print('\tLinking working directory...') if verbose: @@ -376,7 +376,7 @@ def link_job_workdir(self, job_id: str, workspace_id: str, session_id: str, veri self.link_folder(workdir_path.strip(), session_id) else: click.secho('\tNo working directory found to link.', fg='yellow') - + except Exception as e: error_msg = str(e) if "not yet available" in error_msg.lower() or "initializing" in error_msg.lower() or "not available" in error_msg.lower() or "deleted" in error_msg.lower() or "removed" in error_msg.lower(): @@ -387,7 +387,7 @@ def link_job_workdir(self, job_id: str, workspace_id: str, session_id: str, veri def link_job_logs(self, job_id: str, workspace_id: str, session_id: str, verify_ssl, verbose: bool = False): """ Link job logs to an interactive session. - + Parameters ---------- job_id : str @@ -400,7 +400,7 @@ def link_job_logs(self, job_id: str, workspace_id: str, session_id: str, verify_ SSL verification setting verbose : bool Whether to print verbose output - + Returns ------- None @@ -409,23 +409,23 @@ def link_job_logs(self, job_id: str, workspace_id: str, session_id: str, verify_ try: if verbose: print('\tFetching job logs...') - + # Create a temporary Cloudos client for API calls cl = Cloudos(self.cloudos_url, self.apikey, None) logs_dict = cl.get_job_logs(job_id, workspace_id, verify_ssl) - + if logs_dict: # Extract the parent logs directory from any log file path first_log_path = next(iter(logs_dict.values())) logs_dir = '/'.join(first_log_path.split('/')[:-1]) - + print('\tLinking logs directory...') if verbose: print(f'\t\tLogs directory: {logs_dir}') self.link_folder(logs_dir, session_id) else: click.secho('\tNo logs found to link.', fg='yellow') - + except Exception as e: error_msg = str(e) if "not yet available" in error_msg.lower() or "initializing" in error_msg.lower() or "not available" in error_msg.lower() or "deleted" in error_msg.lower() or "removed" in error_msg.lower(): @@ -436,7 +436,7 @@ def link_job_logs(self, job_id: str, workspace_id: str, session_id: str, verify_ def link_path_with_validation(self, path: str, session_id: str, verify_ssl, project_name: str = None, verbose: bool = False): """ Link a path (S3 or File Explorer) to an interactive session with validation. - + Parameters ---------- path : str @@ -449,12 +449,12 @@ def link_path_with_validation(self, path: str, session_id: str, verify_ssl, proj SSL verification setting verbose : bool Whether to print verbose output - + Returns ------- None Prints status messages to console - + Raises ------ click.UsageError @@ -465,15 +465,15 @@ def link_path_with_validation(self, path: str, session_id: str, verify_ssl, proj # Check for Azure paths and provide informative error message if path.startswith("az://"): raise click.UsageError("Azure Blob Storage paths (az://) are not supported for linking. Please use S3 paths (s3://) or File Explorer paths instead.") - + # Validate path requirements if not path.startswith("s3://") and not project_name: raise click.UsageError("When using File Explorer paths, '--project-name' must be provided.") - + # Use the same validation logic as datasets link command is_s3 = path.startswith("s3://") is_folder = True - + if is_s3: # S3 path validation try: @@ -528,7 +528,7 @@ def link_path_with_validation(self, path: str, session_id: str, verify_ssl, proj is_folder = "not_found" except Exception: is_folder = None - + if is_folder == "file": if is_s3: raise ValueError("The path appears to point to a file, not a folder. You can only link folders. Please link the parent folder instead.") @@ -547,9 +547,9 @@ def link_path_with_validation(self, path: str, session_id: str, verify_ssl, proj click.secho("Unable to verify the File Explorer path. Proceeding with linking; " + "however, if the operation fails, please verify the path exists and is a folder.", fg='yellow', bold=True) - + if verbose: print('\tLinking {path}...') - + self.link_folder(path, session_id) diff --git a/cloudos_cli/utils/details.py b/cloudos_cli/utils/details.py index 83501e6c..56f500ef 100644 --- a/cloudos_cli/utils/details.py +++ b/cloudos_cli/utils/details.py @@ -451,7 +451,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ current_page = pagination_metadata.get('Pagination-Page', 1) page_size = pagination_metadata.get('Pagination-Limit', 10) total_pages = (total_jobs + page_size - 1) // page_size if total_jobs > 0 else 1 - + console.print(f"\n[cyan]Total jobs matching filter:[/cyan] {total_jobs}") console.print(f"[cyan]Page:[/cyan] {current_page} of {total_pages}") console.print(f"[cyan]Jobs on this page:[/cyan] {len(jobs)}") @@ -485,13 +485,13 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ "N/A": "[bold bright_black]?[/bold bright_black]" # Grey question mark } status = status_symbol_map.get(status_raw.lower(), status_raw) - + # Name name = str(job.get("name", "N/A")) - + # Project project = str(job.get("project", {}).get("name", "N/A")) - + # Owner (compact format for small terminals) user_info = job.get("user", {}) name_part = user_info.get('name', '') @@ -512,7 +512,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ owner = name_part or surname_part else: owner = "N/A" - + # Pipeline pipeline = str(job.get("workflow", {}).get("name", "N/A")) # Only show the first line if pipeline name contains newlines @@ -520,12 +520,12 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ # Truncate to 25 chars with ellipsis if longer if len(pipeline) > 25: pipeline = pipeline[:22] + "..." - + # ID with hyperlink job_id = str(job.get("_id", "N/A")) job_url = f"{cloudos_url}/app/advanced-analytics/analyses/{job_id}" job_id_with_link = f"[link={job_url}]{job_id}[/link]" - + # Submit time (compact format for small terminals) created_at = job.get("createdAt") if created_at: @@ -541,7 +541,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ submit_time = "N/A" else: submit_time = "N/A" - + # End time (compact format for small terminals) end_time_raw = job.get("endTime") if end_time_raw: @@ -557,7 +557,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ end_time = "N/A" else: end_time = "N/A" - + # Run time (calculate from startTime and endTime) start_time_raw = job.get("startTime") if start_time_raw and end_time_raw: @@ -579,7 +579,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ run_time = "N/A" else: run_time = "N/A" - + # Commit revision = job.get("revision", {}) if job.get("jobType") == "dockerAWS": @@ -589,7 +589,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ # Truncate commit to 7 characters if it's longer if commit != "N/A" and len(commit) > 7: commit = commit[:7] - + # Cost cost_raw = job.get("computeCostSpent") or job.get("realInstancesExecutionCost") if cost_raw is not None: @@ -599,13 +599,13 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ cost = "N/A" else: cost = "N/A" - + # Resources (instance type only) master_instance = job.get("masterInstance", {}) used_instance = master_instance.get("usedInstance", {}) instance_type = used_instance.get("type", "N/A") resources = instance_type if instance_type else "N/A" - + # Storage type storage_mode = job.get("storageMode", "N/A") if storage_mode == "regular": @@ -614,7 +614,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ storage_type = "Lustre" else: storage_type = str(storage_mode).capitalize() if storage_mode != "N/A" else "N/A" - + # Map column keys to their values column_values = { 'status': status, @@ -631,7 +631,7 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ 'resources': resources, 'storage_type': storage_type } - + # Add row to table with only selected columns row_values = [column_values[col] for col in columns_to_show] table.add_row(*row_values) @@ -644,5 +644,5 @@ def create_job_list_table(jobs, cloudos_url, pagination_metadata=None, selected_ current_page = pagination_metadata.get('Pagination-Page', 1) page_size = pagination_metadata.get('Pagination-Limit', 10) total_pages = (total_jobs + page_size - 1) // page_size if total_jobs > 0 else 1 - + console.print(f"\n[cyan]Showing {len(jobs)} of {total_jobs} total jobs | Page {current_page} of {total_pages}[/cyan]") diff --git a/cloudos_cli/utils/errors.py b/cloudos_cli/utils/errors.py index eff6ef64..8bdd72e0 100755 --- a/cloudos_cli/utils/errors.py +++ b/cloudos_cli/utils/errors.py @@ -20,13 +20,13 @@ def __init__(self, rv): except (ValueError, AttributeError): # Response is not JSON or doesn't have expected structure pass - + # Prioritize message from response, fallback to reason if error_message: msg = "Server returned status {}. Message: {}".format(rv.status_code, error_message) else: msg = "Server returned status {}. Reason: {}".format(rv.status_code, rv.reason) - + super(BadRequestException, self).__init__(msg) self.rv = rv diff --git a/tests/test_clos/test_archive_integration.py b/tests/test_clos/test_archive_integration.py index f65f95b1..38ee7efb 100644 --- a/tests/test_clos/test_archive_integration.py +++ b/tests/test_clos/test_archive_integration.py @@ -17,28 +17,28 @@ def test_job_archive_successful_flow(): status_code=200, json={"jobs": [], "pagination_metadata": {"Pagination-Count": 0}} ) - + # Mock checking if job exists in unarchived list (should return the job) m.get( "https://cloudos.lifebit.ai/api/v2/jobs?teamId=workspace_123&archived.status=false&page=1&limit=1&id=valid_job_123", status_code=200, json={"jobs": [{"_id": "valid_job_123", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock the archive API call to succeed m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'valid_job_123' ]) - + # Should succeed assert result.exit_code == 0 assert "Job 'valid_job_123' archived successfully." in result.output @@ -58,28 +58,28 @@ def test_job_archive_multiple_jobs_successful_flow(): status_code=200, json={"jobs": [], "pagination_metadata": {"Pagination-Count": 0}} ) - + # Mock checking if job exists in unarchived list (should return the job) m.get( f"https://cloudos.lifebit.ai/api/v2/jobs?teamId=workspace_123&archived.status=false&page=1&limit=1&id={job_id}", status_code=200, json={"jobs": [{"_id": job_id, "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock the archive API call to succeed m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'job1,job2,job3' ]) - + # Should succeed assert result.exit_code == 0 assert "3 jobs archived successfully: job1, job2, job3" in result.output @@ -102,7 +102,7 @@ def test_job_archive_mixed_valid_invalid_jobs(): status_code=200, json={"jobs": [{"_id": "valid_job", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock job status check - invalid job (not in either list) m.get( "https://cloudos.lifebit.ai/api/v2/jobs?teamId=workspace_123&archived.status=true&page=1&limit=1&id=invalid_job", @@ -114,21 +114,21 @@ def test_job_archive_mixed_valid_invalid_jobs(): status_code=200, json={"jobs": [], "pagination_metadata": {"Pagination-Count": 0}} ) - + # Mock the archive API call to succeed for valid jobs m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'valid_job,invalid_job' ]) - + # Command should exit gracefully when encountering invalid job assert result.exit_code == 0 assert "Failed to get status for job invalid_job" in result.output diff --git a/tests/test_clos/test_archive_jobs.py b/tests/test_clos/test_archive_jobs.py index d1ce7c8f..2f39342b 100644 --- a/tests/test_clos/test_archive_jobs.py +++ b/tests/test_clos/test_archive_jobs.py @@ -16,7 +16,7 @@ def test_archive_jobs_correct_response(self): apikey = "test_apikey" workspace_id = "workspace123" job_ids = ["69413101b07d5f5bb46891b4", "another_job_id"] - + with requests_mock.Mocker() as m: # Mock the PUT request to the archive endpoint m.put( @@ -24,10 +24,10 @@ def test_archive_jobs_correct_response(self): status_code=200, json={"success": True} ) - + cl = Cloudos(cloudos_url, apikey, None) response = cl.archive_jobs(job_ids, workspace_id) - + assert response.status_code == 200 # Check that the request was made with correct data request_data = json.loads(m.last_request.text) @@ -45,7 +45,7 @@ def test_archive_jobs_bad_request_response(self): apikey = "test_apikey" workspace_id = "workspace123" job_ids = ["invalid_job_id"] - + with requests_mock.Mocker() as m: # Mock a 400 bad request response m.put( @@ -53,7 +53,7 @@ def test_archive_jobs_bad_request_response(self): status_code=400, json={"error": "Invalid job ID"} ) - + cl = Cloudos(cloudos_url, apikey, None) with pytest.raises(BadRequestException): cl.archive_jobs(job_ids, workspace_id) @@ -64,17 +64,17 @@ def test_archive_jobs_with_ssl_verification(self): apikey = "test_apikey" workspace_id = "workspace123" job_ids = ["69413101b07d5f5bb46891b4"] - + with requests_mock.Mocker() as m: m.put( f"{cloudos_url}/api/v1/jobs?teamId={workspace_id}", status_code=200, json={"success": True} ) - + cl = Cloudos(cloudos_url, apikey, None) response = cl.archive_jobs(job_ids, workspace_id, verify=False) - + assert response.status_code == 200 def test_archive_jobs_single_job(self): @@ -83,17 +83,17 @@ def test_archive_jobs_single_job(self): apikey = "test_apikey" workspace_id = "workspace123" job_ids = ["69413101b07d5f5bb46891b4"] - + with requests_mock.Mocker() as m: m.put( f"{cloudos_url}/api/v1/jobs?teamId={workspace_id}", status_code=200, json={"success": True} ) - + cl = Cloudos(cloudos_url, apikey, None) response = cl.archive_jobs(job_ids, workspace_id) - + assert response.status_code == 200 request_data = json.loads(m.last_request.text) assert len(request_data["jobIds"]) == 1 @@ -105,17 +105,17 @@ def test_archive_jobs_multiple_jobs(self): apikey = "test_apikey" workspace_id = "workspace123" job_ids = ["job1", "job2", "job3"] - + with requests_mock.Mocker() as m: m.put( f"{cloudos_url}/api/v1/jobs?teamId={workspace_id}", status_code=200, json={"success": True} ) - + cl = Cloudos(cloudos_url, apikey, None) response = cl.archive_jobs(job_ids, workspace_id) - + assert response.status_code == 200 request_data = json.loads(m.last_request.text) assert len(request_data["jobIds"]) == 3 diff --git a/tests/test_clos/test_archive_status_checking.py b/tests/test_clos/test_archive_status_checking.py index f2adf7e4..17c15968 100644 --- a/tests/test_clos/test_archive_status_checking.py +++ b/tests/test_clos/test_archive_status_checking.py @@ -17,14 +17,14 @@ def test_archive_already_archived_job(): status_code=200, json={"jobs": [{"_id": "already_archived_job", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'already_archived_job' ]) - + # Should succeed but indicate no action needed assert result.exit_code == 0 assert "is already archived. No action needed." in result.output @@ -44,21 +44,21 @@ def test_unarchive_already_unarchived_job(): status_code=200, json={"jobs": [], "pagination_metadata": {"Pagination-Count": 0}} ) - + # Mock checking if job exists in unarchived list (should return the job) m.get( "https://cloudos.lifebit.ai/api/v2/jobs?teamId=workspace_123&archived.status=false&page=1&limit=1&id=not_archived_job", status_code=200, json={"jobs": [{"_id": "not_archived_job", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'not_archived_job' ]) - + # Should succeed but indicate no action needed assert result.exit_code == 0 assert "is already unarchived. No action needed." in result.output @@ -78,7 +78,7 @@ def test_archive_mixed_status_jobs(): status_code=200, json={"jobs": [{"_id": "already_archived", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + m.get( "https://cloudos.lifebit.ai/api/v2/jobs?teamId=workspace_123&archived.status=true&page=1&limit=1&id=not_archived", status_code=200, @@ -89,21 +89,21 @@ def test_archive_mixed_status_jobs(): status_code=200, json={"jobs": [{"_id": "not_archived", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock the archive API call for the unarchived job m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'already_archived,not_archived' ]) - + # Should succeed and handle both scenarios assert result.exit_code == 0 assert "Job 'not_archived' archived successfully" in result.output @@ -121,7 +121,7 @@ def test_unarchive_mixed_status_jobs(): status_code=200, json={"jobs": [{"_id": "archived_job", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + m.get( "https://cloudos.lifebit.ai/api/v2/jobs?teamId=workspace_123&archived.status=true&page=1&limit=1&id=not_archived", status_code=200, @@ -132,21 +132,21 @@ def test_unarchive_mixed_status_jobs(): status_code=200, json={"jobs": [{"_id": "not_archived", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock the unarchive API call for the archived job m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'archived_job,not_archived' ]) - + # Should succeed and handle both scenarios assert result.exit_code == 0 assert "Job 'archived_job' unarchived successfully" in result.output @@ -164,7 +164,7 @@ def test_archive_verbose_already_archived(): status_code=200, json={"jobs": [{"_id": "already_archived", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', '--apikey', 'test_key', @@ -172,7 +172,7 @@ def test_archive_verbose_already_archived(): '--job-ids', 'already_archived', '--verbose' ]) - + # Should show verbose information about already archived status assert result.exit_code == 0 assert "Job already_archived is already archived" in result.output diff --git a/tests/test_clos/test_cli_archive_command.py b/tests/test_clos/test_cli_archive_command.py index c13750fc..5257f03a 100644 --- a/tests/test_clos/test_cli_archive_command.py +++ b/tests/test_clos/test_cli_archive_command.py @@ -73,14 +73,14 @@ def test_job_archive_invalid_job_ids(): status_code=200, json={"jobs": [], "pagination_metadata": {"Pagination-Count": 0}} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'archive', '--apikey', 'test_key', '--workspace-id', 'test_workspace', '--job-ids', 'invalid_job' ]) - + # The command should handle the error gracefully with exit code 0 assert result.exit_code == 0 # Error message should be present in output diff --git a/tests/test_clos/test_cli_unarchive_command.py b/tests/test_clos/test_cli_unarchive_command.py index 9a664413..83dcc1bf 100644 --- a/tests/test_clos/test_cli_unarchive_command.py +++ b/tests/test_clos/test_cli_unarchive_command.py @@ -73,14 +73,14 @@ def test_job_unarchive_invalid_job_ids(): status_code=200, json={"jobs": [], "pagination_metadata": {"Pagination-Count": 0}} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', '--apikey', 'test_key', '--workspace-id', 'test_workspace', '--job-ids', 'invalid_job' ]) - + # The command should handle the error gracefully with exit code 0 assert result.exit_code == 0 # Error message should be present in output diff --git a/tests/test_clos/test_unarchive_integration.py b/tests/test_clos/test_unarchive_integration.py index 17fbebae..491ae6de 100644 --- a/tests/test_clos/test_unarchive_integration.py +++ b/tests/test_clos/test_unarchive_integration.py @@ -17,21 +17,21 @@ def test_job_unarchive_successful_flow(): status_code=200, json={"jobs": [{"_id": "archived_job_123", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock the unarchive API call to succeed m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'archived_job_123' ]) - + # Should succeed assert result.exit_code == 0 assert "Job 'archived_job_123' unarchived successfully." in result.output @@ -51,21 +51,21 @@ def test_job_unarchive_multiple_jobs_successful_flow(): status_code=200, json={"jobs": [{"_id": job_id, "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock the unarchive API call to succeed m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'job1,job2,job3' ]) - + # Should succeed assert result.exit_code == 0 assert "3 jobs unarchived successfully: job1, job2, job3" in result.output @@ -83,7 +83,7 @@ def test_job_unarchive_mixed_valid_invalid_jobs(): status_code=200, json={"jobs": [{"_id": "valid_archived_job", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock job status check - invalid job (not in archived or unarchived lists) m.get( "https://cloudos.lifebit.ai/api/v2/jobs?teamId=workspace_123&archived.status=true&page=1&limit=1&id=invalid_job", @@ -95,21 +95,21 @@ def test_job_unarchive_mixed_valid_invalid_jobs(): status_code=200, json={"jobs": [], "pagination_metadata": {"Pagination-Count": 0}} ) - + # Mock the unarchive API call to succeed for valid jobs m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', '--apikey', 'test_key', '--workspace-id', 'workspace_123', '--job-ids', 'valid_archived_job,invalid_job' ]) - + # The command should exit gracefully (0) when encountering invalid job assert result.exit_code == 0 assert "Unarchiving jobs..." in result.output @@ -128,14 +128,14 @@ def test_job_unarchive_verbose_output(): status_code=200, json={"jobs": [{"_id": "archived_job", "status": "completed"}], "pagination_metadata": {"Pagination-Count": 1}} ) - + # Mock the unarchive API call to succeed m.put( "https://cloudos.lifebit.ai/api/v1/jobs?teamId=workspace_123", status_code=200, json={"success": True} ) - + result = runner.invoke(run_cloudos_cli, [ 'job', 'unarchive', '--apikey', 'test_key', @@ -143,7 +143,7 @@ def test_job_unarchive_verbose_output(): '--job-ids', 'archived_job', '--verbose' ]) - + # Should succeed with verbose output assert result.exit_code == 0 assert "Unarchiving jobs..." in result.output diff --git a/tests/test_clos/test_unarchive_jobs.py b/tests/test_clos/test_unarchive_jobs.py index 48bd496b..cabbfb64 100644 --- a/tests/test_clos/test_unarchive_jobs.py +++ b/tests/test_clos/test_unarchive_jobs.py @@ -25,7 +25,7 @@ def test_unarchive_jobs_correct_response(self): assert response.status_code == 200 assert m.called assert m.call_count == 1 - + # Verify the request payload request = m.request_history[0] import json @@ -79,7 +79,7 @@ def test_unarchive_jobs_single_job(self): response = cl.unarchive_jobs(job_ids, "test_workspace") assert response.status_code == 200 - + # Verify the request payload request = m.request_history[0] import json @@ -102,7 +102,7 @@ def test_unarchive_jobs_multiple_jobs(self): response = cl.unarchive_jobs(job_ids, "test_workspace") assert response.status_code == 200 - + # Verify the request payload request = m.request_history[0] import json diff --git a/tests/test_cost/test_job_cost.py b/tests/test_cost/test_job_cost.py index 68d85448..ce1b906b 100644 --- a/tests/test_cost/test_job_cost.py +++ b/tests/test_cost/test_job_cost.py @@ -55,15 +55,15 @@ def test_get_job_costs_correct_response(self): headers=header, status=200 ) - + # get mock response response = self.cost_viewer.get_job_costs(JOB_ID, WORKSPACE_ID) - + # check the response structure assert "master" in response assert "workers" in response assert "paginationMetadata" in response - + # check master instance data master = response["master"] assert master["id"] == "i-00e328d0c4fe4bc17" @@ -71,7 +71,7 @@ def test_get_job_costs_correct_response(self): assert master["isCostSaving"] is False assert master["instancePricePerHour"]["amount"] == 0.1 assert master["storage"]["usageQuantity"] == 600 - + # check workers data workers = response["workers"] assert len(workers) == 2 @@ -83,17 +83,17 @@ def test_calculate_runtime(self): """Test runtime calculation between timestamps""" start_time = "2025-09-01T15:23:59.246Z" end_time = "2025-09-01T15:26:15.291Z" - + runtime = self.cost_viewer._calculate_runtime(start_time, end_time) assert runtime == "2m 16s" - + # Test with hours start_time_long = "2025-09-01T13:23:59.246Z" end_time_long = "2025-09-01T15:26:15.291Z" - + runtime_long = self.cost_viewer._calculate_runtime(start_time_long, end_time_long) assert runtime_long == "2h 2m 16s" - + # Test with invalid timestamp runtime_invalid = self.cost_viewer._calculate_runtime("invalid", "invalid") assert runtime_invalid == "N/A" @@ -104,11 +104,11 @@ def test_format_storage(self): storage_info = {"usageQuantity": 600, "usageUnit": "Gb"} formatted = self.cost_viewer._format_storage(storage_info) assert formatted == "600 Gb" - + # Test empty storage info formatted_empty = self.cost_viewer._format_storage({}) assert formatted_empty == "N/A" - + # Test None storage info formatted_none = self.cost_viewer._format_storage(None) assert formatted_none == "N/A" @@ -119,15 +119,15 @@ def test_format_price(self): price_info = {"amount": 0.1, "currencyCode": "USD"} formatted = self.cost_viewer._format_price(price_info) assert formatted == "$0.1000/hr" - + # Test total price formatted_total = self.cost_viewer._format_price(price_info, total=True) assert formatted_total == "$0.1000" - + # Test empty price info formatted_empty = self.cost_viewer._format_price({}) assert formatted_empty == "N/A" - + # Test None price info formatted_none = self.cost_viewer._format_price(None) assert formatted_none == "N/A" @@ -137,7 +137,7 @@ def test_format_lifecycle_type(self): # Test cost saving (spot) formatted_spot = self.cost_viewer._format_lifecycle_type(True) assert formatted_spot == "spot" - + # Test on demand formatted_on_demand = self.cost_viewer._format_lifecycle_type(False) assert formatted_on_demand == "on demand" @@ -150,24 +150,24 @@ def test_csv_output(self, mock_get): mock_response.status_code = 200 mock_response.json.return_value = json.loads(load_json_file(INPUT)) mock_get.return_value = mock_response - + # Change to temp directory original_cwd = os.getcwd() os.chdir(self.temp_dir) - + try: # Test CSV output self.cost_viewer.display_costs(JOB_ID, WORKSPACE_ID, "csv") - + # Check if CSV file was created csv_filename = f"{JOB_ID}_costs.csv" assert os.path.exists(csv_filename) - + # Read and verify CSV content with open(csv_filename, 'r') as csvfile: reader = csv.reader(csvfile) rows = list(reader) - + # Check header expected_headers = [ "Type", "Instance id", "Instance", "Life-cycle type", @@ -175,23 +175,23 @@ def test_csv_output(self, mock_get): "Compute storage price", "Total" ] assert rows[0] == expected_headers - + # Check we have the right number of rows (header + master + 2 workers) assert len(rows) == 4 - + # Check master row master_row = rows[1] assert master_row[0] == "Master" assert master_row[1] == "i-00e328d0c4fe4bc17" assert master_row[2] == "c4.large" assert master_row[3] == "on demand" - + # Check worker rows worker1_row = rows[2] assert worker1_row[0] == "Worker" assert worker1_row[1] == "i-0d1d9e96cda992e74" assert worker1_row[3] == "spot" - + finally: os.chdir(original_cwd) @@ -203,44 +203,44 @@ def test_json_output(self, mock_get): mock_response.status_code = 200 mock_response.json.return_value = json.loads(load_json_file(INPUT)) mock_get.return_value = mock_response - + # Change to temp directory original_cwd = os.getcwd() os.chdir(self.temp_dir) - + try: # Test JSON output self.cost_viewer.display_costs(JOB_ID, WORKSPACE_ID, "json") - + # Check if JSON file was created json_filename = f"{JOB_ID}_costs.json" assert os.path.exists(json_filename) - + # Read and verify JSON content with open(json_filename, 'r') as jsonfile: data = json.load(jsonfile) - + # Check structure assert "job_id" in data assert "cost_table" in data assert "final_cost" in data - + assert data["job_id"] == JOB_ID - + # Check cost table structure cost_table = data["cost_table"] assert len(cost_table) == 3 # master + 2 workers - + # Check first entry (master) master_entry = cost_table[0] assert master_entry["Type"] == "Master" assert master_entry["Instance id"] == "i-00e328d0c4fe4bc17" assert master_entry["Instance"] == "c4.large" assert master_entry["Life-cycle type"] == "on demand" - + # Check final cost is properly formatted assert data["final_cost"].startswith("$") - + finally: os.chdir(original_cwd) @@ -252,7 +252,7 @@ def test_stdout_output_simple(self, mock_get): mock_response.status_code = 200 mock_response.json.return_value = json.loads(load_json_file(INPUT)) mock_get.return_value = mock_response - + # Mock input to avoid actual user interaction with patch('builtins.input', return_value='q'): # This test simply checks that the method runs without errors @@ -272,14 +272,14 @@ def test_error_handling_401(self, mock_get): mock_response.status_code = 401 mock_response.reason = "Forbidden" mock_response.json.return_value = {"error": "Forbidden"} - + # Create a BadRequestException that matches what the real code would produce exception = BadRequestException(mock_response) mock_get.side_effect = exception - + with pytest.raises(ValueError) as excinfo: self.cost_viewer.display_costs(JOB_ID, WORKSPACE_ID, "stdout") - + # Check that the error message contains the expected text error_message = str(excinfo.value) assert "cannot see other user's job details" in error_message or "401" in error_message or "Forbidden" in error_message @@ -292,14 +292,14 @@ def test_error_handling_400(self, mock_get): mock_response.status_code = 400 mock_response.reason = "Bad Request" mock_response.json.return_value = {"error": "Bad Request"} - + # Create a BadRequestException that matches what the real code would produce exception = BadRequestException(mock_response) mock_get.side_effect = exception - + with pytest.raises(ValueError) as excinfo: self.cost_viewer.display_costs(JOB_ID, WORKSPACE_ID, "stdout") - + assert "Job not found or cost data not available" in str(excinfo.value) @@ -330,17 +330,17 @@ def test_cloudos_get_job_costs_correct_response(self): headers=header, status=200 ) - + # get mock response response = self.cost_viewer.get_job_costs(JOB_ID, WORKSPACE_ID) - + # check the response assert isinstance(response, dict) - + # Parse response content result_string = json.dumps(response) result_json = json.loads(result_string) - + # Verify response structure assert "master" in result_json assert "workers" in result_json @@ -357,7 +357,7 @@ def test_cloudos_get_job_costs_with_pagination(self): "Content-type": "application/json", "apikey": APIKEY } - + # mock GET method with the .json responses.add( responses.GET, @@ -366,10 +366,10 @@ def test_cloudos_get_job_costs_with_pagination(self): headers=header, status=200 ) - + # get mock response with custom pagination response = self.cost_viewer.get_job_costs(JOB_ID, WORKSPACE_ID, page=2, limit=50) - + # check the response assert isinstance(response, dict) @@ -383,7 +383,7 @@ def test_cloudos_get_job_costs_error_response(self): "Content-type": "application/json", "apikey": APIKEY } - + # mock GET method with error responses.add( responses.GET, diff --git a/tests/test_related_analyses/test_related_analyses.py b/tests/test_related_analyses/test_related_analyses.py index 65de50ec..34bffe9a 100644 --- a/tests/test_related_analyses/test_related_analyses.py +++ b/tests/test_related_analyses/test_related_analyses.py @@ -24,19 +24,19 @@ def test_save_as_json(self, tmp_path): "computeCostSpent": 1250 } } - + # Create a temporary file path output_file = tmp_path / "test_output.json" - + # Save data save_as_json(test_data, str(output_file)) - + # Verify file was created and contains correct data assert output_file.exists() - + with open(output_file, 'r') as f: loaded_data = json.load(f) - + assert loaded_data == test_data assert loaded_data["job123"]["name"] == "Test Job" assert loaded_data["job123"]["status"] == "completed" @@ -78,15 +78,15 @@ def test_save_as_json_multiple_jobs(self, tmp_path): "computeCostSpent": 1000 } } - + output_file = tmp_path / "multiple_jobs.json" save_as_json(test_data, str(output_file)) - + assert output_file.exists() - + with open(output_file, 'r') as f: loaded_data = json.load(f) - + assert len(loaded_data) == 3 assert "job1" in loaded_data assert "job2" in loaded_data @@ -109,7 +109,7 @@ def test_save_as_stdout_single_job(self, mock_input): "computeCostSpent": 1250 } } - + # Should not raise any exceptions try: save_as_stdout(test_data, "parent_job_id") @@ -123,7 +123,7 @@ def test_save_as_stdout_empty_data(self, mock_input): Test displaying empty related analyses data """ test_data = {} - + # Should not raise any exceptions try: save_as_stdout(test_data, "parent_job_id") @@ -148,7 +148,7 @@ def test_save_as_stdout_with_null_values(self, mock_input): "computeCostSpent": None # No cost yet } } - + # Should not raise any exceptions try: save_as_stdout(test_data, "parent_job_id") @@ -174,7 +174,7 @@ def test_save_as_stdout_pagination(self, mock_input): "runTime": 100.0 + i, "computeCostSpent": 500 + i * 10 } - + # Should not raise any exceptions and should handle pagination try: save_as_stdout(test_data, "parent_job_id") @@ -191,14 +191,14 @@ def test_save_as_json_empty_data(self, tmp_path): """ test_data = {} output_file = tmp_path / "empty_output.json" - + save_as_json(test_data, str(output_file)) - + assert output_file.exists() - + with open(output_file, 'r') as f: loaded_data = json.load(f) - + assert loaded_data == {} assert len(loaded_data) == 0 From 2ec054cf76b11a4ef4456312bbe85ca3f53c2499 Mon Sep 17 00:00:00 2001 From: Daniel Boloc Date: Thu, 12 Feb 2026 15:56:29 +0100 Subject: [PATCH 41/41] ci: temporarily disable filter-queue --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2ab1486..77b66e5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: # Test filtering by only mine cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-only-mine --last-n-jobs 10 # Test filtering by queue - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "cost_saving_standard_nextflow" --last-n-jobs 10 + #cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "cost_saving_standard_nextflow" --last-n-jobs 10 job_details: needs: job_run_and_status runs-on: ubuntu-latest