From 76a1ae11d5caf08367c9428193da262e1bd1c416 Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Thu, 20 Nov 2025 21:53:12 +0100 Subject: [PATCH] Add OpenPBS tests to the CI/CD This commit adds a Docker-based OpenPBS cluster to the set of target architectures for CI tests, ensuring that the `PBSConnector` is properly tested. --- Makefile | 2 +- streamflow/data/remotepath.py | 13 +++--- streamflow/log_handler.py | 2 +- tests/conftest.py | 9 +++- .../deployment/openpbs/docker-compose.yml | 31 +++++++++++++ .../data/deployment/slurm/docker-compose.yml | 4 +- tests/utils/deployment.py | 44 ++++++++++++++++++- tox.ini | 1 + 8 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 tests/data/deployment/openpbs/docker-compose.yml diff --git a/Makefile b/Makefile index b62091007..f51acc11e 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ test: python -m pytest -rs ${PYTEST_EXTRA} testcov: - python -m pytest -rs --cov --junitxml=junit.xml -o junit_family=legacy --cov-report= ${PYTEST_EXTRA} + python -m pytest -s --cov --junitxml=junit.xml -o junit_family=legacy --cov-report= ${PYTEST_EXTRA} typing: mypy streamflow tests \ No newline at end of file diff --git a/streamflow/data/remotepath.py b/streamflow/data/remotepath.py index 7590e9caf..bc94122e2 100644 --- a/streamflow/data/remotepath.py +++ b/streamflow/data/remotepath.py @@ -915,13 +915,14 @@ async def download( f"Downloading {url} failed with status {response.status}:\n{response.content}" ) connector = context.deployment_manager.get_connector(location.deployment) - await connector.run( - location=location, - command=[ - f'if [ command -v curl ]; then curl -L -o "{filepath}" "{url}"; ' - f'else wget -O "{filepath}" "{url}"; fi' - ], + command = [ + f'if command -v curl; then curl -L -o "{filepath}" "{url}"; ' + f'else wget -O "{filepath}" "{url}"; fi' + ] + result, status = await connector.run( + location=location, command=command, capture_output=True ) + _check_status(command, location, result, status) return StreamFlowPath(filepath, context=context, location=location) diff --git a/streamflow/log_handler.py b/streamflow/log_handler.py index 65d1fd42c..8302d1a69 100644 --- a/streamflow/log_handler.py +++ b/streamflow/log_handler.py @@ -126,5 +126,5 @@ def highlight(self, msg: str | Any) -> str: ) defaultStreamHandler.setFormatter(formatter) logger.addHandler(defaultStreamHandler) -logger.setLevel(logging.INFO) +logger.setLevel(logging.DEBUG) logger.propagate = False diff --git a/tests/conftest.py b/tests/conftest.py index 946439c5c..8c4c744c1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -126,7 +126,14 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: def all_deployment_types() -> list[str]: - deployments_ = ["local", "docker", "docker-compose", "docker-wrapper", "slurm"] + deployments_ = [ + "local", + "docker", + "docker-compose", + "docker-wrapper", + "openpbs", + "slurm", + ] if platform.system() == "Linux": deployments_.extend(["kubernetes", "singularity", "ssh"]) return deployments_ diff --git a/tests/data/deployment/openpbs/docker-compose.yml b/tests/data/deployment/openpbs/docker-compose.yml new file mode 100644 index 000000000..e02d202ff --- /dev/null +++ b/tests/data/deployment/openpbs/docker-compose.yml @@ -0,0 +1,31 @@ +services: + server: + image: alphaunito/openpbs-server:23.06.06 + environment: + PBS_EXECUTION_HOST_NAME_PREFIX: ${COMPOSE_PROJECT_NAME}-execution + PBS_EXECUTION_NODES: 2 + hostname: openpbs-server + networks: + - pbsnet + volumes: + - home:/home/hpcuser + - munge:/etc/munge + - pgsql:/usr/local/pgsql/data + execution: + image: alphaunito/openpbs-execution:23.06.06 + deploy: + mode: replicated + replicas: 2 + environment: + PBS_SERVER_HOST_NAME: openpbs-server + networks: + - pbsnet + volumes: + - home:/home/hpcuser + - munge:/etc/munge +networks: + pbsnet: +volumes: + home: + munge: + pgsql: \ No newline at end of file diff --git a/tests/data/deployment/slurm/docker-compose.yml b/tests/data/deployment/slurm/docker-compose.yml index a8ce86e51..fac10e6df 100644 --- a/tests/data/deployment/slurm/docker-compose.yml +++ b/tests/data/deployment/slurm/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.8" services: slurmctld: - image: alphaunito/slurmctld:21.08.5 + image: alphaunito/slurmctld:23.11.4 environment: SLURMD_HOSTNAME_PREFIX: ${COMPOSE_PROJECT_NAME}-slurmd SLURMD_NODES: 2 @@ -13,7 +13,7 @@ services: - munge:/etc/munge - mysql:/var/lib/mysql slurmd: - image: alphaunito/slurmd:21.08.5 + image: alphaunito/slurmd:23.11.4 deploy: mode: replicated replicas: 2 diff --git a/tests/utils/deployment.py b/tests/utils/deployment.py index c525fea06..ecf34d438 100644 --- a/tests/utils/deployment.py +++ b/tests/utils/deployment.py @@ -79,6 +79,8 @@ def get_deployment(deployment_t: str) -> str: return "__LOCAL__" case "local-fs-volatile": return "local-fs-volatile" + case "openpbs": + return "docker-openpbs" case "parameterizable_hardware": return "custom-hardware" case "singularity": @@ -115,6 +117,16 @@ async def get_deployment_config( "test-fs-volatile", ), ) + case "docker": + return get_docker_deployment_config() + case "docker-compose": + return get_docker_compose_deployment_config() + case "docker-wrapper": + return await get_docker_wrapper_deployment_config(_context) + case "kubernetes": + return get_kubernetes_deployment_config() + case "openpbs": + return await get_openpbs_deployment_config(_context) case "parameterizable-hardware": return get_parameterizable_hardware_deployment_config() case "singularity": @@ -231,6 +243,36 @@ async def get_location( return next(iter(locations.values())).location +async def get_openpbs_deployment_config(_context: StreamFlowContext): + docker_compose_config = DeploymentConfig( + name="docker-compose-openpbs", + type="docker-compose", + config={ + "files": [ + str(get_data_path("deployment", "openpbs", "docker-compose.yml")) + ], + "projectName": random_name(), + }, + external=False, + ) + await _context.deployment_manager.deploy(docker_compose_config) + return DeploymentConfig( + name="docker-openpbs", + type="pbs", + config={ + "services": { + "test": { + "destination": "workq", + "resources": {"nodes": "2:ppn=1"}, + } + } + }, + external=False, + lazy=False, + wraps=WrapsConfig(deployment="docker-compose-openpbs", service="server"), + ) + + def get_parameterizable_hardware_deployment_config() -> DeploymentConfig: workdir = os.path.join( os.path.realpath(tempfile.gettempdir()), "streamflow-test", random_name() @@ -263,7 +305,7 @@ def get_service(_context: StreamFlowContext, deployment_t: str) -> str | None: return "alpine" case "kubernetes": return "sf-test" - case "slurm": + case "slurm" | "openpbs": return "test" case _: raise Exception(f"{deployment_t} deployment type not supported") diff --git a/tox.ini b/tox.ini index b1efaf991..14dbe647c 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ asyncio_default_fixture_loop_scope = session asyncio_default_test_loop_scope = session asyncio_mode = strict testpaths = tests +log_cli = True [testenv] allowlist_externals = make