Add OpenPBS tests to the CI/CD

Makefile

@@ -28,7 +28,7 @@ test:
 	python -m pytest -rs ${PYTEST_EXTRA}
 
 testcov:
-	python -m pytest -rs --cov --junitxml=junit.xml -o junit_family=legacy --cov-report= ${PYTEST_EXTRA}
+	python -m pytest -s --cov --junitxml=junit.xml -o junit_family=legacy --cov-report= ${PYTEST_EXTRA}
 
 typing:
 	mypy streamflow tests

streamflow/data/remotepath.py

@@ -915,13 +915,14 @@ async def download(
                         f"Downloading {url} failed with status {response.status}:\n{response.content}"
                     )
         connector = context.deployment_manager.get_connector(location.deployment)
-        await connector.run(
-            location=location,
-            command=[
-                f'if [ command -v curl ]; then curl -L -o "{filepath}" "{url}"; '
-                f'else wget -O "{filepath}" "{url}"; fi'
-            ],
+        command = [
+            f'if command -v curl; then curl -L -o "{filepath}" "{url}"; '
+            f'else wget -O "{filepath}" "{url}"; fi'
+        ]
+        result, status = await connector.run(
+            location=location, command=command, capture_output=True
         )
+        _check_status(command, location, result, status)
     return StreamFlowPath(filepath, context=context, location=location)
 
 

streamflow/log_handler.py

@@ -126,5 +126,5 @@ def highlight(self, msg: str | Any) -> str:
 )
 defaultStreamHandler.setFormatter(formatter)
 logger.addHandler(defaultStreamHandler)
-logger.setLevel(logging.INFO)
+logger.setLevel(logging.DEBUG)
 logger.propagate = False

tests/conftest.py

@@ -126,7 +126,14 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
 
 
 def all_deployment_types() -> list[str]:
-    deployments_ = ["local", "docker", "docker-compose", "docker-wrapper", "slurm"]
+    deployments_ = [
+        "local",
+        "docker",
+        "docker-compose",
+        "docker-wrapper",
+        "openpbs",
+        "slurm",
+    ]
     if platform.system() == "Linux":
         deployments_.extend(["kubernetes", "singularity", "ssh"])
     return deployments_

tests/data/deployment/openpbs/docker-compose.yml

@@ -0,0 +1,31 @@
+services:
+  server:
+    image: alphaunito/openpbs-server:23.06.06
+    environment:
+      PBS_EXECUTION_HOST_NAME_PREFIX: ${COMPOSE_PROJECT_NAME}-execution
+      PBS_EXECUTION_NODES: 2
+    hostname: openpbs-server
+    networks:
+      - pbsnet
+    volumes:
+      - home:/home/hpcuser
+      - munge:/etc/munge
+      - pgsql:/usr/local/pgsql/data
+  execution:
+    image: alphaunito/openpbs-execution:23.06.06
+    deploy:
+      mode: replicated
+      replicas: 2
+    environment:
+      PBS_SERVER_HOST_NAME: openpbs-server
+    networks:
+      - pbsnet
+    volumes:
+      - home:/home/hpcuser
+      - munge:/etc/munge
+networks:
+  pbsnet:
+volumes:
+  home:
+  munge:
+  pgsql:

tests/data/deployment/slurm/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.8"
 services:
   slurmctld:
-    image: alphaunito/slurmctld:21.08.5
+    image: alphaunito/slurmctld:23.11.4
     environment:
       SLURMD_HOSTNAME_PREFIX: ${COMPOSE_PROJECT_NAME}-slurmd
       SLURMD_NODES: 2
@@ -13,7 +13,7 @@ services:
       - munge:/etc/munge
       - mysql:/var/lib/mysql
   slurmd:
-    image: alphaunito/slurmd:21.08.5
+    image: alphaunito/slurmd:23.11.4
     deploy:
       mode: replicated
       replicas: 2

tests/utils/deployment.py

@@ -79,6 +79,8 @@ def get_deployment(deployment_t: str) -> str:
             return "__LOCAL__"
         case "local-fs-volatile":
             return "local-fs-volatile"
+        case "openpbs":
+            return "docker-openpbs"
         case "parameterizable_hardware":
             return "custom-hardware"
         case "singularity":
@@ -115,6 +117,16 @@ async def get_deployment_config(
                     "test-fs-volatile",
                 ),
             )
+        case "docker":
+            return get_docker_deployment_config()
+        case "docker-compose":
+            return get_docker_compose_deployment_config()
+        case "docker-wrapper":
+            return await get_docker_wrapper_deployment_config(_context)
+        case "kubernetes":
+            return get_kubernetes_deployment_config()
+        case "openpbs":
+            return await get_openpbs_deployment_config(_context)
         case "parameterizable-hardware":
             return get_parameterizable_hardware_deployment_config()
         case "singularity":
@@ -231,6 +243,36 @@ async def get_location(
     return next(iter(locations.values())).location
 
 
+async def get_openpbs_deployment_config(_context: StreamFlowContext):
+    docker_compose_config = DeploymentConfig(
+        name="docker-compose-openpbs",
+        type="docker-compose",
+        config={
+            "files": [
+                str(get_data_path("deployment", "openpbs", "docker-compose.yml"))
+            ],
+            "projectName": random_name(),
+        },
+        external=False,
+    )
+    await _context.deployment_manager.deploy(docker_compose_config)
+    return DeploymentConfig(
+        name="docker-openpbs",
+        type="pbs",
+        config={
+            "services": {
+                "test": {
+                    "destination": "workq",
+                    "resources": {"nodes": "2:ppn=1"},
+                }
+            }
+        },
+        external=False,
+        lazy=False,
+        wraps=WrapsConfig(deployment="docker-compose-openpbs", service="server"),
+    )
+
+
 def get_parameterizable_hardware_deployment_config() -> DeploymentConfig:
     workdir = os.path.join(
         os.path.realpath(tempfile.gettempdir()), "streamflow-test", random_name()
@@ -263,7 +305,7 @@ def get_service(_context: StreamFlowContext, deployment_t: str) -> str | None:
             return "alpine"
         case "kubernetes":
             return "sf-test"
-        case "slurm":
+        case "slurm" | "openpbs":
             return "test"
         case _:
             raise Exception(f"{deployment_t} deployment type not supported")

tox.ini

@@ -10,6 +10,7 @@ asyncio_default_fixture_loop_scope = session
 asyncio_default_test_loop_scope = session
 asyncio_mode = strict
 testpaths = tests
+log_cli = True
 
 [testenv]
 allowlist_externals = make