Support running StreamFlow on Windows

.github/workflows/ci-tests.yaml

@@ -231,6 +231,8 @@ jobs:
         include:
           - on: "macos-15-intel"
             python: "3.14"
+          - on: "windows-2025"
+            python: "3.14"
     runs-on: ${{ matrix.on }}
     env:
       TOXENV: ${{ format('py{0}-unit', matrix.python) }}
@@ -272,10 +274,62 @@ jobs:
           kubectl apply -f https://docs.projectcalico.org/v3.25/manifests/calico.yaml
           kubectl -n kube-system set env daemonset/calico-node FELIX_IGNORELOOSERPF=true
         if: ${{ startsWith(matrix.on, 'ubuntu-') }}
+      - name: "Install WSL (Windows)"
+        uses: Vampire/setup-wsl@v6
+        with:
+          distribution: Ubuntu-24.04
+          wsl-conf: |
+            [automount]
+            root = /
+            options = "metadata"
+
+            [boot]
+            systemd=true
+          wsl-version: 2
+        if: ${{ startsWith(matrix.on, 'windows-') }}
+      - name: "Install Docker on WSL (Windows)"
+        shell: wsl-bash {0}
+        run: |
+          install -m 0755 -d /etc/apt/keyrings \
+            && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \
+            && chmod a+r /etc/apt/keyrings/docker.asc
+
+          cat > /etc/apt/sources.list.d/docker.sources <<EOF
+          Types: deb
+          URIs: https://download.docker.com/linux/ubuntu
+          Suites: $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}")
+          Components: stable
+          Signed-By: /etc/apt/keyrings/docker.asc
+          EOF
+
+          apt update \
+            && apt install -y --no-install-recommends docker-ce \
+            && mkdir /etc/systemd/system/docker.service.d
+
+          cat > /etc/systemd/system/docker.service.d/override.conf << EOF
+          [Service]
+          ExecStart=
+          ExecStart=/usr/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375 --containerd=/run/containerd/containerd.sock
+          EOF
+
+          systemctl daemon-reload \
+            && systemctl restart docker \
+            && systemctl status docker \
+            && docker run hello-world
+        if: ${{ startsWith(matrix.on, 'windows-') }}
+      - name: "Configure Docker to use WSL2 (Windows)"
+        run: |
+          docker context create wsl --docker ("host=tcp://$((wsl hostname -I).Split()[0].Trim()):2375")
+          docker context use wsl
+          docker info
+          docker run hello-world
+        if: ${{ startsWith(matrix.on, 'windows-') }}
       - name: "Install Tox"
         run: uv tool install tox --with tox-uv
       - name: "Run StreamFlow tests via Tox"
-        run: tox
+        run: |
+          docker info
+          tox
       - name: "Upload test results"
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v5

cwl-conformance-test.bat

@@ -0,0 +1,103 @@
+echo off
+
+rem Version of the standard to test against
+rem Current options: v1.0, v1.1, v1.2, and v1.3
+if "%VERSION%"=="" set "VERSION=v1.2"
+
+rem Which commit of the standard's repo to use
+rem Defaults to the last commit of the main branch
+if "%COMMIT%"=="" set "COMMIT=main"
+
+rem Comma-separated list of test names that should be excluded from execution
+rem Defaults to "docker_entrypoint, inplace_update_on_file_content"
+if "%EXCLUDE%"=="" set "EXCLUDE=docker_entrypoint,modify_file_content"
+
+rem Name of the CWLDockerTranslator plugin to use for test execution
+rem This parameter allows to test automatic CWL requirements translators
+if "%DOCKER%"=="" set "DOCKER=docker"
+
+rem Additional arguments for the pytest command
+rem Defaults to none
+rem set "PYTEST_EXTRA="
+
+rem The directory where this script resides
+set "SCRIPT_DIRECTORY=%~dp0"
+set "SCRIPT_DIRECTORY=%SCRIPT_DIRECTORY:~0,-1%"
+
+rem Download archive from GitHub
+if "%VERSION%"=="v1.0" (
+    set "REPO=common-workflow-language"
+) else (
+    set "REPO=cwl-%VERSION%"
+)
+
+if not exist "%REPO%-%COMMIT%" (
+    if not exist "%COMMIT%.tar.gz" (
+        echo Downloading %REPO% @ %COMMIT%...
+        pwsh -Command "Invoke-WebRequest -Uri https://github.com/common-workflow-language/%REPO%/archive/%COMMIT%.tar.gz -OutFile %COMMIT%.tar.gz"
+    )
+    tar -xzf "%COMMIT%.tar.gz"
+)
+
+rem Setup environment
+call :venv cwl-conformance-venv
+python -m pip install -U setuptools wheel pip
+python -m pip install -r "%SCRIPT_DIRECTORY%\requirements.txt"
+python -m pip install -r "%SCRIPT_DIRECTORY%\test-requirements.txt"
+if "%VERSION%"=="v1.3" (
+    python -m pip uninstall -y cwl-utils
+    python -m pip install git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/370/head
+)
+
+rem Set conformance test filename
+if "%VERSION%"=="v1.0" (
+    set "CONFORMANCE_TEST=%SCRIPT_DIRECTORY%\%REPO%-%COMMIT%\%VERSION%\conformance_test_v1.0.yaml"
+) else (
+    set "CONFORMANCE_TEST=%SCRIPT_DIRECTORY%\%REPO%-%COMMIT%\conformance_tests.yaml"
+)
+move "%CONFORMANCE_TEST%" "%CONFORMANCE_TEST:.yaml=.cwltest.yaml%"
+set "CONFORMANCE_TEST=%CONFORMANCE_TEST:.yaml=.cwltest.yaml%"
+
+rem Build command
+set "TEST_COMMAND=python -m pytest "%CONFORMANCE_TEST%" -n auto -rs"
+if not "%EXCLUDE%"=="" (
+    set "TEST_COMMAND=%TEST_COMMAND% --cwl-exclude %EXCLUDE%"
+)
+set "TEST_COMMAND=%TEST_COMMAND% --cov --junitxml=junit.xml -o junit_family=legacy --cov-report= %PYTEST_EXTRA%"
+
+rem Cleanup coverage
+if exist "%SCRIPT_DIRECTORY%\.coverage" del "%SCRIPT_DIRECTORY%\.coverage"
+if exist "%SCRIPT_DIRECTORY%\coverage.xml" del "%SCRIPT_DIRECTORY%\coverage.xml"
+if exist "%SCRIPT_DIRECTORY%\junit.xml" del "%SCRIPT_DIRECTORY%\junit.xml"
+
+rem Run test
+copy "%SCRIPT_DIRECTORY%\tests\cwl-conformance\conftest.py" "%~dpn1"
+copy "%SCRIPT_DIRECTORY%\tests\cwl-conformance\streamflow-%DOCKER%.yml" "%~dpn1\streamflow.yml"
+cmd /c "%TEST_COMMAND%"
+set "RETURN_CODE=%ERRORLEVEL%"
+
+rem Coverage report
+if "%RETURN_CODE%"=="0" (
+    coverage report
+    coverage xml
+)
+
+rem Cleanup
+rd /s /q "%SCRIPT_DIRECTORY%\%REPO%-%COMMIT%"
+rd /s /q "%SCRIPT_DIRECTORY%\cwl-conformance-venv"
+del "%COMMIT%.tar.gz" 2>nul
+
+rem Exit
+exit /b %RETURN_CODE%
+goto :eof
+
+:venv
+if not exist "%~1" (
+    where virtualenv >nul 2>&1 && (
+        virtualenv -p python "%~1"
+    ) || (
+        python -m venv "%~1"
+    )
+)
+call "%~1\Scripts\activate.bat"
+goto :eof

streamflow/deployment/aiotarstream.py

@@ -1,9 +1,7 @@
 from __future__ import annotations
 
 import copy
-import grp
 import os
-import pwd
 import re
 import shutil
 import stat
@@ -21,6 +19,15 @@
 from streamflow.core.data import StreamWrapper
 from streamflow.deployment.stream import BaseStreamWrapper
 
+try:
+    import grp
+except ImportError:
+    grp = None
+try:
+    import pwd
+except ImportError:
+    pwd = None
+
 if TYPE_CHECKING:
     StrOrBytesPath: TypeAlias = str | bytes | os.PathLike[str] | os.PathLike[bytes]
 

streamflow/deployment/connector/container.py

@@ -7,6 +7,7 @@
 import os
 import posixpath
 import shlex
+import sys
 from abc import ABC, abstractmethod
 from collections.abc import MutableMapping, MutableSequence
 from importlib.resources import files
@@ -728,7 +729,11 @@ async def _populate_instance(self, name: str) -> None:
             )
         # Check if the container user is the current host user
         if self._wraps_local():
-            host_user = os.getuid()
+            if sys.platform == "win32":
+                # Windows does not support the `getuid` function
+                host_user = -1
+            else:
+                host_user = os.getuid()
         else:
             stdout, returncode = await self.connector.run(
                 location=self._get_inner_location(),

streamflow/deployment/connector/local.py

@@ -28,8 +28,15 @@ def _local_copy(src: str, dst: str, read_only: bool) -> None:
         try:
             os.symlink(src, dst, target_is_directory=os.path.isdir(src))
         except OSError as e:
-            if not e.errno == errno.EEXIST:
-                raise
+            if e.errno != errno.EEXIST:
+                if sys.platform == "win32" and e.errno == errno.EINVAL:
+                    if logger.isEnabledFor(logging.WARNING):
+                        logger.warning(
+                            f"Unable to create a symbolic link from {src} to {dst}: {e.strerror}"
+                        )
+                    shutil.copy(src, dst)
+                else:
+                    raise
     else:
         if os.path.isdir(src):
             os.makedirs(dst, exist_ok=True)
@@ -164,6 +171,7 @@ async def run(
         timeout: int | None = None,
         job_name: str | None = None,
     ) -> tuple[str, int] | None:
+        # Create command
         command = utils.create_command(
             self.__class__.__name__,
             command,

streamflow/log_handler.py

@@ -126,5 +126,5 @@ def highlight(self, msg: str | Any) -> str:
 )
 defaultStreamHandler.setFormatter(formatter)
 logger.addHandler(defaultStreamHandler)
-logger.setLevel(logging.INFO)
+logger.setLevel(logging.DEBUG)
 logger.propagate = False

tests/test_cwl_execution.py

@@ -121,63 +121,65 @@ async def test_initial_workdir(
     # Create input data
     path = None
     try:
-        if file_type == "file":
-            path = StreamFlowPath(
-                "/tmp", "test.txt", context=context, location=execution_location
-            )
-            await path.write_text("Hello world")
-            path = await path.resolve()
-            file_token = CWLFileToken(
-                {
-                    "class": "File",
-                    "path": str(path),
-                }
-            )
-            initial_paths = [str(path)]
-        elif file_type in ("dir", "listing"):
-            path = StreamFlowPath(
-                "/tmp", "land", context=context, location=execution_location
-            )
-            await path.mkdir()
-            path = await path.resolve()
-            await (path / "lvl1").mkdir()
-            await (path / "lvl2").mkdir()
-            await (path / "lvl1" / "text.txt").write_text("Hello dir")
-            file_token = CWLFileToken(
-                {
-                    "class": "Directory",
-                    "path": str(path),
-                    "listing": [
-                        {
-                            "class": "Directory",
-                            "path": str(path / "lvl1"),
-                            "listing": [
-                                {
-                                    "class": "File",
-                                    "path": str(path / "lvl1" / "text.txt"),
-                                }
-                            ],
-                        },
-                        {"class": "Directory", "path": str(path / "lvl2")},
-                    ],
-                }
-            )
-            initial_paths = (
-                [str(path)]
-                if file_type == "dir"
-                else [str(path / "lvl1"), str(path / "lvl2")]
-            )
-        else:
-            raise NotImplementedError(f"Unsupported file type: {file_type}")
+        match file_type:
+            case "file":
+                path = StreamFlowPath(
+                    "/tmp", "test.txt", context=context, location=execution_location
+                )
+                await path.write_text("Hello world")
+                path = await path.resolve()
+                file_token = CWLFileToken(
+                    {
+                        "class": "File",
+                        "path": str(path),
+                    }
+                )
+                initial_paths = [str(path)]
+            case "dir" | "listing":
+                path = StreamFlowPath(
+                    "/tmp", "land", context=context, location=execution_location
+                )
+                await path.mkdir()
+                path = await path.resolve()
+                await (path / "lvl1").mkdir()
+                await (path / "lvl2").mkdir()
+                await (path / "lvl1" / "text.txt").write_text("Hello dir")
+                file_token = CWLFileToken(
+                    {
+                        "class": "Directory",
+                        "path": str(path),
+                        "listing": [
+                            {
+                                "class": "Directory",
+                                "path": str(path / "lvl1"),
+                                "listing": [
+                                    {
+                                        "class": "File",
+                                        "path": str(path / "lvl1" / "text.txt"),
+                                    }
+                                ],
+                            },
+                            {"class": "Directory", "path": str(path / "lvl2")},
+                        ],
+                    }
+                )
+                initial_paths = (
+                    [str(path)]
+                    if file_type == "dir"
+                    else [str(path / "lvl1"), str(path / "lvl2")]
+                )
+            case _:
+                raise NotImplementedError(f"Unsupported file type: {file_type}")
         # Inject input token
-        if token_type == "file":
-            token_value = file_token
-        elif token_type == "list":
-            token_value = ListToken([file_token])
-        elif token_type == "object":
-            token_value = ObjectToken({"a": file_token})
-        else:
-            raise ValueError(f"Invalid token_type: {token_type}")
+        match token_type:
+            case "file":
+                token_value = file_token
+            case "list":
+                token_value = ListToken([file_token])
+            case "object":
+                token_value = ObjectToken({"a": file_token})
+            case _:
+                raise ValueError(f"Invalid token_type: {token_type}")
         await inject_tokens([token_value], in_port, context)
         # Execute workflow
         await workflow.save(context)