From 6aa84a4a9465178c20a7dff36bd8bea8999e63bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20M=C3=B6ller?= Date: Thu, 2 Apr 2026 07:30:59 +0000 Subject: [PATCH 1/2] chore: devcontainer to run system in "native" environment --- .devcontainer/Dockerfile | 71 +++++++++++++ .devcontainer/devcontainer.json | 41 ++++++++ .devcontainer/postCreate.sh | 10 ++ .devcontainer/postStart.sh | 9 ++ .devcontainer/start-scanomatic.sh | 12 +++ .github/copilot-instructions.md | 28 +++++ .github/workflows/ci.yml | 10 +- .nvmrc | 1 + Dockerfile | 2 +- README.md | 15 ++- scanomatic/io/rpc_client.py | 9 +- tests/system/conftest.py | 164 +++++++++++++++++++++++++++++- tox.ini | 44 ++++---- 13 files changed, 381 insertions(+), 35 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100755 .devcontainer/postCreate.sh create mode 100755 .devcontainer/postStart.sh create mode 100755 .devcontainer/start-scanomatic.sh create mode 100644 .github/copilot-instructions.md create mode 100644 .nvmrc diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000..0bf3ac42 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,71 @@ +FROM python:3.9-bullseye + +ENV DEBIAN_FRONTEND=noninteractive +ARG GECKODRIVER_VERSION=v0.31.0 +ARG CHROME_VERSION=107.0.5304.110-1 +ARG CHROMEDRIVER_VERSION=107.0.5304.62 +ARG NODE_MAJOR=16 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bzip2 \ + ca-certificates \ + curl \ + firefox-esr \ + git \ + gnupg2 \ + iputils-ping \ + libdbus-glib-1-2 \ + libsane \ + libsane-common \ + libxtst6 \ + net-tools \ + nmap \ + sane-utils \ + software-properties-common \ + tzdata \ + unzip \ + usbutils \ + wget \ + xauth \ + xvfb \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_MAJOR}.x | bash - \ + && apt-get update \ + && apt-get install -y --no-install-recommends nodejs \ + && node --version | grep -E "^v${NODE_MAJOR}\\." \ + && rm -rf /var/lib/apt/lists/* + +RUN python3.9 -m pip install --no-cache-dir --upgrade pip \ + && python3.9 -m pip install --no-cache-dir tox==3.27.1 + +# Use distro Firefox ESR to ensure runtime library compatibility with Debian Bullseye. +RUN ln -sf /usr/bin/firefox-esr /usr/local/bin/firefox + +RUN wget -q -O /tmp/geckodriver.tar.gz \ + "https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz" \ + && tar -xzf /tmp/geckodriver.tar.gz -C /usr/local/bin \ + && chmod +x /usr/local/bin/geckodriver \ + && rm -f /tmp/geckodriver.tar.gz + +RUN wget -q -O /tmp/google-chrome.deb \ + "https://mirror.cs.uchicago.edu/google-chrome/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROME_VERSION}_amd64.deb" \ + && apt-get update \ + && apt-get install -y --no-install-recommends /tmp/google-chrome.deb \ + && rm -f /etc/apt/sources.list.d/google-chrome.list \ + && rm -f /tmp/google-chrome.deb \ + && rm -rf /var/lib/apt/lists/* + +RUN wget -q -O /tmp/chromedriver.zip \ + "https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip" \ + && unzip -q /tmp/chromedriver.zip -d /tmp \ + && mv /tmp/chromedriver /usr/local/bin/chromedriver \ + && chmod +x /usr/local/bin/chromedriver \ + && rm -f /tmp/chromedriver.zip + +# Add scanner IDs in case automatic discovery fails. +RUN echo "usb 0x4b8 0x12c" >> /etc/sane.d/epson2.conf \ + && echo "usb 0x4b8 0x151" >> /etc/sane.d/epson2.conf + +WORKDIR /workspaces/scanomatic-standalone diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..9d24b0c7 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,41 @@ +{ + "name": "scanomatic-standalone", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "remoteUser": "root", + "runArgs": [ + "--privileged", + "--device=/dev/bus/usb:/dev/bus/usb" + ], + "containerEnv": { + "LOGGING_LEVEL": "20", + "SOM_PROJECTS_ROOT": "/somprojects", + "SOM_SETTINGS": "/root/.scan-o-matic" + }, + "mounts": [ + "source=scanomatic-settings,target=/root/.scan-o-matic,type=volume", + "source=scanomatic-projects,target=/somprojects,type=volume" + ], + "forwardPorts": [ + 5000 + ], + "portsAttributes": { + "5000": { + "label": "Scan-o-Matic", + "onAutoForward": "openBrowser" + } + }, + "postCreateCommand": "bash .devcontainer/postCreate.sh", + "postStartCommand": "bash .devcontainer/postStart.sh", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-azuretools.vscode-docker" + ] + } + } +} diff --git a/.devcontainer/postCreate.sh b/.devcontainer/postCreate.sh new file mode 100755 index 00000000..261b6a71 --- /dev/null +++ b/.devcontainer/postCreate.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /workspaces/scanomatic-standalone + +python3.9 -m pip install --upgrade pip +python3.9 -m pip install -r requirements.txt + +npm ci +npm run build diff --git a/.devcontainer/postStart.sh b/.devcontainer/postStart.sh new file mode 100755 index 00000000..ac743db3 --- /dev/null +++ b/.devcontainer/postStart.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONFIG_SRC="/workspaces/scanomatic-standalone/data/config" +CONFIG_DST="${SOM_SETTINGS:-/root/.scan-o-matic}/config" + +mkdir -p "$CONFIG_DST" +cp -n "$CONFIG_SRC"/* "$CONFIG_DST"/ 2>/dev/null || true +mkdir -p "${SOM_PROJECTS_ROOT:-/somprojects}" diff --git a/.devcontainer/start-scanomatic.sh b/.devcontainer/start-scanomatic.sh new file mode 100755 index 00000000..34fece0c --- /dev/null +++ b/.devcontainer/start-scanomatic.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /workspaces/scanomatic-standalone + +export PYTHONPATH="/workspaces/scanomatic-standalone${PYTHONPATH:+:$PYTHONPATH}" +export PATH="/workspaces/scanomatic-standalone/scripts${PATH:+:$PATH}" + +exec python3.9 scripts/scan-o-matic \ + --host 0.0.0.0 \ + --port 5000 \ + --no-browser diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..ad279a48 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,28 @@ +# Copilot Instructions for This Repository + +## Environment Overview +- Repository: `scanomatic-standalone` +- Primary runtime: Python (backend) with JavaScript tooling for frontend tests/build. +- Typical development environment: Linux dev container. +- Docker Compose is used for local app startup in normal workflows. + +## Project Layout (High Level) +- `scanomatic/`: Main Python package. +- `tests/`: Unit, integration, and system tests. +- `.github/workflows/`: CI definitions. +- `scripts/`: Entrypoints and helper scripts. + +## Validation Commands +- Backend test and quality environments are run through `tox`. +- Common environments: `tox -e lint`, `tox -e mypy`, `tox -e unit`, `tox -e integration`, `tox -e system`. +- Frontend tests use Karma (`karma.conf.js`), lint uses ESLint. +- For incremental type checks on modified files: `./typecheck-changed.sh`. + +## Runtime Notes +- App startup is commonly done with Docker Compose (`docker-compose up -d`). +- Some system/integration flows may run local services inside the container when Docker daemon access is unavailable. + +## Coding Guidance +- Prefer minimal, targeted changes that match existing style. +- Keep edits scoped; avoid unrelated refactors. +- When changing behavior, update or add tests close to the affected area. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7876dcc..3c6697da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: - name: Install General Dependencies run: | python3.9 -m pip install --upgrade pip - pip install tox + pip install tox==3.27.1 - name: Lint and Types run: | @@ -82,7 +82,7 @@ jobs: - name: Install General Dependencies run: | python3.9 -m pip install --upgrade pip - pip install tox + pip install tox==3.27.1 - name: Lint and Types run: | @@ -101,7 +101,7 @@ jobs: - name: Install General Dependencies run: | python3.9 -m pip install --upgrade pip - pip install tox + pip install tox==3.27.1 - name: Run unit tests run: | @@ -120,7 +120,7 @@ jobs: - name: Install General Dependencies run: | python3.9 -m pip install --upgrade pip - pip install tox + pip install tox==3.27.1 - name: Run integration tests run: | @@ -143,7 +143,7 @@ jobs: - name: Install General Dependencies run: | python3.9 -m pip install --upgrade pip - pip install tox + pip install tox==3.27.1 - name: Run headless system test uses: GabrielBB/xvfb-action@v1 diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 00000000..b6a7d89c --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +16 diff --git a/Dockerfile b/Dockerfile index 0bff5e52..b435c513 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /src RUN npm ci RUN npm run build -FROM python:3.9 +FROM python:3.9-bullseye RUN apt-get update RUN export DEBIAN_FRONTEND=noninteractive \ && ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime \ diff --git a/README.md b/README.md index 4563185a..0d79100b 100644 --- a/README.md +++ b/README.md @@ -45,4 +45,17 @@ Currently this is not required as the code is still riddled with type errors, ho ### System tests System tests require `firefox`, `geckodriver`, `google-chrome` and `chromedriver` to be installed on host system. -After that it run with `tox -e system`. +After that they can be run with `tox -e system`. + +By default the test fixture uses Docker Compose when a working Docker daemon is available, and automatically falls back to a local in-container server when Docker is unavailable (for example inside a dev container without nested Docker). + +You can force mode selection with `SOM_SYSTEM_TEST_MODE`: +- `SOM_SYSTEM_TEST_MODE=docker tox -e system` +- `SOM_SYSTEM_TEST_MODE=local tox -e system` + +The devcontainer installs browser dependencies for system tests, including `firefox-esr`, `geckodriver`, `google-chrome` and `chromedriver`. +To verify installed versions inside the container: +- `firefox --version` +- `geckodriver --version` +- `google-chrome --version` +- `chromedriver --version` diff --git a/scanomatic/io/rpc_client.py b/scanomatic/io/rpc_client.py index 63c47bea..660d5982 100644 --- a/scanomatic/io/rpc_client.py +++ b/scanomatic/io/rpc_client.py @@ -1,5 +1,7 @@ import enum +import shutil import socket +import sys import xmlrpc.client from collections.abc import Callable from http.client import CannotSendRequest, ResponseNotReady @@ -80,7 +82,12 @@ def __init__(self, host: str, port: int, user_id: str): def launch_local(self) -> None: if self.online is False and self.local: self._logger.info("Launching new local server") - Popen(["scan-o-matic_server"]) + launcher = shutil.which("scan-o-matic_server") + if launcher is not None: + # Use the active interpreter to avoid shebang/path issues. + Popen([sys.executable, launcher]) + else: + Popen(["scan-o-matic_server"]) else: self._logger.warning( "Can't launch because server is {0}".format( diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 2d11b07f..0971bbab 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -1,8 +1,15 @@ from pathlib import Path +import os +import shutil +import subprocess +import sys +from tempfile import mkdtemp +from typing import Iterator import pytest import requests -from selenium import webdriver # type: ignore +from selenium import webdriver +from selenium.webdriver.remote.webdriver import WebDriver @pytest.fixture(scope='session') @@ -14,7 +21,15 @@ def docker_compose_file(pytestconfig): @pytest.fixture(scope='session') -def scanomatic(docker_ip, docker_services): +def scanomatic(request): + mode = _system_test_mode() + if mode == 'docker': + return request.getfixturevalue('scanomatic_docker') + return request.getfixturevalue('scanomatic_local') + + +@pytest.fixture(scope='session') +def scanomatic_docker(docker_ip, docker_services): def is_responsive(url: str) -> bool: try: response = requests.get(url) @@ -38,12 +53,151 @@ def is_responsive(url: str) -> bool: return url +@pytest.fixture(scope='session') +def scanomatic_local() -> Iterator[str]: + root = Path(__file__).resolve().parents[2] + local_data_root = Path(mkdtemp(prefix='som-system-tests-')) + scanomatic_data = local_data_root / '.scan-o-matic' + _prepare_local_runtime_data(scanomatic_data) + _prepare_local_projects_root() + + env = os.environ.copy() + env['SCANOMATIC_DATA'] = str(scanomatic_data) + pythonpath = env.get('PYTHONPATH', '').strip() + env['PYTHONPATH'] = f"{root}:{pythonpath}" if pythonpath else str(root) + + server_cmd = [sys.executable, str(root / 'scripts' / 'scan-o-matic_server')] + ui_cmd = [ + sys.executable, + str(root / 'scripts' / 'scan-o-matic'), + '--host', + '127.0.0.1', + '--port', + '5000', + '--no-browser', + ] + + rpc_process = subprocess.Popen(server_cmd, cwd=root, env=env) + ui_process = subprocess.Popen(ui_cmd, cwd=root, env=env) + + url = 'http://127.0.0.1:5000' + _wait_until_responsive(url) + + try: + yield url + finally: + _terminate_process(ui_process) + _terminate_process(rpc_process) + shutil.rmtree(local_data_root, ignore_errors=True) + + +def _system_test_mode() -> str: + requested = os.environ.get('SOM_SYSTEM_TEST_MODE', 'auto').strip().lower() + if requested in ('docker', 'local'): + return requested + return 'docker' if _docker_is_usable() else 'local' + + +def _docker_is_usable() -> bool: + if shutil.which('docker') is None: + return False + try: + result = subprocess.run( + ['docker', 'info'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + check=False, + ) + except (subprocess.SubprocessError, OSError): + return False + return result.returncode == 0 + + +def _prepare_local_runtime_data(scanomatic_data: Path) -> None: + root = Path(__file__).resolve().parents[2] + config_src = root / 'data' / 'config' + config_dst = scanomatic_data / 'config' + shutil.copytree(config_src, config_dst, dirs_exist_ok=True) + + ccc_src = root / 'tests' / 'system' / 'data' / 'TESTUMz.ccc' + ccc_dst = config_dst / 'ccc' / 'TESTUMz.ccc' + ccc_dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ccc_src, ccc_dst) + + +def _prepare_local_projects_root() -> None: + root = Path(__file__).resolve().parents[2] + projects_root = Path('/somprojects') + test_project_dir = projects_root / 'testproject' + upload_dir = projects_root / 'my' / 'project' + shutil.rmtree(test_project_dir, ignore_errors=True) + shutil.rmtree(upload_dir, ignore_errors=True) + test_project_dir.mkdir(parents=True, exist_ok=True) + upload_dir.mkdir(parents=True, exist_ok=True) + + shutil.copyfile( + root / 'tests' / 'system' / 'data' / 'testproject.project.compilation', + test_project_dir / 'testproject.project.compilation', + ) + shutil.copyfile( + ( + root / 'tests' / 'system' / 'data' / + 'testproject.project.compilation.instructions' + ), + test_project_dir / 'testproject.project.compilation.instructions', + ) + + +def _wait_until_responsive(url: str, timeout: int = 30) -> None: + import time + + deadline = time.time() + timeout + while time.time() < deadline: + try: + if ( + requests.get(url + '/fixtures').ok + and requests.get(url + '/api/status/server').ok + ): + return + except requests.RequestException: + pass + time.sleep(0.2) + raise RuntimeError(f'Scan-o-Matic did not become responsive at {url}') + + +def _terminate_process(process: subprocess.Popen) -> None: + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + @pytest.fixture( scope='function', ids=['chrome', 'firefox'], - params=[webdriver.Chrome, webdriver.Firefox], + params=['chrome', 'firefox'], ) def browser(request): - driver = request.param() + driver: WebDriver + if request.param == 'chrome': + chrome_options = webdriver.ChromeOptions() + chrome_options.add_argument('--headless') + chrome_options.add_argument('--no-sandbox') + chrome_options.add_argument('--disable-dev-shm-usage') + chrome_options.add_argument('--disable-gpu') + driver = webdriver.Chrome(options=chrome_options) + else: + firefox_options = webdriver.FirefoxOptions() + firefox_options.add_argument('-headless') + firefox_esr = shutil.which('firefox-esr') + if firefox_esr: + firefox_options.binary_location = firefox_esr + driver = webdriver.Firefox(options=firefox_options) + yield driver - driver.close() + driver.quit() diff --git a/tox.ini b/tox.ini index 47ac5a09..92087854 100644 --- a/tox.ini +++ b/tox.ini @@ -6,12 +6,12 @@ skipsdist = true basepython = python3.9 deps = -rrequirements.txt - mock - pytest - pytest-cov - pytest-flask + mock==4.0.3 + pytest==7.2.0 + pytest-cov==4.0.0 + pytest-flask==1.2.0 commands = - pytest \ + python -m pytest \ --cov scanomatic --cov scripts --cov-report xml \ --junitxml result.xml --ignore dev \ {posargs} tests/unit @@ -20,10 +20,10 @@ commands = basepython = python3.9 deps = -rrequirements.txt - mock - pytest - pytest-cov - pytest-flask + mock==4.0.3 + pytest==7.2.0 + pytest-cov==4.0.0 + pytest-flask==1.2.0 commands = pytest \ --cov scanomatic --cov scripts --cov-report xml \ @@ -34,11 +34,11 @@ commands = basepython = python3.9 deps = -rrequirements.txt - mock - pytest - pytest-docker - pytest-flask - selenium + mock==4.0.3 + pytest==7.2.0 + pytest-docker==1.0.1 + pytest-flask==1.2.0 + selenium==4.6.0 whitelist_externals = chromedriver geckodriver @@ -55,13 +55,13 @@ basepython = python3.9 sitepackages = False deps = -rrequirements.txt - mock - pytest - pytest-cov - pytest-docker - pytest-flask - mypy - selenium + mock==4.0.3 + pytest==7.2.0 + pytest-cov==4.0.0 + pytest-docker==1.0.1 + pytest-flask==1.2.0 + mypy==0.990 + selenium==4.6.0 commands = mypy \ --check-untyped-defs --warn-unused-ignores --no-incremental \ @@ -70,7 +70,7 @@ commands = [testenv:lint] basepython = python3.9 deps = - flake8 + flake8==5.0.4 commands = flake8 From 2b0f6f951487c9f51b333f5f581cf8ebd0beefea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20M=C3=B6ller?= Date: Thu, 2 Apr 2026 07:58:36 +0000 Subject: [PATCH 2/2] chore: ubuntu 24.04 --- .github/workflows/ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c6697da..f0053a91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: [push] jobs: frontend-lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 @@ -17,7 +17,7 @@ jobs: npm run lint frontend-test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 - uses: browser-actions/setup-chrome@latest @@ -38,7 +38,7 @@ jobs: npm test frontend-build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 @@ -51,7 +51,7 @@ jobs: npm run build backend-lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 @@ -70,7 +70,7 @@ jobs: tox -e lint backend-mypy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 @@ -89,7 +89,7 @@ jobs: tox -e mypy backend-test-unit: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 @@ -108,7 +108,7 @@ jobs: tox -e unit backend-test-integration: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 @@ -127,7 +127,7 @@ jobs: tox -e integration test-system: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2 - uses: browser-actions/setup-chrome@latest @@ -151,7 +151,7 @@ jobs: run: tox -e system docker-build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v2