diff --git a/pyproject.toml b/pyproject.toml index bdfecd61..cede0a51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,7 +193,7 @@ bridge = [ # Container Module - Unified container management (Apptainer + Docker) # Use: pip install scitex[container] -container = ["scitex-container"] +container = ["scitex-container>=0.1.6"] # Browser Module - Web automation # Use: pip install scitex[browser] diff --git a/src/scitex/container/__init__.py b/src/scitex/container/__init__.py index a5db0651..86f43081 100755 --- a/src/scitex/container/__init__.py +++ b/src/scitex/container/__init__.py @@ -1,65 +1,22 @@ -#!/usr/bin/env python3 -"""SciTeX container management -- delegates to scitex-container package.""" +"""SciTeX container — thin compatibility shim for scitex-container. -try: - # Re-export subnamespaces so umbrella users can write - # `stx.container.apptainer.build(...)` as well as the flattened - # `stx.container.build(...)`. - from scitex_container import apptainer, docker, env_snapshot, host - from scitex_container.apptainer import ( - build, - build_dev_pythonpath, - build_exec_args, - build_host_mount_binds, - build_srun_command, - cleanup, - deploy, - detect_container_cmd, - find_containers_dir, - freeze, - get_active_version, - is_sandbox, - list_versions, - rollback, - sandbox_create, - sandbox_maintain, - sandbox_to_sif, - status, - switch_version, - verify, - ) +Aliases ``scitex.container`` to the standalone ``scitex_container`` package via +``sys.modules`` so ``scitex.container is scitex_container`` and every +sub-namespace (``scitex.container.apptainer``, ``.docker``, ``.host``, +``.env_snapshot``) keeps resolving. - _BACKEND = "scitex-container" -except ImportError: - from ._build import build - from ._freeze import freeze - from ._status import status - from ._utils import detect_container_cmd, find_containers_dir - from ._versioning import ( - cleanup, - deploy, - get_active_version, - list_versions, - rollback, - switch_version, - ) +Install: ``pip install scitex[container]`` (or ``pip install scitex-container``). +See: https://github.com/ywatanabe1989/scitex-container +""" - _BACKEND = "local" +import sys as _sys -__all__ = [ - "apptainer", - "build", - "cleanup", - "deploy", - "docker", - "env_snapshot", - "freeze", - "get_active_version", - "host", - "list_versions", - "rollback", - "status", - "switch_version", -] +try: + import scitex_container as _real +except ImportError as _e: # pragma: no cover + raise ImportError( + "scitex.container requires the 'scitex-container' package. " + "Install with: pip install scitex[container] (or: pip install scitex-container)" + ) from _e -# EOF +_sys.modules[__name__] = _real diff --git a/src/scitex/container/_build.py b/src/scitex/container/_build.py deleted file mode 100755 index e174e0bf..00000000 --- a/src/scitex/container/_build.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -"""Build Apptainer/Singularity SIF from .def file.""" - -from __future__ import annotations - -import hashlib -import logging -import subprocess -from pathlib import Path - -from ._utils import detect_container_cmd, find_containers_dir - -logger = logging.getLogger("scitex") - - -def build( - def_name: str = "scitex-cloud-shared-v0.1.0", - output_dir: str | Path | None = None, - force: bool = False, -) -> Path: - """Build Apptainer/Singularity SIF from .def file. - - Parameters - ---------- - def_name : str - Name of the .def file (without extension). - output_dir : str or Path, optional - Directory for the output .sif file. Defaults to same dir as .def. - force : bool - Force rebuild even if .def is unchanged. - - Returns - ------- - Path - Path to the built .sif file. - - Raises - ------ - FileNotFoundError - If .def file or container command not found. - RuntimeError - If build fails. - """ - cmd = detect_container_cmd() - containers_dir = find_containers_dir() - def_path = containers_dir / f"{def_name}.def" - - if not def_path.exists(): - raise FileNotFoundError(f"Definition file not found: {def_path}") - - out_dir = Path(output_dir) if output_dir else def_path.parent - sif_path = out_dir / f"{def_name}.sif" - hash_file = out_dir / ".def-hash" - - current_hash = _hash_file(def_path) - - if not force and sif_path.exists() and hash_file.exists(): - stored_hash = hash_file.read_text().strip() - if current_hash == stored_hash: - logger.info("SIF is up-to-date (hash: %s...)", current_hash[:12]) - return sif_path - - logger.info("Building %s from %s", sif_path.name, def_path.name) - result = subprocess.run( - ["sudo", cmd, "build", "--force", str(sif_path), str(def_path)], - capture_output=False, - ) - if result.returncode != 0: - raise RuntimeError(f"Build failed with exit code {result.returncode}") - - hash_file.write_text(current_hash + "\n") - logger.info("Build complete: %s", sif_path) - return sif_path - - -def _hash_file(path: Path) -> str: - """Compute SHA256 hash of a file.""" - h = hashlib.sha256() - h.update(path.read_bytes()) - return h.hexdigest() - - -# EOF diff --git a/src/scitex/container/_freeze.py b/src/scitex/container/_freeze.py deleted file mode 100755 index 0e071f12..00000000 --- a/src/scitex/container/_freeze.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Extract pinned versions from a built SIF for reproducibility.""" - -from __future__ import annotations - -import logging -import subprocess -from pathlib import Path - -from ._utils import detect_container_cmd - -logger = logging.getLogger("scitex") - - -def freeze( - sif_path: str | Path, - output_dir: str | Path | None = None, -) -> dict[str, Path]: - """Extract pinned versions from a built SIF. - - Parameters - ---------- - sif_path : str or Path - Path to the .sif file. - output_dir : str or Path, optional - Directory for lock files. Defaults to same dir as .sif. - - Returns - ------- - dict[str, Path] - Mapping of lock file type to path: {pip, dpkg, node}. - - Raises - ------ - FileNotFoundError - If SIF file or container command not found. - """ - sif_path = Path(sif_path) - if not sif_path.exists(): - raise FileNotFoundError(f"SIF not found: {sif_path}") - - cmd = detect_container_cmd() - out_dir = Path(output_dir) if output_dir else sif_path.parent - out_dir.mkdir(parents=True, exist_ok=True) - - lock_files = {} - - # pip freeze - pip_lock = out_dir / "requirements-lock.txt" - logger.info("Extracting pip freeze...") - result = subprocess.run( - [cmd, "exec", str(sif_path), "pip", "freeze"], - capture_output=True, - text=True, - ) - if result.returncode == 0: - pip_lock.write_text(result.stdout) - lock_files["pip"] = pip_lock - - # dpkg packages - dpkg_lock = out_dir / "dpkg-lock.txt" - logger.info("Extracting dpkg packages...") - result = subprocess.run( - [cmd, "exec", str(sif_path), "dpkg-query", "-W", "-f=${Package}=${Version}\n"], - capture_output=True, - text=True, - ) - if result.returncode == 0: - dpkg_lock.write_text(result.stdout) - lock_files["dpkg"] = dpkg_lock - - # npm global packages - node_lock = out_dir / "node-lock.txt" - logger.info("Extracting npm packages...") - result = subprocess.run( - [cmd, "exec", str(sif_path), "npm", "list", "-g", "--depth=0", "--json"], - capture_output=True, - text=True, - ) - if result.returncode == 0: - node_lock.write_text(result.stdout) - lock_files["node"] = node_lock - - logger.info("Freeze complete: %d lock files", len(lock_files)) - return lock_files - - -# EOF diff --git a/src/scitex/container/_status.py b/src/scitex/container/_status.py deleted file mode 100755 index af1af2ef..00000000 --- a/src/scitex/container/_status.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -"""List available containers and their build status.""" - -from __future__ import annotations - -import hashlib -import logging -from datetime import datetime -from pathlib import Path - -from ._utils import find_containers_dir - -logger = logging.getLogger("scitex") - - -def status(containers_dir: str | Path | None = None) -> list[dict]: - """List available containers and their status. - - Parameters - ---------- - containers_dir : str or Path, optional - Directory containing .def files. Auto-detected if not given. - - Returns - ------- - list[dict] - List of container info dicts with keys: - name, def_path, sif_path, sif_size, sif_date, - hash_current, hash_stored, needs_rebuild. - """ - cdir = Path(containers_dir) if containers_dir else find_containers_dir() - results = [] - - for def_path in sorted(cdir.glob("*.def")): - name = def_path.stem - sif_path = def_path.with_suffix(".sif") - hash_file = cdir / ".def-hash" - - current_hash = _hash_file(def_path) - stored_hash = "" - if hash_file.exists(): - stored_hash = hash_file.read_text().strip() - - info: dict = { - "name": name, - "def_path": str(def_path), - "sif_path": str(sif_path) if sif_path.exists() else None, - "sif_size": None, - "sif_date": None, - "hash_current": current_hash, - "hash_stored": stored_hash or None, - "needs_rebuild": True, - } - - if sif_path.exists(): - stat = sif_path.stat() - info["sif_size"] = _human_size(stat.st_size) - info["sif_date"] = datetime.fromtimestamp(stat.st_mtime).strftime( - "%Y-%m-%d %H:%M" - ) - info["needs_rebuild"] = current_hash != stored_hash - - results.append(info) - - return results - - -def _hash_file(path: Path) -> str: - """Compute SHA256 hash of a file.""" - h = hashlib.sha256() - h.update(path.read_bytes()) - return h.hexdigest() - - -def _human_size(nbytes: int) -> str: - """Convert bytes to human-readable size.""" - for unit in ("B", "KB", "MB", "GB", "TB"): - if abs(nbytes) < 1024: - return f"{nbytes:.1f} {unit}" - nbytes /= 1024 - return f"{nbytes:.1f} PB" - - -# EOF diff --git a/src/scitex/container/_utils.py b/src/scitex/container/_utils.py deleted file mode 100755 index d54a88ed..00000000 --- a/src/scitex/container/_utils.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -"""Shared utilities for container management.""" - -from __future__ import annotations - -import shutil -from pathlib import Path - - -def detect_container_cmd() -> str: - """Detect apptainer or singularity command. - - Returns - ------- - str - The container command name ('apptainer' or 'singularity'). - - Raises - ------ - FileNotFoundError - If neither command is found. - """ - for cmd in ("apptainer", "singularity"): - if shutil.which(cmd): - return cmd - raise FileNotFoundError( - "Neither apptainer nor singularity is installed. " - "Install with: sudo apt-get install apptainer" - ) - - -def find_containers_dir() -> Path: - """Find the containers directory. - - Search order: - 1. ./containers/ (current working directory) - 2. Package-relative containers/ (scitex-python source tree) - 3. ~/.scitex/containers/ (user-managed) - - Returns - ------- - Path - Path to the containers directory. - - Raises - ------ - FileNotFoundError - If no containers directory is found. - """ - # 1. Current working directory - cwd_containers = Path.cwd() / "containers" - if cwd_containers.is_dir() and list(cwd_containers.glob("*.def")): - return cwd_containers - - # 2. Package-relative (scitex-python/containers/) - pkg_root = Path(__file__).resolve().parents[3] # src/scitex/container -> root - pkg_containers = pkg_root / "containers" - if pkg_containers.is_dir() and list(pkg_containers.glob("*.def")): - return pkg_containers - - # 3. User-managed - user_containers = Path.home() / ".scitex" / "containers" - if user_containers.is_dir() and list(user_containers.glob("*.def")): - return user_containers - - raise FileNotFoundError( - "No containers directory found. Searched:\n" - f" - {cwd_containers}\n" - f" - {pkg_containers}\n" - f" - {user_containers}" - ) - - -# EOF diff --git a/src/scitex/container/_versioning.py b/src/scitex/container/_versioning.py deleted file mode 100755 index d3bc9962..00000000 --- a/src/scitex/container/_versioning.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env python3 -"""Container version management: list, switch, rollback, deploy, cleanup.""" - -from __future__ import annotations - -import logging -import re -import subprocess -from datetime import datetime -from pathlib import Path - -logger = logging.getLogger("scitex") - -_VERSION_RE = re.compile(r"^scitex-v(.+)\.sif$") -_BASE_RE = re.compile(r"^scitex-base-v(\d+)\.sif$") - - -def _human_size(nbytes: int) -> str: - """Convert bytes to human-readable size. - - Parameters - ---------- - nbytes : int - Number of bytes. - - Returns - ------- - str - Human-readable size string (e.g. "1.2 GB"). - """ - for unit in ("B", "KB", "MB", "GB", "TB"): - if abs(nbytes) < 1024: - return f"{nbytes:.1f} {unit}" - nbytes /= 1024 - return f"{nbytes:.1f} PB" - - -def _parse_version(path: Path) -> str | None: - """Extract version string from a scitex-v*.sif filename. - - Parameters - ---------- - path : Path - Path to the SIF file. - - Returns - ------- - str or None - Version string if matched, else None. - """ - m = _VERSION_RE.match(path.name) - return m.group(1) if m else None - - -def _versioned_sifs(containers_dir: Path) -> list[Path]: - """Return scitex-v*.sif paths sorted by modification time (newest first). - - Parameters - ---------- - containers_dir : Path - Directory containing SIF files. - - Returns - ------- - list[Path] - Sorted list of versioned SIF paths. - """ - sifs = [ - p - for p in containers_dir.glob("scitex-v*.sif") - if _VERSION_RE.match(p.name) and p.is_file() - ] - sifs.sort(key=lambda p: p.stat().st_mtime, reverse=True) - return sifs - - -def list_versions(containers_dir: Path) -> list[dict]: - """List all versioned SIFs with metadata. - - Parameters - ---------- - containers_dir : Path - Directory containing SIF files. - - Returns - ------- - list[dict] - Each dict contains keys: version, path, size, date, active. - Sorted by modification time (newest first). - """ - containers_dir = Path(containers_dir) - active = get_active_version(containers_dir) - results = [] - - for sif in _versioned_sifs(containers_dir): - version = _parse_version(sif) - if version is None: - continue - stat = sif.stat() - results.append( - { - "version": version, - "path": str(sif), - "size": _human_size(stat.st_size), - "date": datetime.fromtimestamp(stat.st_mtime).strftime( - "%Y-%m-%d %H:%M" - ), - "active": version == active, - } - ) - - return results - - -def get_active_version(containers_dir: Path) -> str | None: - """Read current.sif symlink to determine active version. - - Parameters - ---------- - containers_dir : Path - Directory containing the current.sif symlink. - - Returns - ------- - str or None - Version string of the active SIF, or None if no symlink exists - or it does not point to a valid versioned SIF. - """ - containers_dir = Path(containers_dir) - link = containers_dir / "current.sif" - - if not link.is_symlink(): - return None - - target = link.resolve() - return _parse_version(target) - - -def switch_version( - version: str, - containers_dir: Path, - use_sudo: bool = False, -) -> None: - """Atomically switch current.sif symlink to scitex-v{version}.sif. - - Uses ``ln -sf`` to create a temporary symlink, then ``mv -Tf`` for an - atomic rename on the same filesystem. - - Parameters - ---------- - version : str - Target version string (e.g. "2.19.5"). - containers_dir : Path - Directory containing SIF files. - use_sudo : bool - If True, run ln/mv via sudo (needed for /opt/scitex paths). - - Raises - ------ - FileNotFoundError - If the target SIF does not exist. - RuntimeError - If the symlink switch fails. - """ - containers_dir = Path(containers_dir) - target_name = f"scitex-v{version}.sif" - target_path = containers_dir / target_name - link_path = containers_dir / "current.sif" - - if not target_path.exists(): - raise FileNotFoundError(f"Version {version} not found: {target_path}") - - # Temporary symlink name in the same directory (same filesystem) - tmp_link = containers_dir / f".current.sif.tmp.{id(version)}" - - prefix = ["sudo"] if use_sudo else [] - - try: - # Create temp symlink pointing at target - subprocess.run( - [*prefix, "ln", "-sf", target_name, str(tmp_link)], - check=True, - ) - # Atomic rename over the real symlink - subprocess.run( - [*prefix, "mv", "-Tf", str(tmp_link), str(link_path)], - check=True, - ) - except subprocess.CalledProcessError as exc: - # Clean up temp link on failure - tmp_link.unlink(missing_ok=True) - raise RuntimeError(f"Failed to switch to version {version}: {exc}") from exc - - logger.info("Switched to version %s", version) - - -def rollback( - containers_dir: Path, - use_sudo: bool = False, -) -> str: - """Switch to the version before the current one (by modification time). - - Parameters - ---------- - containers_dir : Path - Directory containing SIF files. - use_sudo : bool - If True, run symlink commands via sudo. - - Returns - ------- - str - Version string that is now active after rollback. - - Raises - ------ - RuntimeError - If there is no current version or no previous version to roll back to. - """ - containers_dir = Path(containers_dir) - active = get_active_version(containers_dir) - - if active is None: - raise RuntimeError("No active version found; cannot rollback") - - sifs = _versioned_sifs(containers_dir) - versions = [_parse_version(s) for s in sifs] - - try: - idx = versions.index(active) - except ValueError: - raise RuntimeError(f"Active version {active} not found in directory") - - if idx + 1 >= len(versions): - raise RuntimeError( - f"No older version available to roll back to (current: {active})" - ) - - previous = versions[idx + 1] - logger.info("Rolling back from %s to %s", active, previous) - switch_version(previous, containers_dir, use_sudo=use_sudo) - return previous - - -def deploy( - source_dir: Path, - target_dir: Path = Path("/opt/scitex/singularity"), -) -> None: - """Copy active SIF and base SIF to target directory. - - Copies the currently active ``scitex-v*.sif`` and the latest - ``scitex-base-v*.sif`` to *target_dir*, then recreates the - ``current.sif`` symlink there. Uses sudo for the copy. - - Parameters - ---------- - source_dir : Path - Directory containing the built SIF files. - target_dir : Path - Deployment target directory (default: /opt/scitex/singularity). - - Raises - ------ - RuntimeError - If no active version is set or copy fails. - FileNotFoundError - If the active SIF or base SIF is missing. - """ - source_dir = Path(source_dir) - target_dir = Path(target_dir) - - active = get_active_version(source_dir) - if active is None: - raise RuntimeError("No active version in source directory") - - active_sif = source_dir / f"scitex-v{active}.sif" - if not active_sif.exists(): - raise FileNotFoundError(f"Active SIF not found: {active_sif}") - - # Find latest base SIF by modification time - base_sifs = sorted( - (p for p in source_dir.glob("scitex-base-v*.sif") if p.is_file()), - key=lambda p: p.stat().st_mtime, - reverse=True, - ) - - # Ensure target directory exists - subprocess.run( - ["sudo", "mkdir", "-p", str(target_dir)], - check=True, - ) - - # Copy active SIF - logger.info("Deploying %s to %s", active_sif.name, target_dir) - subprocess.run( - ["sudo", "cp", str(active_sif), str(target_dir / active_sif.name)], - check=True, - ) - - # Copy base SIF if available - if base_sifs: - base_sif = base_sifs[0] - logger.info("Deploying %s to %s", base_sif.name, target_dir) - subprocess.run( - ["sudo", "cp", str(base_sif), str(target_dir / base_sif.name)], - check=True, - ) - - # Recreate symlink at target - switch_version(active, target_dir, use_sudo=True) - logger.info("Deploy complete: version %s", active) - - -def cleanup( - containers_dir: Path, - keep: int = 3, -) -> list[Path]: - """Remove old scitex-v*.sif files, keeping the N most recent. - - Never removes the active version (current.sif target) or any - ``scitex-base-v*.sif`` base images. - - Parameters - ---------- - containers_dir : Path - Directory containing SIF files. - keep : int - Number of most-recent versioned SIFs to keep (default: 3). - - Returns - ------- - list[Path] - Paths of removed SIF files. - """ - containers_dir = Path(containers_dir) - active = get_active_version(containers_dir) - sifs = _versioned_sifs(containers_dir) # newest first - removed: list[Path] = [] - - protected = set() - if active is not None: - protected.add(f"scitex-v{active}.sif") - - # Keep the top N by modification time - kept = 0 - for sif in sifs: - version = _parse_version(sif) - if version is None: - continue - - if sif.name in protected: - # Active version is always kept and does not count toward limit - continue - - if kept < keep: - kept += 1 - continue - - # Remove - logger.info("Removing old SIF: %s", sif.name) - sif.unlink() - removed.append(sif) - - return removed - - -# EOF