diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e05d95e..6b65448 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -61,7 +61,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python 3.10 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" @@ -94,7 +94,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/CI_CONFIG_GUIDE.md b/CI_CONFIG_GUIDE.md new file mode 100644 index 0000000..34c0e0e --- /dev/null +++ b/CI_CONFIG_GUIDE.md @@ -0,0 +1,379 @@ +# CI Configuration Guide + +## Overview + +**wads** now supports using `pyproject.toml` as the **single source of truth** for CI configuration. This eliminates the need to hardcode project-specific settings in CI workflow files. + +## Quick Start + +### 1. Define CI Configuration in pyproject.toml + +Add a `[tool.wads.ci]` section to your `pyproject.toml`: + +```toml +[tool.wads.ci] +project_name = "myproject" + +[tool.wads.ci.commands] +pre_test = [ + "python scripts/setup_test_data.py", +] + +[tool.wads.ci.env] +required = ["DATABASE_URL"] +defaults = {"LOG_LEVEL" = "DEBUG"} + +[tool.wads.ci.testing] +python_versions = ["3.10", "3.11", "3.12"] +pytest_args = ["-v", "--tb=short", "--cov=myproject"] +coverage_enabled = true +coverage_threshold = 80 +exclude_paths = ["examples", "scrap", "benchmarks"] +``` + +### 2. Generate CI Workflow + +Run `populate` to generate or update your CI workflow: + +```bash +python -m wads.populate /path/to/your/project +``` + +The populate function will: +1. Read your `[tool.wads.ci]` configuration +2. Generate a customized CI workflow based on your settings +3. Create `.github/workflows/ci.yml` with all your configurations applied + +### 3. Validate Environment (Optional) + +Add this step to your CI workflow to validate required environment variables: + +```yaml +- name: Validate Environment + run: python -m wads.scripts.validate_ci_env +``` + +## Configuration Reference + +### Project Settings + +```toml +[tool.wads.ci] +# Project name used throughout CI (defaults to package name) +project_name = "myproject" +``` + +### โš™๏ธ Execution Flow and Commands + +Define commands executed during different CI phases: + +```toml +[tool.wads.ci.commands] +# Pre-test setup - run before testing/linting +pre_test = [ + "python scripts/migrate_db.py", + "python -m playwright install", +] + +# Test commands (defaults to ["pytest"]) +test = [ + "pytest", + "pytest --integration", +] + +# Post-test commands +post_test = [ + "python scripts/upload_coverage.py", +] + +# Custom lint commands (optional) +lint = [ + "ruff check .", +] + +# Custom format commands (optional) +format = [ + "ruff format --check .", +] +``` + +### ๐ŸŒ Environment Variables + +Control required and default environment variables: + +```toml +[tool.wads.ci.env] +# Required variables - CI fails if these aren't set +required = [ + "DATABASE_URL", + "API_KEY", + "AWS_REGION", +] + +# Default variables - set if not provided +defaults = { + "LOG_LEVEL" = "DEBUG", + "TESTING" = "true", + "PYTHONUNBUFFERED" = "1", +} +``` + +### โœ… Code Quality and Formatting + +Configure code quality tools: + +```toml +[tool.wads.ci.quality.ruff] +enabled = true +output_format = "github" # GitHub Actions annotations + +[tool.wads.ci.quality.mypy] +enabled = true +strict = true +python_version = "3.10" +``` + +### ๐Ÿงช Test Configuration + +Control test execution: + +```toml +[tool.wads.ci.testing] +# Python versions to test against +python_versions = ["3.10", "3.11", "3.12"] + +# Pytest arguments +pytest_args = ["-v", "--tb=short", "--cov=myproject"] + +# Coverage settings +coverage_enabled = true +coverage_threshold = 80 # Fail if below this percentage +coverage_report_format = ["term", "xml", "html"] + +# Paths to exclude from testing +exclude_paths = ["examples", "scrap", "benchmarks"] + +# Enable Windows testing (informational, non-blocking) +test_on_windows = true +``` + +### ๐Ÿ“ฆ Build and Publish + +Control build artifacts and publication: + +```toml +[tool.wads.ci.build] +sdist = true +wheel = true + +[tool.wads.ci.publish] +enabled = true +``` + +### ๐Ÿ“„ Documentation + +Configure documentation generation: + +```toml +[tool.wads.ci.docs] +enabled = true +builder = "epythet" # or "sphinx", "mkdocs" +ignore_paths = ["tests/", "scrap/", "examples/"] +``` + +## Advanced Usage + +### Using CI Config Programmatically + +You can read and use CI configuration in your own scripts: + +```python +from wads.ci_config import CIConfig + +# Read config from pyproject.toml +config = CIConfig.from_file(".") + +# Access configuration +print(f"Project: {config.project_name}") +print(f"Python versions: {config.python_versions}") +print(f"Pre-test commands: {config.commands_pre_test}") + +# Check if tools are enabled +if config.is_mypy_enabled(): + print("Mypy is enabled") + +# Get required environment variables +required_vars = config.env_vars_required +print(f"Required env vars: {required_vars}") +``` + +### Validating Environment Variables + +The validation script can be used standalone: + +```bash +# Validate current environment +python -m wads.scripts.validate_ci_env + +# Use in CI (will exit with code 1 if validation fails) +``` + +### Template Substitution + +When `populate` generates CI workflows, it performs these substitutions: + +- `#ENV_BLOCK#` โ†’ Generated environment variables section +- `#PYTHON_VERSIONS#` โ†’ JSON array of Python versions +- `#PRE_TEST_STEPS#` โ†’ YAML steps for pre-test commands +- `#PYTEST_ARGS#` โ†’ Pytest arguments string +- `#COVERAGE_ENABLED#` โ†’ true/false +- `#EXCLUDE_PATHS#` โ†’ Comma-separated exclude paths +- And more... + +## Migration Guide + +### From Hardcoded CI to pyproject.toml + +1. **Extract configuration** from your existing `.github/workflows/ci.yml` +2. **Add to pyproject.toml** under `[tool.wads.ci]` +3. **Re-run populate** to regenerate the CI workflow +4. **Test** that CI still works as expected + +Example migration: + +**Before** (in `.github/workflows/ci.yml`): +```yaml +env: + PROJECT_NAME: myproject + LOG_LEVEL: DEBUG + +strategy: + matrix: + python-version: ["3.10", "3.12"] + +- name: Run Tests + run: pytest -v --cov=myproject +``` + +**After** (in `pyproject.toml`): +```toml +[tool.wads.ci] +project_name = "myproject" + +[tool.wads.ci.env] +defaults = {"LOG_LEVEL" = "DEBUG"} + +[tool.wads.ci.testing] +python_versions = ["3.10", "3.12"] +pytest_args = ["-v", "--cov=myproject"] +``` + +## Benefits + +### โœ… Single Source of Truth +- All CI configuration lives in `pyproject.toml` +- No need to edit YAML files directly +- Configuration is versioned with your code + +### โœ… Type Safety and Validation +- TOML provides structure and validation +- Easier to catch configuration errors + +### โœ… DRY (Don't Repeat Yourself) +- Define once, use everywhere +- No duplication between CI providers + +### โœ… Better Developer Experience +- Configuration is discoverable +- IDE support for TOML +- Clear documentation in one place + +### โœ… Flexibility +- Works with existing CI workflows +- Gradual migration supported +- Falls back to defaults gracefully + +## Examples + +### Minimal Configuration + +```toml +[tool.wads.ci] +# Use all defaults - just specify project name +project_name = "myproject" +``` + +### Database Testing + +```toml +[tool.wads.ci] +project_name = "mydb" + +[tool.wads.ci.commands] +pre_test = [ + "docker-compose up -d postgres", + "python scripts/wait_for_db.py", + "alembic upgrade head", +] +post_test = [ + "docker-compose down", +] + +[tool.wads.ci.env] +required = ["DATABASE_URL"] +defaults = {"POSTGRES_VERSION" = "15"} + +[tool.wads.ci.testing] +pytest_args = ["-v", "--tb=short", "--integration"] +``` + +### Multi-Tool Setup + +```toml +[tool.wads.ci.quality.ruff] +enabled = true + +[tool.wads.ci.quality.mypy] +enabled = true +strict = true + +[tool.wads.ci.quality.black] +enabled = false # Using Ruff formatter instead +``` + +## Troubleshooting + +### CI workflow not updating + +Make sure to run `populate` after changing `pyproject.toml`: + +```bash +python -m wads.populate . +``` + +### Dynamic template not used + +The dynamic template is only used if: +1. `pyproject.toml` exists with `[tool.wads.ci]` section +2. The dynamic template file exists (`github_ci_publish_2025_dynamic.yml`) + +### Environment validation failing + +Check that: +1. Required variables are set in CI secrets +2. Variable names match exactly (case-sensitive) +3. Validation script has access to pyproject.toml + +### Missing dependencies + +If you get import errors: + +```bash +pip install tomli tomli-w # For Python < 3.11 +``` + +## See Also + +- [pyproject.toml specification](https://packaging.python.org/en/latest/specifications/pyproject-toml/) +- [GitHub Actions documentation](https://docs.github.com/en/actions) +- [wads populate documentation](README.md) diff --git a/wads/ci_config.py b/wads/ci_config.py new file mode 100644 index 0000000..616a577 --- /dev/null +++ b/wads/ci_config.py @@ -0,0 +1,414 @@ +""" +Utilities for reading and applying CI configuration from pyproject.toml. + +This module provides the infrastructure for using pyproject.toml as the single +source of truth for CI configuration, eliminating hardcoded project-specific +settings in CI workflow files. +""" + +from typing import Any, Optional +from pathlib import Path +import sys + +if sys.version_info >= (3, 11): + import tomllib +else: + try: + import tomli as tomllib + except ImportError: + raise ImportError("tomli package required for Python < 3.11") + + +class CIConfig: + """Represents CI configuration extracted from pyproject.toml.""" + + def __init__(self, pyproject_data: dict, project_name: str = None): + """ + Initialize CI configuration from pyproject.toml data. + + Args: + pyproject_data: Parsed TOML data from pyproject.toml + project_name: Project name (defaults to project.name from pyproject) + """ + self.data = pyproject_data + self.ci_config = pyproject_data.get("tool", {}).get("wads", {}).get("ci", {}) + self._project_name = ( + project_name or self.ci_config.get("project_name") + or pyproject_data.get("project", {}).get("name", "") + ) + + @classmethod + def from_file(cls, pyproject_path: str | Path) -> "CIConfig": + """ + Load CI configuration from a pyproject.toml file. + + Args: + pyproject_path: Path to pyproject.toml file or directory containing it + + Returns: + CIConfig instance + """ + pyproject_path = Path(pyproject_path) + if pyproject_path.is_dir(): + pyproject_path = pyproject_path / "pyproject.toml" + + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + + return cls(data) + + @property + def project_name(self) -> str: + """Get the project name for CI.""" + return self._project_name + + # โš™๏ธ EXECUTION FLOW AND COMMANDS + @property + def commands_pre_test(self) -> list[str]: + """Get pre-test setup commands.""" + return self.ci_config.get("commands", {}).get("pre_test", []) + + @property + def commands_test(self) -> list[str]: + """Get test commands (defaults to ['pytest']).""" + return self.ci_config.get("commands", {}).get("test", ["pytest"]) + + @property + def commands_post_test(self) -> list[str]: + """Get post-test commands.""" + return self.ci_config.get("commands", {}).get("post_test", []) + + @property + def commands_lint(self) -> list[str]: + """Get lint commands.""" + return self.ci_config.get("commands", {}).get("lint", []) + + @property + def commands_format(self) -> list[str]: + """Get format commands.""" + return self.ci_config.get("commands", {}).get("format", []) + + # ๐ŸŒ ENVIRONMENT VARIABLES + @property + def env_vars_required(self) -> list[str]: + """Get required environment variable names.""" + return self.ci_config.get("env", {}).get("required", []) + + @property + def env_vars_defaults(self) -> dict[str, str]: + """Get default environment variables.""" + return self.ci_config.get("env", {}).get("defaults", {}) + + # โœ… CODE QUALITY AND FORMATTING + @property + def quality_config(self) -> dict: + """Get code quality tool configuration.""" + return self.ci_config.get("quality", {}) + + def is_ruff_enabled(self) -> bool: + """Check if Ruff linter is enabled.""" + return self.quality_config.get("ruff", {}).get("enabled", True) + + def is_black_enabled(self) -> bool: + """Check if Black formatter is enabled.""" + return self.quality_config.get("black", {}).get("enabled", False) + + def is_mypy_enabled(self) -> bool: + """Check if Mypy type checker is enabled.""" + return self.quality_config.get("mypy", {}).get("enabled", False) + + # ๐Ÿงช TEST CONFIGURATION + @property + def testing_config(self) -> dict: + """Get testing configuration.""" + return self.ci_config.get("testing", {}) + + @property + def python_versions(self) -> list[str]: + """Get Python versions to test against.""" + return self.testing_config.get("python_versions", ["3.10", "3.12"]) + + @property + def pytest_args(self) -> list[str]: + """Get pytest arguments.""" + return self.testing_config.get("pytest_args", ["-v", "--tb=short"]) + + @property + def coverage_enabled(self) -> bool: + """Check if coverage is enabled.""" + return self.testing_config.get("coverage_enabled", True) + + @property + def coverage_threshold(self) -> int: + """Get minimum coverage threshold (0 = no enforcement).""" + return self.testing_config.get("coverage_threshold", 0) + + @property + def exclude_paths(self) -> list[str]: + """Get test paths to exclude.""" + return self.testing_config.get("exclude_paths", ["examples", "scrap"]) + + @property + def test_on_windows(self) -> bool: + """Check if Windows testing is enabled.""" + return self.testing_config.get("test_on_windows", True) + + # ๐Ÿ“ฆ BUILD AND PUBLISH SETTINGS + @property + def build_config(self) -> dict: + """Get build configuration.""" + return self.ci_config.get("build", {}) + + @property + def build_sdist(self) -> bool: + """Check if source distribution should be built.""" + return self.build_config.get("sdist", True) + + @property + def build_wheel(self) -> bool: + """Check if wheel should be built.""" + return self.build_config.get("wheel", True) + + @property + def publish_config(self) -> dict: + """Get publish configuration.""" + return self.ci_config.get("publish", {}) + + @property + def publish_enabled(self) -> bool: + """Check if publishing is enabled.""" + return self.publish_config.get("enabled", True) + + # ๐Ÿ“„ DOCUMENTATION SETTINGS + @property + def docs_config(self) -> dict: + """Get documentation configuration.""" + return self.ci_config.get("docs", {}) + + @property + def docs_enabled(self) -> bool: + """Check if documentation generation is enabled.""" + return self.docs_config.get("enabled", True) + + @property + def docs_builder(self) -> str: + """Get documentation builder name.""" + return self.docs_config.get("builder", "epythet") + + @property + def docs_ignore_paths(self) -> list[str]: + """Get paths to ignore during documentation generation.""" + return self.docs_config.get("ignore_paths", ["tests/", "scrap/", "examples/"]) + + def to_ci_env_block(self) -> str: + """ + Generate YAML env block for GitHub Actions. + + Returns: + YAML string for env section + """ + lines = ["env:"] + lines.append(f" PROJECT_NAME: {self.project_name}") + + # Add default environment variables + for key, value in self.env_vars_defaults.items(): + lines.append(f" {key}: {value}") + + return "\n".join(lines) + + def to_pre_test_step(self) -> Optional[str]: + """ + Generate YAML step for pre-test commands. + + Returns: + YAML string for pre-test step, or None if no commands + """ + if not self.commands_pre_test: + return None + + lines = [" - name: Pre-test Setup"] + lines.append(" run: |") + for cmd in self.commands_pre_test: + lines.append(f" {cmd}") + + return "\n".join(lines) + + def has_ci_config(self) -> bool: + """Check if any CI configuration is present.""" + return bool(self.ci_config) + + def generate_env_block(self) -> str: + """ + Generate YAML env block for GitHub Actions. + + Returns: + YAML string for env section + """ + lines = ["env:"] + lines.append(f" PROJECT_NAME: {self.project_name}") + + # Add default environment variables + for key, value in self.env_vars_defaults.items(): + lines.append(f" {key}: {value}") + + return "\n".join(lines) + + def generate_pre_test_steps(self) -> str: + """ + Generate YAML steps for pre-test commands. + + Returns: + YAML string for pre-test steps, or empty string if no commands + """ + if not self.commands_pre_test: + return "" + + lines = [" - name: Pre-test Setup", " run: |"] + for cmd in self.commands_pre_test: + lines.append(f" {cmd}") + + return "\n".join(lines) + + def generate_windows_validation_job(self) -> str: + """ + Generate YAML for Windows validation job. + + Returns: + YAML string for Windows job, or empty string if disabled + """ + if not self.test_on_windows: + return "" + + template = """ + windows-validation: + name: Windows Tests (Informational) + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: windows-latest + continue-on-error: true # Don't fail the entire workflow if Windows tests fail + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v6 + with: + python-version: "3.10" + + - name: Install Dependencies + uses: i2mint/wads/actions/install-deps@master + with: + dependency-files: pyproject.toml + extras: dev,test + + - name: Run Windows Tests + uses: i2mint/wads/actions/windows-tests@master + with: + root-dir: ${{ env.PROJECT_NAME }} + exclude: {exclude} + pytest-args: {pytest_args} +""" + exclude = ",".join(self.exclude_paths) + pytest_args = " ".join(self.pytest_args) + return template.format(exclude=exclude, pytest_args=pytest_args) + + def generate_github_pages_job(self) -> str: + """ + Generate YAML for GitHub Pages job. + + Returns: + YAML string for GitHub Pages job, or empty string if disabled + """ + if not self.docs_enabled: + return "" + + ignore_paths = ",".join(self.docs_ignore_paths) + template = f""" + github-pages: + name: Publish GitHub Pages + + permissions: + contents: write + pages: write + id-token: write + + if: "!contains(github.event.head_commit.message, '[skip ci]') && github.ref == format('refs/heads/{{0}}', github.event.repository.default_branch)" + needs: publish + runs-on: ubuntu-latest + + steps: + - uses: i2mint/epythet/actions/publish-github-pages@master + with: + github-token: ${{{{ secrets.GITHUB_TOKEN }}}} + ignore: "{ignore_paths}" +""" + return template + + def to_ci_template_substitutions(self) -> dict[str, str]: + """ + Generate all template substitutions for CI workflow generation. + + Returns: + Dictionary mapping placeholder names to their values + """ + import json + + return { + "#ENV_BLOCK#": self.generate_env_block(), + "#PYTHON_VERSIONS#": json.dumps(self.python_versions), + "#PRE_TEST_STEPS#": self.generate_pre_test_steps(), + "#EXCLUDE_PATHS#": ",".join(self.exclude_paths), + "#COVERAGE_ENABLED#": str(self.coverage_enabled).lower(), + "#PYTEST_ARGS#": " ".join(self.pytest_args), + "#WINDOWS_VALIDATION_JOB#": self.generate_windows_validation_job(), + "#GITHUB_PAGES_JOB#": self.generate_github_pages_job(), + "#BUILD_SDIST#": str(self.build_sdist).lower(), + "#BUILD_WHEEL#": str(self.build_wheel).lower(), + "#PROJECT_NAME#": self.project_name, + } + + def __repr__(self) -> str: + return f"CIConfig(project_name={self.project_name!r}, has_config={self.has_ci_config()})" + + +def read_ci_config(pyproject_path: str | Path) -> CIConfig: + """ + Read CI configuration from pyproject.toml. + + Args: + pyproject_path: Path to pyproject.toml file or directory containing it + + Returns: + CIConfig instance + + Example: + >>> config = read_ci_config(".") + >>> config.project_name + 'myproject' + >>> config.python_versions + ['3.10', '3.12'] + """ + return CIConfig.from_file(pyproject_path) + + +def get_ci_config_or_defaults( + pyproject_path: str | Path, project_name: str = None +) -> CIConfig: + """ + Read CI configuration from pyproject.toml, using defaults if file doesn't exist. + + Args: + pyproject_path: Path to pyproject.toml file or directory containing it + project_name: Default project name if not found in config + + Returns: + CIConfig instance with defaults if file doesn't exist + """ + pyproject_path = Path(pyproject_path) + if pyproject_path.is_dir(): + pyproject_path = pyproject_path / "pyproject.toml" + + if not pyproject_path.exists(): + # Return empty config with defaults + return CIConfig({"project": {"name": project_name or ""}}, project_name) + + return read_ci_config(pyproject_path) diff --git a/wads/data/github_ci_publish_2025.yml b/wads/data/github_ci_publish_2025.yml index af41bf9..d5da03a 100644 --- a/wads/data/github_ci_publish_2025.yml +++ b/wads/data/github_ci_publish_2025.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -61,7 +61,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python 3.10 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" @@ -103,7 +103,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} diff --git a/wads/data/github_ci_publish_2025_dynamic.yml b/wads/data/github_ci_publish_2025_dynamic.yml new file mode 100644 index 0000000..a51da30 --- /dev/null +++ b/wads/data/github_ci_publish_2025_dynamic.yml @@ -0,0 +1,121 @@ +name: Continuous Integration (Modern) +on: [push, pull_request] + +#ENV_BLOCK# + +jobs: + validation: + name: Validation + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ubuntu-latest + strategy: + matrix: + python-version: #PYTHON_VERSIONS# + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Dependencies + uses: i2mint/wads/actions/install-deps@master + with: + dependency-files: pyproject.toml + extras: dev,test + # Fallback for projects still using setup.cfg: + # dependency-files: setup.cfg + # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} # Uncomment for private dependencies + +#PRE_TEST_STEPS# + + - name: Format Source Code + uses: i2mint/wads/actions/ruff-format@master + with: + line-length: 88 + target-path: . + + - name: Lint Validation + uses: i2mint/wads/actions/ruff-lint@master + with: + root-dir: ${{ env.PROJECT_NAME }} + output-format: github + # Ruff will read configuration from pyproject.toml + + - name: Run Tests + uses: i2mint/wads/actions/run-tests@master + with: + root-dir: ${{ env.PROJECT_NAME }} + exclude: #EXCLUDE_PATHS# + coverage: #COVERAGE_ENABLED# + pytest-args: #PYTEST_ARGS# + +#WINDOWS_VALIDATION_JOB# + + publish: + name: Publish + if: "!contains(github.event.head_commit.message, '[skip ci]') && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main')" + needs: validation + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Format Source Code + uses: i2mint/wads/actions/ruff-format@master + + - name: Update Version Number + id: version + uses: i2mint/isee/actions/bump-version-number@master + + - name: Build Distribution + uses: i2mint/wads/actions/build-dist@master + with: + sdist: #BUILD_SDIST# + wheel: #BUILD_WHEEL# + # Uncomment for private dependencies: + # with: + # ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Publish to PyPI + uses: i2mint/wads/actions/pypi-upload@master + with: + pypi-username: ${{ secrets.PYPI_USERNAME }} + pypi-password: ${{ secrets.PYPI_PASSWORD }} + skip-existing: false + + - name: Track Code Metrics + uses: i2mint/umpyre/actions/track-metrics@master + continue-on-error: true # Don't fail CI if metrics collection fails + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + config-path: .github/umpyre-config.yml # Optional: defaults to .umpyre.yml + + - name: Commit Changes + uses: i2mint/wads/actions/git-commit@master + with: + commit-message: "**CI** Formatted code + Updated version to ${{ env.VERSION }} [skip ci]" + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + push: true + + - name: Tag Repository + uses: i2mint/wads/actions/git-tag@master + with: + tag: ${{ env.VERSION }} + message: "Release version ${{ env.VERSION }}" + push: true + +#GITHUB_PAGES_JOB# diff --git a/wads/data/pyproject_toml_tpl.toml b/wads/data/pyproject_toml_tpl.toml index 598b95e..f722c18 100644 --- a/wads/data/pyproject_toml_tpl.toml +++ b/wads/data/pyproject_toml_tpl.toml @@ -15,6 +15,26 @@ dependencies = [] [project.urls] Homepage = "" +# Repository = "" # URL to the source code repository +# Documentation = "" # URL to the documentation (e.g., Read the Docs, GitHub Pages) +# Changelog = "" # URL to the changelog or release notes +# Issues = "" # URL to the issue tracker + +# [project.scripts] +# # Console script entry points - creates command-line executables +# # Format: "command-name = package.module:function" +# # Example: my-tool = "mypackage.cli:main" + +# [project.gui-scripts] +# # GUI application entry points - similar to scripts but for GUI apps +# # Format: "app-name = package.module:function" +# # Example: my-app = "mypackage.gui:main" + +# [project.entry-points."some.group"] +# # Plugin-style entry points for extensibility +# # Allows other packages to discover and load plugins +# # Format: "plugin-name = package.module:object" +# # Example: json = "mypackage.serializers:JSONSerializer" [project.optional-dependencies] dev = [ @@ -82,3 +102,134 @@ convention = "google" # or "numpy" or "pep257" minversion = "6.0" testpaths = ["tests"] doctest_optionflags = ["NORMALIZE_WHITESPACE", "ELLIPSIS"] + +# ============================================================================ +# CI Configuration (Single Source of Truth) +# ============================================================================ +# This section defines all CI-specific configuration, eliminating the need +# to hardcode project-specific settings in CI workflow files. + +[tool.wads.ci] +# Project name used in CI workflows (defaults to package name if not specified) +project_name = "" + +# โš™๏ธ EXECUTION FLOW AND COMMANDS +# Commands executed during different CI phases + +[tool.wads.ci.commands] +# Pre-test setup commands - run before testing/linting begins +# Examples: database migrations, data generation, environment setup +pre_test = [ + # "python scripts/setup_test_data.py", + # "python -m playwright install", +] + +# Test commands - primary test suite execution +# If not specified, defaults to ["pytest"] +test = [ + # "pytest", + # "pytest --cov=mypackage --cov-report=xml", +] + +# Post-test commands - run after tests complete (e.g., cleanup, reports) +post_test = [ + # "python scripts/generate_coverage_report.py", +] + +# Lint commands - code quality checks +# If not specified, uses Ruff configuration from [tool.ruff] +lint = [ + # "ruff check .", +] + +# Format commands - code formatting +# If not specified, uses Ruff formatter +format = [ + # "ruff format --check .", +] + +# ๐ŸŒ ENVIRONMENT VARIABLES +# Control which environment variables are required or have defaults + +[tool.wads.ci.env] +# Required environment variables - CI will fail if these are not set +# Format: list of variable names +required = [ + # "DATABASE_URL", + # "API_KEY", +] + +# Default environment variables - set these if not provided by CI system +# Format: key-value pairs (inline table syntax) +# Note: These are template defaults; actual values should be in CI secrets +# Example: defaults = {"LOG_LEVEL" = "DEBUG", "TESTING" = "true", "PYTHONUNBUFFERED" = "1"} +defaults = {} + +# โœ… CODE QUALITY AND FORMATTING +# Tool-specific configuration for CI enforcement + +[tool.wads.ci.quality] +# Ruff linter settings for CI (overrides/supplements [tool.ruff]) +[tool.wads.ci.quality.ruff] +enabled = true +# line_length = 88 # Uncomment to override [tool.ruff] setting +# output_format = "github" # For GitHub Actions annotations + +# Black formatter settings (if using Black instead of Ruff format) +[tool.wads.ci.quality.black] +enabled = false +# line_length = 88 +# target_version = ["py310", "py311", "py312"] + +# Mypy type checker settings +[tool.wads.ci.quality.mypy] +enabled = false +# strict = true +# python_version = "3.10" +# ignore_missing_imports = true + +# ๐Ÿงช TEST CONFIGURATION +# Test-specific settings for CI + +[tool.wads.ci.testing] +# Python versions to test against +python_versions = ["3.10", "3.12"] + +# Pytest arguments (supplements [tool.pytest.ini_options]) +pytest_args = ["-v", "--tb=short"] + +# Coverage settings +coverage_enabled = true +coverage_threshold = 0 # Minimum coverage percentage (0 = no enforcement) +coverage_report_format = ["term", "xml"] # terminal, xml, html, etc. + +# Test paths to exclude (in addition to pytest configuration) +exclude_paths = ["examples", "scrap"] + +# Run tests on Windows (informational only, may fail) +test_on_windows = true + +# ๐Ÿ“ฆ BUILD AND PUBLISH SETTINGS +# Control build and publication behavior + +[tool.wads.ci.build] +# Build artifacts to create +sdist = true +wheel = true + +# Publication settings +[tool.wads.ci.publish] +# Publish to PyPI on main/master branch +enabled = true +# skip_existing = false # Skip upload if version already exists + +# ๐Ÿ“„ DOCUMENTATION SETTINGS +# Control documentation generation and publishing + +[tool.wads.ci.docs] +# Generate and publish GitHub Pages +enabled = true +# Builder to use: "epythet", "sphinx", "mkdocs" +builder = "epythet" +# Paths to ignore during documentation generation +ignore_paths = ["tests/", "scrap/", "examples/"] diff --git a/wads/populate.py b/wads/populate.py index 9bba2c0..d93b63d 100755 --- a/wads/populate.py +++ b/wads/populate.py @@ -195,12 +195,30 @@ def write_pyproject_configs(pkg_dir: str, configs: dict): data["project"]["version"] = version data["project"]["description"] = description - # Handle URLs - update homepage and repository + # Handle URLs - update homepage, repository, and documentation if url: if "urls" not in data["project"]: data["project"]["urls"] = {} data["project"]["urls"]["Homepage"] = url - data["project"]["urls"]["Repository"] = url + + # Add repository URL - use explicit value if provided, otherwise use homepage + repository_url = configs.get("repository") or configs.get("repository_url") or url + data["project"]["urls"]["Repository"] = repository_url + + # Add documentation URL + # Default to GitHub Pages if homepage is a GitHub URL + documentation_url = configs.get("documentation") or configs.get("documentation_url") + if not documentation_url and "github.com" in url: + # Parse GitHub URL to extract org and repo + # Expected format: https://github.com/{org}/{repo} + import re + match = re.search(r'github\.com[/:]([^/]+)/([^/\s]+?)(?:\.git)?/?$', url) + if match: + github_org, github_repo = match.groups() + documentation_url = f"https://{github_org}.github.io/{github_repo}" + + if documentation_url: + data["project"]["urls"]["Documentation"] = documentation_url # Update license using inline table syntax data["project"]["license"] = {"text": license_name} @@ -637,13 +655,13 @@ def save_txt_to_pkg(resource_name, content): # No old CI to migrate, create new one from template user_email = kwargs.get("user_email", "thorwhalen1@gmail.com") _add_ci_def( - ci_def_path, ci_tpl_path, root_url, name, _clog, user_email + ci_def_path, ci_tpl_path, root_url, name, _clog, user_email, pkg_dir ) tracker.add(ci_def_path.replace(pkg_dir + os.sep, "")) else: # Not migrating or not github, use template user_email = kwargs.get("user_email", "thorwhalen1@gmail.com") - _add_ci_def(ci_def_path, ci_tpl_path, root_url, name, _clog, user_email) + _add_ci_def(ci_def_path, ci_tpl_path, root_url, name, _clog, user_email, pkg_dir) tracker.add(ci_def_path.replace(pkg_dir + os.sep, "")) else: tracker.skip(ci_def_path.replace(pkg_dir + os.sep, "")) @@ -896,14 +914,57 @@ def _resolve_ci_def_and_tpl_path( return ci_def_path, ci_tpl_path -def _add_ci_def(ci_def_path, ci_tpl_path, root_url, name, clog, user_email): +def _add_ci_def(ci_def_path, ci_tpl_path, root_url, name, clog, user_email, pkg_dir=None): + """ + Generate CI definition file from template. + + If pkg_dir is provided and contains a pyproject.toml with [tool.wads.ci] + configuration, uses the dynamic template with values from CI config. + Otherwise, uses the static template with basic substitutions. + """ clog(f"... making a {ci_def_path}") + + # Try to load CI config from pyproject.toml if pkg_dir provided + ci_config = None + use_dynamic_template = False + if pkg_dir: + pyproject_path = os.path.join(pkg_dir, "pyproject.toml") + if os.path.exists(pyproject_path): + try: + from wads.ci_config import CIConfig + + ci_config = CIConfig.from_file(pyproject_path) + use_dynamic_template = ci_config.has_ci_config() + if use_dynamic_template: + clog("... using CI configuration from pyproject.toml [tool.wads.ci]") + except Exception as e: + clog(f"... could not read CI config from pyproject.toml: {e}") + + # Use dynamic template if CI config is present + if use_dynamic_template and ci_config: + # Use the dynamic template + dynamic_tpl_path = ci_tpl_path.replace(".yml", "_dynamic.yml") + if os.path.exists(dynamic_tpl_path): + ci_tpl_path = dynamic_tpl_path + clog(f"... using dynamic CI template: {os.path.basename(ci_tpl_path)}") + with open(ci_tpl_path) as f_in: ci_def = f_in.read() - ci_def = ci_def.replace("#PROJECT_NAME#", name) + + # Apply CI config substitutions if available + if use_dynamic_template and ci_config: + substitutions = ci_config.to_ci_template_substitutions() + for placeholder, value in substitutions.items(): + ci_def = ci_def.replace(placeholder, value) + else: + # Basic substitutions for static template + ci_def = ci_def.replace("#PROJECT_NAME#", name) + + # Common substitutions for all templates hostname = urlparse(root_url).netloc ci_def = ci_def.replace("#GITLAB_HOSTNAME#", hostname) ci_def = ci_def.replace("#USER_EMAIL#", user_email) + os.makedirs(os.path.dirname(ci_def_path), exist_ok=True) with open(ci_def_path, "w") as f_out: f_out.write(ci_def) diff --git a/wads/scripts/__init__.py b/wads/scripts/__init__.py new file mode 100644 index 0000000..67135cd --- /dev/null +++ b/wads/scripts/__init__.py @@ -0,0 +1,3 @@ +""" +CI and automation scripts for wads. +""" diff --git a/wads/scripts/validate_ci_env.py b/wads/scripts/validate_ci_env.py new file mode 100644 index 0000000..73b398f --- /dev/null +++ b/wads/scripts/validate_ci_env.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +CI Environment Validation Script + +This script validates that all required environment variables are set +based on configuration in pyproject.toml [tool.wads.ci.env]. + +Usage: + python -m wads.scripts.validate_ci_env + +Exit codes: + 0 - All required environment variables are set + 1 - One or more required environment variables are missing +""" + +import os +import sys +from pathlib import Path + + +def validate_ci_environment(pyproject_path: str | Path = ".") -> tuple[bool, list[str]]: + """ + Validate that all required CI environment variables are set. + + Args: + pyproject_path: Path to directory containing pyproject.toml + + Returns: + Tuple of (success, missing_vars) + """ + try: + from wads.ci_config import CIConfig + + config = CIConfig.from_file(pyproject_path) + required_vars = config.env_vars_required + + if not required_vars: + return True, [] + + missing_vars = [var for var in required_vars if var not in os.environ] + + return len(missing_vars) == 0, missing_vars + + except FileNotFoundError: + print("โŒ pyproject.toml not found", file=sys.stderr) + return False, [] + except Exception as e: + print(f"โŒ Error reading CI config: {e}", file=sys.stderr) + return False, [] + + +def main(): + """Main entry point for CI environment validation.""" + print("๐Ÿ” Validating CI environment variables...") + + success, missing_vars = validate_ci_environment() + + if success: + print("โœ… All required environment variables are set") + return 0 + else: + print("\nโŒ Missing required environment variables:", file=sys.stderr) + for var in missing_vars: + print(f" - {var}", file=sys.stderr) + print( + "\nPlease configure these in your CI secrets or environment.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wads/tests/conftest.py b/wads/tests/conftest.py new file mode 100644 index 0000000..2e9f576 --- /dev/null +++ b/wads/tests/conftest.py @@ -0,0 +1,35 @@ +""" +Pytest configuration and hooks for wads tests. +""" + +import pytest +from collections import Counter + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + """ + Custom hook to count and display error messages in the summary. + Provides a grouped view of errors to make logs more readable. + """ + terminalreporter.section("Error Summary Analysis") + + # Collect all error messages + error_counts = Counter() + for report in terminalreporter.stats.get("failed", []): + if report.longrepr: + # Extract the last line of the error (usually the Exception message) + msg = str(report.longrepr).split("\n")[-1] + error_counts[msg] += 1 + + for report in terminalreporter.stats.get("error", []): + if report.longrepr: + msg = str(report.longrepr).split("\n")[-1] + error_counts[msg] += 1 + + # Print the counts + if error_counts: + terminalreporter.write_line("Counts of error types:") + for error, count in error_counts.most_common(): + terminalreporter.write_line(f" {count}x: {error}") + else: + terminalreporter.write_line("No distinct errors found to summarize.")