diff --git a/.github/workflows/update-python-docker.yml b/.github/workflows/update-python-docker.yml new file mode 100644 index 0000000..26baad2 --- /dev/null +++ b/.github/workflows/update-python-docker.yml @@ -0,0 +1,341 @@ +name: Update Python and Docker versions + +on: + # Run every 2 months (first day at 3 AM UTC) + schedule: + - cron: '0 3 1 */2 *' + # Allow manual triggering for testing + workflow_dispatch: {} + +concurrency: + group: python-docker-update + cancel-in-progress: true + +jobs: + update-python: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + issues: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get current Python version + id: current-version + run: | + # Extract current Python version from build-ci-image.yml + CURRENT_VERSION=$(grep -m1 "python-version:" .github/workflows/build-ci-image.yml | sed -E "s/.*python-version:[[:space:]]*['\"]?([0-9.]+)['\"]?.*/\1/") + echo "version=$CURRENT_VERSION" >> $GITHUB_OUTPUT + echo "Using Python $CURRENT_VERSION for workflow execution" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ steps.current-version.outputs.version }} + + - name: Check for Python updates + id: check-update + run: | + # Run the update script; if it fails, this step (and workflow) will fail. + set -euo pipefail + python3 scripts/update_python_version.py + + # Determine whether any tracked files were modified by the script. + # git diff exits with 1 if there are differences, so disable pipefail for this check + set +e + git diff --quiet + DIFF_EXIT=$? + set -e + + if [ $DIFF_EXIT -eq 0 ]; then + echo "update_available=false" >> "$GITHUB_OUTPUT" + else + echo "update_available=true" >> "$GITHUB_OUTPUT" + fi + + - name: Get updated Python version + id: get-version + if: steps.check-update.outputs.update_available == 'true' + run: | + # Extract the new Python version from build-ci-image.yml + NEW_VERSION=$(grep -m1 "python-version:" .github/workflows/build-ci-image.yml | sed -E "s/.*python-version:[[:space:]]*['\"]?([0-9.]+)['\"]?.*/\1/") + echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT + echo "Updated to Python $NEW_VERSION" + + - name: Set up new Python version for testing + if: steps.check-update.outputs.update_available == 'true' + uses: actions/setup-python@v5 + with: + python-version: ${{ steps.get-version.outputs.new_version }} + + - name: Create venv and install deps + if: steps.check-update.outputs.update_available == 'true' + run: | + set -euo pipefail + python3 -m venv .venv + ./.venv/bin/python -m pip install --upgrade pip + ./.venv/bin/python -m pip install -r scripts/requirements.txt -r scripts/requirements-dev.txt + + - name: Run linting tests + id: lint-test + if: steps.check-update.outputs.update_available == 'true' + continue-on-error: true + run: | + set -euo pipefail + echo "Running flake8..." + ./.venv/bin/python -m flake8 -q + echo "lint_passed=true" >> $GITHUB_OUTPUT + + - name: Run pytest tests + id: pytest-test + if: steps.check-update.outputs.update_available == 'true' + continue-on-error: true + run: | + set -euo pipefail + echo "Running pytest..." + ./.venv/bin/python -m pytest -q + echo "tests_passed=true" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + if: steps.check-update.outputs.update_available == 'true' + uses: docker/setup-buildx-action@v3 + + - name: Test build Docker images + id: docker-test + if: steps.check-update.outputs.update_available == 'true' + continue-on-error: true + run: | + set -euo pipefail + echo "Testing QR Dockerfile build..." + docker build -f .github/ci/Dockerfile.qr -t test-qr:latest . + echo "Testing Infra Dockerfile build..." + docker build -f .github/ci/Dockerfile.infra -t test-infra:latest . + echo "docker_build_passed=true" >> $GITHUB_OUTPUT + + - name: Run tests in Docker container + id: docker-pytest + if: steps.check-update.outputs.update_available == 'true' + continue-on-error: true + run: | + set -euo pipefail + echo "Running pytest in Docker container..." + docker run --rm --user 0 -v "$PWD:/workspace" -w /workspace test-infra:latest bash -c "python -m venv --system-site-packages .venv && ./.venv/bin/python -m pytest -q" + echo "docker_tests_passed=true" >> $GITHUB_OUTPUT + + - name: Check for existing update issue + id: check-issue + if: steps.check-update.outputs.update_available == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Checking for existing Python update issues..." + EXISTING_ISSUES=$(gh issue list --state open --label "python-update" --json number,title --limit 1) + if [ "$EXISTING_ISSUES" != "[]" ]; then + ISSUE_NUMBER=$(echo "$EXISTING_ISSUES" | jq -r '.[0].number') + echo "Found existing open issue #$ISSUE_NUMBER" + echo "has_existing=true" >> $GITHUB_OUTPUT + echo "issue_number=$ISSUE_NUMBER" >> $GITHUB_OUTPUT + else + echo "No existing open Python update issue found" + echo "has_existing=false" >> $GITHUB_OUTPUT + fi + + - name: Check for existing update PR + id: check-pr + if: steps.check-update.outputs.update_available == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Checking for existing Python update PRs..." + EXISTING_PRS=$(gh pr list --state open --label "python-update" --json number,title --limit 1) + if [ "$EXISTING_PRS" != "[]" ]; then + PR_NUMBER=$(echo "$EXISTING_PRS" | jq -r '.[0].number') + echo "Found existing open PR #$PR_NUMBER" + echo "has_existing=true" >> $GITHUB_OUTPUT + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + else + echo "No existing open Python update PR found" + echo "has_existing=false" >> $GITHUB_OUTPUT + fi + + - name: Create issue for update + id: create-issue + if: | + steps.check-update.outputs.update_available == 'true' && + steps.lint-test.outcome == 'success' && + steps.pytest-test.outcome == 'success' && + steps.docker-test.outcome == 'success' && + steps.docker-pytest.outcome == 'success' && + steps.check-issue.outputs.has_existing != 'true' && + steps.check-pr.outputs.has_existing != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + NEW_VERSION="${{ steps.get-version.outputs.new_version }}" + + # Ensure the 'python-update' label exists + if ! gh label view "python-update" >/dev/null 2>&1; then + echo "Creating missing label 'python-update'" + gh label create "python-update" --description "Automated Python version updates" --color 0366d6 || true + fi + + # Create issue with heredoc + cat > /tmp/issue_body.txt << 'ISSUE_EOF' + ## Python and Docker Version Update Available + + A new stable Python version is available. + + ### Proposed Changes + - Update `.github/ci/Dockerfile.infra` to use new Python slim image + - Update `.github/ci/Dockerfile.qr` to use new Python slim image + - Update `.github/workflows/build-ci-image.yml` to use new Python version + - Update `.github/workflows/check-todo.yml` to use new Python version + + ### Automated Test Results + All automated tests have passed with the new version: + - ✅ Linting (flake8) + - ✅ Unit tests (pytest) + - ✅ Docker image builds + - ✅ Tests in Docker containers + + ### Next Steps + A pull request will be automatically created and linked to this issue with the necessary changes. + ISSUE_EOF + + ISSUE_NUMBER=$(gh issue create \ + --title "Update Python to version ${NEW_VERSION}" \ + --body-file /tmp/issue_body.txt \ + --label "python-update" \ + --json number -q '.number') + + echo "issue_number=$ISSUE_NUMBER" >> $GITHUB_OUTPUT + echo "Created issue #$ISSUE_NUMBER" + + - name: Create Pull Request + if: | + steps.check-update.outputs.update_available == 'true' && + steps.lint-test.outcome == 'success' && + steps.pytest-test.outcome == 'success' && + steps.docker-test.outcome == 'success' && + steps.docker-pytest.outcome == 'success' && + steps.check-issue.outputs.has_existing != 'true' && + steps.check-pr.outputs.has_existing != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Configure git + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + NEW_VERSION="${{ steps.get-version.outputs.new_version }}" + ISSUE_NUMBER="${{ steps.create-issue.outputs.issue_number }}" + BRANCH_NAME="automated/update-python-${NEW_VERSION}" + + # Create and switch to new branch + git checkout -b "$BRANCH_NAME" + + # Add changes + git add .github/ci/Dockerfile.infra + git add .github/ci/Dockerfile.qr + git add .github/workflows/build-ci-image.yml + git add .github/workflows/check-todo.yml + + # Commit changes with proper formatting + git commit \ + -m "chore: update Python to version ${NEW_VERSION}" \ + -m "" \ + -m "- Update Dockerfile.infra to use python:${NEW_VERSION}-slim" \ + -m "- Update Dockerfile.qr to use python:${NEW_VERSION}-slim" \ + -m "- Update build-ci-image.yml to use Python ${NEW_VERSION}" \ + -m "- Update check-todo.yml to use Python ${NEW_VERSION}" \ + -m "" \ + -m "All tests passed:" \ + -m "- Linting: ✓" \ + -m "- Pytest: ✓" \ + -m "- Docker builds: ✓" \ + -m "- Docker tests: ✓" \ + -m "" \ + -m "Fixes #${ISSUE_NUMBER}" + + # Push branch (use force-with-lease to allow updates if no remote changes) + if ! git push --force-with-lease origin "$BRANCH_NAME"; then + echo "⚠️ Force-with-lease failed - remote branch has been updated by another process" + echo "Attempting regular push (will fail if branch exists with different history)" + git push origin "$BRANCH_NAME" + fi + + # Create PR body with heredoc + cat > /tmp/pr_body.txt << 'PR_EOF' + ## Automated Python and Docker Version Update + + This PR updates the Python version used across the repository. + + ### Changes Made + - Updated `.github/ci/Dockerfile.infra` to use new Python slim image + - Updated `.github/ci/Dockerfile.qr` to use new Python slim image + - Updated `.github/workflows/build-ci-image.yml` to use new Python version + - Updated `.github/workflows/check-todo.yml` to use new Python version + + ### Test Results + All automated tests have passed: + - ✅ Linting (flake8) + - ✅ Unit tests (pytest) + - ✅ Docker image builds + - ✅ Tests in Docker containers + + ### What to Review + - Verify that all CI workflows pass successfully + - Check that no new deprecation warnings are introduced + - Ensure Docker images build and push correctly + + This PR was automatically created by the `update-python-docker` workflow. + PR_EOF + + # Add issue reference to PR body + echo "" >> /tmp/pr_body.txt + echo "Closes #${ISSUE_NUMBER}" >> /tmp/pr_body.txt + + # Create PR linked to the issue + gh pr create \ + --title "chore: Update Python to version ${NEW_VERSION}" \ + --body-file /tmp/pr_body.txt \ + --label "python-update" \ + --base main \ + --head "$BRANCH_NAME" + + - name: Handle test failures + if: | + !cancelled() && + steps.check-update.outputs.update_available == 'true' && + (steps.lint-test.outcome == 'failure' || + steps.pytest-test.outcome == 'failure' || + steps.docker-test.outcome == 'failure' || + steps.docker-pytest.outcome == 'failure') + run: | + echo "❌ Tests failed with the new Python version" + echo "Lint outcome: ${{ steps.lint-test.outcome }}" + echo "Pytest outcome: ${{ steps.pytest-test.outcome }}" + echo "Docker build outcome: ${{ steps.docker-test.outcome }}" + echo "Docker tests outcome: ${{ steps.docker-pytest.outcome }}" + echo "" + echo "Manual intervention required to fix compatibility issues." + echo "The version update has been applied but not committed." + exit 1 + + - name: No update needed + if: steps.check-update.outputs.update_available != 'true' + run: | + echo "✅ Already using the latest Python version" + + - name: Skip - existing issue or PR found + if: steps.check-update.outputs.update_available == 'true' && (steps.check-issue.outputs.has_existing == 'true' || steps.check-pr.outputs.has_existing == 'true') + run: | + echo "ℹ️ Skipping creation - existing Python update issue/PR found" + if [ "${{ steps.check-issue.outputs.has_existing }}" == "true" ]; then + echo "Existing issue: #${{ steps.check-issue.outputs.issue_number }}" + fi + if [ "${{ steps.check-pr.outputs.has_existing }}" == "true" ]; then + echo "Existing PR: #${{ steps.check-pr.outputs.pr_number }}" + fi diff --git a/README.md b/README.md index 0d7e808..3d9d649 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,21 @@ This repository includes automated workflows: - **Lint** — Runs flake8 on Python code - **pytest** — Runs pytest test suite - **Generate QR** — Automatically generates and commits QR codes +- **Update Python/Docker** — Periodically checks for Python updates and creates issues with linked PRs + +**Python and Docker Version Updates** + +- The repository includes a scheduled workflow (`.github/workflows/update-python-docker.yml`) that automatically keeps Python and Docker images up-to-date. +- High-level behavior: + - Runs every 2 months (first day at 3 AM UTC) and can be manually triggered via `workflow_dispatch` + - Detects the latest stable Python version from Docker Hub (supports Python 3.8+) + - Updates all Python version references in Dockerfiles and workflow files + - Runs comprehensive tests (linting, pytest, Docker builds) to verify compatibility + - Creates an issue documenting the available update + - Creates a linked PR with the updates if all tests pass + - Checks for existing update issues and PRs to avoid duplicates using the `python-update` label +- Files updated: `.github/ci/Dockerfile.infra`, `.github/ci/Dockerfile.qr`, `.github/workflows/build-ci-image.yml`, `.github/workflows/check-todo.yml` +- Why this helps: Keeps the repository secure and up-to-date with the latest Python releases without manual intervention, ensuring compatibility is tested before merging. **Copilot Trigger & Automation Label** diff --git a/scripts/update_python_version.py b/scripts/update_python_version.py new file mode 100755 index 0000000..e883506 --- /dev/null +++ b/scripts/update_python_version.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +""" +Script to detect the latest stable Python version and update all references +in the repository to use it. +""" +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Optional + + +def get_latest_python_version() -> Optional[str]: + """ + Determine the latest stable Python 3.x version by querying Docker Hub + `library/python` tags (e.g. "3.12-slim"). + Returns a version string like "3.12" or None on failure. + """ + try: + # Try to use the Docker Hub API to find available Python versions + # This is more reliable in CI environments + result = subprocess.run( + ['curl', '-s', 'https://registry.hub.docker.com/v2/repositories/library/python/tags?page_size=100'], + capture_output=True, + text=True, + timeout=30 + ) + if result.returncode != 0: + print(f"Failed to fetch Docker Hub tags: {result.stderr}") + return None + + # Parse JSON response + data = json.loads(result.stdout) + + # Look for tags like "3.12-slim", "3.11-slim", and also patch tags like + # "3.12.1-slim". We normalize to the major.minor part (e.g. "3.12"). + pattern = r'^(\d+\.\d+)(?:\.\d+)?-slim$' + versions = [] + + for tag_info in data.get('results', []): + tag_name = tag_info.get('name', '') + match = re.match(pattern, tag_name) + if match: + version = match.group(1) + # We assume versions 3.8 and above are relevant for this project; + # the workflow will test compatibility with newer Python 3 releases + # automatically. Limit to Python 3.x only to prevent automatic + # adoption of Python 4.x which may have breaking changes. + major, minor = map(int, version.split('.')) + if major == 3 and minor >= 8: + versions.append(version) + + if not versions: + print("Could not find Python versions in Docker Hub tags") + return None + + # Sort and get the highest version + versions = sorted(set(versions), key=lambda v: tuple(map(int, v.split('.')))) + latest = versions[-1] + print(f"Latest stable Python version: {latest}") + return latest + except Exception as e: + print(f"Error getting latest Python version: {e}") + return None + + +def get_latest_docker_slim_tag(python_version: str) -> Optional[str]: + """ + Get the latest Docker slim image tag for the given Python version. + Returns tag like '3.12-slim' or None on failure. + """ + # Validate input format + if not re.match(r'^\d+\.\d+$', python_version): + print(f"Invalid Python version format: {python_version}") + return None + + try: + # Check if docker command is available + docker_check = subprocess.run( + ['docker', '--version'], + capture_output=True, + timeout=10 + ) + if docker_check.returncode != 0: + print("Docker is not available in the environment") + return None + + # Verify the tag exists by trying to pull image metadata + slim_tag = f"{python_version}-slim" + result = subprocess.run( + ['docker', 'manifest', 'inspect', f'python:{slim_tag}'], + capture_output=True, + text=True, + timeout=30 + ) + + if result.returncode == 0: + print(f"Found Docker image: python:{slim_tag}") + return slim_tag + else: + print(f"Docker image python:{slim_tag} not found") + return None + except FileNotFoundError: + print("Docker command not found in PATH") + return None + except Exception as e: + print(f"Error getting Docker slim tag: {e}") + return None + + +def update_file_content( + file_path: Path, + old_version: str, + new_version: str, + old_docker_tag: str, + new_docker_tag: str +) -> bool: + """ + Update Python version references in a file. + Returns True if changes were made, False otherwise. + """ + try: + content = file_path.read_text(encoding='utf-8') + original_content = content + + # Update python-version: references in YAML files + content = re.sub( + rf"python-version:\s*['\"]?{re.escape(old_version)}['\"]?", + f"python-version: '{new_version}'", + content + ) + + # Update FROM python:X.XX-slim in Dockerfiles + content = re.sub( + rf"FROM python:{re.escape(old_docker_tag)}", + f"FROM python:{new_docker_tag}", + content + ) + + if content != original_content: + file_path.write_text(content, encoding='utf-8') + print(f"Updated {file_path}") + return True + else: + print(f"No changes needed in {file_path}") + return False + except Exception as e: + print(f"Error updating {file_path}: {e}") + return False + + +def get_current_python_version(repo_root: Path) -> Optional[str]: + """ + Get the current Python version used in the repository. + Checks the build-ci-image.yml workflow file. + """ + workflow_file = repo_root / ".github" / "workflows" / "build-ci-image.yml" + try: + content = workflow_file.read_text(encoding='utf-8') + match = re.search(r"python-version:\s*['\"]?(\d+\.\d+)['\"]?", content) + if match: + return match.group(1) + except Exception as e: + print(f"Error reading current version: {e}") + return None + + +def main(): + """Main function to update Python and Docker versions.""" + repo_root = Path(__file__).parent.parent + + # Get current version + current_version = get_current_python_version(repo_root) + if not current_version: + print("Error: Could not determine current Python version") + sys.exit(1) + + print(f"Current Python version: {current_version}") + current_docker_tag = f"{current_version}-slim" + + # Get latest version + latest_version = get_latest_python_version() + if not latest_version: + print("Error: Could not determine latest Python version") + sys.exit(1) + + # Check if update is needed + if latest_version == current_version: + print(f"Already using latest Python version: {current_version}") + sys.exit(0) + + print(f"Update available: {current_version} -> {latest_version}") + + # Get latest Docker slim tag + latest_docker_tag = get_latest_docker_slim_tag(latest_version) + if not latest_docker_tag: + print("Error: Could not verify Docker slim image availability") + sys.exit(1) + + # Files to update + files_to_update = [ + repo_root / ".github" / "ci" / "Dockerfile.infra", + repo_root / ".github" / "ci" / "Dockerfile.qr", + repo_root / ".github" / "workflows" / "build-ci-image.yml", + repo_root / ".github" / "workflows" / "check-todo.yml", + ] + + # Update files + changes_made = False + failed_files = [] + for file_path in files_to_update: + if not file_path.exists(): + print(f"Error: File not found: {file_path}") + failed_files.append(str(file_path)) + continue + + result = update_file_content( + file_path, + current_version, + latest_version, + current_docker_tag, + latest_docker_tag + ) + if result: + changes_made = True + else: + # Check if file still contains the exact old patterns (update failed) + # Use word boundaries and specific patterns to avoid false positives + try: + content = file_path.read_text(encoding='utf-8') + # Check for old version patterns that should have been replaced + old_version_pattern = rf"python-version:\s*['\"]?{re.escape(current_version)}['\"]?" + old_docker_pattern = rf"FROM python:{re.escape(current_docker_tag)}" + + if re.search(old_version_pattern, content) or re.search(old_docker_pattern, content): + print(f"Error: Failed to update {file_path} - old version patterns still present") + failed_files.append(str(file_path)) + except Exception as e: + print(f"Error checking {file_path}: {e}") + failed_files.append(str(file_path)) + + if failed_files: + print(f"\nError: Failed to update the following files: {', '.join(failed_files)}") + sys.exit(1) + + if changes_made: + print("\nSuccessfully updated Python version references") + print(f"Updated from {current_version} to {latest_version}") + sys.exit(0) + else: + print("\nNo changes were made") + sys.exit(1) + + +if __name__ == "__main__": + main()