diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 6054aeb6..75f308ff 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -2,11 +2,10 @@ Only these workflows are active: -1. `pr-gate.yml` - - Required merge gate for PRs into `main`. -2. `main-validation.yml` - - Deeper validation on `main` after merge. -3. `security-deep.yml` +1. `ci.yml` + - Required validation gate for pull requests and pushes to the repository default branch. + - Also supports manual execution via `workflow_dispatch`. +2. `security-deep.yml` - Scheduled/manual deep security scans. If a proposed workflow does not define a distinct decision boundary, do not add it. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..a2ab2cdf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,230 @@ +name: CI Validation + +on: + pull_request: + branches: + - '**' + types: + - opened + - synchronize + - reopened + - ready_for_review + push: + branches: + - '**' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-validation-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + python-quality: + name: Python quality + tests + build + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) || (github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Python environment + uses: ./.github/actions/setup-tradepulse + with: + python-version: '3.11' + cache-prefix: ci-python + + - name: Verify toolchain + run: | + set -euo pipefail + .venv/bin/python --version + .venv/bin/python -m pip --version + + - name: Format check + run: .venv/bin/black --check . + + - name: Lint + run: .venv/bin/ruff check . + + - name: Type check + run: .venv/bin/mypy --config-file=mypy.ini . + + - name: Unit tests with coverage + run: | + .venv/bin/pytest tests/ -m "not slow and not heavy_math and not nightly and not flaky" \ + --cov=core --cov=backtest --cov=execution \ + --cov-report=term-missing \ + --cov-report=xml + + - name: Coverage guardrail + run: | + .venv/bin/python -m tools.coverage.guardrail \ + --config configs/quality/critical_surface.toml \ + --coverage coverage.xml + + - name: Build package artifacts + run: .venv/bin/python -m build --sdist --wheel --outdir dist + + - name: Upload coverage XML + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 + with: + name: python-coverage-xml + path: coverage.xml + if-no-files-found: error + + dependency-security: + name: Dependency security gates + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) || (github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Python environment + uses: ./.github/actions/setup-tradepulse + with: + python-version: '3.11' + cache-prefix: ci-security + + - name: Run pip-audit HIGH/CRITICAL gate + run: | + set -euo pipefail + .venv/bin/pip-audit -r requirements.lock -f json -o /tmp/pip-audit-runtime.json || true + .venv/bin/pip-audit -r requirements-dev.lock -f json -o /tmp/pip-audit-dev.json || true + .venv/bin/python .github/scripts/pip_audit_high_gate.py /tmp/pip-audit-runtime.json /tmp/pip-audit-dev.json + + - name: Audit frontend dependencies + working-directory: apps/web + run: | + set -euo pipefail + npm ci + npm audit --audit-level=high --omit=dev + + web-frontend: + name: Web frontend checks + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) || (github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: apps/web/package-lock.json + + - name: Install dependencies + working-directory: apps/web + run: npm ci + + - name: Format check + working-directory: apps/web + run: npm run format:check + + - name: Lint + working-directory: apps/web + run: npm run lint + + - name: Type check + working-directory: apps/web + run: npm run typecheck + + - name: Unit tests + working-directory: apps/web + run: npm run test -- --ci + + - name: Build + working-directory: apps/web + run: npm run build + + dashboard-frontend: + name: Dashboard frontend checks + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) || (github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '20' + cache: npm + cache-dependency-path: ui/dashboard/package-lock.json + + - name: Install dependencies + working-directory: ui/dashboard + run: npm ci + + - name: Lint + working-directory: ui/dashboard + run: npm run lint + + - name: Unit tests + working-directory: ui/dashboard + run: npm run test + + go-validation: + name: Go tests + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) || (github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v5 + with: + go-version-file: go.mod + cache: true + + - name: Run Go tests + run: go test ./... + + rust-validation: + name: Rust fmt + clippy + tests + build + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) || (github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Rust toolchain + run: rustup toolchain install stable --profile minimal --component rustfmt clippy + + - name: Cargo fmt check + run: cargo fmt --manifest-path rust/tradepulse-accel/Cargo.toml --all --check + + - name: Cargo clippy + run: cargo clippy --manifest-path rust/tradepulse-accel/Cargo.toml --all-targets -- -D warnings + + - name: Cargo tests + run: cargo test --manifest-path rust/tradepulse-accel/Cargo.toml + + - name: Cargo build + run: cargo build --manifest-path rust/tradepulse-accel/Cargo.toml --locked + + docker-build-validation: + name: Docker build validation + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) || (github.event_name == 'pull_request' && github.base_ref == github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Build root image + run: docker build -f Dockerfile . + + - name: Build cortex_service image + run: docker build -f cortex_service/Dockerfile cortex_service diff --git a/.github/workflows/main-validation.yml b/.github/workflows/main-validation.yml deleted file mode 100644 index eebea09b..00000000 --- a/.github/workflows/main-validation.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Main Validation - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: main-validation-${{ github.ref }} - cancel-in-progress: false - -jobs: - python-full-validation: - name: python-full-validation - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - - name: Setup Python environment - uses: ./.github/actions/setup-tradepulse - with: - python-version: '3.11' - cache-prefix: main-full-tests - - - name: Verify pip executable - run: .venv/bin/python -m pip --version - - - name: Run broader post-merge validation suite - run: | - .venv/bin/pytest tests/ -m "not nightly and not flaky" \ - --cov=core \ - --cov=backtest \ - --cov=execution \ - --cov-config=configs/quality/critical_surface.coveragerc \ - --cov-report=term-missing \ - --cov-report=xml - - - name: Enforce coverage guardrail - run: | - .venv/bin/python -m tools.coverage.guardrail \ - --config configs/quality/critical_surface.toml \ - --coverage coverage.xml - - packaging-smoke: - name: packaging-smoke - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - - name: Setup Python environment - uses: ./.github/actions/setup-tradepulse - with: - python-version: '3.11' - cache-prefix: main-package - - - name: Verify pip executable - run: .venv/bin/python -m pip --version - - - name: Build Python package artifacts - run: .venv/bin/python -m build --sdist --wheel --outdir dist - - frontend-build-smoke: - name: frontend-build-smoke - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - - name: Setup Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: '20' - cache: npm - cache-dependency-path: apps/web/package-lock.json - - - name: Build frontend - working-directory: apps/web - run: | - npm ci - npm run build diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml deleted file mode 100644 index e9132995..00000000 --- a/.github/workflows/pr-gate.yml +++ /dev/null @@ -1,346 +0,0 @@ -name: PR Gate - -on: - pull_request: - branches: [main] - merge_group: - -permissions: - contents: read - pull-requests: read - -concurrency: - group: pr-gate-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: true - -jobs: - repo-policy: - name: repo-policy - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - - - name: Validate workflow policy invariants - run: | - set -euo pipefail - for wf in .github/workflows/*.yml; do - grep -Eq '^permissions:' "$wf" || { echo "ERROR: $wf missing permissions"; exit 1; } - if grep -Eq '^\s*pull_request_target:' "$wf"; then - echo "ERROR: $wf uses forbidden pull_request_target" - exit 1 - fi - done - - - name: Enforce pinned actions policy - run: | - set -euo pipefail - python - <<'PY' - import re - from pathlib import Path - - pattern = re.compile(r'^\s*uses:\s*([^\s#]+)') - violations = [] - for wf in sorted(Path('.github/workflows').glob('*.yml')): - for lineno, line in enumerate(wf.read_text(encoding='utf-8').splitlines(), start=1): - m = pattern.search(line) - if not m: - continue - ref = m.group(1) - if ref.startswith('./') or ref.startswith('docker://'): - continue - if not re.search(r'@[a-f0-9]{40}$', ref): - violations.append(f"{wf}:{lineno}: unpinned action reference: {ref}") - - if violations: - print('\n'.join(violations)) - raise SystemExit(1) - print('Pinned action policy passed.') - PY - - - name: Actionlint validation - run: | - set -euo pipefail - docker run --rm -v "$PWD":/repo -w /repo rhysd/actionlint:1.7.8 -color - - python-quality: - name: python-quality - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - fetch-depth: 0 - - - name: Setup Python environment - uses: ./.github/actions/setup-tradepulse - with: - python-version: '3.11' - cache-prefix: pr-quality - - - name: Verify pip executable and Python contract - run: | - set -euo pipefail - .venv/bin/python -m pip --version - .venv/bin/python - <<'PY' - import tomllib - from pathlib import Path - - requires = tomllib.loads(Path('pyproject.toml').read_text(encoding='utf-8'))['project']['requires-python'] - if requires != '>=3.11,<3.13': - raise SystemExit(f'Unexpected requires-python: {requires}') - print(f'requires-python validated: {requires}') - PY - - - name: Run lint/type gate on changed Python files - run: | - set -euo pipefail - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - base_ref="origin/${{ github.base_ref }}" - git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" - changed_files=$(git diff --name-only "$base_ref...$GITHUB_SHA") - else - changed_files=$(git diff-tree --no-commit-id --name-only -r "$GITHUB_SHA") - fi - - mapfile -t py_files < <(echo "$changed_files" | grep -E '\.py$' || true) - if [[ ${#py_files[@]} -eq 0 ]]; then - echo "No Python file changes detected; python-quality passes." - exit 0 - fi - - .venv/bin/ruff check "${py_files[@]}" - .venv/bin/black --check "${py_files[@]}" - .venv/bin/mypy "${py_files[@]}" - - python-fast-tests: - name: python-fast-tests - runs-on: ubuntu-latest - needs: python-quality - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - fetch-depth: 0 - - - name: Setup Python environment - uses: ./.github/actions/setup-tradepulse - with: - python-version: '3.11' - cache-prefix: pr-fast-tests - - - name: Verify pip executable - run: .venv/bin/python -m pip --version - - - name: Run fast deterministic pytest gate - run: | - set -euo pipefail - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - base_ref="origin/${{ github.base_ref }}" - git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" - changed_files=$(git diff --name-only "$base_ref...$GITHUB_SHA") - else - changed_files=$(git diff-tree --no-commit-id --name-only -r "$GITHUB_SHA") - fi - - if ! echo "$changed_files" | grep -Eq '^(tests/|core/|backtest/|execution/|src/|pyproject\.toml|requirements(\-dev)?\.(txt|lock)|pytest\.ini)'; then - echo "No Python runtime/test surface changes detected; python-fast-tests passes." - exit 0 - fi - - .venv/bin/pytest tests/ -m "not slow and not heavy_math and not nightly and not flaky" -q - - frontend-gate: - name: frontend-gate - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - fetch-depth: 0 - - - name: Determine if frontend gate applies - id: changes - run: | - set -euo pipefail - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - base_ref="origin/${{ github.base_ref }}" - git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" - changed_files=$(git diff --name-only "$base_ref...$GITHUB_SHA") - else - changed_files=$(git diff-tree --no-commit-id --name-only -r "$GITHUB_SHA") - fi - - if echo "$changed_files" | grep -Eq '^(apps/web/|package\.json|package-lock\.json)$'; then - echo "frontend_changed=true" >> "$GITHUB_OUTPUT" - else - echo "frontend_changed=false" >> "$GITHUB_OUTPUT" - fi - - - name: Verify frontend lockfile - if: steps.changes.outputs.frontend_changed == 'true' - run: test -f apps/web/package-lock.json - - - name: Setup Node.js - if: steps.changes.outputs.frontend_changed == 'true' - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: '20' - cache: npm - cache-dependency-path: apps/web/package-lock.json - - - name: Run frontend quality and tests - if: steps.changes.outputs.frontend_changed == 'true' - working-directory: apps/web - run: | - npm ci - npm run format:check - npm run lint - npm run typecheck - npm run test -- --ci - - - name: Frontend gate not applicable - if: steps.changes.outputs.frontend_changed != 'true' - run: echo 'No frontend changes detected; frontend gate passed deterministically.' - - dependency-review: - name: dependency-review - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - fetch-depth: 0 - - - name: Setup Python environment - uses: ./.github/actions/setup-tradepulse - with: - python-version: '3.11' - cache-prefix: pr-deps - - - name: Verify pip executable - run: .venv/bin/python -m pip --version - - - name: Ensure dependency-audit tooling - run: | - set -euo pipefail - if ! .venv/bin/python -m pip show pip-audit >/dev/null 2>&1; then - .venv/bin/python -m pip install -c constraints/security.txt pip-audit - fi - - - name: Dependency review gate - run: | - set -euo pipefail - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - base_ref="origin/${{ github.base_ref }}" - git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" - changed_files=$(git diff --name-only "$base_ref...$GITHUB_SHA") - else - changed_files=$(git diff-tree --no-commit-id --name-only -r "$GITHUB_SHA") - fi - - python_dep_changed=false - frontend_dep_changed=false - - if echo "$changed_files" | grep -Eq '^(requirements(\-dev)?\.(txt|lock)|constraints/security\.txt|pyproject\.toml)$'; then - python_dep_changed=true - fi - - if echo "$changed_files" | grep -Eq '^(apps/web/package(-lock)?\.json|package\.json|package-lock\.json)$'; then - frontend_dep_changed=true - fi - - if [[ "$python_dep_changed" == "true" ]]; then - echo "Python dependency manifests changed; running HIGH/CRITICAL pip-audit." - .venv/bin/pip-audit -r requirements.lock -f json -o /tmp/pip-audit-runtime.json || true - .venv/bin/pip-audit -r requirements-dev.lock -f json -o /tmp/pip-audit-dev.json || true - .venv/bin/python .github/scripts/pip_audit_high_gate.py /tmp/pip-audit-runtime.json /tmp/pip-audit-dev.json - fi - - if [[ "$frontend_dep_changed" == "true" ]]; then - echo "Frontend dependency manifests changed; running npm audit HIGH gate." - test -f apps/web/package-lock.json - pushd apps/web >/dev/null - npm ci - npm audit --audit-level=high --omit=dev - popd >/dev/null - fi - - if [[ "$python_dep_changed" == "false" && "$frontend_dep_changed" == "false" ]]; then - echo "No dependency-manifest changes detected; dependency-review passes." - fi - - secrets-supply-chain: - name: secrets-supply-chain - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 - with: - fetch-depth: 0 - - - name: Setup Python environment - uses: ./.github/actions/setup-tradepulse - with: - python-version: '3.11' - cache-prefix: pr-security - - - name: Verify pip executable - run: .venv/bin/python -m pip --version - - - name: Ensure security tooling - run: | - set -euo pipefail - missing=() - .venv/bin/python -m pip show bandit >/dev/null 2>&1 || missing+=(bandit) - .venv/bin/python -m pip show pip-audit >/dev/null 2>&1 || missing+=(pip-audit) - .venv/bin/python -m pip show detect-secrets >/dev/null 2>&1 || missing+=(detect-secrets) - if [[ ${#missing[@]} -gt 0 ]]; then - .venv/bin/python -m pip install -c constraints/security.txt "${missing[@]}" - fi - - - name: Collect changed files - id: changed - run: | - set -euo pipefail - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - base_ref="origin/${{ github.base_ref }}" - git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" - git diff --name-only "$base_ref...$GITHUB_SHA" > /tmp/changed-files.txt - else - git diff-tree --no-commit-id --name-only -r "$GITHUB_SHA" > /tmp/changed-files.txt - fi - cat /tmp/changed-files.txt - - - name: Bandit on changed security-critical Python files - run: | - set -euo pipefail - mapfile -t bandit_targets < <(grep -E '^(core|backtest|execution|src)/.*\.py$' /tmp/changed-files.txt || true) - if [[ ${#bandit_targets[@]} -eq 0 ]]; then - echo "No security-critical Python changes detected; Bandit gate passes." - else - .venv/bin/bandit -ll "${bandit_targets[@]}" - fi - - - name: Detect-secrets using baseline - run: | - set -euo pipefail - mapfile -t existing_changed_files < <(while IFS= read -r f; do [[ -f "$f" ]] && echo "$f"; done < /tmp/changed-files.txt) - if [[ ${#existing_changed_files[@]} -eq 0 ]]; then - echo "No file additions/modifications to scan; secrets gate passes." - exit 0 - fi - - .venv/bin/detect-secrets-hook --baseline .github/detect-secrets.baseline "${existing_changed_files[@]}" - - - name: HIGH/CRITICAL dependency audit guard - run: | - .venv/bin/pip-audit -r requirements.lock -f json -o /tmp/pip-audit-runtime.json || true - .venv/bin/pip-audit -r requirements-dev.lock -f json -o /tmp/pip-audit-dev.json || true - .venv/bin/python .github/scripts/pip_audit_high_gate.py /tmp/pip-audit-runtime.json /tmp/pip-audit-dev.json