diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index fbce968697..f99831da32 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -5,9 +5,6 @@ on: - main - dev pull_request: - branches: - - main - - dev release: types: - published @@ -16,8 +13,130 @@ on: permissions: {} jobs: - build: - name: Build image + test-e2e: + name: End-to-end test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + + - name: Build database container + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/database/Dockerfile + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository_owner }}/augur_database:test + cache-from: type=gha,scope=container-database + cache-to: type=gha,scope=container-database,mode=min + load: true + + - name: Build keyman container + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/keyman/Dockerfile + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository_owner }}/augur_keyman:test + cache-from: type=gha,scope=container-keyman + cache-to: type=gha,scope=container-keyman,mode=min + load: true + + - name: Build rabbitmq container + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/rabbitmq/Dockerfile + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test + cache-from: type=gha,scope=container-rabbitmq + cache-to: type=gha,scope=container-rabbitmq,mode=min + load: true + + - name: Build backend container + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/backend/Dockerfile + platforms: linux/amd64 + tags: ghcr.io/${{ github.repository_owner }}/augur_backend:test + cache-from: type=gha,scope=container-backend + cache-to: type=gha,scope=container-backend,mode=min + load: true + + - name: Prepare compose file + run: | + yq eval -i '.services.augur.image = "ghcr.io/${{ github.repository_owner }}/augur_backend:test"' docker-compose.yml + yq eval -i '.services.augur.pull_policy = "never"' docker-compose.yml + yq eval -i '.services.augur.restart = "no"' docker-compose.yml + + yq eval -i '.services.augur-db.image = "ghcr.io/${{ github.repository_owner }}/augur_database:test"' docker-compose.yml + yq eval -i '.services.augur-db.pull_policy = "never"' docker-compose.yml + yq eval -i '.services.augur-db.restart = "no"' docker-compose.yml + + yq eval -i '.services.augur-keyman.image = "ghcr.io/${{ github.repository_owner }}/augur_keyman:test"' docker-compose.yml + yq eval -i '.services.augur-keyman.pull_policy = "never"' docker-compose.yml + yq eval -i '.services.augur-keyman.restart = "no"' docker-compose.yml + + yq eval -i '.services.rabbitmq.image = "ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test"' docker-compose.yml + yq eval -i '.services.rabbitmq.pull_policy = "never"' docker-compose.yml + yq eval -i '.services.rabbitmq.restart = "no"' docker-compose.yml + + - name: Setup Docker Compose + uses: docker/setup-compose-action@v1 + with: + version: latest + + - name: Set up list of log lines to match + run: | + cat < /tmp/regex_matches.txt + Gunicorn webserver started + Starting core worker processes + Starting secondary worker processes + Starting facade worker processes + Retrieved \\d+ github api keys for use + Fetching new repos \\(complete\\) + Inserting \\d+ contributors + Inserting \\d+ issues + Inserting prs of length: \\d+ + Querying committers count + Done generating scc data for repo + Sending due task + EOF + + - name: Start services & wait for output + # This starts the system and sends the output to "await_all.py" which + # scans for the regex matches from above. Once all matches are seen at + # least once, the `compose down` will run to shut down the system. If + # this all doesn't happen before the timeout, the job will fail. + run: | + docker compose -f docker-compose.yml up --no-build 2>&1 \ + | (./scripts/ci/await_all.py /tmp/regex_matches.txt \ + && docker compose -f docker-compose.yml down) + timeout-minutes: 3 + env: + AUGUR_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }} + AUGUR_GITHUB_USERNAME: ${{ github.repository_owner }} + AUGUR_GITLAB_API_KEY: dummy + AUGUR_GITLAB_USERNAME: dummy + + - name: Dump logs + # Always run this step to get logs, even if the previous step fails + if: always() + # We use tail so that we can see the name of each file as it's printed + run: "docker run -t --rm -v augur_logs:/logs bash -c 'find /logs -type f | xargs tail -n +0'" + + + + push-image: + name: Push image + needs: test-e2e + # We don't push images on pull requests + if: github.event_name != 'pull_request' permissions: contents: read # to fetch code (actions/checkout) packages: write # to push docker image @@ -57,8 +176,9 @@ jobs: images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }} # Pushes to the dev branch update the *:devel-latest tag # Releases update the *:latest tag and the *: tag + # Main does not update any tags tags: | - type=raw,value=devel-latest,enable=${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/dev' }} + type=raw,value=devel-latest,enable=${{ github.ref == 'refs/heads/dev' }} type=raw,value=latest,enable=${{ github.event_name == 'release' }} type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }} @@ -72,7 +192,8 @@ jobs: labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64 # Only push if we've tagged the image in the metadata step - push: ${{ github.event_name != 'pull_request' && steps.meta.outputs.tags != '' }} + push: ${{ steps.meta.outputs.tags != '' }} tags: ${{ steps.meta.outputs.tags }} + # Use the same cache as the build step cache-from: type=gha,scope=container-${{ matrix.image }} - cache-to: type=gha,scope=container-${{ matrix.image }},mode=max + cache-to: type=gha,scope=container-${{ matrix.image }},mode=min diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 16bfb415f8..bcf7380039 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -109,6 +109,7 @@ ENV SCORECARD_DIR=/scorecard COPY --from=golang-builder --chmod=u=rw,u+X,go=r,go+X "/scorecard" "/scorecard/scorecard" RUN mkdir -p repos/ logs/ /augur/facade/ +RUN ln -s /cache /augur/augur/static/cache COPY --chmod=u=rwx,go=rx ./docker/backend/entrypoint.sh / COPY --chmod=u=rwx,go=rx ./docker/backend/init.sh / diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index 9d677fd2be..f5f954749c 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -4,9 +4,9 @@ FROM postgres:16 LABEL maintainer="outdoors@acm.org" LABEL version="0.86.1" -ENV POSTGRES_DB "test" -ENV POSTGRES_USER "augur" -ENV POSTGRES_PASSWORD "augur" +ENV POSTGRES_DB="test" +ENV POSTGRES_USER="augur" +ENV POSTGRES_PASSWORD="augur" EXPOSE 5432 diff --git a/scripts/ci/await_all.py b/scripts/ci/await_all.py new file mode 100755 index 0000000000..2da8590f39 --- /dev/null +++ b/scripts/ci/await_all.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +""" +await_all.py: Waits for all regex patterns to match from stdin input. + +Usage: + python await_all.py + +- should be a text file with one regex pattern per line. +- Reads lines from stdin, echoing each line to stdout. +- Prints a match message to stdout when a pattern matches a line. +- Exits with success (0) when all patterns have matched at least once. +- Exits with failure (1) if end of input is reached and not all patterns matched. + +Example: + python await_all.py patterns.txt < input.txt + + where patterns.txt contains: + ^ERROR + ^WARNING + ^INFO + + and input.txt contains: + INFO: All systems operational + ... other lines ... + WARNING: Low disk space + ERROR: Disk failure +""" + +import sys +import re + +if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(2) + +patterns_file = sys.argv[1] + +# Read regex patterns from the file +with open(patterns_file, 'r') as f: + pattern_lines = [line.strip() for line in f if line.strip()] + +if not pattern_lines: + print("No patterns found in the file.", file=sys.stderr) + sys.exit(2) + +patterns = [(i, re.compile(p)) for i, p in enumerate(pattern_lines)] +matched = set() + +try: + for line in sys.stdin: + print(line, end='') # Copy to stdout immediately + for idx, regex in patterns: + if regex.search(line): + print(f"✅✅✅ MATCH pattern {idx+1}: '{pattern_lines[idx]}'") + matched.add(idx) + if len(matched) == len(patterns): + # All patterns matched, exit early + break +except KeyboardInterrupt: + pass + +unmatched = [pattern_lines[i] for i in range(len(patterns)) if i not in matched] +if unmatched: + print("❌❌❌❌❌ Did not match all patterns. ❌❌❌❌❌\nUnmatched patterns:") + for p in unmatched: + print(f"❌ - {p}") + sys.exit(1) +else: + print("✅✅✅✅✅ All patterns matched. ✅✅✅✅✅") + sys.exit(0)