Release Candidate PR for v0.86.3

.github/workflows/build_docker.yml

@@ -5,9 +5,6 @@ on:
       - main
       - dev
   pull_request:
-    branches:
-      - main
-      - dev
   release:
     types:
       - published
@@ -16,8 +13,130 @@ on:
 permissions: {}
 
 jobs:
-  build:
-    name: Build image
+  test-e2e:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        id: setup-buildx
+
+      - name: Build database container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/database/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_database:test
+          cache-from: type=gha,scope=container-database
+          cache-to: type=gha,scope=container-database,mode=min
+          load: true
+
+      - name: Build keyman container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/keyman/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_keyman:test
+          cache-from: type=gha,scope=container-keyman
+          cache-to: type=gha,scope=container-keyman,mode=min
+          load: true
+
+      - name: Build rabbitmq container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/rabbitmq/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test
+          cache-from: type=gha,scope=container-rabbitmq
+          cache-to: type=gha,scope=container-rabbitmq,mode=min
+          load: true
+
+      - name: Build backend container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/backend/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_backend:test
+          cache-from: type=gha,scope=container-backend
+          cache-to: type=gha,scope=container-backend,mode=min
+          load: true
+
+      - name: Prepare compose file
+        run: |
+          yq eval -i '.services.augur.image = "ghcr.io/${{ github.repository_owner }}/augur_backend:test"' docker-compose.yml
+          yq eval -i '.services.augur.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.augur-db.image = "ghcr.io/${{ github.repository_owner }}/augur_database:test"' docker-compose.yml
+          yq eval -i '.services.augur-db.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur-db.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.augur-keyman.image = "ghcr.io/${{ github.repository_owner }}/augur_keyman:test"' docker-compose.yml
+          yq eval -i '.services.augur-keyman.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur-keyman.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.rabbitmq.image = "ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test"' docker-compose.yml
+          yq eval -i '.services.rabbitmq.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.rabbitmq.restart = "no"' docker-compose.yml
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@v1
+        with:
+          version: latest
+
+      - name: Set up list of log lines to match
+        run: |
+          cat <<EOF > /tmp/regex_matches.txt
+          Gunicorn webserver started
+          Starting core worker processes
+          Starting secondary worker processes
+          Starting facade worker processes
+          Retrieved \\d+ github api keys for use
+          Fetching new repos \\(complete\\)
+          Inserting \\d+ contributors
+          Inserting \\d+ issues
+          Inserting prs of length: \\d+
+          Querying committers count
+          Done generating scc data for repo
+          Sending due task
+          EOF
+
+      - name: Start services & wait for output
+        # This starts the system and sends the output to "await_all.py" which
+        # scans for the regex matches from above. Once all matches are seen at
+        # least once, the `compose down` will run to shut down the system. If
+        # this all doesn't happen before the timeout, the job will fail.
+        run: |
+          docker compose -f docker-compose.yml up --no-build 2>&1 \
+            | (./scripts/ci/await_all.py /tmp/regex_matches.txt \
+                && docker compose -f docker-compose.yml down)
+        timeout-minutes: 3
+        env:
+          AUGUR_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
+          AUGUR_GITHUB_USERNAME: ${{ github.repository_owner }}
+          AUGUR_GITLAB_API_KEY: dummy
+          AUGUR_GITLAB_USERNAME: dummy
+
+      - name: Dump logs
+        # Always run this step to get logs, even if the previous step fails
+        if: always()
+        # We use tail so that we can see the name of each file as it's printed
+        run: "docker run -t --rm -v augur_logs:/logs bash -c 'find /logs -type f | xargs tail -n +0'"
+
+
+
+  push-image:
+    name: Push image
+    needs: test-e2e
+    # We don't push images on pull requests
+    if: github.event_name != 'pull_request'
     permissions:
       contents: read #  to fetch code (actions/checkout)
       packages: write #  to push docker image
@@ -57,8 +176,9 @@ jobs:
           images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }}
           # Pushes to the dev branch update the *:devel-latest tag
           # Releases update the *:latest tag and the *:<version> tag
+          # Main does not update any tags
           tags: |
-            type=raw,value=devel-latest,enable=${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/dev' }}
+            type=raw,value=devel-latest,enable=${{ github.ref == 'refs/heads/dev' }}
             type=raw,value=latest,enable=${{ github.event_name == 'release' }}
             type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }}
 
@@ -72,7 +192,8 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           platforms: linux/amd64
           # Only push if we've tagged the image in the metadata step
-          push: ${{ github.event_name != 'pull_request' && steps.meta.outputs.tags != '' }}
+          push: ${{ steps.meta.outputs.tags != '' }}
           tags: ${{ steps.meta.outputs.tags }}
+          # Use the same cache as the build step
           cache-from: type=gha,scope=container-${{ matrix.image }}
-          cache-to: type=gha,scope=container-${{ matrix.image }},mode=max
+          cache-to: type=gha,scope=container-${{ matrix.image }},mode=min

docker/backend/Dockerfile

@@ -109,6 +109,7 @@ ENV SCORECARD_DIR=/scorecard
 COPY --from=golang-builder --chmod=u=rw,u+X,go=r,go+X "/scorecard" "/scorecard/scorecard"
 
 RUN mkdir -p repos/ logs/ /augur/facade/
+RUN ln -s /cache /augur/augur/static/cache
 
 COPY --chmod=u=rwx,go=rx ./docker/backend/entrypoint.sh /
 COPY --chmod=u=rwx,go=rx ./docker/backend/init.sh /

docker/database/Dockerfile

@@ -4,9 +4,9 @@
 LABEL maintainer="outdoors@acm.org"
 LABEL version="0.86.1"
 
-ENV POSTGRES_DB "test"
-ENV POSTGRES_USER "augur"
-ENV POSTGRES_PASSWORD "augur"
+ENV POSTGRES_DB="test"
+ENV POSTGRES_USER="augur"
+ENV POSTGRES_PASSWORD="augur"
 
 EXPOSE 5432
 

scripts/ci/await_all.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""
+await_all.py: Waits for all regex patterns to match from stdin input.
+
+Usage:
+    python await_all.py <patterns_file>
+
+- <patterns_file> should be a text file with one regex pattern per line.
+- Reads lines from stdin, echoing each line to stdout.
+- Prints a match message to stdout when a pattern matches a line.
+- Exits with success (0) when all patterns have matched at least once.
+- Exits with failure (1) if end of input is reached and not all patterns matched.
+
+Example:
+    python await_all.py patterns.txt < input.txt
+
+    where patterns.txt contains:
+      ^ERROR
+      ^WARNING
+      ^INFO
+
+    and input.txt contains:
+      INFO: All systems operational
+      ... other lines ...
+      WARNING: Low disk space
+      ERROR: Disk failure
+"""
+
+import sys
+import re
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} <patterns_file>", file=sys.stderr)
+    sys.exit(2)
+
+patterns_file = sys.argv[1]
+
+# Read regex patterns from the file
+with open(patterns_file, 'r') as f:
+    pattern_lines = [line.strip() for line in f if line.strip()]
+
+if not pattern_lines:
+    print("No patterns found in the file.", file=sys.stderr)
+    sys.exit(2)
+
+patterns = [(i, re.compile(p)) for i, p in enumerate(pattern_lines)]
+matched = set()
+
+try:
+    for line in sys.stdin:
+        print(line, end='')  # Copy to stdout immediately
+        for idx, regex in patterns:
+            if regex.search(line):
+                print(f"✅✅✅ MATCH pattern {idx+1}: '{pattern_lines[idx]}'")
+                matched.add(idx)
+        if len(matched) == len(patterns):
+            # All patterns matched, exit early
+            break
+except KeyboardInterrupt:
+    pass
+
+unmatched = [pattern_lines[i] for i in range(len(patterns)) if i not in matched]
+if unmatched:
+    print("❌❌❌❌❌ Did not match all patterns. ❌❌❌❌❌\nUnmatched patterns:")
+    for p in unmatched:
+        print(f"❌ - {p}")
+    sys.exit(1)
+else:
+    print("✅✅✅✅✅ All patterns matched. ✅✅✅✅✅")
+    sys.exit(0)