Dev to main update

.github/workflows/build_docker.yml

@@ -3,51 +3,197 @@ on:
   push:
     branches:
       - main
+      - dev
   pull_request:
-    branches:
-      - main
   release:
     types:
       - published
+  workflow_dispatch:
 
 permissions: {}
 
 jobs:
-  build:
+  test-e2e:
+    name: End-to-end test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        id: setup-buildx
+
+      - name: Build database container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/database/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_database:test
+          cache-from: type=gha,scope=container-database
+          cache-to: type=gha,scope=container-database,mode=min
+          load: true
+
+      - name: Build keyman container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/keyman/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_keyman:test
+          cache-from: type=gha,scope=container-keyman
+          cache-to: type=gha,scope=container-keyman,mode=min
+          load: true
+
+      - name: Build rabbitmq container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/rabbitmq/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test
+          cache-from: type=gha,scope=container-rabbitmq
+          cache-to: type=gha,scope=container-rabbitmq,mode=min
+          load: true
+
+      - name: Build backend container
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/backend/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_backend:test
+          cache-from: type=gha,scope=container-backend
+          cache-to: type=gha,scope=container-backend,mode=min
+          load: true
+
+      - name: Prepare compose file
+        run: |
+          yq eval -i '.services.augur.image = "ghcr.io/${{ github.repository_owner }}/augur_backend:test"' docker-compose.yml
+          yq eval -i '.services.augur.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.augur-db.image = "ghcr.io/${{ github.repository_owner }}/augur_database:test"' docker-compose.yml
+          yq eval -i '.services.augur-db.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur-db.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.augur-keyman.image = "ghcr.io/${{ github.repository_owner }}/augur_keyman:test"' docker-compose.yml
+          yq eval -i '.services.augur-keyman.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur-keyman.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.rabbitmq.image = "ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test"' docker-compose.yml
+          yq eval -i '.services.rabbitmq.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.rabbitmq.restart = "no"' docker-compose.yml
+
+      - name: Setup Docker Compose
+        uses: docker/setup-compose-action@v1
+        with:
+          version: latest
+
+      - name: Set up list of log lines to match
+        run: |
+          cat <<EOF > /tmp/regex_matches.txt
+          Gunicorn webserver started
+          Starting core worker processes
+          Starting secondary worker processes
+          Starting facade worker processes
+          Retrieved \\d+ github api keys for use
+          Fetching new repos \\(complete\\)
+          Inserting \\d+ contributors
+          Inserting \\d+ issues
+          Inserting prs of length: \\d+
+          Querying committers count
+          Done generating scc data for repo
+          Sending due task
+          EOF
+
+      - name: Start services & wait for output
+        # This starts the system and sends the output to "await_all.py" which
+        # scans for the regex matches from above. Once all matches are seen at
+        # least once, the `compose down` will run to shut down the system. If
+        # this all doesn't happen before the timeout, the job will fail.
+        run: |
+          docker compose -f docker-compose.yml up --no-build 2>&1 \
+            | (./scripts/ci/await_all.py /tmp/regex_matches.txt \
+                && docker compose -f docker-compose.yml down)
+        timeout-minutes: 3
+        env:
+          AUGUR_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
+          AUGUR_GITHUB_USERNAME: ${{ github.repository_owner }}
+          AUGUR_GITLAB_API_KEY: dummy
+          AUGUR_GITLAB_USERNAME: dummy
+
+      - name: Dump logs
+        # Always run this step to get logs, even if the previous step fails
+        if: always()
+        # We use tail so that we can see the name of each file as it's printed
+        run: "docker run -t --rm -v augur_logs:/logs bash -c 'find /logs -type f | xargs tail -n +0'"
+
+
+
+  push-image:
+    name: Push image
+    needs: test-e2e
+    # We don't push images on pull requests
+    if: github.event_name != 'pull_request'
     permissions:
       contents: read #  to fetch code (actions/checkout)
       packages: write #  to push docker image
-
-    name: Build image
+    strategy:
+      matrix:
+        image:
+          - backend
+          - database
+          - rabbitmq
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout main
-        uses: actions/checkout@v2
-      - name: Run the build
-        run: |
-          set -ex
-          # use that here since the variable are not present before start, so can't be in env
-          export LOGIN=$GITHUB_REPOSITORY_OWNER
-
-          echo $PASSWORD | docker login $REGISTRY -u $LOGIN --password-stdin
-
-          for i in docker/* ; do
-              CONTAINER=$(basename $i)
-              echo "Building $CONTAINER"
-              export IMAGE=$LOGIN/augur_$CONTAINER
-              DOCKERFILE=${i}/Dockerfile
-
-              docker build . -f $DOCKERFILE --tag $REGISTRY/$IMAGE:latest
-              if [[ $GITHUB_EVENT_NAME == 'release' ]]; then
-                TAG=$(basename $GITHUB_REF)
-                docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:$TAG
-                docker push $REGISTRY/$IMAGE:latest
-                docker push $REGISTRY/$IMAGE:$TAG
-              elif [[ $GITHUB_EVENT_NAME == 'push' ]]; then
-                docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:devel-latest
-                docker push $REGISTRY/$IMAGE:devel-latest
-              fi
-          done
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        id: setup-buildx
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set container metadata
+        uses: docker/metadata-action@v5
+        id: meta
         env:
-          REGISTRY: ghcr.io
-          PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest
+        with:
+          annotations: |
+            org.opencontainers.image.title=augur_${{ matrix.image}}
+          labels: |
+            org.opencontainers.image.title=augur_${{ matrix.image}}
+          images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }}
+          # Pushes to the dev branch update the *:devel-latest tag
+          # Releases update the *:latest tag and the *:<version> tag
+          # Main does not update any tags
+          tags: |
+            type=raw,value=devel-latest,enable=${{ github.ref == 'refs/heads/dev' }}
+            type=raw,value=latest,enable=${{ github.event_name == 'release' }}
+            type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }}
+
+      - name: Build and push
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          annotations: ${{ steps.meta.outputs.annotations }}
+          context: .
+          file: ./docker/${{ matrix.image }}/Dockerfile
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+          # Only push if we've tagged the image in the metadata step
+          push: ${{ steps.meta.outputs.tags != '' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          # Use the same cache as the build step
+          cache-from: type=gha,scope=container-${{ matrix.image }}
+          cache-to: type=gha,scope=container-${{ matrix.image }},mode=min

.github/workflows/checks.yml

@@ -28,4 +28,15 @@
         uses: reviewdog/action-misspell@v1
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          locale: "US"
+          locale: "US"
+
+  uv-lock:
+    name: runner / uv-lock
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Ensure uv lockfile is up to date
+        run: uv lock --check

.gitignore

@@ -9,6 +9,8 @@ augur_export_env.sh
 !docker.config.json
 config.yml
 reports.yml
+*.pid
+*.sock
 
 node_modules/
 .idea/
@@ -104,10 +106,6 @@ target/
 profile_default/
 ipython_config.py
 
-# pyenv
-.python-version
-.python-version-hash
-
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies

.pylintrc

@@ -3,16 +3,6 @@
 # go here to check pylint codes if not explained
 #https://vald-phoenix.github.io/pylint-errors/
 
-#doc string checkers
-#enable=C0112,C0114,C0115,C0116
-
-# checks for black listed names being used
-#enable=C0102
-
-#refactoring checker
-#enable=R
-
-disable=E0611,E1101,W1203,R0801,W0614,W0611,C0411,C0103,C0301,C0303,C0304,C0305,W0311,E0401,C0116
 
 
 # Analyse import fallback blocks. This can be used to support both Python 2 and
@@ -150,29 +140,9 @@ confidence=HIGH,
            INFERENCE_FAILURE,
            UNDEFINED
 
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once). You can also use "--disable=all" to
-# disable everything first and then re-enable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use "--disable=all --enable=classes
-# --disable=W".
-disable=raw-checker-failed,
-        bad-inline-option,
-        locally-disabled,
-        file-ignored,
-        suppressed-message,
-        useless-suppression,
-        deprecated-pragma,
-        use-symbolic-message-instead
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member
+# Only enable specific messages
+disable=all
+enable=unused-import,redefined-outer-name,E1206,E1205,E0704,E0107,E4702,E1101,E0211,E0213,E0103,E1133,E1120,E3102,E0602,E1123,E0001,W0702,W1404,W0706,W0101,W0120,W0718,R1737,R1705,R1720,R1724,R1723,R0401,R1701,C1802,C0200,C0501,C0201,W1001,E1102,R0923
 
 
 [LOGGING]

.python-version

@@ -0,0 +1 @@
+3.11

.readthedocs.yml

@@ -5,10 +5,15 @@
 
 # Required
 version: 2
-build: 
+build:
   os: ubuntu-22.04  # <- add this line
   tools:
     python: "3.10"
+  jobs:
+    post_create_environment:
+      # Use uv to create a requirements file that RTD can install
+      - pip install uv
+      - uv export --format requirements.txt --only-group docs -o requirements.txt
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
@@ -24,12 +29,8 @@ formats: all
 # Optionally set the version of Python and requirements required to build your docs
 python:
    install:
-      - method: pip
-        path: .
-        extra_requirements:
-            - dev
-      - method: setuptools
-        path: .
+      # Install the requirements file created during the post_create_environment job
+      - requirements: requirements.txt
 
 #   build: 
 #     os: ubuntu-22.04

CITATION.cff

@@ -0,0 +1,14 @@
+# This CITATION.cff reference content was generated from Zotero.
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+    - family-names: Goggins
+      given-names: Sean
+    - family-names: Lumbard
+      given-names: Kevin
+    - family-names: Germonprez
+      given-names: Matt
+title: "Open Source Community Health: Analytical Metrics and Their Corresponding Narratives"
+doi: 10.1109/SoHeal52568.2021.00010
+date-released: 2021
+url: https://www.seangoggins.net/wp-content/plugins/zotpress/lib/request/request.dl.php?api_user_id=655145&dlkey=HNG22ZSU&content_type=application/pdf