Include hot fixes in dev

.github/workflows/build_docker.yml

@@ -11,45 +11,68 @@ on:
   release:
     types:
       - published
+  workflow_dispatch:
 
 permissions: {}
 
 jobs:
   build:
+    name: Build image
     permissions:
       contents: read #  to fetch code (actions/checkout)
       packages: write #  to push docker image
-
-    name: Build image
+    strategy:
+      matrix:
+        image:
+          - backend
+          - database
+          - rabbitmq
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout main
-        uses: actions/checkout@v2
-      - name: Run the build
-        run: |
-          set -ex
-          # use that here since the variable are not present before start, so can't be in env
-          export LOGIN=$GITHUB_REPOSITORY_OWNER
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-          echo $PASSWORD | docker login $REGISTRY -u $LOGIN --password-stdin
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        id: setup-buildx
 
-          for i in docker/* ; do
-              CONTAINER=$(basename $i)
-              echo "Building $CONTAINER"
-              export IMAGE=$LOGIN/augur_$CONTAINER
-              DOCKERFILE=${i}/Dockerfile
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-              docker build . -f $DOCKERFILE --tag $REGISTRY/$IMAGE:latest
-              if [[ $GITHUB_EVENT_NAME == 'release' ]]; then
-                TAG=$(basename $GITHUB_REF)
-                docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:$TAG
-                docker push $REGISTRY/$IMAGE:latest
-                docker push $REGISTRY/$IMAGE:$TAG
-              elif [[ $GITHUB_EVENT_NAME == 'push' ]]; then
-                docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:devel-latest
-                docker push $REGISTRY/$IMAGE:devel-latest
-              fi
-          done
+      - name: Set container metadata
+        uses: docker/metadata-action@v5
+        id: meta
         env:
-          REGISTRY: ghcr.io
-          PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest
+        with:
+          annotations: |
+            org.opencontainers.image.title=augur_${{ matrix.image}}
+          labels: |
+            org.opencontainers.image.title=augur_${{ matrix.image}}
+          images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }}
+          # Pushes to the dev branch update the *:devel-latest tag
+          # Releases update the *:latest tag and the *:<version> tag
+          tags: |
+            type=raw,value=devel-latest,enable=${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/dev' }}
+            type=raw,value=latest,enable=${{ github.event_name == 'release' }}
+            type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }}
+
+      - name: Build and push
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          annotations: ${{ steps.meta.outputs.annotations }}
+          context: .
+          file: ./docker/${{ matrix.image }}/Dockerfile
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+          # Only push if we've tagged the image in the metadata step
+          push: ${{ github.event_name != 'pull_request' && steps.meta.outputs.tags != '' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha,scope=container-${{ matrix.image }}
+          cache-to: type=gha,scope=container-${{ matrix.image }},mode=max

CONTRIBUTING.md

@@ -92,7 +92,7 @@ git push origin master
 ### Augur
 - [Stable documentation (`main` branch)](https://oss-augur.readthedocs.io/en/main/)
 - [Nightly/developer build documentation (`dev` branch)](https://oss-augur.readthedocs.io/en/dev/) (warning: this is should be considered an unstable branch and should not be used for production)
-- [Live Augur demo](http://zephyr.osshealth.io/)
+- [Live Augur demo](https://ai.chaoss.io)
 
 ### CHAOSS
 - [Website](https://chaoss.community/)

README.md

@@ -1,4 +1,4 @@
-# Augur NEW Release v0.81.0
+# Augur NEW Release v0.86.0
 
 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! 
 The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). 
@@ -11,8 +11,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o
 ## NEW RELEASE ALERT!
 **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**.
 
-<<<<<<< HEAD
-Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.81.0).
+Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.86.0).
 
 
 - The `main` branch is a stable version of our new architecture, which features:
@@ -44,9 +43,9 @@ For more information on [how to get involved on the CHAOSS website](https://chao
 
 ## Collecting Data
 
-Augur supports ```Python3.6``` through ```Python3.9``` on all platforms. ```Python3.10``` and above do not yet work because of machine learning worker dependencies. On OSX, you can create a ```Python3.9``` environment, by running:
+Augur supports ```Python3.7``` through ```Python3.11``` on all platforms. ```Python3.12``` and above do not yet work because of machine learning worker dependencies. On OSX, you can create a ```Python3.11``` environment, by running:
 ```
-$ python3.9 -m venv path/to/venv
+$ python3.11 -m venv path/to/venv
 ```
 
 Augur's main focus is to measure the overall health and sustainability of open source projects.
@@ -84,7 +83,7 @@ We strongly believe that much of what makes open source so great is the incredib
 
 ## License, Copyright, and Funding
 
-Copyright © 2023 University of Nebraska at Omaha, University of Missouri, Brian Warner, and the CHAOSS Project.
+Copyright © 2025 University of Nebraska at Omaha, University of Missouri, Brian Warner, and the CHAOSS Project.
 
 Augur is free software: you can redistribute it and/or modify it under the terms of the MIT License as published by the Open Source Initiative. See the [LICENSE](LICENSE) file for more details.
 

augur/application/cli/__init__.py

@@ -6,6 +6,7 @@
 import re
 import json
 import httpx
+import traceback
 
 from augur.application.db.engine import DatabaseEngine
 from augur.application.db import get_engine, dispose_database_engine
@@ -16,23 +17,32 @@ def test_connection(function_internet_connection):
     @click.pass_context
     def new_func(ctx, *args, **kwargs):
         usage = re.search(r"Usage:\s(.*)\s\[OPTIONS\]", str(ctx.get_usage())).groups()[0]
+        success = False
         with httpx.Client() as client:
             try:
                 _ = client.request(
                     method="GET", url="http://chaoss.community", timeout=10, follow_redirects=True)
-
-                return ctx.invoke(function_internet_connection, *args, **kwargs)
+                success = True
             except (TimeoutError, httpx.TimeoutException):
                 print("Request timed out.")
-            except httpx.NetworkError:
+            except httpx.NetworkError as e:
                 print(f"Network Error: {httpx.NetworkError}")
-            except httpx.ProtocolError:
+                print(traceback.format_exc())
+            except httpx.ProtocolError as e:
                 print(f"Protocol Error: {httpx.ProtocolError}")
-            print(f"\n\n{usage} command setup failed\n \
-                You are not connected to the internet.\n \
-                Please connect to the internet to run Augur\n \
-                Consider setting http_proxy variables for limited access installations.")
-            sys.exit(-1)        
+                print(traceback.format_exc())
+
+            if not success:
+                print(
+                    f"""
+                    \n\n{usage} command setup failed.
+                    There was an error while testing for network connectivity
+                    Please check your connection to the internet to run Augur
+                    Consider setting http_proxy variables for limited access installations."""
+                )
+                sys.exit(-1)
+
+        return ctx.invoke(function_internet_connection, *args, **kwargs)
 
     return update_wrapper(new_func, function_internet_connection)
 

augur/application/cli/backend.py

@@ -30,7 +30,7 @@
 import sqlalchemy as s
 
 from keyman.KeyClient import KeyClient, KeyPublisher
 
 logger = AugurLogger("augur", reset_logfiles=True).get_logger()
 
 
@@ -86,6 +86,9 @@
 
     worker_vmem_cap = get_value("Celery", 'worker_process_vmem_cap')
 
+    # create rabbit messages so if it failed on shutdown the queues are clean
+    cleanup_collection_status_and_rabbit(logger, ctx.obj.engine)
+
     gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} augur.api.server:app --log-file gunicorn.log"
     server = subprocess.Popen(gunicorn_command.split(" "))
 
@@ -110,7 +113,7 @@
     logger.info(f"The API is available at '{api_response.json()['route']}'")
 
     processes = start_celery_worker_processes(float(worker_vmem_cap), disable_collection)
 
     if os.path.exists("celerybeat-schedule.db"):
             logger.info("Deleting old task schedule")
             os.remove("celerybeat-schedule.db")
@@ -180,7 +183,7 @@
 
             try:
                 keypub.shutdown()
-                cleanup_after_collection_halt(logger, ctx.obj.engine)
+                cleanup_collection_status_and_rabbit(logger, ctx.obj.engine)
             except RedisConnectionError:
                 pass
 
@@ -255,7 +258,7 @@
     logger = logging.getLogger("augur.cli")
 
     augur_stop(signal.SIGTERM, logger, ctx.obj.engine)
 
 @cli.command('stop-collection-blocking')
 @test_connection
 @test_db_connection
@@ -268,7 +271,7 @@
     processes = get_augur_processes()
 
     stopped = []
 
     p: psutil.Process
     for p in processes:
         if p.name() == "celery":
@@ -278,7 +281,7 @@
     if not len(stopped):
         logger.info("No collection processes found")
         return
 
     _, alive = psutil.wait_procs(stopped, 5,
                                  lambda p: logger.info(f"STOPPED: {p.pid}"))
 
@@ -287,7 +290,7 @@
         for i in range(len(alive)):
             if alive[i].status() == psutil.STATUS_ZOMBIE:
                 logger.info(f"KILLING ZOMBIE: {alive[i].pid}")
                 alive[i].kill()
                 killed.append(i)
             elif not alive[i].is_running():
                 logger.info(f"STOPPED: {p.pid}")
@@ -302,7 +305,7 @@
         logger.info(f"Waiting on [{', '.join(str(p.pid for p in alive))}]")
         time.sleep(0.5)
 
-    cleanup_after_collection_halt(logger, ctx.obj.engine)
+    cleanup_collection_status_and_rabbit(logger, ctx.obj.engine)
 
 @cli.command('kill')
 @test_connection
@@ -316,11 +319,11 @@
     logger = logging.getLogger("augur.cli")
     augur_stop(signal.SIGKILL, logger, ctx.obj.engine)
 
 
 def augur_stop(signal, logger, engine):
     """
     Stops augur with the given signal, 
     and cleans up collection if it was running
     """
 
     augur_processes = get_augur_processes()
@@ -330,10 +333,10 @@
     _broadcast_signal_to_processes(augur_processes, broadcast_signal=signal, given_logger=logger)
 
     if "celery" in process_names:
-        cleanup_after_collection_halt(logger, engine)
+        cleanup_collection_status_and_rabbit(logger, engine)
 
 
-def cleanup_after_collection_halt(logger, engine):
+def cleanup_collection_status_and_rabbit(logger, engine):
     clear_redis_caches()
 
     connection_string = get_value("RabbitMQ", "connection_string")
@@ -485,7 +488,7 @@
 def _broadcast_signal_to_processes(processes, broadcast_signal=signal.SIGTERM, given_logger=None):
     if given_logger is None:
         _logger = logger
     else:
         _logger = given_logger
     for process in processes:
         if process.pid != os.getpid():

augur/application/cli/github.py

@@ -32,7 +32,7 @@ def update_api_key():
             """
             SELECT value as github_key from config Where section_name='Keys' AND setting_name='github_api_key'
             UNION All
-            SELECT access_token as github_key from worker_oauth ORDER BY github_key DESC;
+            SELECT access_token as github_key from worker_oauth where platform='github' ORDER BY github_key DESC;
             """
         )
 

augur/application/db/data_parse.py

@@ -291,7 +291,7 @@ def extract_pr_event_data(event: dict, pr_id: int, gh_src_id: int, platform_id:
         'action': event['event'],
         'action_commit_hash': None,
         'created_at': event['created_at'],
-        'issue_event_src_id': gh_src_id,
+        'issue_event_src_id': event["id"],
         'node_id': event['node_id'],
         'node_url': event['url'],
         'tool_source': tool_source,