diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index b29ab2ed89..fbce968697 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -11,45 +11,68 @@ on: release: types: - published + workflow_dispatch: permissions: {} jobs: build: + name: Build image permissions: contents: read # to fetch code (actions/checkout) packages: write # to push docker image - - name: Build image + strategy: + matrix: + image: + - backend + - database + - rabbitmq runs-on: ubuntu-latest steps: - - name: Checkout main - uses: actions/checkout@v2 - - name: Run the build - run: | - set -ex - # use that here since the variable are not present before start, so can't be in env - export LOGIN=$GITHUB_REPOSITORY_OWNER + - name: Checkout repository + uses: actions/checkout@v4 - echo $PASSWORD | docker login $REGISTRY -u $LOGIN --password-stdin + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx - for i in docker/* ; do - CONTAINER=$(basename $i) - echo "Building $CONTAINER" - export IMAGE=$LOGIN/augur_$CONTAINER - DOCKERFILE=${i}/Dockerfile + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: github.event_name != 'pull_request' + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - docker build . -f $DOCKERFILE --tag $REGISTRY/$IMAGE:latest - if [[ $GITHUB_EVENT_NAME == 'release' ]]; then - TAG=$(basename $GITHUB_REF) - docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:$TAG - docker push $REGISTRY/$IMAGE:latest - docker push $REGISTRY/$IMAGE:$TAG - elif [[ $GITHUB_EVENT_NAME == 'push' ]]; then - docker tag $REGISTRY/$IMAGE:latest $REGISTRY/$IMAGE:devel-latest - docker push $REGISTRY/$IMAGE:devel-latest - fi - done + - name: Set container metadata + uses: docker/metadata-action@v5 + id: meta env: - REGISTRY: ghcr.io - PASSWORD: ${{ secrets.GITHUB_TOKEN }} + DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest + with: + annotations: | + org.opencontainers.image.title=augur_${{ matrix.image}} + labels: | + org.opencontainers.image.title=augur_${{ matrix.image}} + images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }} + # Pushes to the dev branch update the *:devel-latest tag + # Releases update the *:latest tag and the *: tag + tags: | + type=raw,value=devel-latest,enable=${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/dev' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' }} + type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }} + + - name: Build and push + id: push + uses: docker/build-push-action@v6 + with: + annotations: ${{ steps.meta.outputs.annotations }} + context: . + file: ./docker/${{ matrix.image }}/Dockerfile + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + # Only push if we've tagged the image in the metadata step + push: ${{ github.event_name != 'pull_request' && steps.meta.outputs.tags != '' }} + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha,scope=container-${{ matrix.image }} + cache-to: type=gha,scope=container-${{ matrix.image }},mode=max diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 3f19c12343..0000000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Build the Docker image - run: | - ls -l - docker build . --file docker/backend/Dockerfile --tag my-image-name:$(date +%s) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6368d58bf5..6fed03731b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,7 @@ git push origin master ### Augur - [Stable documentation (`main` branch)](https://oss-augur.readthedocs.io/en/main/) - [Nightly/developer build documentation (`dev` branch)](https://oss-augur.readthedocs.io/en/dev/) (warning: this is should be considered an unstable branch and should not be used for production) -- [Live Augur demo](http://zephyr.osshealth.io/) +- [Live Augur demo](https://ai.chaoss.io) ### CHAOSS - [Website](https://chaoss.community/) diff --git a/README.md b/README.md index 50db605109..33e41bcf15 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.85.1 +# Augur NEW Release v0.86.0 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). @@ -11,7 +11,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o ## NEW RELEASE ALERT! **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**. -Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.85.1). +Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.86.0). - The `main` branch is a stable version of our new architecture, which features: diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 95de27ce02..0000000000 --- a/Vagrantfile +++ /dev/null @@ -1,108 +0,0 @@ -$script = <<-'SCRIPT' -set -euxo pipefail - -sudo apt-get -y update -sudo apt-get -y install --no-install-recommends \ - build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev \ - git gcc gfortran \ - python3 python3-pip python3.8-venv \ - postgresql postgresql-contrib \ - libomp-dev \ - golang libgomp1 -sudo pg_ctlcluster 12 main start - -go get -u github.com/boyter/scc/ - -# # install Go -# installGo() ( -# cd "$(mktemp -d)" -# wget https://golang.org/dl/go1.16.5.linux-amd64.tar.gz -# rm -rf /usr/local/go && tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz -# ) -# sudo installGo -# export PATH=$PATH:/usr/local/go/bin - - -########################################################################################## -# see: https://oss-augur.readthedocs.io/en/master/getting-started/database.html -cat < /tmp/init.psql -CREATE DATABASE augur; -CREATE USER augur WITH ENCRYPTED PASSWORD 'password'; -GRANT ALL PRIVILEGES ON DATABASE augur TO augur; -EOF -sudo -u postgres psql -U postgres -f /tmp/init.psql - - -########################################################################################## -# see: https://oss-augur.readthedocs.io/en/master/getting-started/installation.html - -mkdir -p "$HOME/augur/" "$HOME/augur/logs/" "$HOME/augur/repos/" -cat < "$HOME/augur/config.json" -{ - "Database": { - "host": "localhost", - "password": "password" - }, - "Server": { - "host": "0.0.0.0" - }, - "Logging": { - "logs_directory": "$HOME/augur/logs/", - "log_level": "INFO", - "verbose": 0, - "quiet": 0, - "debug": 1 - }, - "Workers": { - "facade_worker": { - "repo_directory": "$HOME/augur/repos/", - "switch": 1 - }, - "github_worker": { - "switch": 1 - }, - "insight_worker": { - "switch": 1 - }, - "linux_badge_worker": { - "switch": 1 - }, - "pull_request_worker": { - "switch": 1 - }, - "repo_info_worker": { - "switch": 1 - }, - "release_worker": { - "switch": 1 - } - } -} -EOF - - -python3 -m venv $HOME/.virtualenvs/augur_env -source $HOME/.virtualenvs/augur_env/bin/activate -pip install wheel - -cd /vagrant -python setup.py bdist_wheel -make clean -make install-dev - -augur config init --rc-config-file "$HOME/config.json" -augur db create-schema -augur backend start" - -SCRIPT - -Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/focal64" - - config.vm.provider "virtualbox" do |v| - v.memory = 20480 - v.cpus = 4 - end - - config.vm.provision "shell", privileged: false, inline: $script -end diff --git a/augur/application/db/data_parse.py b/augur/application/db/data_parse.py index 71bde0aa21..6f20fa35a7 100644 --- a/augur/application/db/data_parse.py +++ b/augur/application/db/data_parse.py @@ -291,7 +291,7 @@ def extract_pr_event_data(event: dict, pr_id: int, gh_src_id: int, platform_id: 'action': event['event'], 'action_commit_hash': None, 'created_at': event['created_at'], - 'issue_event_src_id': gh_src_id, + 'issue_event_src_id': event["id"], 'node_id': event['node_id'], 'node_url': event['url'], 'tool_source': tool_source, diff --git a/augur/application/schema/alembic/versions/31_update_pr_events_unique.py b/augur/application/schema/alembic/versions/31_update_pr_events_unique.py new file mode 100644 index 0000000000..b55b60a09a --- /dev/null +++ b/augur/application/schema/alembic/versions/31_update_pr_events_unique.py @@ -0,0 +1,83 @@ +"""Update pr events unique + +Revision ID: 31 +Revises: 30 +Create Date: 2025-03-08 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import text +from augur.application.db import create_database_engine, get_database_string + + +# revision identifiers, used by Alembic. +revision = '31' +down_revision = '30' +branch_labels = None +depends_on = None + + + # conn = op.get_bind() + # conn.execute(text(""" + # UPDATE pull_request_events + # SET issue_event_src_id = substring(node_url FROM '.*/([0-9]+)$')::BIGINT; + # """)) + + +def upgrade(): + + connection_string = get_database_string() + engine = create_database_engine(connection_string) + + with engine.connect() as conn: + + result = conn.execute(text("SELECT COUNT(*) FROM pull_request_events WHERE issue_event_src_id=pr_platform_event_id")) + total_rows = result.scalar() + if total_rows != 0: + print(f"Rows needing updated: {total_rows}") + print(f"0.0% complete") + total_updated = 0 + + while True: + result = conn.execute(text(""" + WITH cte AS ( + SELECT pr_event_id + FROM pull_request_events + WHERE issue_event_src_id=pr_platform_event_id + LIMIT 250000 + ) + UPDATE pull_request_events + SET issue_event_src_id = substring(node_url FROM '.*/([0-9]+)$')::BIGINT + FROM cte + WHERE pull_request_events.pr_event_id = cte.pr_event_id + RETURNING 1; + """)) + + conn.commit() + + rows_updated = result.rowcount + total_updated += rows_updated + + if rows_updated == 0: + print(f"Update complete") + break + + percentage_updated = (total_updated / total_rows) * 100 + + print(f"{percentage_updated:.1f}% complete ({total_rows-total_updated} rows left)") + + print("Creating (repo_id, issue_event_src_id) index") + op.create_unique_constraint('pr_events_repo_id_event_src_id_unique', 'pull_request_events', ['repo_id', 'issue_event_src_id'], schema='augur_data') + + +def downgrade(): + op.drop_constraint('pr_events_repo_id_event_src_id_unique', 'pull_request_events', schema='augur_data', type_='unique') + + print("Please run in background. This downgrade will take a very *very* long time") + conn = op.get_bind() + conn.execute(text(""" + UPDATE pull_request_events + SET issue_event_src_id = pr_platform_event_id + WHERE issue_event_src_id <> pr_platform_event_id; + """)) \ No newline at end of file diff --git a/augur/tasks/data_analysis/clustering_worker/setup.py b/augur/tasks/data_analysis/clustering_worker/setup.py index a197b21568..bd591554c3 100644 --- a/augur/tasks/data_analysis/clustering_worker/setup.py +++ b/augur/tasks/data_analysis/clustering_worker/setup.py @@ -26,7 +26,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', #'sklearn==0.0.0', 'scikit-learn==1.5.0', 'numpy==1.26.0', diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/setup.py b/augur/tasks/data_analysis/contributor_breadth_worker/setup.py index 805edfb36b..70a4cd6312 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/setup.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/setup.py @@ -27,7 +27,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3' + 'psycopg2-binary==2.9.9' ], entry_points={ 'console_scripts': [ diff --git a/augur/tasks/data_analysis/discourse_analysis/setup.py b/augur/tasks/data_analysis/discourse_analysis/setup.py index ca936a6000..67cb4dd7e7 100644 --- a/augur/tasks/data_analysis/discourse_analysis/setup.py +++ b/augur/tasks/data_analysis/discourse_analysis/setup.py @@ -26,7 +26,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'scipy>=1.10.0', 'nltk==3.6.6', diff --git a/augur/tasks/data_analysis/insight_worker/setup.py b/augur/tasks/data_analysis/insight_worker/setup.py index 92d663e3ae..b28b4bf5ab 100644 --- a/augur/tasks/data_analysis/insight_worker/setup.py +++ b/augur/tasks/data_analysis/insight_worker/setup.py @@ -27,7 +27,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'scipy>=1.10.0', 'sklearn==0.0', diff --git a/augur/tasks/data_analysis/message_insights/setup.py b/augur/tasks/data_analysis/message_insights/setup.py index 2f86701619..bd136fdfec 100644 --- a/augur/tasks/data_analysis/message_insights/setup.py +++ b/augur/tasks/data_analysis/message_insights/setup.py @@ -28,7 +28,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'scipy>=1.10.0', 'scikit-learn==1.5.0', #0.24.2', diff --git a/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py b/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py index 63ccbec1de..391b368f32 100644 --- a/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py +++ b/augur/tasks/data_analysis/pull_request_analysis_worker/setup.py @@ -26,7 +26,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'sklearn==0.0', 'nltk==3.6.6', 'numpy==1.26.0', diff --git a/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py b/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py index 1e2e04ac12..5f8bcd5772 100644 --- a/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py +++ b/augur/tasks/git/util/facade_worker/facade_worker/analyzecommit.py @@ -32,6 +32,7 @@ import sqlalchemy as s from augur.application.db.lib import execute_sql, fetchall_data_from_sql_text +from augur.tasks.init import get_rabbitmq_conn_string def analyze_commit(logger, repo_id, repo_loc, commit): diff --git a/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py b/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py index 40f3a29e0e..caae6c02ba 100644 --- a/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py +++ b/augur/tasks/git/util/facade_worker/facade_worker/utilitymethods.py @@ -26,7 +26,7 @@ # repos. It also rebuilds analysis data, checks any changed affiliations and # aliases, and caches data for display. import subprocess -from subprocess import check_output +from subprocess import check_output, CalledProcessError import os import sqlalchemy as s from augur.application.db.models import * @@ -154,7 +154,15 @@ def get_repo_commit_count(logger, facade_helper, repo_git): if count_branches(repo_loc) == 0: return 0 - check_commit_count_cmd = check_output(["git", "--git-dir", repo_loc, "rev-list", "--count", "HEAD"]) + try: + check_commit_count_cmd = check_output( + ["git", "--git-dir", repo_loc, "rev-list", "--count", "HEAD"], + stderr=subprocess.PIPE) + except CalledProcessError as e: + logger.error(f"Ran into {e}: {e.output} {e.stderr} \n With return code {e.returncode}") + raise e + + commit_count = int(check_commit_count_cmd) return commit_count diff --git a/augur/tasks/git/util/facade_worker/setup.py b/augur/tasks/git/util/facade_worker/setup.py index e2a1af8b75..ef65f223a2 100644 --- a/augur/tasks/git/util/facade_worker/setup.py +++ b/augur/tasks/git/util/facade_worker/setup.py @@ -27,7 +27,7 @@ def read(filename): 'Flask-Login==0.5.0', 'Flask-WTF==1.0.0', 'requests==2.32.0', - 'psycopg2-binary==2.9.3', + 'psycopg2-binary==2.9.9', 'click==8.0.3', 'XlsxWriter==1.3.7' ], diff --git a/augur/tasks/github/events.py b/augur/tasks/github/events.py index 5c79691c26..654dc68c6a 100644 --- a/augur/tasks/github/events.py +++ b/augur/tasks/github/events.py @@ -78,7 +78,7 @@ def _insert_issue_events(self, events): bulk_insert_dicts(self._logger, events, IssueEvent, issue_event_natural_keys) def _insert_pr_events(self, events): - pr_event_natural_keys = ["node_id"] + pr_event_natural_keys = ["repo_id", "issue_event_src_id"] bulk_insert_dicts(self._logger, events, PullRequestEvent, pr_event_natural_keys) def _insert_contributors(self, contributors): diff --git a/augur/tasks/github/util/github_data_access.py b/augur/tasks/github/util/github_data_access.py index a648f990c6..0eda1c8462 100644 --- a/augur/tasks/github/util/github_data_access.py +++ b/augur/tasks/github/util/github_data_access.py @@ -5,6 +5,8 @@ from urllib.parse import urlparse, parse_qs, urlencode from keyman.KeyClient import KeyClient +GITHUB_RATELIMIT_REMAINING_CAP = 50 + class RatelimitException(Exception): @@ -110,6 +112,13 @@ def make_request(self, url, method="GET", timeout=100): response.raise_for_status() + try: + if "X-RateLimit-Remaining" in response.headers and int(response.headers["X-RateLimit-Remaining"]) < GITHUB_RATELIMIT_REMAINING_CAP: + raise RatelimitException(response) + except ValueError: + self.logger.warning(f"X-RateLimit-Remaining was not an integer. Value: {response.headers['X-RateLimit-Remaining']}") + + return response def make_request_with_retries(self, url, method="GET", timeout=100): @@ -148,7 +157,7 @@ def __handle_github_ratelimit_response(self, response): f'\n\n\n\nSleeping for {retry_after} seconds due to secondary rate limit issue.\n\n\n\n') time.sleep(retry_after) - elif "X-RateLimit-Remaining" in headers and int(headers["X-RateLimit-Remaining"]) == 0: + elif "X-RateLimit-Remaining" in headers and int(headers["X-RateLimit-Remaining"]) < GITHUB_RATELIMIT_REMAINING_CAP: current_epoch = int(time.time()) epoch_when_key_resets = int(headers["X-RateLimit-Reset"]) key_reset_time = epoch_when_key_resets - current_epoch diff --git a/docker-compose.yml b/docker-compose.yml index e077260ed7..81452bd09d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,14 +10,16 @@ services: - "POSTGRES_PASSWORD=${AUGUR_DB_PASSWORD:-augur}" - "PGDATA=/var/lib/postgresql/data/pgdata" ports: - - "127.0.0.1:${AUGUR_DB_PORT:-5432}:5432" + - "${AUGUR_DB_PORT:-5432}:5432" volumes: - augurpostgres:/var/lib/postgresql/data + networks: + - augur redis: image: "redis:alpine" - ports: - - 6379:6379 + networks: + - augur rabbitmq: image: augur-rabbitmq @@ -28,12 +30,8 @@ services: - RABBIT_MQ_DEFAULT_USER=${AUGUR_RABBITMQ_USERNAME:-augur} - RABBIT_MQ_DEFAULT_PASSWORD=${AUGUR_RABBITMQ_PASSWORD:-password123} - RABBIT_MQ_DEFAULT_VHOST=${AUGUR_RABBITMQ_VHOST:-augur_vhost} - # ports for amqp connections / management api - ports: - - 5671:5671 - - 5672:5672 - - 15671:15671 - - 15672:15672 + networks: + - augur augur: image: augur-new:latest @@ -45,8 +43,8 @@ services: restart: unless-stopped ports: - 5002:5000 - extra_hosts: - - "host.docker.internal:host-gateway" #Be able to ping services on the local machine + #extra_hosts: + # - "host.docker.internal:host-gateway" #Be able to ping services on the local machine environment: - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@augur-db:5432/augur" - "AUGUR_DB_SCHEMA_BUILD=1" @@ -61,6 +59,30 @@ services: - augur-db - redis - rabbitmq + networks: + - augur + + # Flower is a UI that helps more easily monitor running tasks for celery workers. + # This wont show you every error from the entire augur system, but it should show most of them. + # uncomment the section below to use flower + #flower: + # image: augur-new:latest + # restart: unless-stopped + # command: + # [ "celery", "-A", "augur.tasks.init.celery_app.celery_app", "flower", "--max-tasks=1000000" ] + # ports: + # - 5555:5555 + # environment: + # - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@augur-db:5432/augur" + # - REDIS_CONN_STRING=redis://redis:6379 + # - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${AUGUR_RABBITMQ_VHOST:-augur_vhost} + # depends_on: + # - augur + # - augur-db + # - redis + # - rabbitmq + # networks: + # - augur volumes: facade: @@ -68,4 +90,5 @@ volumes: augurpostgres: driver: local - +networks: + augur: \ No newline at end of file diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index df6e952b05..959d3952f5 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -32,7 +32,7 @@ RUN ./scripts/docker/install-golang-deps.sh FROM python:3.11-slim-bullseye LABEL maintainer="outdoors@acm.org" -LABEL version="0.85.1" +LABEL version="0.86.0" ENV DEBIAN_FRONTEND=noninteractive ENV PATH="/usr/bin/:/usr/local/bin:/usr/lib:${PATH}" @@ -120,8 +120,8 @@ RUN set -x \ RUN ./scripts/docker/install-workers-deps.sh -COPY --from=golang "/root/scc" "./scc" -COPY --from=golang "/root/scorecard/scorecard" "./scorecard" +COPY --from=golang "/root/scc" "/root/scc" +COPY --from=golang "/root/scorecard/scorecard" "/root/scorecard" # RUN ./scripts/install/workers.sh diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index a1a59b0c89..e5825ca7bf 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:16 LABEL maintainer="outdoors@acm.org" -LABEL version="0.85.1" +LABEL version="0.86.0" ENV POSTGRES_DB "test" ENV POSTGRES_USER "augur" diff --git a/docker/rabbitmq/Dockerfile b/docker/rabbitmq/Dockerfile index b4ec1948ba..9b0b3c98c3 100644 --- a/docker/rabbitmq/Dockerfile +++ b/docker/rabbitmq/Dockerfile @@ -1,7 +1,7 @@ FROM rabbitmq:3.12-management-alpine LABEL maintainer="574/augur@simplelogin.com" -LABEL version="0.85.1" +LABEL version="0.86.0" ARG RABBIT_MQ_DEFAULT_USER=augur ARG RABBIT_MQ_DEFAULT_PASSWORD=password123 diff --git a/docs/dev-osx-install.md b/docs/dev-osx-install.md index c70bac7714..aa5b67b7fa 100644 --- a/docs/dev-osx-install.md +++ b/docs/dev-osx-install.md @@ -31,7 +31,7 @@ export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" ## Git Platform Requirements (Things to have setup prior to initiating installation.) 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens ### Fork and Clone Augur 1. Fork https://github.com/chaoss/augur diff --git a/docs/new-install-ubuntu-python-3.10.md b/docs/new-install-ubuntu-python-3.10.md index 3cd7a5a38a..6fa2ee78fc 100644 --- a/docs/new-install-ubuntu-python-3.10.md +++ b/docs/new-install-ubuntu-python-3.10.md @@ -5,7 +5,7 @@ We default to this version of Ubuntu for the moment because Augur does not yet s ## Git Platform Requirements (Things to have setup prior to initiating installation.) 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens ### Fork and Clone Augur 1. Fork https://github.com/chaoss/augur diff --git a/docs/new-install.md b/docs/new-install.md index 9ec8960857..017cb41322 100644 --- a/docs/new-install.md +++ b/docs/new-install.md @@ -5,7 +5,7 @@ We default to this version of Ubuntu for the moment because Augur does not yet s ## Git Platform Requirements (Things to have setup prior to initiating installation.) 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens ### Fork and Clone Augur 1. Fork https://github.com/chaoss/augur diff --git a/docs/source/development-guide/installation.rst b/docs/source/development-guide/installation.rst index d98a909874..e1dc27cc88 100644 --- a/docs/source/development-guide/installation.rst +++ b/docs/source/development-guide/installation.rst @@ -23,8 +23,9 @@ The process for installing Augur's source code for development is essentially th .. code-block:: bash - $ make install-dev - + $ make install + # Note: The `make install-dev` command has been removed. + This will install a few extra dependencies for testing and documentation, as well as install all the Python packages in `editable mode `_. This means you will not have to reinstall the package every time you make a change to the Python source code. This command will also create your ``augur.config.json`` file in the root of your cloned source code directory **instead of** the default location in ``$HOME/.augur/``. This is purely for convenience's sake, as it will allow you to open this file in your text editor with all the other source code files, and also allows you to have multiple developer installations of Augur on the same machine if needed. If Augur finds a config file in both the root of the cloned directory AND in the default location, it will always use the one in the root of the cloned directory. diff --git a/docs/source/development-guide/make/installation.rst b/docs/source/development-guide/make/installation.rst index 7dfa7180a8..7f850d4513 100644 --- a/docs/source/development-guide/make/installation.rst +++ b/docs/source/development-guide/make/installation.rst @@ -25,13 +25,22 @@ Example\: ``make install-dev`` --------------------- -The same as ``make install``, except it installs the additional developer dependencies and installs the packages in editable mode. +**Note:** The `make install-dev` command has been **deprecated and removed** from the Makefile. -Example\: +It was previously used to install developer-specific dependencies and set up Augur in editable mode. These steps are now handled by the standard `make install` command, which installs all necessary components for development and usage. + +Please use the following instead: + +.. code-block:: bash + + $ make install + +For additional developer dependencies (if needed), you can also run: .. code-block:: bash - $ make install-dev + $ pip install -e .[dev] + --------------- diff --git a/docs/source/getting-started/dev-osx-install.rst b/docs/source/getting-started/dev-osx-install.rst index 05285da207..05d16c29be 100644 --- a/docs/source/getting-started/dev-osx-install.rst +++ b/docs/source/getting-started/dev-osx-install.rst @@ -47,8 +47,7 @@ Git Platform Requirements (Things to have setup prior to initiating installation ---------------------------------------------------------------------------------- 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: - https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens Fork and Clone Augur ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/getting-started/installation.rst b/docs/source/getting-started/installation.rst index 5c2e2f62f7..4e367958c1 100644 --- a/docs/source/getting-started/installation.rst +++ b/docs/source/getting-started/installation.rst @@ -216,11 +216,6 @@ your installation of Python 3: on most systems, this is ``python3``, but yours m # run the install script $ make install -.. code-block:: bash - - # If you want to develop with Augur, use this command instead - $ make install-dev - If you think something went wrong, check the log files in ``logs/``. If you want to try again, you can use ``make clean`` to delete any build files before running ``make install`` again. MacOS users: diff --git a/docs/source/getting-started/new-install.rst b/docs/source/getting-started/new-install.rst index 3fd45f4a97..b10aa0b41e 100644 --- a/docs/source/getting-started/new-install.rst +++ b/docs/source/getting-started/new-install.rst @@ -9,8 +9,7 @@ Git Platform Requirements (Things to have setup prior to initiating installation ---------------------------------------------------------------------------------- 1. Obtain a GitHub Access Token: https://github.com/settings/tokens -2. Obtain a GitLab Access Token: - https://gitlab.com/-/profile/personal_access_tokens +2. Obtain a GitLab Access Token: https://gitlab.com/-/user_settings/personal_access_tokens Fork and Clone Augur ~~~~~~~~~~~~~~~~~~~~ diff --git a/gsoc-ideas.md b/gsoc-ideas.md index cea5dff355..3bc2748811 100644 --- a/gsoc-ideas.md +++ b/gsoc-ideas.md @@ -5,6 +5,34 @@ [Micro-tasks and place for questions](https://github.com/chaoss/augur/issues/1640) +## Microtasks + +For becoming familiar with Augur, you can start by reading some documentation. You can find useful information at in the links, below. Grimoirelab also has a set of installation instructions and documentation here: https://chaoss.github.io/grimoirelab-tutorial/ + +#### GSoC Students : +Once you're familiar with Augur, you can have a look at the following microtasks. + +Microtask 0: + Download and configure Augur, creating a dev environment using the general cautions noted here: +Augur + https://oss-augur.readthedocs.io/en/dev/getting-started/installation.html and the full documentation here: + https://oss-augur.readthedocs.io/en/dev/development-guide/toc.html +Grimoirelab + https://chaoss.github.io/grimoirelab-tutorial/ + +Microstask 1: + Work on any Augur or Grimoirelab Issue that's Open + +Microtask 2: + Identify new issues you encounter during installation. + +Microstask 3: + Explore data presently captured, develop an experimental visualization using tools of your choice. If Jupyter Notebooks against an Augur database/API endpoint collection, use https://github.com/chaoss/augur-community-reports for development. + +Microtask 4: + Anything you want to show us. Even if you find bugs in our documentation and want to issue a PR for those! + + This project will add GenSIM logic, and other capabilities to the Clustering Worker inside of Augur Software, and be extended into a generalized Open Source Software Conversational Topic Modeling Instrument. CHOASS/augur has several workers that store machine learning information derived from computational linguistic analysis of data in the `message` table. The message table includes messages from issue, pull request, pull request review, and email messages. They are related to their origin with bridge tables like `pull_request_message_ref`. The ML/CL workers are all run against all the messages, regardless of origin. diff --git a/metadata.py b/metadata.py index 184caa4f70..740db52f3d 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.85.1" -__release__ = "v0.85.1 (Eurovision: Fire Saga)" +__version__ = "0.86.0" +__release__ = "v0.86.0 (Pod People)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Sean Goggins, Brian Warner & Augurlabs 2025" diff --git a/podman-compose.yml b/podman-compose.yml deleted file mode 100644 index 9970f81325..0000000000 --- a/podman-compose.yml +++ /dev/null @@ -1,68 +0,0 @@ -#SPDX-License-Identifier: MIT -version: '3' -services: - augur-db: - image: postgres:14 - restart: unless-stopped - environment: - - "POSTGRES_DB=augur" - - "POSTGRES_USER=${AUGUR_DB_USER:-augur}" - - "POSTGRES_PASSWORD=${AUGUR_DB_PASSWORD:-augur}" - - "PGDATA=/var/lib/postgresql/data/pgdata" - ports: - - "${AUGUR_DB_PORT:-5432}:5432" - volumes: - - augurpostgres:/var/lib/postgresql/data - - redis: - image: "redis:alpine" - ports: - - 6379:6379 - - rabbitmq: - image: augur-rabbitmq - build: - context: . - dockerfile: ./docker/rabbitmq/Dockerfile - args: - - RABBIT_MQ_DEFAULT_USER=${AUGUR_RABBITMQ_USERNAME:-augur} - - RABBIT_MQ_DEFAULT_PASSWORD=${AUGUR_RABBITMQ_PASSWORD:-password123} - - RABBIT_MQ_DEFAULT_VHOST=${AUGUR_RABBITMQ_VHOST:-augur_vhost} - # ports for amqp connections / management api - ports: - - 5671:5671 - - 5672:5672 - - 15671:15671 - - 15672:15672 - - augur: - image: augur-new:latest - build: - context: . - dockerfile: ./docker/backend/Dockerfile - volumes: - - facade:/augur/facade - restart: unless-stopped - ports: - - 5002:5000 - environment: - - "AUGUR_DB=postgresql+psycopg2://${AUGUR_DB_USER:-augur}:${AUGUR_DB_PASSWORD:-augur}@augur-db:5432/augur" - - "AUGUR_DB_SCHEMA_BUILD=1" - - "AUGUR_GITHUB_API_KEY=${AUGUR_GITHUB_API_KEY}" - - "AUGUR_GITLAB_API_KEY=${AUGUR_GITLAB_API_KEY}" - - "AUGUR_GITHUB_USERNAME=${AUGUR_GITHUB_USERNAME}" - - "AUGUR_GITLAB_USERNAME=${AUGUR_GITLAB_USERNAME}" - - REDIS_CONN_STRING=redis://redis:6379 - - RABBITMQ_CONN_STRING=amqp://${AUGUR_RABBITMQ_USERNAME:-augur}:${AUGUR_RABBITMQ_PASSWORD:-password123}@rabbitmq:5672/${AUGUR_RABBITMQ_VHOST:-augur_vhost} - depends_on: - - augur-db - - redis - - rabbitmq - -volumes: - facade: - driver: local - augurpostgres: - driver: local - - diff --git a/setup.py b/setup.py index 8591f483e2..a356fef858 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ "pandas==1.5.3", # 1.4.3 "numpy==1.26.0", # 1.23.2 "requests==2.32.0", # 2.28.1 - "psycopg2-binary==2.9.3", #2.9.3 what is pscopg-binary 3.0.16 + "psycopg2-binary==2.9.9", #2.9.3 what is pscopg-binary 3.0.16 "click==8.0.3", # 8.1.3 "psutil==5.8.0", # 5.9.1 "gunicorn==22.0.0", # 20.1.0