diff --git a/.dockerignore b/.dockerignore index bb0224d..3eab569 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ .github/ +.git/ diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index fee4f10..b4530e7 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -1,20 +1,13 @@ --- name: build main -# Actions that take place after every commit the master/main branch. -# Here every commit is built, tagged as 'latest' and tested. +# Actions that take place after a workflow trigger on the master/main branch. # If a DOCKERHUB_USERNAME secret is defined the image is pushed. # If a TRIGGER_AWX secret is defined the image is deployed to Kubernetes. -# + # Actions also run if the repository is tagged. # Every tag is deployed to staging and every production-grade tag -# (of the form N.N.N) is deployed to production.defaults: -# -# Actions also run on a schedule - the the container is built, tested, -# pushed and deployed (if the relevant secrets are set) based on -# a defined schedule. -# -# Actions also run on external trigger (workflow-dispatch). +# (of the form N.N.N) is deployed to production. # --------------- # Control secrets @@ -24,9 +17,9 @@ name: build main # have the following GitHub 'Repository Secrets' defined # (i.e. via 'Settings -> Secrets'): - # -# BE_IMAGE_TAG optional - default latest +# BE_IMAGE_TAG optional - default is a valid version # BE_NAMESPACE optional - default xchem -# FE_BRANCH optional - default master +# FE_IMAGE_TAG optional - default is a valid version # FE_NAMESPACE optional - default xchem # STACK_NAMESPACE optional - default xchem # @@ -42,7 +35,8 @@ name: build main # 'awx/fragalysis-developer'. # You should not set this and TRIGGER_AWX. # -# SLACK_NOTIFY_WEBHOOK optional - required for Slack notifications +# SLACK_NOTIFY_STAGING_WEBHOOK optional - required for Slack notifications +# SLACK_NOTIFY_PRODUCTION_WEBHOOK optional - required for Slack notifications # # ----------- # Environment (GitHub Environments) @@ -62,10 +56,6 @@ name: build main on: push: - branches: - - 'master' - - 'main' - - '703' tags: - '**' # Build if triggered externally. @@ -76,16 +66,19 @@ on: description: The fragalysis-backend namespace (to pull from) required: false be_image_tag: - description: The fragalysis-backend image contaienr tag (to pull from) + description: The fragalysis-backend image container tag (to pull from) required: false fe_namespace: description: The fragalysis-frontend namespace (to clone from) required: false - fe_branch: - description: The fragalysis-frontend branch (to clone from) + fe_image_tag: + description: The fragalysis-frontend image container tag (to pull from) required: false stack_namespace: - description: The fragalysis-stack namespace (to publish to) + description: The fragalysis-stack Docker Hub namespace (to publish to) + required: false + stack_image_tag: + description: The image tag to apply to the fragalysis-stack image required: false env: @@ -96,34 +89,39 @@ env: # # For Jobs conditional on the presence of a secret see this Gist... # https://gist.github.com/jonico/24ffebee6d2fa2e679389fac8aef50a3 - BE_IMAGE_TAG: latest + # + # New (tagged) production stack builds should always be preceded by a change to one + # or both of the Backend or Frontend tags. i.e. before we make a production + # release the author needs to change one or both of: - + # + # - BE_IMAGE_TAG + # - FE_IMAGE_TAG + # + # + # Commit the changes and then tag or make a release from the stack repository. + BE_IMAGE_TAG: 2026.02.2 + FE_IMAGE_TAG: 2026.01.1 BE_NAMESPACE: xchem - FE_BRANCH: master FE_NAMESPACE: xchem STACK_NAMESPACE: xchem - # Common slack notification variables. - # Used in the rtCamp/action-slack-notify Action. - SLACK_USERNAME: ActionBot - SLACK_ICON: https://github.com/InformaticsMatters/dls-fragalysis-stack-kubernetes/raw/master/icons/094-robot-face-3-512.png?size=48 jobs: build: runs-on: ubuntu-latest outputs: deploy: ${{ steps.vars.outputs.deploy }} - deploy-developer: ${{ steps.vars.outputs.deploy-developer }} - production-tag: ${{ steps.vars.outputs.production-tag }} + deploy_developer: ${{ steps.vars.outputs.deploy_developer }} + production_tag: ${{ steps.vars.outputs.production_tag }} push: ${{ steps.vars.outputs.push }} tag: ${{ steps.vars.outputs.tag }} version: ${{ steps.vars.outputs.version }} steps: - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v3.x + uses: rlespinasse/github-slug-action@v4 - name: Initialise workflow variables id: vars env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - SLACK_NOTIFY_WEBHOOK: ${{ secrets.SLACK_NOTIFY_WEBHOOK }} TRIGGER_AWX: ${{ secrets.TRIGGER_AWX }} TRIGGER_DEVELOPER_AWX: ${{ secrets.TRIGGER_DEVELOPER_AWX }} run: | @@ -131,162 +129,265 @@ jobs: BE_NAMESPACE="${{ env.BE_NAMESPACE }}" if [ -n "${{ github.event.inputs.be_namespace }}" ]; then BE_NAMESPACE="${{ github.event.inputs.be_namespace }}"; elif [ -n "${{ secrets.BE_NAMESPACE }}" ]; then BE_NAMESPACE="${{ secrets.BE_NAMESPACE }}"; fi - echo set-output name=BE_NAMESPACE::${BE_NAMESPACE} - echo ::set-output name=BE_NAMESPACE::${BE_NAMESPACE} + echo BE_NAMESPACE=${BE_NAMESPACE} + echo "BE_NAMESPACE=${BE_NAMESPACE}" >> $GITHUB_OUTPUT # BE_IMAGE_TAG BE_IMAGE_TAG="${{ env.BE_IMAGE_TAG }}" if [ -n "${{ github.event.inputs.be_image_tag }}" ]; then BE_IMAGE_TAG="${{ github.event.inputs.be_image_tag }}"; elif [ -n "${{ secrets.BE_IMAGE_TAG }}" ]; then BE_IMAGE_TAG="${{ secrets.BE_IMAGE_TAG }}"; fi - echo set-output name=BE_IMAGE_TAG::${BE_IMAGE_TAG} - echo ::set-output name=BE_IMAGE_TAG::${BE_IMAGE_TAG} + echo BE_IMAGE_TAG=${BE_IMAGE_TAG} + echo "BE_IMAGE_TAG=${BE_IMAGE_TAG}" >> $GITHUB_OUTPUT # FE_NAMESPACE FE_NAMESPACE="${{ env.FE_NAMESPACE }}" if [ -n "${{ github.event.inputs.fe_namespace }}" ]; then FE_NAMESPACE="${{ github.event.inputs.fe_namespace }}"; elif [ -n "${{ secrets.FE_NAMESPACE }}" ]; then FE_NAMESPACE="${{ secrets.FE_NAMESPACE }}"; fi - echo set-output name=FE_NAMESPACE::${FE_NAMESPACE} - echo ::set-output name=FE_NAMESPACE::${FE_NAMESPACE} + echo FE_NAMESPACE=${FE_NAMESPACE} + echo "FE_NAMESPACE=${FE_NAMESPACE}" >> $GITHUB_OUTPUT - # FE_BRANCH - FE_BRANCH="${{ env.FE_BRANCH }}" - if [ -n "${{ github.event.inputs.fe_branch }}" ]; then FE_BRANCH="${{ github.event.inputs.fe_branch }}"; - elif [ -n "${{ secrets.FE_BRANCH }}" ]; then FE_BRANCH="${{ secrets.FE_BRANCH }}"; fi - echo set-output name=FE_BRANCH::${FE_BRANCH} - echo ::set-output name=FE_BRANCH::${FE_BRANCH} + # FE_IMAGE_TAG + FE_IMAGE_TAG="${{ env.FE_IMAGE_TAG }}" + if [ -n "${{ github.event.inputs.fe_image_tag }}" ]; then FE_IMAGE_TAG="${{ github.event.inputs.fe_image_tag }}"; + elif [ -n "${{ secrets.FE_IMAGE_TAG }}" ]; then FE_IMAGE_TAG="${{ secrets.FE_IMAGE_TAG }}"; fi + echo FE_IMAGE_TAG=${FE_IMAGE_TAG} + echo "FE_IMAGE_TAG=${FE_IMAGE_TAG}" >> $GITHUB_OUTPUT # STACK_NAMESPACE STACK_NAMESPACE="${{ env.STACK_NAMESPACE }}" if [ -n "${{ github.event.inputs.stack_namespace }}" ]; then STACK_NAMESPACE="${{ github.event.inputs.stack_namespace }}"; elif [ -n "${{ secrets.STACK_NAMESPACE }}" ]; then STACK_NAMESPACE="${{ secrets.STACK_NAMESPACE }}"; fi - echo set-output name=STACK_NAMESPACE::${STACK_NAMESPACE} - echo ::set-output name=STACK_NAMESPACE::${STACK_NAMESPACE} + echo STACK_NAMESPACE=${STACK_NAMESPACE} + echo "STACK_NAMESPACE=${STACK_NAMESPACE}" >> $GITHUB_OUTPUT + + # Set a version + STACK_VERSION="0.0.0" + if [[ "${{ github.ref }}" =~ ^refs/tags/ ]]; then STACK_VERSION="${{ env.GITHUB_REF_SLUG }}"; + else STACK_VERSION="${{ github.ref_name }}.${{ github.run_number }}"; fi + echo STACK_VERSION=${STACK_VERSION} + echo "STACK_VERSION=${STACK_VERSION}" >> $GITHUB_OUTPUT - # What image tag are we using? 'latest' (if not tagged) or a GitHub tag? + # What image tag are we creating? 'latest' (if not tagged) or a GitHub tag? TAG="latest" if [[ "${{ github.ref }}" =~ ^refs/tags/ ]]; then TAG="${{ env.GITHUB_REF_SLUG }}"; fi - echo set-output name=tag::${TAG} - echo ::set-output name=tag::${TAG} - - # Set a version - VERSION="0.0.0" - if [[ "${{ github.ref }}" =~ ^refs/tags/ ]]; then VERSION="${{ env.GITHUB_REF_SLUG }}"; - else VERSION="${{ github.ref_name }}.${{ github.run_number }}"; fi - echo set-output name=version::${VERSION} - echo ::set-output name=version::${VERSION} + echo tag=${TAG} + echo "tag=${TAG}" >> $GITHUB_OUTPUT # Do we push, i.e. is DOCKERHUB_USERNAME defined? - echo set-output name=push::${{ env.DOCKERHUB_USERNAME != '' }} - echo ::set-output name=push::${{ env.DOCKERHUB_USERNAME != '' }} + echo push=${{ env.DOCKERHUB_USERNAME != '' }} + echo "push=${{ env.DOCKERHUB_USERNAME != '' }}" >> $GITHUB_OUTPUT # Do we deploy official images, i.e. is TRIGGER_AWX 'yes'? - echo set-output name=deploy::${{ env.TRIGGER_AWX == 'yes' }} - echo ::set-output name=deploy::${{ env.TRIGGER_AWX == 'yes' }} + echo deploy=${{ env.TRIGGER_AWX == 'yes' }} + echo "deploy=${{ env.TRIGGER_AWX == 'yes' }}" >> $GITHUB_OUTPUT # Do we deploy developer images, i.e. is TRIGGER_DEVELOPER_AWX 'yes'? - echo set-output name=deploy-developer::${{ env.TRIGGER_DEVELOPER_AWX == 'yes' }} - echo ::set-output name=deploy-developer::${{ env.TRIGGER_DEVELOPER_AWX == 'yes' }} + echo deploy_developer=${{ env.TRIGGER_DEVELOPER_AWX == 'yes' }} + echo "deploy_developer=${{ env.TRIGGER_DEVELOPER_AWX == 'yes' }}" >> $GITHUB_OUTPUT # Do we deploy to production, i.e. is there a TAG of the form N.N.N? HAS_PRODUCTION_TAG=false if [[ ${{ env.GITHUB_REF_SLUG }} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then HAS_PRODUCTION_TAG=true; fi - echo set-output name=production-tag::${HAS_PRODUCTION_TAG} - echo ::set-output name=production-tag::${HAS_PRODUCTION_TAG} + echo production_tag=${HAS_PRODUCTION_TAG} + echo "production_tag=${HAS_PRODUCTION_TAG}" >> $GITHUB_OUTPUT # Do we send Slack notifications, i.e. is SLACK_NOTIFY_WEBHOOK defined? - echo set-output name=notify::${{ env.SLACK_NOTIFY_WEBHOOK != '' }} - echo ::set-output name=notify::${{ env.SLACK_NOTIFY_WEBHOOK != '' }} + echo notify=${{ env.SLACK_NOTIFY_WEBHOOK != '' }} + echo "notify=${{ env.SLACK_NOTIFY_WEBHOOK != '' }}" >> $GITHUB_OUTPUT - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Display build args run: | echo BE_NAMESPACE=${{ steps.vars.outputs.BE_NAMESPACE }} echo BE_IMAGE_TAG=${{ steps.vars.outputs.BE_IMAGE_TAG }} echo FE_NAMESPACE=${{ steps.vars.outputs.FE_NAMESPACE }} - echo FE_BRANCH=${{ steps.vars.outputs.FE_BRANCH }} - - name: Set VERSION file - run: | - echo VERSION="${{ steps.vars.outputs.version }}" - echo "${{ steps.vars.outputs.version }}" > VERSION + echo FE_IMAGE_TAG=${{ steps.vars.outputs.FE_IMAGE_TAG }} + echo STACK_NAMESPACE=${{ steps.vars.outputs.STACK_NAMESPACE }} + echo STACK_VERSION=${{ steps.vars.outputs.STACK_VERSION }} - name: Build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: tags: ${{ steps.vars.outputs.STACK_NAMESPACE }}/fragalysis-stack:${{ steps.vars.outputs.tag }} build-args: | BE_NAMESPACE=${{ steps.vars.outputs.BE_NAMESPACE }} BE_IMAGE_TAG=${{ steps.vars.outputs.BE_IMAGE_TAG }} FE_NAMESPACE=${{ steps.vars.outputs.FE_NAMESPACE }} - FE_BRANCH=${{ steps.vars.outputs.FE_BRANCH }} - STACK_NAMESPACE: ${{ steps.vars.outputs.STACK_NAMESPACE }} - - name: Test - run: > - docker-compose -f docker-compose.test.yml up - --exit-code-from tests - --abort-on-container-exit - env: - STACK_NAMESPACE: ${{ steps.vars.outputs.STACK_NAMESPACE }} - STACK_TAG: ${{ steps.vars.outputs.tag }} + FE_IMAGE_TAG=${{ steps.vars.outputs.FE_IMAGE_TAG }} + STACK_NAMESPACE=${{ steps.vars.outputs.STACK_NAMESPACE }} + STACK_VERSION=${{ steps.vars.outputs.tag }} + - name: Vulnerability Scan (OSV) + # The vulnerability scan (using Google's OSV) + # Allowed to fail (we force true) as it's used for information only + # The generated "Common Vulnerability Scoring System (CVSS)"" values + # are interpreted as... + # + # - None: 0.0 (No impact) + # - Low: 0.1–3.9 (Minor risk) + # - Medium: 4.0–6.9 (Moderate risk) + # - High: 7.0–8.9 (Serious risk) + # - Critical: 9.0–10.0 (Grave, severe risk) + run: | + wget https://github.com/google/osv-scanner/releases/latest/download/osv-scanner_linux_amd64 -O osv-scanner + chmod +x osv-scanner + ./osv-scanner scan image ${{ steps.vars.outputs.STACK_NAMESPACE }}/fragalysis-stack:${{ steps.vars.outputs.tag }} \ + --format markdown --output scan.md || true + cat scan.md >> $GITHUB_STEP_SUMMARY - name: Login to DockerHub if: steps.vars.outputs.push == 'true' - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Push if: steps.vars.outputs.push == 'true' run: docker push ${{ steps.vars.outputs.STACK_NAMESPACE }}/fragalysis-stack:${{ steps.vars.outputs.tag }} - - name: Notify build - if: steps.vars.outputs.notify == 'true' + +# Behaviour tests are too fragile. +# Disabled for now (achristie) +# +# trigger-behaviour-tests: +# # Trigger behaviour tests +# # (if deploying, but not to production) +# needs: build +# if: | +# needs.build.outputs.push == 'true' && +# needs.build.outputs.deploy == 'true' && +# needs.build.outputs.production_tag == 'false' +# runs-on: ubuntu-latest +# steps: +# - name: Trigger behaviour tests +# uses: informaticsmatters/trigger-ci-action@v1 +# with: +# ci-owner: xchem +# ci-repository: fragalysis-stack-behaviour-tests +# ci-name: latest stack test +# ci-user: ${{ secrets.STACK_USER }} +# ci-user-token: ${{ secrets.STACK_USER_TOKEN }} + + deploy-staging: + # A fixed job that deploys to the Fragalysis Staging Kubernetes Namespace + # using a pre-defined AWX Job Template name, + # and the awx/fragalysis-production environment. + # + # All builds, tagged or otherwise, are deployed to staging. + needs: build + if: | + needs.build.outputs.push == 'true' && + needs.build.outputs.deploy == 'true' + runs-on: ubuntu-latest + environment: awx/fragalysis-production + env: + slack_notify_staging_webhook: ${{ secrets.SLACK_NOTIFY_STAGING_WEBHOOK }} + steps: + - name: Notify staging deployment started + if: ${{ env.slack_notify_staging_webhook != '' }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ env.slack_notify_staging_webhook }} + SLACK_TITLE: A new STAGING deployment has begun + SLACK_MESSAGE: Image tag is "${{ needs.build.outputs.tag }}" + SLACK_FOOTER: '' + MSG_MINIMAL: true + - name: Deploy staging + id: deploy_staging + uses: informaticsmatters/trigger-awx-action@v3 + with: + template: Staging Fragalysis Stack + template-host: ${{ secrets.AWX_HOST }} + template-user: ${{ secrets.AWX_USER }} + template-user-password: ${{ secrets.AWX_USER_PASSWORD }} + template-var: stack_image_tag + template-var-value: ${{ needs.build.outputs.tag }} + continue-on-error: true + - name: Notify staging deployment failure + if: ${{ env.slack_notify_staging_webhook != '' && steps.deploy_staging.outcome == 'failure' }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ env.slack_notify_staging_webhook }} + SLACK_TITLE: The STAGING deployment FAILED + SLACK_MESSAGE: Please review the corresponding fragalysis-stack GitHuib Action Log + SLACK_FOOTER: '' + MSG_MINIMAL: true + - name: Notify staging deployment complete + if: ${{ env.slack_notify_staging_webhook != '' && steps.deploy_staging.outcome == 'success' }} uses: rtCamp/action-slack-notify@v2 env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY_WEBHOOK }} - SLACK_TITLE: Build Complete - SLACK_MESSAGE: Built image ${{ steps.vars.outputs.STACK_NAMESPACE }}/fragalysis-stack:${{ steps.vars.outputs.tag }} + SLACK_WEBHOOK: ${{ env.slack_notify_staging_webhook }} + SLACK_TITLE: A new STAGING deployment is ready + SLACK_MESSAGE: Image tag is "${{ needs.build.outputs.tag }}" + SLACK_FOOTER: '' + MSG_MINIMAL: true deploy-production: - # A fixed job that "deploys to the Fragalysis Staging" Kubernetes Namespace - # using a pre-defined AWX Job Template name + # A fixed job that deploys to the Fragalysis Production Kubernetes Namespace + # using a pre-defined AWX Job Template name, # and the awx/fragalysis-production environment. + # + # Only builds triggered by production-grade tags are deployed to production. needs: build if: | needs.build.outputs.push == 'true' && needs.build.outputs.deploy == 'true' && - needs.build.outputs.production-tag == 'true' + needs.build.outputs.production_tag == 'true' runs-on: ubuntu-latest environment: awx/fragalysis-production + env: + slack_notify_production_webhook: ${{ secrets.SLACK_NOTIFY_PRODUCTION_WEBHOOK }} steps: + - name: Notify production deployment started + if: ${{ env.slack_notify_production_webhook != '' }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ env.slack_notify_production_webhook }} + SLACK_TITLE: A new PRODUCTION deployment has begun + SLACK_MESSAGE: Image tag is "${{ needs.build.outputs.tag }}" + SLACK_FOOTER: '' + MSG_MINIMAL: true - name: Deploy production - uses: informaticsmatters/trigger-awx-action@v1 + id: deploy_production + uses: informaticsmatters/trigger-awx-action@v3 with: - template: Production Fragalysis Stack (Version Change) + template: Production Fragalysis Stack template-host: ${{ secrets.AWX_HOST }} template-user: ${{ secrets.AWX_USER }} template-user-password: ${{ secrets.AWX_USER_PASSWORD }} template-var: stack_image_tag template-var-value: ${{ needs.build.outputs.tag }} - - name: Notify production deployment - if: steps.vars.outputs.notify == 'true' + continue-on-error: true + - name: Notify production deployment failure + if: ${{ env.slack_notify_production_webhook != '' && steps.deploy_production.outcome == 'failure' }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ env.slack_notify_production_webhook }} + SLACK_TITLE: The PRODUCTION deployment FAILED + SLACK_MESSAGE: Please review the corresponding fragalysis-stack GitHuib Action Log + SLACK_FOOTER: '' + MSG_MINIMAL: true + - name: Notify production deployment complete + if: ${{ env.slack_notify_production_webhook != '' && steps.deploy_production.outcome == 'success'}} uses: rtCamp/action-slack-notify@v2 env: - SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFY_WEBHOOK }} - SLACK_TITLE: Deployment Complete - SLACK_MESSAGE: Deployed to awx/fragalysis-production + SLACK_WEBHOOK: ${{ env.slack_notify_production_webhook }} + SLACK_TITLE: A new PRODUCTION deployment is ready + SLACK_MESSAGE: Image tag is "${{ needs.build.outputs.tag }}" + SLACK_FOOTER: '' + MSG_MINIMAL: true deploy-developer: - # A "deploy to a developer's Fragalysis" Kubernetes Namespace - # using an environment-defined AWX Job Template name + # Deploys to a developer's Fragalysis Kubernetes Namespace + # using an environment-defined AWX Job Template name, # and the awx/fragalysis-developer environment. needs: build if: | needs.build.outputs.push == 'true' && - needs.build.outputs.deploy-developer == 'true' + needs.build.outputs.deploy_developer == 'true' runs-on: ubuntu-latest environment: awx/fragalysis-developer steps: - name: Deploy developer - uses: informaticsmatters/trigger-awx-action@v1 + uses: informaticsmatters/trigger-awx-action@v3 with: template: ${{ secrets.AWX_TEMPLATE_NAME }} template-host: ${{ secrets.AWX_HOST }} diff --git a/.gitignore b/.gitignore index 723ef36..52e533e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -.idea \ No newline at end of file +.idea + +data/ diff --git a/Dockerfile b/Dockerfile index 9499b66..ae5bebc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,72 +1,59 @@ ARG BE_NAMESPACE=xchem ARG BE_IMAGE_TAG=latest -FROM ${BE_NAMESPACE}/fragalysis-backend:${BE_IMAGE_TAG} +ARG FE_NAMESPACE=xchem +ARG FE_IMAGE_TAG=latest +ARG STACK_NAMESPACE=xchem +ARG STACK_VERSION=0.0.0 +# Start with the frontend container image AS 'frontend'. +# As part of the build we will copy the contents of its '/frontend' directory +# into the backend image that we also pull in. +FROM ${FE_NAMESPACE}/fragalysis-frontend:${FE_IMAGE_TAG} AS frontend # We have to repeat the ARG assignments... # ARGs are reset during the FROM action +# See https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact # Us -ARG STACK_NAMESPACE=xchem -# Backend origin (a container) -ARG BE_NAMESPACE=xchem -ARG BE_IMAGE_TAG=latest -# By default this is hosted on the xchem project's master branch -# but it can be redirected with a couple of build-args. -ARG FE_NAMESPACE=xchem -ARG FE_BRANCH=master +ARG STACK_NAMESPACE +ARG STACK_VERSION +# Backend image identity +ARG BE_NAMESPACE +ARG BE_IMAGE_TAG +# Frontend image identity +ARG FE_NAMESPACE +ARG FE_IMAGE_TAG + +# Get the backend image +# (we'll copy the pre-compiled frontend into it) +FROM ${BE_NAMESPACE}/fragalysis-backend:${BE_IMAGE_TAG} + +# We have to repeat the ARG assignments... +ARG STACK_NAMESPACE +ARG STACK_VERSION +ARG BE_NAMESPACE +ARG BE_IMAGE_TAG +ARG FE_NAMESPACE +ARG FE_IMAGE_TAG # Set the container ENV to record the origin of the b/e and f/e +# (for diagnostic purposes) ENV BE_NAMESPACE ${BE_NAMESPACE} ENV BE_IMAGE_TAG ${BE_IMAGE_TAG} ENV FE_NAMESPACE ${FE_NAMESPACE} -ENV FE_BRANCH ${FE_BRANCH} +ENV FE_IMAGE_TAG ${FE_IMAGE_TAG} ENV STACK_NAMESPACE ${STACK_NAMESPACE} +ENV STACK_VERSION ${STACK_VERSION} ENV APP_ROOT /code -ENV APP_USER_ID 2000 -RUN useradd -c 'Container user' --user-group --uid ${APP_USER_ID} --home-dir ${APP_ROOT} -s /bin/bash frag -RUN apt-get update -y &&\ - apt-get install -y wget gnupg bzip2 &&\ - apt-get clean +# Copy the frontend code from the frontend container +WORKDIR ${APP_ROOT}/frontend +WORKDIR ${APP_ROOT}/static -# Install yarn (instead of npm) -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -RUN apt-get update -y && apt-get install -y yarn && apt-get clean - -# Install nodejs -RUN wget -q https://nodejs.org/download/release/v12.22.11/node-v12.22.11-linux-x64.tar.gz &&\ - mkdir -p /usr/local/lib/nodejs &&\ - tar -xf node-v12.22.11-linux-x64.tar.gz -C /usr/local/lib/nodejs &&\ - rm node-v12.22.11-linux-x64.tar.gz -ENV PATH /usr/local/lib/nodejs/node-v12.22.11-linux-x64/bin:$PATH - -# Add in the frontend code -RUN git clone https://github.com/${FE_NAMESPACE}/fragalysis-frontend ${APP_ROOT}/frontend -RUN cd ${APP_ROOT}/frontend && git checkout ${FE_BRANCH} - -# Now build the code -RUN cd ${APP_ROOT}/frontend && yarn install -RUN cd ${APP_ROOT}/frontend && yarn run build - -ADD docker-entrypoint.sh ${APP_ROOT}/docker-entrypoint.sh - -# Symlink these -RUN mkdir ${APP_ROOT}/static +COPY --from=frontend /frontend ${APP_ROOT}/frontend RUN ln -s ${APP_ROOT}/frontend/bundles/ ${APP_ROOT}/static/bundles -RUN chmod 755 ${APP_ROOT}/docker-entrypoint.sh -RUN chmod 755 ${APP_ROOT}/makemigrations.sh -RUN chmod 755 ${APP_ROOT}/launch-stack.sh - -#RUN chown -R ${APP_USER_ID} ${APP_ROOT} /run /var - -# The VSERION file records the origin of the stack -# and is set by the corresponding GitHub Action -COPY LICENSE /LICENSE -COPY README.md /README.md -COPY VERSION /code/VERSION - WORKDIR ${APP_ROOT} +# The entrypoint is a responsibility of the backend image + CMD ["./docker-entrypoint.sh"] diff --git a/LICENSE b/LICENSE index 8dada3e..864bfbd 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2025 Diamond Light Source Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index a64e9ce..acb01e1 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,81 @@ [![build main](https://github.com/xchem/fragalysis-stack/actions/workflows/build-main.yaml/badge.svg)](https://github.com/xchem/fragalysis-stack/actions/workflows/build-main.yaml) -[![experimental](http://badges.github.io/stability-badges/dist/experimental.svg)](http://github.com/xchem/fragalysis-stack) -[![Version](http://img.shields.io/badge/version-0.0.1-blue.svg?style=flat)](https://github.com/xchem/fragalysis-stack) +![GitHub tag (latest SemVer pre-release)](https://img.shields.io/github/v/tag/xchem/fragalysis-stack) [![License](http://img.shields.io/badge/license-Apache%202.0-blue.svg?style=flat)](https://github.com/xchem/fragalysis-stack/blob/master/LICENSE.txt) # Fragalysis stack -Docker setup for building a Django, RDKit and Postgres stack with neo4j -Simply run docker-compose up + +Docker setup for building a Django, RDKit and Postgres stack with neo4j. + +> There is no application code in this repository, it is a repository where the + stack application is *assembled* using environment variables that define the + origin of the [Backend] and [Frontend] container images. + +The stack is built and orchestrated using GitHub Actions and is deployed +to *staging* and *production* installations (**Namespaces** in a designated +Kubernetes cluster). + +The build variables `DOCKERHUB_USERNAME` and `TRIGGER_AWX` must be defined +for orchestration to take place. Staging deployments take place on every downstream +build (frontend and backend) and tag, and production deployments take place on every tag. + +You **MUST** make sure the Action variables that select the backend and frontend +container images are updated prior to every production release so the stack +uses the appropriate backend and frontend code. You will find these variables +in the `.github/workflows/build-main.yaml` action file: - + +- `BE_IMAGE_TAG` +- `FE_IMAGE_TAG` + +[More information on pushing to production](README.md#pushing-a-release-to-production) + +## Local development + +A docker-compose file provides a convenient way of launching the stack locally. +The suitability of the various docker-compose files is the responsibility of +the developer. + +Check the compose file, adjust accordingly, then: - + + docker-compose up -d + +> Containers in the docker-compose generally store persistent data in + the `./data` directory of this repository. These directories + are created automatically if they do not exist. + +When you're done you can tear everything down with: - + + docker-compose down + +## Pushing a release to production + +1. Create new releases for the [Frontend] and [Backend]. + + * [Create a new release for the frontend](https://github.com/xchem/fragalysis-frontend/releases/new) + * Create a new tag with the format: `YYYY.MM.#` where: + * `YYYY` is the current year + * `MM` is the current month + * `#` is the patch number (positive integer) + * Choose a target: `staging` or `production` + * Title the release + * Describe the release + * [Create a new release for the backend](https://github.com/xchem/fragalysis-backend/releases/new) + * Same as the frontend + + 2. Update [build-main.yaml](.github/workflows/build-main.yaml) with the new tags + * Change `FE_IMAGE_TAG` to the desired Frontend tag + * Change `BE_IMAGE_TAG` to the desired Backend tag + * Commit the changes to a new branch and start a pull request + * Wait for review and approval + * Wait for the [GitHub action](https://github.com/xchem/fragalysis-stack/actions) to complete (~10-20mins) + +> N.B. you can get the current Frontend, Backend, and Stack tags from the bottom of the Fragalysis menu + +3. Create a new release for [fragalysis-stack](https://github.com/xchem/fragalysis-stack/releases/new) + * Use the same tag convention as for the Frontend and Backend. + * Tags do not need to agree across the three repositories! + +--- + +[backend]: https://github.com/xchem/fragalysis-backend +[frontend]: https://github.com/xchem/fragalysis-frontend diff --git a/VERSION b/VERSION deleted file mode 100644 index bd52db8..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.0 \ No newline at end of file diff --git a/changelog.md b/changelog.md index 0fb31c4..919da23 100644 --- a/changelog.md +++ b/changelog.md @@ -1,4 +1,70 @@ # Changelog (from 22/07/2021) + + +## 16/12/2022 - version 2022.12.1 +Job launcher release +### Frontend ([PR #335](https://github.com/xchem/fragalysis-frontend/pull/320)). +:heavy_plus_sign: Job table shows running jobs, and autoreloads +:heavy_plus_sign: Improved snapshot loading + +### Backend ([PR #349-#361 #365 #367](https://github.com/xchem/fragalysis-backend/pull/347)). +:heavy_plus_sign: Bug fixing: backend timeout, error report + +### Resolved tickets. +[#977](https://github.com/m2ms/fragalysis-frontend/issues/917),[#996](https://github.com/m2ms/fragalysis-frontend/issues/996),[#979](https://github.com/m2ms/fragalysis-frontend/issues/979),[#1003](https://github.com/m2ms/fragalysis-frontend/issues/1003),[#1011](https://github.com/m2ms/fragalysis-frontend/issues/1011),[#935](https://github.com/m2ms/fragalysis-frontend/issues/935),[#995](https://github.com/m2ms/fragalysis-frontend/issues/995), + + +## 29/09/2022 - version 2022.09.0 +Job launcher release +### Frontend ([PR #300 #302 #306-#312 #314-#320](https://github.com/xchem/fragalysis-frontend/pull/320)). +:heavy_plus_sign: New functionality: Job Launcher +:heavy_plus_sign: Improved performance +:heavy_plus_sign: Bug fixing: Layout, filters, snapshots + +### Backend ([PR #330 #331 #334 #336-#347](https://github.com/xchem/fragalysis-backend/pull/347)). +:heavy_plus_sign: New functionality: Job Launcher +:heavy_plus_sign: Improved container building + +### Resolved tickets. +[#917](https://github.com/m2ms/fragalysis-frontend/issues/917), [#862](https://github.com/m2ms/fragalysis-frontend/issues/862), [#936](https://github.com/m2ms/fragalysis-frontend/issues/936), [#859](https://github.com/m2ms/fragalysis-frontend/issues/859), [#959](https://github.com/m2ms/fragalysis-frontend/issues/959), [#960](https://github.com/m2ms/fragalysis-frontend/issues/960), [#870](https://github.com/m2ms/fragalysis-frontend/issues/870), [#949](https://github.com/m2ms/fragalysis-frontend/issues/949), [#932](https://github.com/m2ms/fragalysis-frontend/issues/932), [#867](https://github.com/m2ms/fragalysis-frontend/issues/867), [#933](https://github.com/m2ms/fragalysis-frontend/issues/933), [#934](https://github.com/m2ms/fragalysis-frontend/issues/934), [#906](https://github.com/m2ms/fragalysis-frontend/issues/906), [#878](https://github.com/m2ms/fragalysis-frontend/issues/978), [#911](https://github.com/m2ms/fragalysis-frontend/issues/911), [#991](https://github.com/m2ms/fragalysis-frontend/issues/991), [#703](https://github.com/m2ms/fragalysis-frontend/issues/703), [#875](https://github.com/m2ms/fragalysis-frontend/issues/875), [#868](https://github.com/m2ms/fragalysis-frontend/issues/868), [#942](https://github.com/m2ms/fragalysis-frontend/issues/942), [#887](https://github.com/m2ms/fragalysis-frontend/issues/887), [#966](https://github.com/m2ms/fragalysis-frontend/issues/966), [#889](https://github.com/m2ms/fragalysis-frontend/issues/889), [#939](https://github.com/m2ms/fragalysis-frontend/issues/939), [#926](https://github.com/m2ms/fragalysis-frontend/issues/926), [#796](https://github.com/m2ms/fragalysis-frontend/issues/796), [#869](https://github.com/m2ms/fragalysis-frontend/issues/869), [#885](https://github.com/m2ms/fragalysis-frontend/issues/885), [#883](https://github.com/m2ms/fragalysis-frontend/issues/883), [#904](https://github.com/m2ms/fragalysis-frontend/issues/904), [#958](https://github.com/m2ms/fragalysis-frontend/issues/958), [#946](https://github.com/m2ms/fragalysis-frontend/issues/946), [#973](https://github.com/m2ms/fragalysis-frontend/issues/973), [#957](https://github.com/m2ms/fragalysis-frontend/issues/957), [#947](https://github.com/m2ms/fragalysis-frontend/issues/947), [#969](https://github.com/m2ms/fragalysis-frontend/issues/969), [#864](https://github.com/m2ms/fragalysis-frontend/issues/864), [#873](https://github.com/m2ms/fragalysis-frontend/issues/873), ..... + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## 20/07/2022 - version 2.7.1. @@ -18,7 +84,7 @@ Quick fix ### Frontend ([PR #299](https://github.com/xchem/fragalysis-frontend/pull/299)). :heavy_plus_sign: Bug fixed: RHS F button crashing -### Backend ([PR #325,#327,#328,#329](https://github.com/xchem/fragalysis-backend/pull/314)). +### Backend ([PR #325,#327,#328,#329](https://github.com/xchem/fragalysis-backend/pull/329)). :heavy_plus_sign: Bug fixed: download sdf files :heavy_plus_sign: Crystal ids in the sdf files diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dd3f896..90c70dc 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,3 +1,4 @@ +--- version: '3' services: diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index e5713a9..0000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: '3' - -services: - database: - image: postgres:12.2 - volumes: - - ../django_data:/var/lib/postgresql/data - environment: - POSTGRES_PASSWORD: fragalysis - POSTGRES_DB: frag - PGDATA: /var/lib/postgresql/data/pgdata - ports: - - "5432:5432" - tests: - image: ${STACK_NAMESPACE:-xchem}/fragalysis-stack:${STACK_TAG:-latest} - command: - - /code/wait-for-it.sh - - database:5432 - - -- - - /bin/bash - - /code/test_entry.sh - volumes: - - ../logs:/code/logs/ - - ../media:/code/media/ - environment: - POSTGRESQL_DATABASE: frag - POSTGRESQL_USER: postgres - POSTGRESQL_PASSWORD: fragalysis - POSTGRESQL_HOST: database - POSTGRESQL_PORT: 5432 - ports: - - "80:80" - depends_on: - - database diff --git a/docker-compose.yml b/docker-compose.yml index 344369f..bf213c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,29 +1,32 @@ +--- version: '3' services: + database: - image: postgres:12.2 + image: postgres:12.16-alpine3.18 + container_name: database volumes: - - ../django_data:/var/lib/postgresql/data + - ./data/postgresql/data:/var/lib/postgresql/data environment: POSTGRES_PASSWORD: fragalysis POSTGRES_DB: frag PGDATA: /var/lib/postgresql/data/pgdata ports: - "5432:5432" - cartridge: - image: informaticsmatters/rdkit_cartridge:Release_2017_09_1 - user: root - environment: - - POSTGRES_PASSWORD=password - volumes: - - ../cartridge_data:/var/lib/postgresql/data + healthcheck: + test: pg_isready -U postgres -d frag + interval: 10s + timeout: 2s + retries: 5 + start_period: 10s + + # The graph graph: - container_name: neo4j - image: neo4j - command: /bin/bash -c "sleep 1000" + image: neo4j:4.4.2 + container_name: graph ports: -# Comment these two out in produciton + # These are not used in production - "7474:7474" - "7687:7687" ulimits: @@ -31,26 +34,73 @@ services: soft: 40000 hard: 40000 volumes: - - ../neo4j/data:/data - - ../neo4j/logs:/logs + - ./data/neo4j/data:/data + - ./data/neo4j/logs:/logs environment: - NEO4J_AUTH=none - NEO4J_dbms_memory_pagecache_size=4G - web: - image: xchem/fragalysis-stack:latest + healthcheck: + test: wget http://localhost:7474 || exit 1 + interval: 10s + timeout: 10s + retries: 20 + start_period: 10s + + # Redis (Celery/Worker Broker) + redis: + image: redis:7.2.3-alpine3.18 + container_name: redis + ports: + - "6379:6379" + healthcheck: + test: redis-cli ping + interval: 10s + timeout: 2s + retries: 5 + start_period: 10s + + stack: + image: ${STACK_NAMESPACE:-xchem}/fragalysis-stack:${STACK_TAG:-latest} + container_name: stack + build: + context: . + dockerfile: Dockerfile command: /bin/bash /code/launch-stack.sh volumes: - - ../logs:/code/logs/ - - ../media:/code/media/ + - ./data/logs:/code/logs/ + - ./data/media:/code/media/ + - .:/code/ environment: - POSTGRESQL_DATABASE: frag POSTGRESQL_USER: postgres - POSTGRESQL_PASSWORD: fragalysis - POSTGRESQL_HOST: database - POSTGRESQL_PORT: 5432 + # Celery tasks need to run synchronously + CELERY_TASK_ALWAYS_EAGER: 'True' + # Error reporting and default/root log-level + FRAGALYSIS_BACKEND_SENTRY_DNS: ${FRAGALYSIS_BACKEND_SENTRY_DNS} + LOGGING_FRAMEWORK_ROOT_LEVEL: ${LOGGING_FRAMEWORK_ROOT_LEVEL:-INFO} + # Keycloak configuration + OIDC_KEYCLOAK_REALM: ${OIDC_KEYCLOAK_REALM} + OIDC_RP_CLIENT_ID: ${OIDC_RP_CLIENT_ID:-fragalysis-local} + OIDC_RP_CLIENT_SECRET: ${OIDC_RP_CLIENT_SECRET} + OIDC_AS_CLIENT_ID: ${OIDC_AS_CLIENT_ID:-account-server-api} + OIDC_DM_CLIENT_ID: ${OIDC_DM_CLIENT_ID:-data-manager-api} + OIDC_RENEW_ID_TOKEN_EXPIRY_MINUTES: '210' + # Squonk configuration + SQUONK2_VERIFY_CERTIFICATES: 'No' + SQUONK2_UNIT_BILLING_DAY: 3 + SQUONK2_PRODUCT_FLAVOUR: BRONZE + SQUONK2_SLUG: fs-local + SQUONK2_ORG_OWNER: ${SQUONK2_ORG_OWNER} + SQUONK2_ORG_OWNER_PASSWORD: ${SQUONK2_ORG_OWNER_PASSWORD} + SQUONK2_ORG_UUID: ${SQUONK2_ORG_UUID} + SQUONK2_UI_URL: ${SQUONK2_UI_URL} + SQUONK2_DMAPI_URL: ${SQUONK2_DMAPI_URL} + SQUONK2_ASAPI_URL: ${SQUONK2_ASAPI_URL} ports: - - "80:80" + - "8080:80" depends_on: - - database - - graph - - cartridge + database: + condition: service_healthy + redis: + condition: service_healthy + graph: + condition: service_healthy diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100644 index d733620..0000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -/code/launch-stack.sh