Merge branch 'develop' into ks_standard_mem_env

Author: kshakir

.circleci/config.yml

@@ -0,0 +1,47 @@
+#This is a series of steps that will setup Java/sbt/Cromwell on the local runner.
+#These steps are meant to be re-used and invoked by other Github Action Workflows
+name: 'Set Up Cromwell Steps'
+description: Specific steps that will set up git secrets, java, sbt, and Cromwell on the local machine. 
+inputs:
+  cromwell_repo_token: #As an input to this action, you are required to pass in a token that can be used to authenticate while checking out Cromwell.
+    required: true
+
+runs:
+  using: "composite" # <-- this allows these steps to be used by other workflows.
+  steps:
+      #Allows this github action to use a cache to store stuff like Java and sbt files between runs.
+      - uses: actions/checkout@v3
+        name: Checkout Coursier Cache
+      - uses: coursier/cache-action@v6  
+        name: Enable Coursier Cache
+
+      #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script.
+      - name: Git secrets setup 
+        run: |
+          git clone https://github.com/awslabs/git-secrets.git ~/git-secrets
+          cd ~/git-secrets
+          git checkout ad82d68ee924906a0401dfd48de5057731a9bc84
+          sudo make install
+        shell: bash
+
+      - name: Secrets check
+        run: |
+          sudo ln -s "$(which echo)" /usr/local/bin/say
+          ./minnie-kenny.sh --force
+          git secrets --scan-history
+        shell: bash
+
+      #Clone the cromwell repo to this VM.
+      - name: Clone Cromwell
+        uses: actions/checkout@v3
+        with:
+          repository: broadinstitute/cromwell
+          token: ${{ inputs.cromwell_repo_token }}
+
+      #Install Java to this VM. This Java version and distribution is compatible with Cromwell.
+      - name: Setup JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: 11
+

.github/set_up_cromwell_action/action.yml

@@ -41,19 +41,33 @@ jobs:
         echo "CROMWELL_NUMBER=$((previous_version + 1))" >> $GITHUB_ENV
     - name: Save complete image ID
       run: |
-        echo "CROMWELL_SNAP_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA-SNAP"`" >> $GITHUB_ENV
+        echo "CROMWELL_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA"`" >> $GITHUB_ENV
     # `DSDEJENKINS_PASSWORD` auto syncs from vault with https://github.com/broadinstitute/terraform-ap-deployments/pull/614
     - name: Login to Docker Hub
       uses: docker/login-action@v1
       with:
         username: dsdejenkins
         password: ${{ secrets.DSDEJENKINS_PASSWORD }}
+    # Build & push `cromwell`, `womtool`, `cromiam`, and `cromwell-drs-localizer`
     - name: Build Cromwell Docker
       run: |
         set -e
         cd cromwell
-        sbt server/docker
-        docker push broadinstitute/cromwell:$CROMWELL_SNAP_VERSION
+        sbt -Dproject.isSnapshot=false -Dproject.isRelease=false dockerBuildAndPush
+    - name: Deploy to dev and board release train (Cromwell)
+      uses: broadinstitute/repository-dispatch@master
+      with:
+        token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+        repository: broadinstitute/terra-helmfile
+        event-type: update-service
+        client-payload: '{"service": "cromwell", "version": "${{ env.CROMWELL_VERSION }}", "dev_only": false}'
+    - name: Deploy to dev and board release train (CromIAM)
+      uses: broadinstitute/repository-dispatch@master
+      with:
+        token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+        repository: broadinstitute/terra-helmfile
+        event-type: update-service
+        client-payload: '{"service": "cromiam", "version": "${{ env.CROMWELL_VERSION }}", "dev_only": false}'
     - name: Edit & push chart
       env:
         BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
@@ -62,10 +76,10 @@ jobs:
         cd cromwhelm
         git checkout main
         ls -la
-        sed -i "s/appVersion.*/appVersion: \"$CROMWELL_SNAP_VERSION\"/" cromwell-helm/Chart.yaml
-        sed -i "s/image: broadinstitute\/cromwell.*/image: broadinstitute\/cromwell:$CROMWELL_SNAP_VERSION/" cromwell-helm/templates/cromwell.yaml
+        sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" terra-batch-libchart/values.yaml
+
         git diff
         git config --global user.name "broadbot"
         git config --global user.email "broadbot@broadinstitute.org"
-        git commit -am "Auto update to Cromwell $CROMWELL_SNAP_VERSION"
+        git commit -am "Auto update to Cromwell $CROMWELL_VERSION"
         git push https://broadbot:$BROADBOT_GITHUB_TOKEN@github.com/broadinstitute/cromwhelm.git main

.github/workflows/chart_update_on_merge.yml

@@ -0,0 +1,33 @@
+name: 'Cromwell unit tests'
+
+#This github action runs all of Cromwell's unit tests.
+
+#This is what shows up in the github workflows page as the title. 
+run-name: ${{ github.actor }} running Cromwell sbt unit tests.
+
+#What will trigger the workflow to run. 
+on:
+  workflow_dispatch: #Manual trigger from GitHub UI
+  push:
+  merge_group:
+
+permissions: 
+  contents: read
+
+jobs:
+  build-and-test: 
+    #This action is using a Github free runner, rather than a Broad self-hosted one.
+    #This is because the Broad ones don't have sbt installed by default.
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3 # checkout the cromwell repo
+    - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell.
+      with:
+        cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+  
+    #Invoke SBT to run all unit tests for Cromwell. 
+    - name: Run tests
+      run: |
+        set -e
+        sbt "test"