Convert pipeline container to a multi-part build

.dockerignore

@@ -1,9 +1,39 @@
-*
-!requirements.txt
-!Cargo.toml
-!Cargo.lock
-!utils
-!openssl
-!uv.lock
-!pyproject.toml
-!Makefile
+# Exclude build and development files
+.git/
+.github/
+*.nf
+workflows/
+tests/
+containers/
+Dockerfile.old
+.dockerignore
+.gitignore
+.pre-commit-config.yaml
+pytest.ini
+.python-version
+RELEASE.rst
+LICENSE
+README.md
+*.md
+
+# Python artifacts
+__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+.coverage
+htmlcov/
+*.egg-info/
+dist/
+build/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db

.github/workflows/main.yaml

@@ -22,7 +22,75 @@ jobs:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          MSG_MINIMAL: true
 
+  wait-for-rust-build:
+    runs-on: ubuntu-latest
+    outputs:
+      rust_changed: ${{ steps.check-rust.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: Check if Rust files changed
+        id: check-rust
+        run: |
+          if git diff --name-only HEAD^ HEAD | grep -qE '^(utils/|Cargo\.(toml|lock))'; then
+            echo "changed=true" >> $GITHUB_OUTPUT
+            echo "Rust files changed, waiting for rust-container workflow..."
+          else
+            echo "changed=false" >> $GITHUB_OUTPUT
+            echo "No Rust changes, skipping wait"
+          fi
+
+      - name: Wait for Rust container build
+        if: steps.check-rust.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "Waiting for rust-container workflow to complete..."
+          sleep 10  # Give workflow time to start
+
+          # Wait up to 30 minutes for rust-container workflow to complete
+          timeout=1800
+          elapsed=0
+          while [ $elapsed -lt $timeout ]; do
+            # Check for running or completed rust-container workflows for this commit
+            status=$(gh run list \
+              --workflow=rust-container.yaml \
+              --commit=${{ github.sha }} \
+              --json status,conclusion \
+              --jq '.[0] | "\(.status):\(.conclusion)"')
+
+            if [ -z "$status" ]; then
+              echo "No rust-container workflow found yet, waiting..."
+              sleep 10
+              elapsed=$((elapsed + 10))
+              continue
+            fi
+
+            workflow_status=$(echo "$status" | cut -d: -f1)
+            workflow_conclusion=$(echo "$status" | cut -d: -f2)
+
+            if [ "$workflow_status" = "completed" ]; then
+              if [ "$workflow_conclusion" = "success" ]; then
+                echo "✓ Rust container build completed successfully"
+                exit 0
+              else
+                echo "✗ Rust container build failed with conclusion: $workflow_conclusion"
+                exit 1
+              fi
+            fi
+
+            echo "Rust container build status: $workflow_status (elapsed: ${elapsed}s)"
+            sleep 15
+            elapsed=$((elapsed + 15))
+          done
+
+          echo "✗ Timeout waiting for rust-container workflow"
+          exit 1
+
   create-docker-image:
+    needs: wait-for-rust-build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/rust-container.yaml

@@ -0,0 +1,47 @@
+# GitHub Actions workflow for building and pushing Rust utilities container
+# Rebuilds when Rust code changes (utils/**, Cargo.toml, Cargo.lock)
+
+name: Build Rust Utilities Container
+
+on:
+  push:
+    branches: ['master', 'dev']
+    paths:
+      - 'utils/**'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - 'containers/rust-utils/**'
+      - '.github/workflows/rust-container.yaml'
+  workflow_dispatch:
+    inputs:
+      force_rebuild:
+        description: 'Force rebuild Rust utilities container'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  build-rust-utils:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Docker login
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+      - name: Build Rust utilities container
+        run: docker build -t rnacentral/rust-utils:latest -f containers/rust-utils/Dockerfile .
+
+      - name: Push to Docker Hub
+        run: docker push rnacentral/rust-utils:latest
+
+      - name: Slack notification
+        if: always()
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_MESSAGE: 'Rust utilities container built and pushed: ${{ job.status }}'
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          MSG_MINIMAL: true

.github/workflows/tool-containers.yaml

@@ -0,0 +1,119 @@
+# GitHub Actions workflow for building and pushing tool containers
+# These containers cache slow-to-build bioinformatics tools (Infernal, Samtools)
+
+name: Build and Push Tool Containers
+
+on:
+  push:
+    branches: ['master', 'dev']
+    paths:
+      - 'containers/infernal/**'
+      - 'containers/samtools/**'
+      - '.github/workflows/tool-containers.yaml'
+  workflow_dispatch:
+    inputs:
+      force_rebuild:
+        description: 'Force rebuild all tool containers'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      infernal: ${{ steps.changes.outputs.infernal }}
+      samtools: ${{ steps.changes.outputs.samtools }}
+      force: ${{ github.event.inputs.force_rebuild == 'true' }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: Detect changed files
+        id: changes
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.force_rebuild }}" = "true" ]; then
+            echo "infernal=true" >> $GITHUB_OUTPUT
+            echo "samtools=true" >> $GITHUB_OUTPUT
+          else
+            # Check if infernal files changed
+            if git diff --name-only HEAD^ HEAD | grep -q "^containers/infernal/"; then
+              echo "infernal=true" >> $GITHUB_OUTPUT
+            else
+              echo "infernal=false" >> $GITHUB_OUTPUT
+            fi
+
+            # Check if samtools files changed
+            if git diff --name-only HEAD^ HEAD | grep -q "^containers/samtools/"; then
+              echo "samtools=true" >> $GITHUB_OUTPUT
+            else
+              echo "samtools=false" >> $GITHUB_OUTPUT
+            fi
+          fi
+
+  build-infernal:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.infernal == 'true' || needs.detect-changes.outputs.force == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Docker login
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+      - name: Build Infernal container
+        run: docker build -t rnacentral/infernal:1.1.2 -f containers/infernal/Dockerfile containers/infernal/
+
+      - name: Tag as latest
+        run: docker tag rnacentral/infernal:1.1.2 rnacentral/infernal:latest
+
+      - name: Push versioned tag
+        run: docker push rnacentral/infernal:1.1.2
+
+      - name: Push latest tag
+        run: docker push rnacentral/infernal:latest
+
+      - name: Slack notification
+        if: always()
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_MESSAGE: 'Infernal container built and pushed: ${{ job.status }}'
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          MSG_MINIMAL: true
+
+  build-samtools:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.samtools == 'true' || needs.detect-changes.outputs.force == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Docker login
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+      - name: Build Samtools container
+        run: docker build -t rnacentral/samtools:1.18 -f containers/samtools/Dockerfile containers/samtools/
+
+      - name: Tag as latest
+        run: docker tag rnacentral/samtools:1.18 rnacentral/samtools:latest
+
+      - name: Push versioned tag
+        run: docker push rnacentral/samtools:1.18
+
+      - name: Push latest tag
+        run: docker push rnacentral/samtools:latest
+
+      - name: Slack notification
+        if: always()
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_MESSAGE: 'Samtools container built and pushed: ${{ job.status }}'
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          MSG_MINIMAL: true

Cargo.toml

@@ -1,4 +1,14 @@
 [workspace]
 members = [
-  "utils/*",
+  "utils/bed-expander",
+  "utils/expand-urs",
+  "utils/ftp-export",
+  "utils/json2dfasta",
+  "utils/json2fasta",
+  "utils/precompute",
+  "utils/rnc-core",
+  "utils/rnc-test-utils",
+  "utils/rnc-utils",
+  "utils/search-export",
+  "utils/split-ena",
 ]