From c9335affce20e5692b79538a5c97540db1cb9394 Mon Sep 17 00:00:00 2001 From: Jay Hesselberth Date: Tue, 14 Jan 2025 06:15:30 -0700 Subject: [PATCH 1/3] Test pipeline on GHA --- .github/workflows/test-pipeline.yaml | 82 +++++++++++++++++++ ...{dl_test_data.sh => download_test_data.sh} | 0 cluster/{ => lsf}/config.yaml | 0 cluster/slurm/config.yaml | 39 +++++++++ 4 files changed, 121 insertions(+) create mode 100644 .github/workflows/test-pipeline.yaml rename .test/{dl_test_data.sh => download_test_data.sh} (100%) rename cluster/{ => lsf}/config.yaml (100%) create mode 100644 cluster/slurm/config.yaml diff --git a/.github/workflows/test-pipeline.yaml b/.github/workflows/test-pipeline.yaml new file mode 100644 index 0000000..4e7c27c --- /dev/null +++ b/.github/workflows/test-pipeline.yaml @@ -0,0 +1,82 @@ +name: Test snakemake 🐍 pipeline +on: + pull_request: + branches: + - main + +jobs: + test-pipeline: + runs-on: ubuntu-latest + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - "8888:3306" + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - name: Setup pipeline 🚰 + uses: actions/checkout@v4 + + - name: Setup slurm 🐌 + uses: koesterlab/setup-slurm-action@v1 + + - name: Cache conda 🏦 + id: cache-conda + uses: actions/cache@v4 + env: + # Increase this value to reset cache if environment.yml has not changed + CACHE_NUMBER_CONDA: 1 + with: + path: ~/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER_CONDA }}-${{hashFiles('environment.yml') }} + + - if: ${{ steps.cache-conda.outputs.cache-hit != 'true' }} + name: Setup conda 🐍 + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: aatrnaseqpipe + environment-file: environment.yml + # https://github.com/conda-incubator/setup-miniconda/issues/267 + # XXX don't use this: use-only-tar-bz2: true + + - if: ${{ steps.cache-conda.outputs.cache-hit == 'true' }} + name: Activate conda 🐍 + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: aatrnaseqpipe + + - run: conda info + - run: conda list + - run: conda config --show + + - name: Cache test data 🏦 + id: cache-test + uses: actions/cache@v4 + env: + # Increase this value to reset cache if environment.yml has not changed + CACHE_NUMBER_TEST_DATA: 0 + with: + path: ./.test + key: + # hash based on the FASTA reference + ${{ runner.os }}-test-data-${{ env.CACHE_NUMBER_TEST_DATA }}-${{hashFiles('./.test/sacCer3-mature-tRNAs-dual-adapt-v2.fa')}} + + - if: ${{ steps.cache-test.outputs.cache-hit != 'true' }} + name: Download test data ⬇️ + working-directory: ./.test + run: | + echo ">> working directory: $(pwd)" + bash download_test_data.sh + echo ">> directory contents:" + ls -l + - if: false + name: Dry-run snakemake + run: | + conda init + conda activate aatrnaseqpipe + snakemake -n \ + --configfile=config/config-test.yaml \ + --profile-cluster/slurm diff --git a/.test/dl_test_data.sh b/.test/download_test_data.sh similarity index 100% rename from .test/dl_test_data.sh rename to .test/download_test_data.sh diff --git a/cluster/config.yaml b/cluster/lsf/config.yaml similarity index 100% rename from cluster/config.yaml rename to cluster/lsf/config.yaml diff --git a/cluster/slurm/config.yaml b/cluster/slurm/config.yaml new file mode 100644 index 0000000..b99111b --- /dev/null +++ b/cluster/slurm/config.yaml @@ -0,0 +1,39 @@ +executor: cluster-generic +cluster-generic-submit-cmd: + sbatch \ + --output="{log}.out" \ + --error="{log}.err" \ + --job-name="{rule}-{wildcards}" \ + --nodes=1 \ + --ntasks={threads} \ + --partition="{resources.queue}" \ + --mem={resources.mem_mb} \ + "{resources.gpu_opts}" + +# note that GB values should be passed to mem_mb +default-resources: + - mem_mb=8 + # - queue="rna" + - gpu_opts="" + +jobs: 50 + +resources: + - ngpu=8 + +# set rule specific requirements +# set-resources: +# - rebasecall:queue="gpu" +# - rebasecall:gpu_opts="-gpu num=1:j_exclusive=yes" +# - rebasecall:ngpu=1 +# - rebasecall:mem_mb=24 +# - remora:mem_mb=24 +# - cca_classify:queue="gpu" +# - cca_classify:gpu_opts="-gpu num=1:j_exclusive=yes" +# - cca_classify:ngpu=1 + + +printshellcmds: True +show-failed-logs: True +latency-wait: 60 +cluster-generic-cancel-cmd: "scancel" From 1c265286c7e1aee891ffa1377aec0934145a12b8 Mon Sep 17 00:00:00 2001 From: Jay Hesselberth Date: Tue, 14 Jan 2025 06:29:22 -0700 Subject: [PATCH 2/3] Update test-pipeline.yaml --- .github/workflows/test-pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-pipeline.yaml b/.github/workflows/test-pipeline.yaml index 4e7c27c..dc93acd 100644 --- a/.github/workflows/test-pipeline.yaml +++ b/.github/workflows/test-pipeline.yaml @@ -48,6 +48,7 @@ jobs: with: activate-environment: aatrnaseqpipe + - run: conda activate aatrnaseqpipe - run: conda info - run: conda list - run: conda config --show From f02161e5485781c4ba371567ee0dc3fabb96fbfe Mon Sep 17 00:00:00 2001 From: Jay Hesselberth Date: Sat, 8 Mar 2025 06:42:45 -0700 Subject: [PATCH 3/3] Revert debug statements --- .github/workflows/test-pipeline.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test-pipeline.yaml b/.github/workflows/test-pipeline.yaml index dc93acd..1a8d028 100644 --- a/.github/workflows/test-pipeline.yaml +++ b/.github/workflows/test-pipeline.yaml @@ -48,11 +48,6 @@ jobs: with: activate-environment: aatrnaseqpipe - - run: conda activate aatrnaseqpipe - - run: conda info - - run: conda list - - run: conda config --show - - name: Cache test data 🏦 id: cache-test uses: actions/cache@v4 @@ -73,6 +68,7 @@ jobs: bash download_test_data.sh echo ">> directory contents:" ls -l + - if: false name: Dry-run snakemake run: |