From 84517a769667ec1d3af774082c8b8b648419a0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zo=C3=AB=20Ma?= Date: Tue, 18 Mar 2025 11:52:15 -0400 Subject: [PATCH 1/3] Update package install instructions for building containers. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - In both the bwa/samtools and the clinvar-report containers, update the package names to be installed by the 'apt-get' commands. - When building external packages from source, refer to the ldconfig command by full path, because under /usr/sbin it may not be under the root user's PATH. - Build bwa 07.7.18 instead of 07.7.17, as the latter could be affected by a bug that makes it fail to build from source (see https://github.com/lh3/bwa/pull/267 and BWA 07.7.18 release notes). Arvados-DCO-1.1-Signed-off-by: Zoë Ma --- WGS-processing/docker/bwa_samtools/Dockerfile | 8 ++++---- WGS-processing/docker/clinvar-report/Dockerfile | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/WGS-processing/docker/bwa_samtools/Dockerfile b/WGS-processing/docker/bwa_samtools/Dockerfile index 5f01b1d..14bfd75 100644 --- a/WGS-processing/docker/bwa_samtools/Dockerfile +++ b/WGS-processing/docker/bwa_samtools/Dockerfile @@ -3,7 +3,7 @@ USER root MAINTAINER Sarah Wait Zaranek RUN apt-get update -qy -RUN apt-get install -qy build-essential wget cmake zlib1g-dev python-pip unzip libbz2-dev liblzma-dev libcurl4-openssl-dev libncurses-dev autoconf git +RUN apt-get install -qy build-essential wget cmake zlib1g-dev python3-pip unzip libbz2-dev liblzma-dev libcurl4-openssl-dev libncurses-dev autoconf git RUN mkdir -p /usr/share/man/man1 RUN apt-get install -qy default-jdk @@ -13,12 +13,12 @@ RUN cd /tmp && git clone 'https://github.com/simongog/sdsl-lite.git' && cd sdsl- RUN apt-get remove libhts-dev # Install htslib 1.10.2 -RUN cd /tmp && wget http://github.com/samtools/htslib/archive/1.10.2.zip && unzip 1.10.2 && cd htslib-1.10.2 && make && make install && cd /tmp && rm -rf htslib-1.10.2 && ldconfig +RUN cd /tmp && wget http://github.com/samtools/htslib/archive/1.10.2.zip && unzip 1.10.2 && cd htslib-1.10.2 && make && make install && cd /tmp && rm -rf htslib-1.10.2 && /sbin/ldconfig # Install samtools 1.10 RUN wget https://github.com/samtools/samtools/archive/1.10.zip && unzip 1.10 && cd samtools-1.10 && autoheader && autoconf && ./configure && make && make install -# Install BWA 07.7.17 -RUN wget https://github.com/lh3/bwa/archive/v0.7.17.zip && unzip v0.7.17 && cd bwa-0.7.17 && make +# Install BWA 07.7.18 +RUN wget https://github.com/lh3/bwa/archive/v0.7.18.zip && unzip v0.7.18 && cd bwa-0.7.18 && make CMD /bin/sh diff --git a/WGS-processing/docker/clinvar-report/Dockerfile b/WGS-processing/docker/clinvar-report/Dockerfile index 2c0976b..32729c0 100644 --- a/WGS-processing/docker/clinvar-report/Dockerfile +++ b/WGS-processing/docker/clinvar-report/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Sarah Wait Zaranek # Install python tools RUN apt-get update && apt-get install -y --no-install-recommends apt-utils -RUN apt-get install -qy apt-utils unzip autoconf wget python-pip python-numpy python-scipy python-matplotlib python-pandas +RUN apt-get install -qy apt-utils unzip autoconf wget python3-pip python3-numpy python3-scipy python3-matplotlib python3-pandas RUN apt-get upgrade -y RUN apt-get clean @@ -18,7 +18,7 @@ RUN cd /tmp && git clone 'https://github.com/simongog/sdsl-lite.git' && cd sdsl- RUN apt-get remove libhts-dev # Install htslib 1.10.2 -RUN cd /tmp && wget http://github.com/samtools/htslib/archive/1.10.2.zip && unzip 1.10.2 && cd htslib-1.10.2 && make && make install && cd /tmp && rm -rf htslib-1.10.2 && ldconfig +RUN cd /tmp && wget http://github.com/samtools/htslib/archive/1.10.2.zip && unzip 1.10.2 && cd htslib-1.10.2 && make && make install && cd /tmp && rm -rf htslib-1.10.2 && /sbin/ldconfig # Install bcftools 1.10.2 RUN wget http://github.com/samtools/bcftools/archive/1.10.2.zip && unzip 1.10.2 && cd bcftools-1.10.2 && autoheader && autoconf && ./configure && make && make install From b689da4a3fa1b6ec7ffa9a311990035ade2596c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zo=C3=AB=20Ma?= Date: Tue, 18 Mar 2025 11:59:34 -0400 Subject: [PATCH 2/3] python3 compatibility for the report generator script. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In more recent versions of Python, the 'unicode()' global builtin is no longer there. In addition the binary IO for generating report doesn't appear necessary in our case anymore. Arvados-DCO-1.1-Signed-off-by: Zoë Ma --- WGS-processing/src/annotation/generatereport.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/WGS-processing/src/annotation/generatereport.py b/WGS-processing/src/annotation/generatereport.py index d07ec93..46921d9 100644 --- a/WGS-processing/src/annotation/generatereport.py +++ b/WGS-processing/src/annotation/generatereport.py @@ -13,11 +13,10 @@ def tablegeneration(reportdata,sectionlabel): str_io = io.StringIO() reportdatasub.to_html(buf=str_io, classes='table table-bordered',index_names=False,index=False) html_str = str_io.getvalue() - html_str_encoded = unicode(html_str).encode('utf8') - html_str_encoded = html_str_encoded.replace('<','<') - html_str_encoded = html_str_encoded.replace('>','>') - html_str_encoded = html_str_encoded.replace('_',' ') - section_html = labelhtml+html_str_encoded + html_str = html_str.replace('<','<') + html_str = html_str.replace('>','>') + html_str = html_str.replace('_',' ') + section_html = labelhtml+html_str return section_html def generatereport(): @@ -90,7 +89,7 @@ def generatereport(): total_html = source_code_head + pathogenic_html + likely_pathogenic_html + drug_html + protective_html + risk_html + affects_html + association_html + other_html + benign_html + likely_benign_html + source_code_tail # write out report html - f = open(samplename+'.html','wb') + f = open(samplename+'.html','w') f.write(total_html) f.close() From 3221b02dd2a58be07435d4a51aafa8ac8c8324fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zo=C3=AB=20Ma?= Date: Tue, 18 Mar 2025 12:02:59 -0400 Subject: [PATCH 3/3] Update CWL tool definitions for WGS tutorial. - For bwa/samtools, update the executable path of 'bwa' to match the updated version 07.7.18 - For bwa/samtools, lower the number of CPU cores hinted from 16 to 6, because with the former it had trouble starting the job at all on systems with no more than 16 cores. - For generate-report, refer to the Python executable by the full path, because in the container built for that step, the bare 'python' command points to the Python instance installed by arvados-cwl-runner for which the Python libraries are not installed (the libraries are installed to the system Python directory) --- WGS-processing/cwl/helper/bwamem-samtools-view.cwl | 6 +++--- WGS-processing/cwl/helper/generate-report.cwl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/WGS-processing/cwl/helper/bwamem-samtools-view.cwl b/WGS-processing/cwl/helper/bwamem-samtools-view.cwl index 71fd365..ea12a55 100644 --- a/WGS-processing/cwl/helper/bwamem-samtools-view.cwl +++ b/WGS-processing/cwl/helper/bwamem-samtools-view.cwl @@ -12,12 +12,12 @@ hints: outputDirType: keep_output_dir ResourceRequirement: ramMin: 30000 - coresMin: 16 + coresMin: 6 SoftwareRequirement: packages: BWA: specs: [ "https://identifiers.org/rrid/RRID:SCR_010910" ] - version: [ "0.7.17" ] + version: [ "0.7.18" ] Samtools: specs: [ "https://identifiers.org/rrid/RRID:SCR_002105" ] version: [ "1.10" ] @@ -58,7 +58,7 @@ outputs: glob: "*bam" arguments: - - /bwa-0.7.17/bwa + - /bwa-0.7.18/bwa - mem - -M - -t diff --git a/WGS-processing/cwl/helper/generate-report.cwl b/WGS-processing/cwl/helper/generate-report.cwl index 3773f7d..a0b86bb 100644 --- a/WGS-processing/cwl/helper/generate-report.cwl +++ b/WGS-processing/cwl/helper/generate-report.cwl @@ -38,7 +38,7 @@ outputs: outputBinding: glob: "*html" -baseCommand: python +baseCommand: /usr/bin/python3 arguments: - $(inputs.reportfunc)