Chore/improve nf base images gh action

.github/workflows/build_and_push_nf_base_images.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
-          path: containers       
+          path: containers
 
       - name: Checkout other private repository
         uses: actions/checkout@v3
@@ -46,16 +46,14 @@ jobs:
               echo "Building an image present in $dir"
               image_name=nextflow-approved/public
               tag_name=$(basename "$dir")
-              docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
-              docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name
+              docker buildx build --push --platform linux/amd64 -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
               echo "Built an image with name --> $image_name:$tag_name"
 
               for dir in containers/nextflow-base-images/*/;do
                   echo "Building an image present in $dir"
                   image_name=nextflow-approved/public
                   tag_name=$(basename "$dir")
-                  docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
-                  docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name
+                  docker buildx build --push --platform linux/amd64 -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
                   echo "Built an image with name --> $image_name:$tag_name"
               done
 

.github/workflows/get_layer_info_for_nf_imgs.yml

@@ -29,7 +29,7 @@ jobs:
           if [ -z "$layer_json" ]; then
             layer_json="{}"
           fi
-          echo "Line: $LINENO layer_json=$layer_json"
+          echo "Line $LINENO: layer_json=$layer_json"
           # Function to fetch the manifest with retry logic for rate limiting
           fetch_manifest() {
             local url=$1
@@ -45,34 +45,67 @@ jobs:
                 echo "Line $LINENO: Rate limit exceeded, waiting for $wait_time seconds before retrying..." >&2
                 sleep $wait_time
                 wait_time=$((wait_time * 2))  # Exponential backoff
-              else
+                continue
+              fi
+
+              mediaType=$(jq -r '.mediaType // empty' <<<"$response")
+
+              if [ "$mediaType" == "application/vnd.oci.image.index.v1+json" ]; then
+                # Pick the amd64/linux image manifest (ignore attestations)
+                digest=$(jq -r '
+                  .manifests[]
+                  | select(.platform.architecture=="amd64" and .platform.os=="linux")
+                  | .digest
+                ' <<<"$response")
+
+                if [ -z "$digest" ]; then
+                  echo "Line $LINENO: No amd64/linux manifest found in OCI index" >&2
+                  return 1
+                fi
+
+                # Replace tag with digest to form concrete manifest URL
+                resolved_url="$(sed "s|/manifests/.*|/manifests/$digest|" <<<"$url")"
+
+                # Fetch and echo the resolved image manifest
+                response=$(curl -s \
+                  -H "Authorization: Bearer $TOKEN" \
+                  -H "Accept: application/vnd.oci.image.manifest.v1+json" \
+                  "$resolved_url")
+
                 echo "$response"
                 return
               fi
+
+              echo "$response"
+              return
+
             done
 
             echo "Line $LINENO: Failed to fetch manifest after $retries attempts due to rate limiting." >&2
             return 1
           }
 
+          echo "Line $LINENO: started reading lines from script"
           while IFS= read -r image_url; do
             # Ignore lines that start with #
-              if [[ "$image_url" == \#* ]]; then
-              echo "Line_$LINENO: Skipping this line -- $image_url"
-                continue
-              fi
+            if [[ "$image_url" == \#* ]]; then
+            echo "Line $LINENO: Skipping this line -- $image_url"
+              continue
+            fi
 
             # Strip the first * if a line starts with *
-              if [[ "$image_url" == \** ]]; then
-              echo "Line_$LINENO: Stripping * from this line -- $image_url"
-                image_url=$(echo "${image_url:1}"| sed 's/^[ \t]*//;s/[ \t]*$//')
-              fi
-            echo "Line $LINENO: started reading lines from script"
+            if [[ "$image_url" == \** ]]; then
+            echo "Line $LINENO: Stripping * from this line -- $image_url"
+              image_url=$(echo "${image_url:1}"| sed 's/^[ \t]*//;s/[ \t]*$//')
+            fi
+
             manifest_url=$(echo "${image_url}" | sed 's|public\.ecr\.aws/\(.*\):\(.*\)|https://public.ecr.aws/v2/\1/manifests/\2|')
             tag_name=$(echo "${image_url}" | sed 's|\(.*\):\(.*\)|\2|')
-            echo "$LINENO: manifest_url = $manifest_url, tag_name = $tag_name"
+            echo "Line $LINENO: manifest_url = $manifest_url - tag_name = $tag_name"
             # Fetch the manifest with retry using exponential backoff
-            response=$(fetch_manifest "$manifest_url" | jq "[.layers[].digest]|.[-1]")
+            response=$(fetch_manifest "$manifest_url")
+            echo "Response from manifest_url: $response"
+            response=$(echo "${response}" | jq "[.layers[].digest]|.[-1]")
 
             if [ $? -ne 0 ] || [ -z "$response" ]; then
               echo "Line $LINENO: Failed to retrieve valid response from manifest_url -- $manifest_url" >&2
@@ -94,7 +127,7 @@ jobs:
           done <<< "$url_list"
 
           if [ -n "$layer_json" ]; then
-            echo "$layer_json" >  $output_file_name
+            echo "$layer_json" > $output_file_name
           fi
       - name: Commit and push changes
         uses: stefanzweifel/git-auto-commit-action@v5

nextflow-base-images/gen3-cuda-11.8-ubuntu22.04-openssl/Dockerfile

@@ -1,36 +1,45 @@
 # Use the specified base image
-FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04
-
-RUN apt-get purge -y --auto-remove openssl && apt-get autoremove && apt-get autoclean
+FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04 AS openssl-builder
 
-RUN apt-get update && apt-get -y upgrade && apt install -y wget
+# Install build deps only
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        wget \
+        ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
 
-RUN apt install -y build-essential && \
-    apt-get install -y python3 && \
-    apt-get install -y python3-pip
-
-# install openssl 3.0.8 as it is required for FIPS compliance.
 WORKDIR /tmp
 RUN wget https://www.openssl.org/source/openssl-3.0.8.tar.gz && \
-    tar -xzvf openssl-3.0.8.tar.gz && \
-    rm openssl-3.0.8.tar.gz
+    tar -xzf openssl-3.0.8.tar.gz
 
 WORKDIR /tmp/openssl-3.0.8
-RUN ./Configure enable-fips && \
-    make && \
-    make install
+RUN ./Configure enable-fips && make -j$(nproc) && make install_sw install_ssldirs
 
-# Changing adding `/usr/local/lib` as a prefix to LD_LIBRARY_PATH will
-# give precedence to OpenSSL 3.0.8 library files over the 3.0.2
-ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH
+FROM nvcr.io/nvidia/cuda:11.8.0-base-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
 
-# Make config changes ti ensure FIPS compliance
-RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf
-RUN sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf
-RUN sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf
-RUN sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf
+# Install runtime deps only
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        python3 \
+        python3-pip \
+        libglib2.0-0 && \
+    rm -rf /var/lib/apt/lists/*
 
+# Copy OpenSSL runtime artifacts only
+COPY --from=openssl-builder /usr/local/ssl /usr/local/ssl
+COPY --from=openssl-builder /usr/local/lib /usr/local/lib
+COPY --from=openssl-builder /usr/local/lib64 /usr/local/lib64
+COPY --from=openssl-builder /usr/local/bin/openssl /usr/local/bin/openssl
+
+ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH
+ENV OPENSSL_CONF=/usr/local/ssl/openssl.cnf
 
-# Clean up the temporary directory
+# FIPS config
+RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf && \
+    sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf && \
+    sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf && \
+    sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf
 WORKDIR /
-RUN rm -rf /tmp/openssl-3.0.8

nextflow-base-images/gen3-cuda-12.3-torch2.2-ubuntu22.04-openssl/Dockerfile

@@ -3,20 +3,19 @@ FROM public.ecr.aws/u5x5h6w3/nextflow-approved/public:gen3-cuda-12.3-ubuntu22.04
 # Set environment variables
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Install system dependencies
+# Install system dependencies + clean up
 RUN apt-get update && \
-    apt-get install -y \
+    apt-get install -y --no-install-recommends \
         git \
         python3-pip \
         python3-dev \
         python3-opencv \
-        libglib2.0-0
-
-# Upgrade pip
-RUN python3 -m pip install --upgrade pip
-
-# Install PyTorch and torchvision
-RUN pip3 install torch -f https://download.pytorch.org/whl/cu123/torch_stable.html
+        libglib2.0-0 && \
+    rm -rf /var/lib/apt/lists/*
+# Upgrade pip and install torch (NO CACHE)
+RUN python3 -m pip install --upgrade pip && \
+    pip3 install --no-cache-dir torch \
+        -f https://download.pytorch.org/whl/cu123/torch_stable.html
 
 # Set the working directory
 WORKDIR /app

nextflow-base-images/gen3-cuda-12.3-ubuntu22.04-openssl/Dockerfile

@@ -1,36 +1,45 @@
 # Use the specified base image
-FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04
-
-RUN apt-get purge -y --auto-remove openssl && apt-get autoremove && apt-get autoclean
+FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04 AS openssl-builder
 
-RUN apt-get update && apt-get -y upgrade && apt install -y wget
+# Install build deps only
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        wget \
+        ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
 
-RUN apt install -y build-essential && \
-    apt-get install -y python3 && \
-    apt-get install -y python3-pip
-
-# install openssl 3.0.8 as it is required for FIPS compliance.
 WORKDIR /tmp
 RUN wget https://www.openssl.org/source/openssl-3.0.8.tar.gz && \
-    tar -xzvf openssl-3.0.8.tar.gz && \
-    rm openssl-3.0.8.tar.gz
+    tar -xzf openssl-3.0.8.tar.gz
 
 WORKDIR /tmp/openssl-3.0.8
-RUN ./Configure enable-fips && \
-    make && \
-    make install
+RUN ./Configure enable-fips && make -j$(nproc) && make install_sw install_ssldirs
 
-# Changing adding `/usr/local/lib` as a prefix to LD_LIBRARY_PATH will
-# give precedence to OpenSSL 3.0.8 library files over the 3.0.2
-ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH
+FROM nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
 
-# Make config changes ti ensure FIPS compliance
-RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf
-RUN sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf
-RUN sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf
-RUN sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf
+# Install runtime deps only
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        python3 \
+        python3-pip \
+        libglib2.0-0 && \
+    rm -rf /var/lib/apt/lists/*
 
+# Copy OpenSSL runtime artifacts only
+COPY --from=openssl-builder /usr/local/ssl /usr/local/ssl
+COPY --from=openssl-builder /usr/local/lib /usr/local/lib
+COPY --from=openssl-builder /usr/local/lib64 /usr/local/lib64
+COPY --from=openssl-builder /usr/local/bin/openssl /usr/local/bin/openssl
+
+ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH
+ENV OPENSSL_CONF=/usr/local/ssl/openssl.cnf
 
-# Clean up the temporary directory
+# FIPS config
+RUN sed -i 's$# .include fipsmodule.cnf$.include /usr/local/ssl/fipsmodule.cnf$g' /usr/local/ssl/openssl.cnf && \
+    sed -i 's$providers = provider_sect$providers = provider_sect\nalg_section = algorithm_sect$g' /usr/local/ssl/openssl.cnf && \
+    sed -i 's$# fips = fips_sect$fips = fips_sect$g' /usr/local/ssl/openssl.cnf && \
+    sed -i -e 's$# activate = 1$activate = 1 \n\n[algorithm_sect]\ndefault_properties = fips=yes$g' /usr/local/ssl/openssl.cnf
 WORKDIR /
-RUN rm -rf /tmp/openssl-3.0.8