Run CI in Docker containers

.github/workflows/build-ci-docker-images.yml

@@ -0,0 +1,60 @@
+name: COSP - Build CI Docker Images
+run-name: CI Image Build
+
+on: [workflow_dispatch]
+
+jobs:
+  docker:
+    strategy:
+      fail-fast: false
+      matrix:
+        toolchain:  [oneapi, gfortran]
+        include:
+        - toolchain: oneapi
+          rttov_arch: ifx
+        - toolchain: gfortran
+          rttov_arch: gfortran
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Build minimal-toolchain:${{ matrix.toolchain }}
+        uses: docker/build-push-action@v6
+        with:
+          file: containers/Dockerfile-${{ matrix.toolchain }}-minimal
+          build-args: TOOLCHAIN=${{ matrix.toolchain }}
+          tags: minimal-toolchain:${{ matrix.toolchain }}
+
+      - name: Build add-netcdf:${{ matrix.toolchain }}
+        uses: docker/build-push-action@v6
+        with:
+          file: containers/Dockerfile-add-netcdf
+          build-args: TOOLCHAIN=${{ matrix.toolchain }}
+          tags: add-netcdf:${{ matrix.toolchain }}
+
+      - name: Build add-python:${{ matrix.toolchain }}
+        uses: docker/build-push-action@v6
+        with:
+          file: containers/Dockerfile-add-python
+          build-args: TOOLCHAIN=${{ matrix.toolchain }}
+          tags: add-python:${{ matrix.toolchain }}
+
+      - name: Log in to the Github container registry
+        if: ${{ github.ref == 'refs/heads/add-containers' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: CFMIP
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Build add-RTTOV:${{ matrix.toolchain }}
+        uses: docker/build-push-action@v6
+        with:
+          file: containers/Dockerfile-add-RTTOV
+          build-args: |
+            TOOLCHAIN=${{ matrix.toolchain }}
+            RTTOV_ARCH=${{ matrix.rttov_arch }}
+          secrets: |
+            "RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }}"
+          tags: ghcr.io/cfmip/cospv2.0-ci:${{ matrix.toolchain }}
+          push: True

.github/workflows/containerized-ci.yml

@@ -7,25 +7,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        compiler: [ifx]
+        toolchain: [oneapi, gfortran]
         include:
         # Flags and KGOs for Intel Fortran Compiler
-        - compiler: ifx
+        - toolchain: oneapi
+          compiler: ifx
           fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08
-          image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:oneapi
+        - toolchain: gfortran
+          compiler: gfortran
+          fcflags: -O3 -ffree-line-length-none -fcheck=bounds -finit-real=nan
        # Common variables
         - kgo_version: v007
     defaults:
       run:
         shell: bash -el {0}
     container:
-      image: ${{ matrix.image }}
+      image: ghcr.io/cfmip/cospv2.0-ci:${{ matrix.toolchain }}
     env:
       F90: ${{ matrix.compiler }}
       FC:  ${{ matrix.compiler }}
       F90FLAGS:  ${{ matrix.fcflags }}
       # Make variables:
-      NFHOME: /opt/netcdf-fortran
+      NFHOME: /usr
       # KGO tests variables
       ATOL: 0.0
       RTOL: 0.0
@@ -35,15 +38,6 @@ jobs:
     # Checks-out repository under $GITHUB_WORKSPACE
     #
     - uses: actions/checkout@v6
-    # Set up conda environment
-    - name: Setup conda environment
-      uses: conda-incubator/setup-miniconda@v3.2.0
-      with:
-        auto-update-conda: true
-        activate-environment: ci-env
-        environment-file: build/environment.yml
-        python-version: ${{ matrix.python-version }}
-        auto-activate-base: false
 
     ###############################################################################
     # Build COSP and retrieve input and test files

.github/workflows/continuous_integration.yml

@@ -27,7 +27,8 @@
 
 # Workflow for continuous integration tests
 name: Continuous integration gfortran compilers
-on: [push, pull_request, workflow_dispatch]
+# on: [push, pull_request, workflow_dispatch]
+on: [pull_request, workflow_dispatch]
 
 jobs:
   # This workflow contains a single job called "ci_gfortran"

containers/Dockerfile-add-RTTOV

@@ -0,0 +1,41 @@
+ARG TOOLCHAIN
+FROM add-python:$TOOLCHAIN
+
+# gpg and xz needed for unpacking the tar file 
+# hdf5 (serial) needed for RTTOV
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \
+     gpg xz-utils \
+     libhdf5-dev
+
+#
+# Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball
+#   De-armor RTTOV tarball with passkey kept in Github secrets and passed in to the Docker build 
+#
+ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" 
+RUN --mount=type=secret,id=RTTOV_TARFILE_KEY,env=RTTOV_TARFILE_KEY \
+    curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \
+    gpg --quiet --batch --yes --decrypt \
+        --passphrase=$RTTOV_TARFILE_KEY --output ${filename} ${filename}.gpg && \
+    rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg
+
+#
+# Copy in a new ifx arch file (which might or might not get used); append to Makefile.inc
+#
+WORKDIR /RTTOV_src/
+RUN --mount=type=bind,source=containers/rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \
+    cat /tmp/Makefile-inc-patch >> build/Makefile.inc && \
+    rm -f build/arch/ifx 
+ADD containers/rttov-ifx-arch-file build/arch/ifx
+
+WORKDIR /RTTOV_src/src/
+ENV installdir="/usr/" lapack=0 f2py=0 gui=0 hdf5=0
+
+RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack}
+ARG RTTOV_ARCH
+RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags 
+RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir       $makeflags
+
+# RUN wget -np -l1 \ 
+#  https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \
+#  -P../../RTTOV_coefs/

containers/Dockerfile-add-netcdf

@@ -0,0 +1,31 @@
+#
+# Install NetCDF Fortran and its dependencies
+#
+
+ARG TOOLCHAIN
+FROM minimal-toolchain:$TOOLCHAIN
+
+# Install the dependencies
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \
+      curl \
+      libnetcdf-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install NetCDF Fortran
+#   The version must be compitible with NetCDF C installed above
+ARG NFVERSION=4.6.2
+RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-fortran-$NFVERSION.tar.gz | tar xz \
+  && cd netcdf-fortran-$NFVERSION \
+  && { ./configure \
+         CFLAGS='-O2' \
+         FCFLAGS='-O2 -fPIC' \
+         --disable-static \ 
+         --prefix=/usr || \
+       { cat ./config.log; exit 1; } } 
+RUN cd netcdf-fortran-$NFVERSION \
+  && make -j \
+  && make install \
+  && cd .. \
+  && rm -rf netcdf-fortran-$NFVERSION \
+  && ldconfig

containers/Dockerfile-add-python

@@ -0,0 +1,20 @@
+#
+# Install Python3 and a small set of packages
+#
+
+ARG TOOLCHAIN
+FROM add-netcdf:$TOOLCHAIN
+
+# Install Python
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \
+      python-is-python3 \
+      python3 \
+      python3-pip \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install packages from build/environment.yml 
+RUN pip3 install  --break-system-packages \
+      netCDF4 \
+      cartopy \ 
+      matplotlib

containers/Dockerfile-gfortran-minimal

@@ -0,0 +1,18 @@
+FROM ubuntu:24.04
+
+# Extend and update the package registry
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \
+      ca-certificates \
+      curl \
+      wget \
+      gpg \
+      binutils \
+      g++ \
+      gcc \
+      libc-dev \
+      make \
+      git \
+      gfortran>=14 
+
+ENV FC=gfortran CC=gcc F77=gfortran

containers/Dockerfile-oneapi-minimal

@@ -0,0 +1,37 @@
+FROM ubuntu:24.04
+
+ARG ONEAPI_VERSION='2025.3'
+
+# Extend and update the package registry
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \
+      ca-certificates \
+      curl \
+      wget \
+      gpg \
+      binutils \
+      g++ \
+      gcc \
+      libc-dev \
+      make \
+      cmake \
+      git 
+
+# See https://www.intel.com/content/www/us/en/docs/oneapi/installation-guide-linux/2025-2/hpc-apt.html#HPC-APT 
+
+RUN wget -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+     | gpg --dearmor \
+     | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null \
+ && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
+     |  tee /etc/apt/sources.list.d/oneAPI.list \
+ && apt-get update \
+ && apt-get install --yes --no-install-recommends \
+     intel-oneapi-compiler-dpcpp-cpp-${ONEAPI_VERSION} \
+     intel-oneapi-compiler-fortran-${ONEAPI_VERSION} \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/opt/intel/oneapi/compiler/${ONEAPI_VERSION}/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/${ONEAPI_VERSION}/lib:${LD_LIBRARY_PATH}"
+
+# Set default compiler executables
+ENV FC=ifx CC=icx

containers/rttov-Makefile-inc

@@ -0,0 +1,7 @@
+# HDF5_PREFIX  = /usr
+# FFLAGS_HDF5  = -D_RTTOV_HDF $(FFLAG_MOD)$(HDF5_PREFIX)/include
+# LDFLAGS_HDF5 = -L$(HDF5_PREFIX)/lib -lhdf5_hl_fortran -lhdf5_hl -lhdf5_fortran -lhdf5 -lz
+
+NETCDF_PREFIX  = /usr
+FFLAGS_NETCDF  = -D_RTTOV_NETCDF -I$(NETCDF_PREFIX)/include
+LDFLAGS_NETCDF = -L$(NETCDF_PREFIX)/lib -lnetcdff

containers/rttov-ifx-arch-file

@@ -0,0 +1,19 @@
+FC=ifx
+FC77=ifx
+CC=gcc
+LDFLAGS_ARCH=
+CFLAGS_ARCH=
+FFLAGS_ARCH=-fPIC -O3 -fp-model source
+AR=ar r
+
+# Loop unrolling causes ifort v13 and later to take a long time to compile these subroutines
+FFLAGS_ARCH_rttov_opdep_9_ad=-fPIC -O3 -unroll0 -fp-model source
+FFLAGS_ARCH_rttov_opdep_9_k=-fPIC -O3 -unroll0 -fp-model source
+FFLAGS_ARCH_rttov_opdep_13_ad=-fPIC -O3 -unroll0 -fp-model source
+FFLAGS_ARCH_rttov_opdep_13_k=-fPIC -O3 -unroll0 -fp-model source
+
+# -fp-model source ensures more consistent floating point results
+
+F2PY=f2py --fcompiler=intelem
+F2PYFLAGS_ARCH="-fPIC"
+F2PYLDFLAGS_ARCH=