From cffab0924c3166aaa46c2dbeb9e1f8ebf9e70d6d Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Thu, 4 Dec 2025 22:37:56 -0500 Subject: [PATCH 01/47] Rough start to making containers --- .../workflows/build-gfortran-container.yml | 16 +++++++++ containers/Dockerfile-gfortran | 33 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 .github/workflows/build-gfortran-container.yml create mode 100644 containers/Dockerfile-gfortran diff --git a/.github/workflows/build-gfortran-container.yml b/.github/workflows/build-gfortran-container.yml new file mode 100644 index 0000000000..99682ba0b9 --- /dev/null +++ b/.github/workflows/build-gfortran-container.yml @@ -0,0 +1,16 @@ +name: COSP Docker Images Build +run-name: Docker Image Build + +on: workflow_dispatch + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build gfortran toolchain + uses: docker/build-push-action@v6 + with: + file: containers/Dockerfile-gfortran + tags: toolchain:gfortran \ No newline at end of file diff --git a/containers/Dockerfile-gfortran b/containers/Dockerfile-gfortran new file mode 100644 index 0000000000..ed39dc3549 --- /dev/null +++ b/containers/Dockerfile-gfortran @@ -0,0 +1,33 @@ +FROM ubuntu:24.04 + +# Extend and update the package registry +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + ca-certificates \ + curl \ + wget \ + gpg \ + binutils \ + g++ \ + gcc \ + libc-dev \ + make \ + git \ + gfortran-12 \ + gfortran-13 \ + gfortran-14 \ + libnetcdff-dev + +# Install Python +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + python-is-python3 \ + python3 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Install packages from build/environment.yml +RUN pip3 install --break-system-packages \ + netCDF4 \ + cartopy \ + matplotlib From 518993d947dd23aecf54977d620ec092df43ffac Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Thu, 8 Jan 2026 20:10:00 -0500 Subject: [PATCH 02/47] Files needed to build Docker image with RTTOV compiled with Intel OneAPI --- containers/Dockerfile-oneapi | 41 ++++++++++++++++++++++++++++++++++ containers/rttov-Makefile-inc | 7 ++++++ containers/rttov-ifx-arch-file | 19 ++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 containers/Dockerfile-oneapi create mode 100644 containers/rttov-Makefile-inc create mode 100644 containers/rttov-ifx-arch-file diff --git a/containers/Dockerfile-oneapi b/containers/Dockerfile-oneapi new file mode 100644 index 0000000000..aaa83de684 --- /dev/null +++ b/containers/Dockerfile-oneapi @@ -0,0 +1,41 @@ +# Base Docker image from earth-system-radiation includes +# OneAPI compiler chain, netCDF, Python +FROM earthsystemradiation/rte-rrtmgp-ci:oneapi + +ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" + +# gpg and xz needed for unpacking the tar file +# hdf5 (serial) needed for RTTOV +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + gpg xz-utils \ + libhdf5-dev + +# +# Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball +# De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} +# +RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ + gpg --quiet --batch --yes --decrypt \ + --passphrase=${{ secrets.RTTOV_TARFILE_KEY }} --output ${filename} ${filename}.gpg && \ + rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg + +# +# Copy in a new ifx arch file; append to Makefile.inc +# +WORKDIR /RTTOV_src/ +RUN --mount=type=bind,source=rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ + cat /tmp/Makefile-inc-patch >> build/Makefile.inc && \ + rm -f build/arch/ifx +ADD rttov-ifx-arch-file build/arch/ifx + +WORKDIR src/ +ENV myarch="ifx" clean="n" installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 + +RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} && \ + make ARCH=$myarch INSTALLDIR=$installdir clean $makeflags && \ + make ARCH=$myarch INSTALLDIR=$installdir $makeflags + +RUN wget -np -l1 \ + https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ + -P../../RTTOV_coefs/ diff --git a/containers/rttov-Makefile-inc b/containers/rttov-Makefile-inc new file mode 100644 index 0000000000..021e08018c --- /dev/null +++ b/containers/rttov-Makefile-inc @@ -0,0 +1,7 @@ +# HDF5_PREFIX = /usr/ +# FFLAGS_HDF5 = -D_RTTOV_HDF $(FFLAG_MOD)$(HDF5_PREFIX)/include +# LDFLAGS_HDF5 = -L$(HDF5_PREFIX)/lib -lhdf5_hl_fortran -lhdf5_hl -lhdf5_fortran -lhdf5 -lz + +NETCDF_PREFIX = /usr/ +FFLAGS_NETCDF = -D_RTTOV_NETCDF -I$(NETCDF_PREFIX)/include +LDFLAGS_NETCDF = -L$(NETCDF_PREFIX)/lib -lnetcdff diff --git a/containers/rttov-ifx-arch-file b/containers/rttov-ifx-arch-file new file mode 100644 index 0000000000..da8a4a917f --- /dev/null +++ b/containers/rttov-ifx-arch-file @@ -0,0 +1,19 @@ +FC=ifx +FC77=ifx +CC=gcc +LDFLAGS_ARCH= +CFLAGS_ARCH= +FFLAGS_ARCH=-fPIC -O3 -fp-model source +AR=ar r + +# Loop unrolling causes ifort v13 and later to take a long time to compile these subroutines +FFLAGS_ARCH_rttov_opdep_9_ad=-fPIC -O3 -unroll0 -fp-model source +FFLAGS_ARCH_rttov_opdep_9_k=-fPIC -O3 -unroll0 -fp-model source +FFLAGS_ARCH_rttov_opdep_13_ad=-fPIC -O3 -unroll0 -fp-model source +FFLAGS_ARCH_rttov_opdep_13_k=-fPIC -O3 -unroll0 -fp-model source + +# -fp-model source ensures more consistent floating point results + +F2PY=f2py --fcompiler=intelem +F2PYFLAGS_ARCH="-fPIC" +F2PYLDFLAGS_ARCH= \ No newline at end of file From 7dca4557478e41849b891b3a09a9eab758141b19 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Thu, 8 Jan 2026 20:29:39 -0500 Subject: [PATCH 03/47] Draft Github Action to build Docker image with ifx --- .../workflows/build-gfortran-container.yml | 4 +-- .github/workflows/build-oneapi-container.yml | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-oneapi-container.yml diff --git a/.github/workflows/build-gfortran-container.yml b/.github/workflows/build-gfortran-container.yml index 99682ba0b9..495ed3e2f0 100644 --- a/.github/workflows/build-gfortran-container.yml +++ b/.github/workflows/build-gfortran-container.yml @@ -1,5 +1,5 @@ -name: COSP Docker Images Build -run-name: Docker Image Build +name: COSP gfortran Docker Images Build +run-name: gfortran Docker Image Build on: workflow_dispatch diff --git a/.github/workflows/build-oneapi-container.yml b/.github/workflows/build-oneapi-container.yml new file mode 100644 index 0000000000..f605630229 --- /dev/null +++ b/.github/workflows/build-oneapi-container.yml @@ -0,0 +1,27 @@ +name: COSP ifx Docker Images Build +run-name: ifx Docker Image Build + +on: workflow_dispatch + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Log in to the Container registry + if: ${{ github.ref == 'refs/heads/main' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push rttov:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: containers/Dockerfile-oneapi + push: ${{ github.ref == 'refs/heads/main' }} + tags: | + earthsystemradiation/rttov:oneapi + ghcr.io/earth-system-radiation/rttov:oneapi From 82e048571f284d6362acc179e218bfca82bce4c5 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:03:35 -0500 Subject: [PATCH 04/47] Progress on general building of CI containers --- .github/workflows/build-ci-docker-images.yml | 46 ++++++++++++++++++++ containers/Dockerfile-add-RTTOV | 40 +++++++++++++++++ containers/Dockerfile-add-python | 20 +++++++++ containers/Dockerfile-gfortran | 33 -------------- containers/Dockerfile-gfortran-minimal | 20 +++++++++ containers/Dockerfile-oneapi-minimal | 37 ++++++++++++++++ 6 files changed, 163 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/build-ci-docker-images.yml create mode 100644 containers/Dockerfile-add-RTTOV create mode 100644 containers/Dockerfile-add-python delete mode 100644 containers/Dockerfile-gfortran create mode 100644 containers/Dockerfile-gfortran-minimal create mode 100644 containers/Dockerfile-oneapi-minimal diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml new file mode 100644 index 0000000000..48e131680d --- /dev/null +++ b/.github/workflows/build-ci-docker-images.yml @@ -0,0 +1,46 @@ +name: COSP - Build CI Docker Images +run-name: CI Image Build + +on: [push] + +jobs: + docker: + strategy: + fail-fast: false + matrix: + toolchain: [oneapi, gfortran] + include: + - toolchain: oneapi + rttov_arch: ifx + - toolchain: gfortran + rttov_arch: gfortran + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build minimal-toolchain:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: Dockerfile-${{ matrix.toolchain }}-minimal + tags: minimal-toolchain:${{ matrix.toolchain }} + + - name: Build add-netcdf:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: Dockerfile-add-netcdf + build-args: TOOLCHAIN=${{ matrix.toolchain }} + tags: add-netcdf:${{ matrix.toolchain }} + + - name: Build add-python:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: Dockerfile-add-python + build-args: TOOLCHAIN=${{ matrix.toolchain }} + tags: add-python:${{ matrix.toolchain }} + + - name: Build add-RTTOV:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: Dockerfile-add-RTTOV + build-args: TOOLCHAIN=${{ matrix.toolchain }} + tags: add-rttov:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV new file mode 100644 index 0000000000..35e39b2cf3 --- /dev/null +++ b/containers/Dockerfile-add-RTTOV @@ -0,0 +1,40 @@ +ARG TOOLCHAIN +FROM add-netcdf:$TOOLCHAIN + +ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" + +# gpg and xz needed for unpacking the tar file +# hdf5 (serial) needed for RTTOV +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + gpg xz-utils \ + libhdf5-dev + +# +# Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball +# De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} +# +RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ + gpg --quiet --batch --yes --decrypt \ + --passphrase=${{ secrets.RTTOV_TARFILE_KEY }} --output ${filename} ${filename}.gpg && \ + rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg + +# +# Copy in a new ifx arch file (which might or might not get used); append to Makefile.inc +# +WORKDIR /RTTOV_src/ +RUN --mount=type=bind,source=rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ + cat /tmp/Makefile-inc-patch >> build/Makefile.inc && \ + rm -f build/arch/ifx +ADD rttov-ifx-arch-file build/arch/ifx + +WORKDIR src/ +ENV myarch=$rttov_arch installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 + +RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} && \ + make ARCH=$myarch INSTALLDIR=$installdir clean $makeflags && \ + make ARCH=$myarch INSTALLDIR=$installdir $makeflags + +RUN wget -np -l1 \ + https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ + -P../../RTTOV_coefs/ diff --git a/containers/Dockerfile-add-python b/containers/Dockerfile-add-python new file mode 100644 index 0000000000..82f72fc527 --- /dev/null +++ b/containers/Dockerfile-add-python @@ -0,0 +1,20 @@ +# +# Install Python3 and a small set of packages +# + +ARG TOOLCHAIN +FROM add-netcdf:$TOOLCHAIN + +# Install Python +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + python-is-python3 \ + python3 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Install packages from build/environment.yml +RUN pip3 install --break-system-packages \ + netCDF4 \ + cartopy \ + matplotlib diff --git a/containers/Dockerfile-gfortran b/containers/Dockerfile-gfortran deleted file mode 100644 index ed39dc3549..0000000000 --- a/containers/Dockerfile-gfortran +++ /dev/null @@ -1,33 +0,0 @@ -FROM ubuntu:24.04 - -# Extend and update the package registry -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ - ca-certificates \ - curl \ - wget \ - gpg \ - binutils \ - g++ \ - gcc \ - libc-dev \ - make \ - git \ - gfortran-12 \ - gfortran-13 \ - gfortran-14 \ - libnetcdff-dev - -# Install Python -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ - python-is-python3 \ - python3 \ - python3-pip \ - && rm -rf /var/lib/apt/lists/* - -# Install packages from build/environment.yml -RUN pip3 install --break-system-packages \ - netCDF4 \ - cartopy \ - matplotlib diff --git a/containers/Dockerfile-gfortran-minimal b/containers/Dockerfile-gfortran-minimal new file mode 100644 index 0000000000..faf840fc70 --- /dev/null +++ b/containers/Dockerfile-gfortran-minimal @@ -0,0 +1,20 @@ +FROM ubuntu:24.04 + +# Extend and update the package registry +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + ca-certificates \ + curl \ + wget \ + gpg \ + binutils \ + g++ \ + gcc \ + libc-dev \ + make \ + git \ + gfortran-12 \ + gfortran-13 \ + gfortran-14 + +ENV FC=gfortran CC=gcc \ No newline at end of file diff --git a/containers/Dockerfile-oneapi-minimal b/containers/Dockerfile-oneapi-minimal new file mode 100644 index 0000000000..0b96c526ff --- /dev/null +++ b/containers/Dockerfile-oneapi-minimal @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 + +ARG ONEAPI_VERSION='2025.3' + +# Extend and update the package registry +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + ca-certificates \ + curl \ + wget \ + gpg \ + binutils \ + g++ \ + gcc \ + libc-dev \ + make \ + cmake \ + git + +# See https://www.intel.com/content/www/us/en/docs/oneapi/installation-guide-linux/2025-2/hpc-apt.html#HPC-APT + +RUN wget -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor \ + | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null \ + && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | tee /etc/apt/sources.list.d/oneAPI.list \ + && apt-get update \ + && apt-get install --yes --no-install-recommends \ + intel-oneapi-compiler-dpcpp-cpp-${ONEAPI_VERSION} \ + intel-oneapi-compiler-fortran-${ONEAPI_VERSION} \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/opt/intel/oneapi/compiler/${ONEAPI_VERSION}/bin:${PATH}" +ENV LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/${ONEAPI_VERSION}/lib:${LD_LIBRARY_PATH}" + +# Set default compiler executables +ENV FC=ifx CC=icx From 2a3f45bbd053385e4947ca5417c3dfb50ba94bb9 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:05:31 -0500 Subject: [PATCH 05/47] Dockerfiles in subdir --- .github/workflows/build-ci-docker-images.yml | 8 ++++---- .github/workflows/containerized-ci.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 48e131680d..25fabdff78 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -21,26 +21,26 @@ jobs: - name: Build minimal-toolchain:${{ matrix.toolchain }} uses: docker/build-push-action@v6 with: - file: Dockerfile-${{ matrix.toolchain }}-minimal + file: containers/Dockerfile-${{ matrix.toolchain }}-minimal tags: minimal-toolchain:${{ matrix.toolchain }} - name: Build add-netcdf:${{ matrix.toolchain }} uses: docker/build-push-action@v6 with: - file: Dockerfile-add-netcdf + file: containers/Dockerfile-add-netcdf build-args: TOOLCHAIN=${{ matrix.toolchain }} tags: add-netcdf:${{ matrix.toolchain }} - name: Build add-python:${{ matrix.toolchain }} uses: docker/build-push-action@v6 with: - file: Dockerfile-add-python + file: containers/Dockerfile-add-python build-args: TOOLCHAIN=${{ matrix.toolchain }} tags: add-python:${{ matrix.toolchain }} - name: Build add-RTTOV:${{ matrix.toolchain }} uses: docker/build-push-action@v6 with: - file: Dockerfile-add-RTTOV + file: containers/Dockerfile-add-RTTOV build-args: TOOLCHAIN=${{ matrix.toolchain }} tags: add-rttov:${{ matrix.toolchain }} diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml index 2d3cb1fdad..596224faf7 100644 --- a/.github/workflows/containerized-ci.yml +++ b/.github/workflows/containerized-ci.yml @@ -1,5 +1,5 @@ name: Continuous integration in containers -on: [push, pull_request, workflow_dispatch] +on: [workflow_dispatch] jobs: Containerized-CI: From 705f1a7a5e9e1d7be350f3c17c365e42fd5f72ef Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:07:34 -0500 Subject: [PATCH 06/47] Missed a file --- containers/Dockerfile-add-netcdf | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 containers/Dockerfile-add-netcdf diff --git a/containers/Dockerfile-add-netcdf b/containers/Dockerfile-add-netcdf new file mode 100644 index 0000000000..46bbbeafe6 --- /dev/null +++ b/containers/Dockerfile-add-netcdf @@ -0,0 +1,29 @@ +# +# Install NetCDF Fortran and its dependencies +# + +ARG TOOLCHAIN +FROM minimal-toolchain:$TOOLCHAIN + +# Install the dependencies +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + curl \ + libnetcdf-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install NetCDF Fortran +# The version must be compitible with NetCDF C installed above +ARG NFVERSION=4.6.2 +RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-fortran-$NFVERSION.tar.gz | tar xz \ + && cd netcdf-fortran-$NFVERSION \ + && { ./configure \ + CFLAGS='-O2' \ + FCFLAGS='-O2 -fPIC' \ + --disable-static || \ + { cat ./config.log; exit 1; } } \ + && make -j \ + && make install \ + && cd .. \ + && rm -rf netcdf-fortran-$NFVERSION \ + && ldconfig From 21b12583818295fc2af09fc138855b3c1bc65e2a Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:15:52 -0500 Subject: [PATCH 07/47] Older netcdf Fortran? --- .github/workflows/build-ci-docker-images.yml | 1 + containers/Dockerfile-add-netcdf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 25fabdff78..402b0cf491 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -22,6 +22,7 @@ jobs: uses: docker/build-push-action@v6 with: file: containers/Dockerfile-${{ matrix.toolchain }}-minimal + build-args: TOOLCHAIN=${{ matrix.toolchain }} tags: minimal-toolchain:${{ matrix.toolchain }} - name: Build add-netcdf:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-netcdf b/containers/Dockerfile-add-netcdf index 46bbbeafe6..56a38f6008 100644 --- a/containers/Dockerfile-add-netcdf +++ b/containers/Dockerfile-add-netcdf @@ -14,7 +14,7 @@ RUN apt-get update \ # Install NetCDF Fortran # The version must be compitible with NetCDF C installed above -ARG NFVERSION=4.6.2 +ARG NFVERSION=4.5.4 RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-fortran-$NFVERSION.tar.gz | tar xz \ && cd netcdf-fortran-$NFVERSION \ && { ./configure \ From 493b1d341658335737938b8cd82f7255a1b35609 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:18:00 -0500 Subject: [PATCH 08/47] Debug in steps --- containers/Dockerfile-add-netcdf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/Dockerfile-add-netcdf b/containers/Dockerfile-add-netcdf index 56a38f6008..e014a64033 100644 --- a/containers/Dockerfile-add-netcdf +++ b/containers/Dockerfile-add-netcdf @@ -21,8 +21,8 @@ RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-for CFLAGS='-O2' \ FCFLAGS='-O2 -fPIC' \ --disable-static || \ - { cat ./config.log; exit 1; } } \ - && make -j \ + { cat ./config.log; exit 1; } } +RUN make -j \ && make install \ && cd .. \ && rm -rf netcdf-fortran-$NFVERSION \ From 9f0ee51c1ab59ec464cd1005d2ab05a78dde15ab Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:25:25 -0500 Subject: [PATCH 09/47] more debugging --- containers/Dockerfile-add-netcdf | 3 ++- containers/Dockerfile-gfortran-minimal | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/containers/Dockerfile-add-netcdf b/containers/Dockerfile-add-netcdf index e014a64033..9fc6304743 100644 --- a/containers/Dockerfile-add-netcdf +++ b/containers/Dockerfile-add-netcdf @@ -22,7 +22,8 @@ RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-for FCFLAGS='-O2 -fPIC' \ --disable-static || \ { cat ./config.log; exit 1; } } -RUN make -j \ +RUN cd netcdf-fortran-$NFVERSION \ + && make -j \ && make install \ && cd .. \ && rm -rf netcdf-fortran-$NFVERSION \ diff --git a/containers/Dockerfile-gfortran-minimal b/containers/Dockerfile-gfortran-minimal index faf840fc70..8ae7b4cb15 100644 --- a/containers/Dockerfile-gfortran-minimal +++ b/containers/Dockerfile-gfortran-minimal @@ -17,4 +17,4 @@ RUN apt-get update \ gfortran-13 \ gfortran-14 -ENV FC=gfortran CC=gcc \ No newline at end of file +ENV FC=gfortran CC=gcc From 4bdb06ce59d330d86da68522b24464969bdda560 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:32:11 -0500 Subject: [PATCH 10/47] arguments for rttov_arch --- .github/workflows/build-ci-docker-images.yml | 2 +- containers/Dockerfile-add-RTTOV | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 402b0cf491..8c5693b06f 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -36,7 +36,7 @@ jobs: uses: docker/build-push-action@v6 with: file: containers/Dockerfile-add-python - build-args: TOOLCHAIN=${{ matrix.toolchain }} + build-args: [TOOLCHAIN=${{ matrix.toolchain }}, rttov_arch=${{ matrix.rttov_arch }}] tags: add-python:${{ matrix.toolchain }} - name: Build add-RTTOV:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 35e39b2cf3..41126d84a9 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -1,4 +1,5 @@ -ARG TOOLCHAIN +ARG TOOLCHAIN +ARG rttov_arch FROM add-netcdf:$TOOLCHAIN ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" From 7f299405df4b8cab5ef4230469ca6def884a0675 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:33:34 -0500 Subject: [PATCH 11/47] using environment variable? --- .github/workflows/build-ci-docker-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 8c5693b06f..62ed915ca6 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -36,7 +36,8 @@ jobs: uses: docker/build-push-action@v6 with: file: containers/Dockerfile-add-python - build-args: [TOOLCHAIN=${{ matrix.toolchain }}, rttov_arch=${{ matrix.rttov_arch }}] + env: rttov_arch=${{ matrix.rttov_arch }} + build-args: TOOLCHAIN=${{ matrix.toolchain }} tags: add-python:${{ matrix.toolchain }} - name: Build add-RTTOV:${{ matrix.toolchain }} From 88ecada1408a2055e68bd019dbfbdf2168b1bdcb Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:38:01 -0500 Subject: [PATCH 12/47] file location outside container? --- containers/Dockerfile-add-RTTOV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 41126d84a9..ffd34fe243 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -27,7 +27,7 @@ WORKDIR /RTTOV_src/ RUN --mount=type=bind,source=rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ cat /tmp/Makefile-inc-patch >> build/Makefile.inc && \ rm -f build/arch/ifx -ADD rttov-ifx-arch-file build/arch/ifx +ADD containers/rttov-ifx-arch-file build/arch/ifx WORKDIR src/ ENV myarch=$rttov_arch installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 From 109db181ca92cbe522460500f7307fb850a6a12f Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:46:54 -0500 Subject: [PATCH 13/47] how to get arch in? --- .github/workflows/build-ci-docker-images.yml | 3 +-- containers/Dockerfile-add-RTTOV | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 62ed915ca6..aef2509b46 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -36,8 +36,7 @@ jobs: uses: docker/build-push-action@v6 with: file: containers/Dockerfile-add-python - env: rttov_arch=${{ matrix.rttov_arch }} - build-args: TOOLCHAIN=${{ matrix.toolchain }} + build-args: TOOLCHAIN=${{ matrix.toolchain }}, rttov_arch=${{ matrix.rttov_arch }} tags: add-python:${{ matrix.toolchain }} - name: Build add-RTTOV:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index ffd34fe243..b15aa006f5 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -1,6 +1,6 @@ -ARG TOOLCHAIN +ARG TOOLCHAIN ARG rttov_arch -FROM add-netcdf:$TOOLCHAIN +FROM add-python:$TOOLCHAIN ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" @@ -30,7 +30,7 @@ RUN --mount=type=bind,source=rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ ADD containers/rttov-ifx-arch-file build/arch/ifx WORKDIR src/ -ENV myarch=$rttov_arch installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 +ENV myarch=${rttov_arch} installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} && \ make ARCH=$myarch INSTALLDIR=$installdir clean $makeflags && \ From 359681003d578f9913571fa14a5888de0eae6fed Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:50:48 -0500 Subject: [PATCH 14/47] Args in the right places --- .github/workflows/build-ci-docker-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index aef2509b46..4ff931a8d7 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -36,12 +36,12 @@ jobs: uses: docker/build-push-action@v6 with: file: containers/Dockerfile-add-python - build-args: TOOLCHAIN=${{ matrix.toolchain }}, rttov_arch=${{ matrix.rttov_arch }} + build-args: TOOLCHAIN=${{ matrix.toolchain }} tags: add-python:${{ matrix.toolchain }} - name: Build add-RTTOV:${{ matrix.toolchain }} uses: docker/build-push-action@v6 with: file: containers/Dockerfile-add-RTTOV - build-args: TOOLCHAIN=${{ matrix.toolchain }} + build-args: TOOLCHAIN=${{ matrix.toolchain }}, rttov_arch=${{ matrix.rttov_arch }} tags: add-rttov:${{ matrix.toolchain }} From 44444ba30064c61aa7f5199f0146863de3c8c1c2 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 21:58:01 -0500 Subject: [PATCH 15/47] Another way to do a list? --- .github/workflows/build-ci-docker-images.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 4ff931a8d7..cc6aed4710 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -43,5 +43,7 @@ jobs: uses: docker/build-push-action@v6 with: file: containers/Dockerfile-add-RTTOV - build-args: TOOLCHAIN=${{ matrix.toolchain }}, rttov_arch=${{ matrix.rttov_arch }} + build-args: | + TOOLCHAIN=${{ matrix.toolchain }} + rttov_arch=${{ matrix.rttov_arch }} tags: add-rttov:${{ matrix.toolchain }} From db7de882d09d57863a127c501a5dbb620f9d2fe2 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 22:07:10 -0500 Subject: [PATCH 16/47] Where are all those files --- .github/workflows/build-ci-docker-images.yml | 2 +- containers/Dockerfile-add-RTTOV | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index cc6aed4710..5050f64e30 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -45,5 +45,5 @@ jobs: file: containers/Dockerfile-add-RTTOV build-args: | TOOLCHAIN=${{ matrix.toolchain }} - rttov_arch=${{ matrix.rttov_arch }} + RTTOV_ARCH=${{ matrix.rttov_arch }} tags: add-rttov:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index b15aa006f5..8c4e9146fd 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -1,5 +1,5 @@ ARG TOOLCHAIN -ARG rttov_arch +ARG RTTOV_ARCH FROM add-python:$TOOLCHAIN ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" @@ -24,17 +24,17 @@ RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm= # Copy in a new ifx arch file (which might or might not get used); append to Makefile.inc # WORKDIR /RTTOV_src/ -RUN --mount=type=bind,source=rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ +RUN --mount=type=bind,source=containers/rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ cat /tmp/Makefile-inc-patch >> build/Makefile.inc && \ rm -f build/arch/ifx ADD containers/rttov-ifx-arch-file build/arch/ifx WORKDIR src/ -ENV myarch=${rttov_arch} installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 +ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} && \ - make ARCH=$myarch INSTALLDIR=$installdir clean $makeflags && \ - make ARCH=$myarch INSTALLDIR=$installdir $makeflags + make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags && \ + make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir $makeflags RUN wget -np -l1 \ https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ From c3452b99c30e657634e8c31c3b4ad0202d63d2cc Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 22:11:59 -0500 Subject: [PATCH 17/47] Does configure really want F77? --- containers/Dockerfile-gfortran-minimal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/Dockerfile-gfortran-minimal b/containers/Dockerfile-gfortran-minimal index 8ae7b4cb15..d0b81ee1f2 100644 --- a/containers/Dockerfile-gfortran-minimal +++ b/containers/Dockerfile-gfortran-minimal @@ -17,4 +17,4 @@ RUN apt-get update \ gfortran-13 \ gfortran-14 -ENV FC=gfortran CC=gcc +ENV FC=gfortran CC=gcc F77=gfortran From 19b7fa6b64edeee7c42ccec3d0743dee791fdfc3 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 22:18:30 -0500 Subject: [PATCH 18/47] generic gfortran? --- containers/Dockerfile-gfortran-minimal | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/containers/Dockerfile-gfortran-minimal b/containers/Dockerfile-gfortran-minimal index d0b81ee1f2..fa676dbe93 100644 --- a/containers/Dockerfile-gfortran-minimal +++ b/containers/Dockerfile-gfortran-minimal @@ -13,8 +13,6 @@ RUN apt-get update \ libc-dev \ make \ git \ - gfortran-12 \ - gfortran-13 \ - gfortran-14 + gfortran>=14 ENV FC=gfortran CC=gcc F77=gfortran From 805b275c844bcd109339277c413cf3487feb6c69 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Mon, 12 Jan 2026 22:24:01 -0500 Subject: [PATCH 19/47] Revise netcdf version --- containers/Dockerfile-add-netcdf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/Dockerfile-add-netcdf b/containers/Dockerfile-add-netcdf index 9fc6304743..09695a3b1b 100644 --- a/containers/Dockerfile-add-netcdf +++ b/containers/Dockerfile-add-netcdf @@ -14,7 +14,7 @@ RUN apt-get update \ # Install NetCDF Fortran # The version must be compitible with NetCDF C installed above -ARG NFVERSION=4.5.4 +ARG NFVERSION=4.6.2 RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-fortran-$NFVERSION.tar.gz | tar xz \ && cd netcdf-fortran-$NFVERSION \ && { ./configure \ From 9366a5201a01bc3ee83d7c970fe88c3c39b259b3 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:10:50 -0500 Subject: [PATCH 20/47] Debug in steps --- containers/Dockerfile-add-RTTOV | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 8c4e9146fd..3ad1cf4d7e 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -15,8 +15,8 @@ RUN apt-get update \ # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball # De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} # -RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ - gpg --quiet --batch --yes --decrypt \ +RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg +RUN gpg --quiet --batch --yes --decrypt \ --passphrase=${{ secrets.RTTOV_TARFILE_KEY }} --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg From d060ab82dad2a94c1f56da6a5e6b77eace882a51 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:18:43 -0500 Subject: [PATCH 21/47] RTFM --- .github/workflows/build-ci-docker-images.yml | 2 ++ containers/Dockerfile-add-RTTOV | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 5050f64e30..ae33dd3018 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -41,6 +41,8 @@ jobs: - name: Build add-RTTOV:${{ matrix.toolchain }} uses: docker/build-push-action@v6 + env: + rttov_tarfile_key: ${{ secrets.RTTOV_TARFILE_KEY }} with: file: containers/Dockerfile-add-RTTOV build-args: | diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 3ad1cf4d7e..b107e1a788 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -17,7 +17,7 @@ RUN apt-get update \ # RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg RUN gpg --quiet --batch --yes --decrypt \ - --passphrase=${{ secrets.RTTOV_TARFILE_KEY }} --output ${filename} ${filename}.gpg && \ + --passphrase="$rttov_tarfile_key" --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From 448be1d35f5061c5b736df137bfe3c975b7e7d2c Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:23:10 -0500 Subject: [PATCH 22/47] Maybe braces? --- containers/Dockerfile-add-RTTOV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index b107e1a788..a15924150f 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -17,7 +17,7 @@ RUN apt-get update \ # RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg RUN gpg --quiet --batch --yes --decrypt \ - --passphrase="$rttov_tarfile_key" --output ${filename} ${filename}.gpg && \ + --passphrase="${rttov_tarfile_key}" --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From bc6fa1b42f9448806af8f63341c9caf8e57864d8 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:26:33 -0500 Subject: [PATCH 23/47] No quotes? Seems unlikely --- containers/Dockerfile-add-RTTOV | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index a15924150f..632e864e12 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -15,9 +15,9 @@ RUN apt-get update \ # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball # De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} # -RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg -RUN gpg --quiet --batch --yes --decrypt \ - --passphrase="${rttov_tarfile_key}" --output ${filename} ${filename}.gpg && \ +RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ + gpg --quiet --batch --yes --decrypt \ + --passphrase=${rttov_tarfile_key} --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From 8048f8edd3057aa7db53d9f02fb1e2fb7a21e5f3 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:30:55 -0500 Subject: [PATCH 24/47] with? --- .github/workflows/build-ci-docker-images.yml | 1 + containers/Dockerfile-add-RTTOV | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index ae33dd3018..88ca971374 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -44,6 +44,7 @@ jobs: env: rttov_tarfile_key: ${{ secrets.RTTOV_TARFILE_KEY }} with: + rttov_tarfile_key: ${{ secrets.RTTOV_TARFILE_KEY }} file: containers/Dockerfile-add-RTTOV build-args: | TOOLCHAIN=${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 632e864e12..28e9b32b90 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -17,7 +17,7 @@ RUN apt-get update \ # RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ - --passphrase=${rttov_tarfile_key} --output ${filename} ${filename}.gpg && \ + --passphrase="${rttov_tarfile_key}" --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From ba6440c4da92d85721f979dcf1e7e097622e4e56 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:40:01 -0500 Subject: [PATCH 25/47] using input --- containers/Dockerfile-add-RTTOV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 28e9b32b90..5f88814fb5 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -17,7 +17,7 @@ RUN apt-get update \ # RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ - --passphrase="${rttov_tarfile_key}" --output ${filename} ${filename}.gpg && \ + --passphrase="${{ github.event.inputs.rttov_tarfile_key }}" --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From d172ba60a1a5f19da72d0803d4a190ca14915215 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:47:02 -0500 Subject: [PATCH 26/47] debug --- .github/workflows/build-ci-docker-images.yml | 3 +-- containers/Dockerfile-add-RTTOV | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 88ca971374..f624057a9d 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -44,9 +44,8 @@ jobs: env: rttov_tarfile_key: ${{ secrets.RTTOV_TARFILE_KEY }} with: - rttov_tarfile_key: ${{ secrets.RTTOV_TARFILE_KEY }} file: containers/Dockerfile-add-RTTOV build-args: | - TOOLCHAIN=${{ matrix.toolchain }} + TOOLCHAIN= ${{ matrix.toolchain }} RTTOV_ARCH=${{ matrix.rttov_arch }} tags: add-rttov:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 5f88814fb5..261fc022ea 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -15,9 +15,10 @@ RUN apt-get update \ # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball # De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} # +RUN cat "${rttov_tarfile_key}" && cat ${rttov_tarfile_key} RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ - --passphrase="${{ github.event.inputs.rttov_tarfile_key }}" --output ${filename} ${filename}.gpg && \ + --passphrase="${rttov_tarfile_key}" --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From 3eb9d680c9b19074ac4afe3dce444da2526e644a Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:49:16 -0500 Subject: [PATCH 27/47] Syntax --- .github/workflows/build-ci-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index f624057a9d..ae33dd3018 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -46,6 +46,6 @@ jobs: with: file: containers/Dockerfile-add-RTTOV build-args: | - TOOLCHAIN= ${{ matrix.toolchain }} + TOOLCHAIN=${{ matrix.toolchain }} RTTOV_ARCH=${{ matrix.rttov_arch }} tags: add-rttov:${{ matrix.toolchain }} From 0ddc4561e6643cfae7230aae37d1606b2a5452fb Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 18:55:01 -0500 Subject: [PATCH 28/47] Debug (also updated secrets --- containers/Dockerfile-add-RTTOV | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 261fc022ea..17d8bee0a9 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -15,7 +15,10 @@ RUN apt-get update \ # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball # De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} # -RUN cat "${rttov_tarfile_key}" && cat ${rttov_tarfile_key} +RUN echo "${rttov_tarfile_key}" +RUN echo ${rttov_tarfile_key} +RUN echo ${{ secrets.RTTOV_TARFILE_KEY }} + RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ --passphrase="${rttov_tarfile_key}" --output ${filename} ${filename}.gpg && \ From b49cce467c85a4410033ac8a92bef080a2ce27dc Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:03:34 -0500 Subject: [PATCH 29/47] Pass secret to Docker build --- .github/workflows/build-ci-docker-images.yml | 3 +-- containers/Dockerfile-add-RTTOV | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index ae33dd3018..0955106e79 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -41,11 +41,10 @@ jobs: - name: Build add-RTTOV:${{ matrix.toolchain }} uses: docker/build-push-action@v6 - env: - rttov_tarfile_key: ${{ secrets.RTTOV_TARFILE_KEY }} with: file: containers/Dockerfile-add-RTTOV build-args: | TOOLCHAIN=${{ matrix.toolchain }} RTTOV_ARCH=${{ matrix.rttov_arch }} + RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }} tags: add-rttov:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 17d8bee0a9..f5f7efc038 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -1,8 +1,9 @@ ARG TOOLCHAIN ARG RTTOV_ARCH +ARG RTTOV_TARFILE_KEY FROM add-python:$TOOLCHAIN -ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" +ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" # gpg and xz needed for unpacking the tar file # hdf5 (serial) needed for RTTOV @@ -15,13 +16,10 @@ RUN apt-get update \ # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball # De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} # -RUN echo "${rttov_tarfile_key}" -RUN echo ${rttov_tarfile_key} -RUN echo ${{ secrets.RTTOV_TARFILE_KEY }} RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ - --passphrase="${rttov_tarfile_key}" --output ${filename} ${filename}.gpg && \ + --passphrase="$RTTOV_TARFILE_KEY" --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From 0396cf3fd24c970f416956de399fac1ee4d2d940 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:07:10 -0500 Subject: [PATCH 30/47] No quotes? --- containers/Dockerfile-add-RTTOV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index f5f7efc038..adc0af72ed 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -19,7 +19,7 @@ RUN apt-get update \ RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ - --passphrase="$RTTOV_TARFILE_KEY" --output ${filename} ${filename}.gpg && \ + --passphrase=$RTTOV_TARFILE_KEY --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg # From 723125c3abe3d8d9eaa81b2d60dabae9b6e70f7f Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:15:40 -0500 Subject: [PATCH 31/47] Using secrets --- .github/workflows/build-ci-docker-images.yml | 1 + containers/Dockerfile-add-RTTOV | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 0955106e79..fe3fa05585 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -46,5 +46,6 @@ jobs: build-args: | TOOLCHAIN=${{ matrix.toolchain }} RTTOV_ARCH=${{ matrix.rttov_arch }} + secrets: | RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }} tags: add-rttov:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index adc0af72ed..f04d4fc8f7 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -1,6 +1,5 @@ ARG TOOLCHAIN ARG RTTOV_ARCH -ARG RTTOV_TARFILE_KEY FROM add-python:$TOOLCHAIN ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" @@ -16,7 +15,7 @@ RUN apt-get update \ # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball # De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} # - +RUN --mount=type=secret,id=RTTOV_TARFILE_KEY,env=RTTOV_TARFILE_KEY RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ --passphrase=$RTTOV_TARFILE_KEY --output ${filename} ${filename}.gpg && \ From 1872ec719c4e1170feb0c3e3d094057df18e65db Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:20:36 -0500 Subject: [PATCH 32/47] Quotes in Actions file? --- .github/workflows/build-ci-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index fe3fa05585..b80a9b117a 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -47,5 +47,5 @@ jobs: TOOLCHAIN=${{ matrix.toolchain }} RTTOV_ARCH=${{ matrix.rttov_arch }} secrets: | - RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }} + "RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }}" tags: add-rttov:${{ matrix.toolchain }} From a0d909cedc466198766cf5822e55912cc58ad32c Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:25:21 -0500 Subject: [PATCH 33/47] Secret last for one RUN? --- containers/Dockerfile-add-RTTOV | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index f04d4fc8f7..6ee9409d4f 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -15,8 +15,8 @@ RUN apt-get update \ # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball # De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} # -RUN --mount=type=secret,id=RTTOV_TARFILE_KEY,env=RTTOV_TARFILE_KEY -RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ +RUN --mount=type=secret,id=RTTOV_TARFILE_KEY,env=RTTOV_TARFILE_KEY \ + curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ --passphrase=$RTTOV_TARFILE_KEY --output ${filename} ${filename}.gpg && \ rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg From 1ce1eabb0ada7c7a6491c5b7ca7e5669ecf0d02c Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:33:34 -0500 Subject: [PATCH 34/47] Debug in steps --- containers/Dockerfile-add-RTTOV | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 6ee9409d4f..e53fc17dcd 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -2,8 +2,6 @@ ARG TOOLCHAIN ARG RTTOV_ARCH FROM add-python:$TOOLCHAIN -ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" - # gpg and xz needed for unpacking the tar file # hdf5 (serial) needed for RTTOV RUN apt-get update \ @@ -13,8 +11,9 @@ RUN apt-get update \ # # Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball -# De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} +# De-armor RTTOV tarball with passkey kept in Github secrets and passed in to the Docker build # +ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" RUN --mount=type=secret,id=RTTOV_TARFILE_KEY,env=RTTOV_TARFILE_KEY \ curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ gpg --quiet --batch --yes --decrypt \ @@ -30,12 +29,12 @@ RUN --mount=type=bind,source=containers/rttov-Makefile-inc,target=/tmp/Makefile- rm -f build/arch/ifx ADD containers/rttov-ifx-arch-file build/arch/ifx -WORKDIR src/ -ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 +WORKDIR /RTTOV_src/src/ +ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 -RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} && \ - make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags && \ - make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir $makeflags +RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} +RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags +RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir $makeflags RUN wget -np -l1 \ https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ From 98eec60988bd24eaa494422a4d1ad3c5685ad87d Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:38:11 -0500 Subject: [PATCH 35/47] import ARG as env variable --- containers/Dockerfile-add-RTTOV | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index e53fc17dcd..4600773d2a 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -30,11 +30,11 @@ RUN --mount=type=bind,source=containers/rttov-Makefile-inc,target=/tmp/Makefile- ADD containers/rttov-ifx-arch-file build/arch/ifx WORKDIR /RTTOV_src/src/ -ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 +ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 arch=$RTTOV_ARCH RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} -RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags -RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir $makeflags +RUN make ARCH=$arch INSTALLDIR=$installdir clean $makeflags +RUN make ARCH=$arch INSTALLDIR=$installdir $makeflags RUN wget -np -l1 \ https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ From 4ea03d093eb6aeb6110d756fe209b813a18e1e6f Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 19:44:05 -0500 Subject: [PATCH 36/47] Ordering matters, it turns out --- containers/Dockerfile-add-RTTOV | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 4600773d2a..2a59631850 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -1,5 +1,4 @@ ARG TOOLCHAIN -ARG RTTOV_ARCH FROM add-python:$TOOLCHAIN # gpg and xz needed for unpacking the tar file @@ -30,11 +29,12 @@ RUN --mount=type=bind,source=containers/rttov-Makefile-inc,target=/tmp/Makefile- ADD containers/rttov-ifx-arch-file build/arch/ifx WORKDIR /RTTOV_src/src/ -ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 arch=$RTTOV_ARCH +ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} -RUN make ARCH=$arch INSTALLDIR=$installdir clean $makeflags -RUN make ARCH=$arch INSTALLDIR=$installdir $makeflags +ARG RTTOV_ARCH +RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags +RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir $makeflags RUN wget -np -l1 \ https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ From 6491dec5241a0838787f38b95ee71b60c41b9c7b Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 22:49:13 -0500 Subject: [PATCH 37/47] Specify installation dir for netCDF Fortran --- .github/workflows/build-ci-docker-images.yml | 10 ++++- containers/Dockerfile-add-netcdf | 3 +- containers/Dockerfile-oneapi | 41 -------------------- containers/rttov-Makefile-inc | 4 +- 4 files changed, 13 insertions(+), 45 deletions(-) delete mode 100644 containers/Dockerfile-oneapi diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index b80a9b117a..dd798eeef8 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -39,6 +39,14 @@ jobs: build-args: TOOLCHAIN=${{ matrix.toolchain }} tags: add-python:${{ matrix.toolchain }} + - name: Log in to the Github container registry + if: ${{ github.ref == 'refs/heads/master' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build add-RTTOV:${{ matrix.toolchain }} uses: docker/build-push-action@v6 with: @@ -48,4 +56,4 @@ jobs: RTTOV_ARCH=${{ matrix.rttov_arch }} secrets: | "RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }}" - tags: add-rttov:${{ matrix.toolchain }} + tags: ghcr.io/cfmip/cospv2.0-ci:${{ matrix.toolchain }} diff --git a/containers/Dockerfile-add-netcdf b/containers/Dockerfile-add-netcdf index 09695a3b1b..993c2dd9ec 100644 --- a/containers/Dockerfile-add-netcdf +++ b/containers/Dockerfile-add-netcdf @@ -20,7 +20,8 @@ RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-for && { ./configure \ CFLAGS='-O2' \ FCFLAGS='-O2 -fPIC' \ - --disable-static || \ + --disable-static \ + --prefix=/usr || \ { cat ./config.log; exit 1; } } RUN cd netcdf-fortran-$NFVERSION \ && make -j \ diff --git a/containers/Dockerfile-oneapi b/containers/Dockerfile-oneapi deleted file mode 100644 index aaa83de684..0000000000 --- a/containers/Dockerfile-oneapi +++ /dev/null @@ -1,41 +0,0 @@ -# Base Docker image from earth-system-radiation includes -# OneAPI compiler chain, netCDF, Python -FROM earthsystemradiation/rte-rrtmgp-ci:oneapi - -ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" - -# gpg and xz needed for unpacking the tar file -# hdf5 (serial) needed for RTTOV -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ - gpg xz-utils \ - libhdf5-dev - -# -# Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball -# De-armor RTTOV tarball with passkey (should be hidden with github secrets) ${{ secrets.RTTOV_TARFILE_KEY }} -# -RUN curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ - gpg --quiet --batch --yes --decrypt \ - --passphrase=${{ secrets.RTTOV_TARFILE_KEY }} --output ${filename} ${filename}.gpg && \ - rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg - -# -# Copy in a new ifx arch file; append to Makefile.inc -# -WORKDIR /RTTOV_src/ -RUN --mount=type=bind,source=rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ - cat /tmp/Makefile-inc-patch >> build/Makefile.inc && \ - rm -f build/arch/ifx -ADD rttov-ifx-arch-file build/arch/ifx - -WORKDIR src/ -ENV myarch="ifx" clean="n" installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 netcdf=1 - -RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} && \ - make ARCH=$myarch INSTALLDIR=$installdir clean $makeflags && \ - make ARCH=$myarch INSTALLDIR=$installdir $makeflags - -RUN wget -np -l1 \ - https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ - -P../../RTTOV_coefs/ diff --git a/containers/rttov-Makefile-inc b/containers/rttov-Makefile-inc index 021e08018c..89ba8b1bba 100644 --- a/containers/rttov-Makefile-inc +++ b/containers/rttov-Makefile-inc @@ -1,7 +1,7 @@ -# HDF5_PREFIX = /usr/ +# HDF5_PREFIX = /usr # FFLAGS_HDF5 = -D_RTTOV_HDF $(FFLAG_MOD)$(HDF5_PREFIX)/include # LDFLAGS_HDF5 = -L$(HDF5_PREFIX)/lib -lhdf5_hl_fortran -lhdf5_hl -lhdf5_fortran -lhdf5 -lz -NETCDF_PREFIX = /usr/ +NETCDF_PREFIX = /usr FFLAGS_NETCDF = -D_RTTOV_NETCDF -I$(NETCDF_PREFIX)/include LDFLAGS_NETCDF = -L$(NETCDF_PREFIX)/lib -lnetcdff From 82fefc373b3eeab63c75e5eba66c4c772caf85b8 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 22:57:04 -0500 Subject: [PATCH 38/47] Test pushing to ghcr.io --- .github/workflows/build-ci-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index dd798eeef8..865afd436a 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -40,7 +40,7 @@ jobs: tags: add-python:${{ matrix.toolchain }} - name: Log in to the Github container registry - if: ${{ github.ref == 'refs/heads/master' }} + if: ${{ github.ref == 'refs/heads/add-containers' }} uses: docker/login-action@v3 with: registry: ghcr.io From df8ae2db4405042a8e67db7b0893dbd75da22d16 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Tue, 13 Jan 2026 23:07:41 -0500 Subject: [PATCH 39/47] Build containers on workflow_dispatch, containerized-CI on push --- .github/workflows/build-ci-docker-images.yml | 2 +- .../workflows/build-gfortran-container.yml | 16 ----------- .github/workflows/build-oneapi-container.yml | 27 ------------------- .github/workflows/containerized-ci.yml | 24 +++++++---------- containers/Dockerfile-add-RTTOV | 6 ++--- 5 files changed, 13 insertions(+), 62 deletions(-) delete mode 100644 .github/workflows/build-gfortran-container.yml delete mode 100644 .github/workflows/build-oneapi-container.yml diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 865afd436a..b40fea7e41 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -1,7 +1,7 @@ name: COSP - Build CI Docker Images run-name: CI Image Build -on: [push] +on: [workflow_dispatch] jobs: docker: diff --git a/.github/workflows/build-gfortran-container.yml b/.github/workflows/build-gfortran-container.yml deleted file mode 100644 index 495ed3e2f0..0000000000 --- a/.github/workflows/build-gfortran-container.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: COSP gfortran Docker Images Build -run-name: gfortran Docker Image Build - -on: workflow_dispatch - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Build gfortran toolchain - uses: docker/build-push-action@v6 - with: - file: containers/Dockerfile-gfortran - tags: toolchain:gfortran \ No newline at end of file diff --git a/.github/workflows/build-oneapi-container.yml b/.github/workflows/build-oneapi-container.yml deleted file mode 100644 index f605630229..0000000000 --- a/.github/workflows/build-oneapi-container.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: COSP ifx Docker Images Build -run-name: ifx Docker Image Build - -on: workflow_dispatch - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - - name: Log in to the Container registry - if: ${{ github.ref == 'refs/heads/main' }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push rttov:${{ matrix.toolchain }} - uses: docker/build-push-action@v6 - with: - file: containers/Dockerfile-oneapi - push: ${{ github.ref == 'refs/heads/main' }} - tags: | - earthsystemradiation/rttov:oneapi - ghcr.io/earth-system-radiation/rttov:oneapi diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml index 596224faf7..a445213d56 100644 --- a/.github/workflows/containerized-ci.yml +++ b/.github/workflows/containerized-ci.yml @@ -1,5 +1,5 @@ name: Continuous integration in containers -on: [workflow_dispatch] +on: [push, workflow_dispatch] jobs: Containerized-CI: @@ -7,25 +7,28 @@ jobs: strategy: fail-fast: false matrix: - compiler: [ifx] + toolchain: [oneapi, gfortran] include: # Flags and KGOs for Intel Fortran Compiler - - compiler: ifx + - toolchain: oneapi + compiler: ifx fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08 - image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:oneapi + - toolchain: gfortran + compiler: gfortran + fcflags: -O3 -ffree-line-length-none -fcheck=bounds -finit-real=nan # Common variables - kgo_version: v007 defaults: run: shell: bash -el {0} container: - image: ${{ matrix.image }} + image: ghcr.io/cfmip/cospv2.0-ci:${{ matrix.toolchain }} env: F90: ${{ matrix.compiler }} FC: ${{ matrix.compiler }} F90FLAGS: ${{ matrix.fcflags }} # Make variables: - NFHOME: /opt/netcdf-fortran + NFHOME: /usr # KGO tests variables ATOL: 0.0 RTOL: 0.0 @@ -35,15 +38,6 @@ jobs: # Checks-out repository under $GITHUB_WORKSPACE # - uses: actions/checkout@v6 - # Set up conda environment - - name: Setup conda environment - uses: conda-incubator/setup-miniconda@v3.2.0 - with: - auto-update-conda: true - activate-environment: ci-env - environment-file: build/environment.yml - python-version: ${{ matrix.python-version }} - auto-activate-base: false ############################################################################### # Build COSP and retrieve input and test files diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index 2a59631850..d9a395a30a 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -36,6 +36,6 @@ ARG RTTOV_ARCH RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir $makeflags -RUN wget -np -l1 \ - https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ - -P../../RTTOV_coefs/ +# RUN wget -np -l1 \ +# https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ +# -P../../RTTOV_coefs/ From fb946a60f872797f362b34fa96122f60d26c0799 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 13 Jan 2026 22:32:25 -0700 Subject: [PATCH 40/47] Username used to log into ghcr.io needs to match organization in tag name used in subsequent step. Otherwise, package is not published. --- .github/workflows/build-ci-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index b40fea7e41..2ecae21769 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -44,7 +44,7 @@ jobs: uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ github.actor }} + username: CFMIP password: ${{ secrets.GITHUB_TOKEN }} - name: Build add-RTTOV:${{ matrix.toolchain }} From 55e8edb0bf78be3c9031baa450df6321a5d5b48f Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 13 Jan 2026 22:41:42 -0700 Subject: [PATCH 41/47] Add push to trigger workflow --- .github/workflows/build-ci-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 2ecae21769..cb019da120 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -1,7 +1,7 @@ name: COSP - Build CI Docker Images run-name: CI Image Build -on: [workflow_dispatch] +on: [push, workflow_dispatch] jobs: docker: From 9a6d38525e0fa871a003ecc31889ce817a885ceb Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 13 Jan 2026 22:51:14 -0700 Subject: [PATCH 42/47] Add push to trigger workflow --- .github/workflows/build-ci-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index cb019da120..fc7edfaefd 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -45,7 +45,7 @@ jobs: with: registry: ghcr.io username: CFMIP - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GHCR_TOKEN }} - name: Build add-RTTOV:${{ matrix.toolchain }} uses: docker/build-push-action@v6 From 49b79176ffa4f074da14817a2721e001520fba00 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Wed, 14 Jan 2026 19:40:42 -0500 Subject: [PATCH 43/47] Change back to building and pushing containers (no actual CI) --- .github/workflows/build-ci-docker-images.yml | 2 +- .github/workflows/containerized-ci.yml | 2 +- .github/workflows/continuous_integration.yml | 1 + containers/Dockerfile-add-RTTOV | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index fc7edfaefd..722845fbf5 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -1,7 +1,7 @@ name: COSP - Build CI Docker Images run-name: CI Image Build -on: [push, workflow_dispatch] +on: [push] jobs: docker: diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml index a445213d56..3cc62ad314 100644 --- a/.github/workflows/containerized-ci.yml +++ b/.github/workflows/containerized-ci.yml @@ -1,5 +1,5 @@ name: Continuous integration in containers -on: [push, workflow_dispatch] +on: [workflow_dispatch] jobs: Containerized-CI: diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index a209db3549..496d3504e3 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -28,6 +28,7 @@ # Workflow for continuous integration tests name: Continuous integration gfortran compilers on: [push, pull_request, workflow_dispatch] +on: [pull_request, workflow_dispatch] jobs: # This workflow contains a single job called "ci_gfortran" diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV index d9a395a30a..b3cd2103c5 100644 --- a/containers/Dockerfile-add-RTTOV +++ b/containers/Dockerfile-add-RTTOV @@ -29,7 +29,7 @@ RUN --mount=type=bind,source=containers/rttov-Makefile-inc,target=/tmp/Makefile- ADD containers/rttov-ifx-arch-file build/arch/ifx WORKDIR /RTTOV_src/src/ -ENV installdir="/RTTOV_build/" lapack=0 f2py=0 gui=0 hdf5=0 +ENV installdir="/usr/" lapack=0 f2py=0 gui=0 hdf5=0 RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} ARG RTTOV_ARCH From 9f1c43bc0f5d5252124446dc2e1b88a5bd4788bd Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Wed, 14 Jan 2026 20:08:20 -0500 Subject: [PATCH 44/47] Correct syntax error (images not being pushed?) --- .github/workflows/continuous_integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 496d3504e3..6e65651c69 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -27,7 +27,7 @@ # Workflow for continuous integration tests name: Continuous integration gfortran compilers -on: [push, pull_request, workflow_dispatch] +# on: [push, pull_request, workflow_dispatch] on: [pull_request, workflow_dispatch] jobs: From 5966b0c92935a7b609d11c26699158f778db308d Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Thu, 15 Jan 2026 21:57:37 -0500 Subject: [PATCH 45/47] Missing flag? --- .github/workflows/build-ci-docker-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 722845fbf5..0ef4d24ada 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -57,3 +57,4 @@ jobs: secrets: | "RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }}" tags: ghcr.io/cfmip/cospv2.0-ci:${{ matrix.toolchain }} + push: True From 9d50fa2f0d289e71bd58cfd9e92fcc65642df364 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Thu, 15 Jan 2026 22:08:27 -0500 Subject: [PATCH 46/47] So does the containerized CI run in both containers? --- .github/workflows/build-ci-docker-images.yml | 2 +- .github/workflows/containerized-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml index 0ef4d24ada..f4d3401928 100644 --- a/.github/workflows/build-ci-docker-images.yml +++ b/.github/workflows/build-ci-docker-images.yml @@ -1,7 +1,7 @@ name: COSP - Build CI Docker Images run-name: CI Image Build -on: [push] +on: [workflow_dispatch] jobs: docker: diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml index 3cc62ad314..911521950a 100644 --- a/.github/workflows/containerized-ci.yml +++ b/.github/workflows/containerized-ci.yml @@ -1,5 +1,5 @@ name: Continuous integration in containers -on: [workflow_dispatch] +on: [push] jobs: Containerized-CI: From 16c6fe7ff55e013537bf32a6d414508ad301b5a2 Mon Sep 17 00:00:00 2001 From: Robert Pincus Date: Fri, 16 Jan 2026 11:38:00 -0500 Subject: [PATCH 47/47] Expand use of containerized CI --- .github/workflows/containerized-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml index 911521950a..124383e1ea 100644 --- a/.github/workflows/containerized-ci.yml +++ b/.github/workflows/containerized-ci.yml @@ -1,5 +1,5 @@ name: Continuous integration in containers -on: [push] +on: [push, pull_request, workflow_dispatch] jobs: Containerized-CI: