diff --git a/.github/workflows/build-ci-docker-images.yml b/.github/workflows/build-ci-docker-images.yml new file mode 100644 index 0000000000..f4d3401928 --- /dev/null +++ b/.github/workflows/build-ci-docker-images.yml @@ -0,0 +1,60 @@ +name: COSP - Build CI Docker Images +run-name: CI Image Build + +on: [workflow_dispatch] + +jobs: + docker: + strategy: + fail-fast: false + matrix: + toolchain: [oneapi, gfortran] + include: + - toolchain: oneapi + rttov_arch: ifx + - toolchain: gfortran + rttov_arch: gfortran + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build minimal-toolchain:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: containers/Dockerfile-${{ matrix.toolchain }}-minimal + build-args: TOOLCHAIN=${{ matrix.toolchain }} + tags: minimal-toolchain:${{ matrix.toolchain }} + + - name: Build add-netcdf:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: containers/Dockerfile-add-netcdf + build-args: TOOLCHAIN=${{ matrix.toolchain }} + tags: add-netcdf:${{ matrix.toolchain }} + + - name: Build add-python:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: containers/Dockerfile-add-python + build-args: TOOLCHAIN=${{ matrix.toolchain }} + tags: add-python:${{ matrix.toolchain }} + + - name: Log in to the Github container registry + if: ${{ github.ref == 'refs/heads/add-containers' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: CFMIP + password: ${{ secrets.GHCR_TOKEN }} + + - name: Build add-RTTOV:${{ matrix.toolchain }} + uses: docker/build-push-action@v6 + with: + file: containers/Dockerfile-add-RTTOV + build-args: | + TOOLCHAIN=${{ matrix.toolchain }} + RTTOV_ARCH=${{ matrix.rttov_arch }} + secrets: | + "RTTOV_TARFILE_KEY=${{ secrets.RTTOV_TARFILE_KEY }}" + tags: ghcr.io/cfmip/cospv2.0-ci:${{ matrix.toolchain }} + push: True diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml index 2d3cb1fdad..124383e1ea 100644 --- a/.github/workflows/containerized-ci.yml +++ b/.github/workflows/containerized-ci.yml @@ -7,25 +7,28 @@ jobs: strategy: fail-fast: false matrix: - compiler: [ifx] + toolchain: [oneapi, gfortran] include: # Flags and KGOs for Intel Fortran Compiler - - compiler: ifx + - toolchain: oneapi + compiler: ifx fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08 - image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:oneapi + - toolchain: gfortran + compiler: gfortran + fcflags: -O3 -ffree-line-length-none -fcheck=bounds -finit-real=nan # Common variables - kgo_version: v007 defaults: run: shell: bash -el {0} container: - image: ${{ matrix.image }} + image: ghcr.io/cfmip/cospv2.0-ci:${{ matrix.toolchain }} env: F90: ${{ matrix.compiler }} FC: ${{ matrix.compiler }} F90FLAGS: ${{ matrix.fcflags }} # Make variables: - NFHOME: /opt/netcdf-fortran + NFHOME: /usr # KGO tests variables ATOL: 0.0 RTOL: 0.0 @@ -35,15 +38,6 @@ jobs: # Checks-out repository under $GITHUB_WORKSPACE # - uses: actions/checkout@v6 - # Set up conda environment - - name: Setup conda environment - uses: conda-incubator/setup-miniconda@v3.2.0 - with: - auto-update-conda: true - activate-environment: ci-env - environment-file: build/environment.yml - python-version: ${{ matrix.python-version }} - auto-activate-base: false ############################################################################### # Build COSP and retrieve input and test files diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index a209db3549..6e65651c69 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -27,7 +27,8 @@ # Workflow for continuous integration tests name: Continuous integration gfortran compilers -on: [push, pull_request, workflow_dispatch] +# on: [push, pull_request, workflow_dispatch] +on: [pull_request, workflow_dispatch] jobs: # This workflow contains a single job called "ci_gfortran" diff --git a/containers/Dockerfile-add-RTTOV b/containers/Dockerfile-add-RTTOV new file mode 100644 index 0000000000..b3cd2103c5 --- /dev/null +++ b/containers/Dockerfile-add-RTTOV @@ -0,0 +1,41 @@ +ARG TOOLCHAIN +FROM add-python:$TOOLCHAIN + +# gpg and xz needed for unpacking the tar file +# hdf5 (serial) needed for RTTOV +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + gpg xz-utils \ + libhdf5-dev + +# +# Pull an encrypted RTTOV v13.2 tarball from Google Drive; dearmor and unpack; delete original tarball +# De-armor RTTOV tarball with passkey kept in Github secrets and passed in to the Docker build +# +ENV fileid="1g61nacsXMgXn9KG0xSUODs-qX4mkuqmb" filename="rttov132.tar.xz" +RUN --mount=type=secret,id=RTTOV_TARFILE_KEY,env=RTTOV_TARFILE_KEY \ + curl -L "https://drive.usercontent.google.com/download?id=${fileid}&confirm=xxx" -o ${filename}.gpg && \ + gpg --quiet --batch --yes --decrypt \ + --passphrase=$RTTOV_TARFILE_KEY --output ${filename} ${filename}.gpg && \ + rm -f ./RTTOV_src && mkdir ./RTTOV_src && tar -xf ${filename} -C ./RTTOV_src/ && rm ${filename} ${filename}.gpg + +# +# Copy in a new ifx arch file (which might or might not get used); append to Makefile.inc +# +WORKDIR /RTTOV_src/ +RUN --mount=type=bind,source=containers/rttov-Makefile-inc,target=/tmp/Makefile-inc-patch \ + cat /tmp/Makefile-inc-patch >> build/Makefile.inc && \ + rm -f build/arch/ifx +ADD containers/rttov-ifx-arch-file build/arch/ifx + +WORKDIR /RTTOV_src/src/ +ENV installdir="/usr/" lapack=0 f2py=0 gui=0 hdf5=0 + +RUN ../build/Makefile.PL RTTOV_HDF=${hdf5} RTTOV_F2PY=${f2py} RTTOV_USER_LAPACK=${lapack} +ARG RTTOV_ARCH +RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir clean $makeflags +RUN make ARCH=$RTTOV_ARCH INSTALLDIR=$installdir $makeflags + +# RUN wget -np -l1 \ +# https://nwp-saf.eumetsat.int/downloads/rtcoef_rttov13/rttov13pred101L/rtcoef_eos_2_airs_l1c_7gas.H5 \ +# -P../../RTTOV_coefs/ diff --git a/containers/Dockerfile-add-netcdf b/containers/Dockerfile-add-netcdf new file mode 100644 index 0000000000..993c2dd9ec --- /dev/null +++ b/containers/Dockerfile-add-netcdf @@ -0,0 +1,31 @@ +# +# Install NetCDF Fortran and its dependencies +# + +ARG TOOLCHAIN +FROM minimal-toolchain:$TOOLCHAIN + +# Install the dependencies +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + curl \ + libnetcdf-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install NetCDF Fortran +# The version must be compitible with NetCDF C installed above +ARG NFVERSION=4.6.2 +RUN curl https://downloads.unidata.ucar.edu/netcdf-fortran/$NFVERSION/netcdf-fortran-$NFVERSION.tar.gz | tar xz \ + && cd netcdf-fortran-$NFVERSION \ + && { ./configure \ + CFLAGS='-O2' \ + FCFLAGS='-O2 -fPIC' \ + --disable-static \ + --prefix=/usr || \ + { cat ./config.log; exit 1; } } +RUN cd netcdf-fortran-$NFVERSION \ + && make -j \ + && make install \ + && cd .. \ + && rm -rf netcdf-fortran-$NFVERSION \ + && ldconfig diff --git a/containers/Dockerfile-add-python b/containers/Dockerfile-add-python new file mode 100644 index 0000000000..82f72fc527 --- /dev/null +++ b/containers/Dockerfile-add-python @@ -0,0 +1,20 @@ +# +# Install Python3 and a small set of packages +# + +ARG TOOLCHAIN +FROM add-netcdf:$TOOLCHAIN + +# Install Python +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + python-is-python3 \ + python3 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# Install packages from build/environment.yml +RUN pip3 install --break-system-packages \ + netCDF4 \ + cartopy \ + matplotlib diff --git a/containers/Dockerfile-gfortran-minimal b/containers/Dockerfile-gfortran-minimal new file mode 100644 index 0000000000..fa676dbe93 --- /dev/null +++ b/containers/Dockerfile-gfortran-minimal @@ -0,0 +1,18 @@ +FROM ubuntu:24.04 + +# Extend and update the package registry +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + ca-certificates \ + curl \ + wget \ + gpg \ + binutils \ + g++ \ + gcc \ + libc-dev \ + make \ + git \ + gfortran>=14 + +ENV FC=gfortran CC=gcc F77=gfortran diff --git a/containers/Dockerfile-oneapi-minimal b/containers/Dockerfile-oneapi-minimal new file mode 100644 index 0000000000..0b96c526ff --- /dev/null +++ b/containers/Dockerfile-oneapi-minimal @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 + +ARG ONEAPI_VERSION='2025.3' + +# Extend and update the package registry +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get --yes install --no-install-recommends \ + ca-certificates \ + curl \ + wget \ + gpg \ + binutils \ + g++ \ + gcc \ + libc-dev \ + make \ + cmake \ + git + +# See https://www.intel.com/content/www/us/en/docs/oneapi/installation-guide-linux/2025-2/hpc-apt.html#HPC-APT + +RUN wget -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor \ + | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null \ + && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | tee /etc/apt/sources.list.d/oneAPI.list \ + && apt-get update \ + && apt-get install --yes --no-install-recommends \ + intel-oneapi-compiler-dpcpp-cpp-${ONEAPI_VERSION} \ + intel-oneapi-compiler-fortran-${ONEAPI_VERSION} \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/opt/intel/oneapi/compiler/${ONEAPI_VERSION}/bin:${PATH}" +ENV LD_LIBRARY_PATH="/opt/intel/oneapi/compiler/${ONEAPI_VERSION}/lib:${LD_LIBRARY_PATH}" + +# Set default compiler executables +ENV FC=ifx CC=icx diff --git a/containers/rttov-Makefile-inc b/containers/rttov-Makefile-inc new file mode 100644 index 0000000000..89ba8b1bba --- /dev/null +++ b/containers/rttov-Makefile-inc @@ -0,0 +1,7 @@ +# HDF5_PREFIX = /usr +# FFLAGS_HDF5 = -D_RTTOV_HDF $(FFLAG_MOD)$(HDF5_PREFIX)/include +# LDFLAGS_HDF5 = -L$(HDF5_PREFIX)/lib -lhdf5_hl_fortran -lhdf5_hl -lhdf5_fortran -lhdf5 -lz + +NETCDF_PREFIX = /usr +FFLAGS_NETCDF = -D_RTTOV_NETCDF -I$(NETCDF_PREFIX)/include +LDFLAGS_NETCDF = -L$(NETCDF_PREFIX)/lib -lnetcdff diff --git a/containers/rttov-ifx-arch-file b/containers/rttov-ifx-arch-file new file mode 100644 index 0000000000..da8a4a917f --- /dev/null +++ b/containers/rttov-ifx-arch-file @@ -0,0 +1,19 @@ +FC=ifx +FC77=ifx +CC=gcc +LDFLAGS_ARCH= +CFLAGS_ARCH= +FFLAGS_ARCH=-fPIC -O3 -fp-model source +AR=ar r + +# Loop unrolling causes ifort v13 and later to take a long time to compile these subroutines +FFLAGS_ARCH_rttov_opdep_9_ad=-fPIC -O3 -unroll0 -fp-model source +FFLAGS_ARCH_rttov_opdep_9_k=-fPIC -O3 -unroll0 -fp-model source +FFLAGS_ARCH_rttov_opdep_13_ad=-fPIC -O3 -unroll0 -fp-model source +FFLAGS_ARCH_rttov_opdep_13_k=-fPIC -O3 -unroll0 -fp-model source + +# -fp-model source ensures more consistent floating point results + +F2PY=f2py --fcompiler=intelem +F2PYFLAGS_ARCH="-fPIC" +F2PYLDFLAGS_ARCH= \ No newline at end of file