EMS build tooling

.dockerignore

@@ -0,0 +1,6 @@
+.env
+.gitlab_ci.yaml
+*.log
+*.md
+docker-compose.yml
+hooks

.gitignore

@@ -22,3 +22,17 @@ libjigsaw64r.so
 # Misc
 sediments-us
 
+# Common test output files
+# (Test directories may also have their own .gitignore files)
+model/tests/ems_test.log
+model/tests/**/alert.*
+model/tests/**/core.*
+model/tests/**/diag.txt
+model/tests/**/in*.nc
+model/tests/**/loc*.ts
+model/tests/**/obc_spec.txt
+model/tests/**/out*.nc
+model/tests/**/runlog
+model/tests/**/setup.txt
+model/tests/**/out/*
+!model/tests/**/out/.empty

Dockerfile

@@ -0,0 +1,69 @@
+#------------------------------------------------------------------------------
+# CSIRO EMS Image Build Script
+#------------------------------------------------------------------------------
+# The recommended base image is one of the onaci/ereefs-netcdf-base variants,
+# as those have been designed to include all the EMS dependencies.
+# Allow it to be overridden in order to choose *which* variant, or even
+# something completely different.
+ARG BASE_IMAGE="onaci/ereefs-netcdf-base:python-3.11-slim-bullseye"
+FROM ${BASE_IMAGE}
+
+# Record the actual base image used from the FROM command as a label.
+ARG BASE_IMAGE
+LABEL org.opencontainers.image.base.name=${BASE_IMAGE}
+
+# Enable Bash in RUN commands, and ensure that any commands with
+# pipes exit on the first failure.
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Prepare a base directory for all the EMS code
+ARG EMS_BASE="/usr/local/src/ems"
+ENV EMS_BASE="${EMS_BASE}"
+WORKDIR ${EMS_BASE}
+
+# Install all the EMS Source Code
+COPY ./ ./
+
+# Selectively remove some subdirectories that we want to omit
+# (List can be overridden by a build argument)
+ARG EMS_RM_DIRS=""
+RUN if [ -n "${EMS_RM_DIRS}" ]; then for d in $EMS_RM_DIRS; do rm -rf "${EMS_BASE}/${d}"; done; fi
+
+# Compile the EMS components
+# We build with OpenMP support by default for use in a docker container
+# Different options can be specified by overriding the EMS_CONFIGURE_OPTS build argument
+# (The base image *does* include NetCDF and HDF5 with Open MPI support, so that is an option)
+ENV EMS_BUILD_LOG="${EMS_BASE}/ems_build.log"
+ARG EMS_CONFIGURE_OPTS="--enable-omp"
+RUN make distclean || true \
+    && conf/configure ${EMS_CONFIGURE_OPTS} 2>&1 | tee -a "${EMS_BUILD_LOG}"
+RUN make clean \
+    && make 2>&1 | tee -a "${EMS_BUILD_LOG}"
+RUN make check install 2>&1 | tee -a "${EMS_BUILD_LOG}"
+
+# Symlink the EMS executable(s) into the default path
+# (This will fail if the executable did not build for some reason)
+ENV SHOC="${EMS_BASE}/model/hd/shoc" \
+    COMPAS="${EMS_BASE}/model/hd-us/compas"
+RUN if [ -n "${SHOC}" ]; then chmod 0755 "${SHOC}" && ln -s "${SHOC}" /usr/local/bin/shoc; fi; \
+    if [ -n "${COMPAS}" ]; then chmod 0755 "${COMPAS}" && ln -s "${COMPAS}" /usr/local/bin/compas; fi; \
+    ln -s "${EMS_BASE}/model/tests/run-tests.sh" /usr/local/bin/run-ems-tests
+
+# Optionally run all available unit tests
+# (Override the EMS_TEST_RUN build argument to have a value of 0 to skip testing at build time)
+ARG EMS_TEST_RUN=1
+ARG EMS_TEST_INCLUDE="${EMS_BASE}/model/tests"
+ENV EMS_TEST_INCLUDE="${EMS_TEST_INCLUDE}"
+ARG EMS_TEST_EXCLUDE=""
+ENV EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE}"
+ENV EMS_TEST_LOG="${EMS_BASE}/model/tests/ems_test.log"
+RUN if [ $EMS_TEST_RUN -eq 1 ]; then run-ems-tests; fi
+
+# Encode EMS metadata in labels
+LABEL au.csiro.ems.base=${EMS_BASE} \
+      au.csiro.ems.shoc=${SHOC} \
+      au.csiro.ems.compas=${COMPAS}
+
+# Configure the default entrypoint to be a default EMS executable
+ENTRYPOINT ["/bin/bash", "-c" ]
+CMD [ "${SHOC:-$COMPAS}", "-v"]

conf/ems_version.h

@@ -1,4 +1,4 @@
-#define EMS_VERSION "v1.5.1"
+#define EMS_VERSION "v1.5.2"
 
                 #define EMS_IS_RELEASE 1
 

docker-compose.test.yml

@@ -0,0 +1,22 @@
+---
+#-------------------------------------------------------------------------------
+# Docker *test* environment composition for EMS
+# This exists to support the autotest option for a Dockerhub repository build
+# (See: https://docs.docker.com/docker-hub/builds/automated-testing/)
+#
+# If this repository and rbanch are configured as Dockerhub autotest
+# repositories, this will automatically build and test EMS with all default
+# settings every time a push is made to the repository.
+#
+# The resulting image is not automatically pushed to the DockerHub registry
+# unless the same respository is automatically configured for autobuild.
+#-------------------------------------------------------------------------------
+version: "3.8"
+
+services:
+  ems:
+    build:
+      context: .
+      args:
+        EMS_TEST_RUN: 0
+    command: "run-ems-tests"

docker-compose.yml

@@ -0,0 +1,23 @@
+---
+#-------------------------------------------------------------------------------
+# Docker development environment composition for ems
+# (Not for production use!)
+#
+# This default compose stack is designed for use on a local development host.
+#-------------------------------------------------------------------------------
+version: "3.8"
+
+services:
+  ems:
+    build:
+      context: .
+      args:
+        #EMS_RM_DIRS: "ext/swan model/lib/da model/lib/exchange"
+        EMS_TEST_RUN: 0
+    command: "run-ems-tests"
+    container_name: ems
+    environment:
+      EMS_TEST_INCLUDE: '/usr/local/src/ems/model/tests'
+    image: onaci/ereefs-ems-shoc:dev
+    volumes:
+      - ./model/tests/:/usr/local/src/ems/model/tests/

hooks/README.md

@@ -0,0 +1,7 @@
+# csiro-coasts/EMS/hooks
+
+The files in this directory are custom build phase hooks for the automated dockerhub builder,
+and are used to set up custom build configurations and docker registry push options for when
+this repository is set up as a DockerHub repository with autobuild enabled.
+
+See <https://docs.docker.com/docker-hub/builds/advanced/#custom-build-phase-hooks> for more information about how they work.

hooks/build

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Source environment variables for this repository branch
+if [ -f .env ]; then
+    source .env
+fi
+
+# Derive some more variables from the dockerhub build environment
+BUILD_TIMESTAMP="$(date --rfc-3339=seconds)"
+DOCKER_TAG="${DOCKER_TAG:-}"
+SAFE_TIMESTAMP="$(echo $BUILD_TIMESTAMP | sed 's/ /T/g' | sed 's/:/-/g' | sed 's/\+.*//')"
+SOURCE_URL="$(git remote get-url origin || true)"
+VERSION_TAG="${SOURCE_BRANCH}_v${SAFE_TIMESTAMP}-${SOURCE_COMMIT}"
+
+
+# Treat most DOCKER_TAG values as overrides for the
+# BASE_IMAGE variable sourced from the .env file
+if [[ -n "${DOCKER_TAG}" ]] && [[ "${DOCKER_TAG}" != "latest" ]] && [[ "${DOCKER_TAG}" != "${SOURCE_BRANCH}" ]]; then
+    # Assume the custom docker tag is the tag of our desired BASE_IMAGE.
+    BASE_IMAGE="onaci/ereefs-netcdf-base:${DOCKER_TAG}"
+
+    # Override the VERSION_TAG to include it
+    VERSION_TAG="${SOURCE_BRANCH}_${DOCKER_TAG}_v${SAFE_TIMESTAMP}-${SOURCE_COMMIT}"
+fi
+
+# Build our customised docker image
+docker build --pull \
+    --build-arg "BASE_IMAGE=${BASE_IMAGE:-}" \
+    --build-arg "EMS_RM_DIRS=${EMS_RM_DIRS:-}" \
+    --build-arg "EMS_TEST_RUN=0" \
+    --label "org.opencontainers.image.authors=${BUILD_AUTHORS:-CSIRO Coastal Group}" \
+    --label "org.opencontainers.image.branch=${SOURCE_BRANCH}" \
+    --label "org.opencontainers.image.buildhost=${DOCKER_REPO}" \
+    --label "org.opencontainers.image.created=${BUILD_TIMESTAMP}" \
+    --label "org.opencontainers.image.licenses=https://github.com/csiro-coasts/EMS/blob/master/LICENSE.md" \
+    --label "org.opencontainers.image.revision=${SOURCE_COMMIT}" \
+    --label "org.opencontainers.image.source=${SOURCE_URL}" \
+    --label "org.opencontainers.image.title=${DOCKER_REPO:-csiro-coasts/EMS}" \
+    --label "org.opencontainers.image.url=https://github.com/csiro-coasts/EMS" \
+    --label "org.opencontainers.image.vendor=CSIRO" \
+    --label "org.opencontainers.image.version=${VERSION_TAG}" \
+    -f ${DOCKERFILE_PATH} \
+    -t ${IMAGE_NAME} \
+    .

hooks/post_push

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+VERSION_TAG="$(docker image inspect  --format '{{ index .Config.Labels "org.opencontainers.image.version"}}' $IMAGE_NAME || true)"
+if [ -n "${VERSION_TAG}" ]; then
+    IMAGE_TAGGED="${DOCKER_REPO}:${VERSION_TAG}"
+    docker tag $IMAGE_NAME $IMAGE_TAGGED
+    docker push $IMAGE_TAGGED
+fi

model/hd-us/include/proto.h

@@ -14,7 +14,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: proto.h 7477 2023-12-22 01:31:36Z riz008 $
+ *  $Id: proto.h 7480 2024-02-01 03:49:57Z riz008 $
  *
  */
 
@@ -1587,6 +1587,7 @@ landfillfn_t locate_landfill_function(const char *tag);
 void dump_bathy_mask(dump_data_t *dumpdata, double bathyf);
 void write_text_att(int cdfid, int varid, const char *name,
                     const char *text);
+void write_date_created(int cdfid);
 void dump_windows(master_t *master, geometry_t **window, char *name, char *iname);
 void read_windows(geometry_t *geom, geometry_t **window, char *name);
 void check_window_map_us(geometry_t *geom, geometry_t **window, char *name);

model/hd-us/outputs/writeatts.c

@@ -13,7 +13,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: writeatts.c 7465 2023-12-13 03:52:41Z her127 $
+ *  $Id: writeatts.c 7480 2024-02-01 03:49:57Z riz008 $
  *
  */
 
@@ -516,6 +516,7 @@ void write_dump_attributes(dump_data_t *dumpdata, int cdfid,
   sprintf(buf, "%s/%s", buf, dumpdata->prmname);
   write_text_att(cdfid, NC_GLOBAL, "paramfile", buf);
   write_text_att(cdfid, NC_GLOBAL, "ems_version", version);
+  write_date_created(cdfid);
   write_text_att(cdfid, NC_GLOBAL, "Conventions", "CMR/Timeseries/SHOC");
   if (dumpdata->runno >= 0)
     write_text_att(cdfid, NC_GLOBAL, "Run_ID", dumpdata->runnoc);
@@ -675,6 +676,16 @@ static void write_grid_atts(dump_data_t *dumpdata, int fid, int ilower,
   }
 }
 
+void write_date_created(int cdfid)
+{
+  time_t now = time(NULL);
+  char buf[32];
+  sprintf(buf, "%s", ctime(&now));
+  /* Remove trailing newline */
+  buf[strlen(buf)-1] = '\0';
+  write_text_att(cdfid, NC_GLOBAL, "date_created", buf);
+}
+
 void read_grid_atts(parameters_t *params, int fid)
 {
   char buf[MAXSTRLEN];

model/hd/inputs/readparam.c

@@ -14,7 +14,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: readparam.c 7449 2023-12-13 03:44:20Z her127 $
+ *  $Id: readparam.c 7479 2024-02-01 03:49:06Z riz008 $
  *
  */
 
@@ -1759,7 +1759,6 @@ parameters_t *params_read(FILE *fp)
 	       strcmp(tracer->name, "salt") == 0 || strcmp(tracer->name, "temp") == 0 ) {
 	    params->tdif_v[m] = n;
             m++;
-	    printf("tracer %s %x\n",tracer->name,tracer->type);
 	  }
         } else {
 	  if(!(tracer->type & CLOSURE)) {
@@ -3037,7 +3036,6 @@ parameters_t *auto_params(FILE * fp, int autof)
     for (c = 0; c < params->nvals; c++)
       printf("%d %f\n",c,params->bathy[c]);
   */
-  printf("a %f\n",params->bathy[1]);
   if (!prm_read_double(fp, "BATHYMIN", &params->bmin)) {
     params->bmin = 1e10;
     for (c = 0; c < params->nvals; c++) {

model/lib/ecology/process_library/light_spectral_col.c

@@ -55,7 +55,8 @@
  *         13. Prevent products being calculated with zero Rrs.
  *         26. Give canopy order in the optical_setup.nc
  *         27. Check out-of-bounds reflectance from benthic substrates - some are zero.
- *         30. Coral skeleton reflectance is hardwired - would need another tracer attribute. 
+ *         30. Coral skeleton reflectance is hardwired - need another tracer attribute.
+ *         31. Suspended macroalgae is hardwired - need another tracer attribute.
  *         39. Colour_of_source_in_water (flu and bio) to add attribute to optical_setup.nc
  *         45. Add potassium decay as light source.
  *         47. Add passive fluorescence for all phytoplankton? Even Symbiodinium?
@@ -76,6 +77,7 @@
  *         72. We don't have a simulated satellite products for Secchi.
  *         73. Light_spectral_col doesn't work for KEYWORD specification of tracers.
  *         74. Likely to be problems with using col->b to identify output columns in fully-coupled version.
+ *         75. Put in ems version into optical_setup.nc file attributes.
  */
 
 #include <stdlib.h>
@@ -1434,14 +1436,7 @@ void light_spectral_col_init(eprocess* p)
 
     write_text_att(ncid1, NC_GLOBAL, "title", "CSIRO Environmental Modelling Suite (EMS) Optical setup file");
     write_text_att(ncid1, NC_GLOBAL, "description", "Optical grid and optical parameter values on the optical grid");
-
-    /* Output time created */
-
-    time_t now = time(NULL);
-    // char buf[32];
-    // sprintf(buf, "%s", ctime(&now));
-    // buf[0] = '\0'; // Remove trailing newline
-    // write_text_att(ncid1, NC_GLOBAL, "date_created", buf);
+    write_date_created(ncid1);
 
     /* Source file and date created */
 
@@ -1488,12 +1483,12 @@ void light_spectral_col_init(eprocess* p)
     nc_put_att_text(ncid1, varid1, "orientation", 35,"First is higher in the water column");
 
     nc_def_var(ncid1,"PARbot",NC_DOUBLE,0,dim_dummy,&varid1);
-    nc_put_att_text(ncid1, varid1, "description", 24,"PAR bot range wavelength");
+    nc_put_att_text(ncid1, varid1, "description", 28,"Lower wavelength edge of PAR");
     nc_put_att_text(ncid1, varid1, "units", 2,"nm");
     nc_put_att_text(ncid1, varid1, "puv_uom",52,"http://vocab.nerc.ac.uk/collection/P06/current/UXNM/");
 
     nc_def_var(ncid1,"PARtop",NC_DOUBLE,0,dim_dummy,&varid1);
-    nc_put_att_text(ncid1, varid1, "description", 24,"PAR top range wavelength");
+    nc_put_att_text(ncid1, varid1, "description", 28,"Upper wavelength edge of PAR");
     nc_put_att_text(ncid1, varid1, "units", 2,"nm");
     nc_put_att_text(ncid1, varid1, "puv_uom",52,"http://vocab.nerc.ac.uk/collection/P06/current/UXNM/");
 
@@ -2201,11 +2196,7 @@ void light_spectral_col_init(eprocess* p)
       write_text_att(ncid, NC_GLOBAL, "title", "CSIRO Environmental Modelling Suite (EMS) optical model output.");
       write_text_att(ncid, NC_GLOBAL, "description", "Spectrally-resolved optical properties in a model column.");
       write_text_att(ncid, NC_GLOBAL, "vertical grid", "Depth is relative to the moving surface, so f(space,time).");
-      time_t now1 = time(NULL); 
-      char buf1[32];
-      // sprintf(buf1, "%s", ctime(&now1));
-      buf1[strlen(buf1)-1] = '\0'; // Remove trailing newline
-      // write_text_att(ncid, NC_GLOBAL, "date_created", buf1);
+      write_date_created(ncid);
 
       nc_def_var(ncid,"i_index",NC_INT,0,0,&varid);
       nc_def_var(ncid,"j_index",NC_INT,0,0,&varid);

model/lib/sediments/sed_init.c

@@ -13,7 +13,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: sed_init.c 7186 2022-08-17 05:48:47Z mar644 $
+ *  $Id: sed_init.c 7481 2024-02-01 03:50:21Z riz008 $
  *
  */
 
@@ -572,7 +572,7 @@ static void sed_tracers_init(FILE * prmfd, sediment_t *sediment)
     np = param->ntr + 30;
     //allocate mem
     param->prmpointS = (double ***)p_alloc_2d(param->ncol,np);
-    fprintf(stderr, "sed_init.c: ncol = %d, np = %d \n", param->ncol, np);
+    // fprintf(stderr, "sed_init.c: ncol = %d, np = %d \n", param->ncol, np);
 
     param->prmnameS = (char **)p_alloc_1d(np);
     param->prmindexS = i_alloc_1d(np);