From cbb701eacf54034294675d5343d618d70c723eb8 Mon Sep 17 00:00:00 2001
From: sa-cem-worker <sa-cem-worker@cem-server2.it.csiro.au>
Date: Thu, 1 Feb 2024 15:05:36 +1100
Subject: [PATCH 01/12] v1.5.2 release based on SVN rev 7482

---
 conf/ems_version.h                            |  2 +-
 model/hd-us/include/proto.h                   |  3 +-
 model/hd-us/outputs/writeatts.c               | 13 +++++++-
 model/hd/inputs/readparam.c                   |  4 +--
 .../process_library/light_spectral_col.c      | 23 +++++---------
 model/lib/sediments/sed_init.c                |  4 +--
 model/lib/waves/waves.c                       | 31 ++++++++++---------
 7 files changed, 41 insertions(+), 39 deletions(-)

diff --git a/conf/ems_version.h b/conf/ems_version.h
index a54eb4a..ca95ccc 100644
--- a/conf/ems_version.h
+++ b/conf/ems_version.h
@@ -1,4 +1,4 @@
-#define EMS_VERSION "v1.5.1"
+#define EMS_VERSION "v1.5.2"
 
                 #define EMS_IS_RELEASE 1
                 
\ No newline at end of file
diff --git a/model/hd-us/include/proto.h b/model/hd-us/include/proto.h
index a9c71ce..18e708e 100644
--- a/model/hd-us/include/proto.h
+++ b/model/hd-us/include/proto.h
@@ -14,7 +14,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: proto.h 7477 2023-12-22 01:31:36Z riz008 $
+ *  $Id: proto.h 7480 2024-02-01 03:49:57Z riz008 $
  *
  */
 
@@ -1587,6 +1587,7 @@ landfillfn_t locate_landfill_function(const char *tag);
 void dump_bathy_mask(dump_data_t *dumpdata, double bathyf);
 void write_text_att(int cdfid, int varid, const char *name,
                     const char *text);
+void write_date_created(int cdfid);
 void dump_windows(master_t *master, geometry_t **window, char *name, char *iname);
 void read_windows(geometry_t *geom, geometry_t **window, char *name);
 void check_window_map_us(geometry_t *geom, geometry_t **window, char *name);
diff --git a/model/hd-us/outputs/writeatts.c b/model/hd-us/outputs/writeatts.c
index 2dca2f2..299e5d0 100644
--- a/model/hd-us/outputs/writeatts.c
+++ b/model/hd-us/outputs/writeatts.c
@@ -13,7 +13,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: writeatts.c 7465 2023-12-13 03:52:41Z her127 $
+ *  $Id: writeatts.c 7480 2024-02-01 03:49:57Z riz008 $
  *
  */
 
@@ -516,6 +516,7 @@ void write_dump_attributes(dump_data_t *dumpdata, int cdfid,
   sprintf(buf, "%s/%s", buf, dumpdata->prmname);
   write_text_att(cdfid, NC_GLOBAL, "paramfile", buf);
   write_text_att(cdfid, NC_GLOBAL, "ems_version", version);
+  write_date_created(cdfid);
   write_text_att(cdfid, NC_GLOBAL, "Conventions", "CMR/Timeseries/SHOC");
   if (dumpdata->runno >= 0)
     write_text_att(cdfid, NC_GLOBAL, "Run_ID", dumpdata->runnoc);
@@ -675,6 +676,16 @@ static void write_grid_atts(dump_data_t *dumpdata, int fid, int ilower,
   }
 }
 
+void write_date_created(int cdfid)
+{
+  time_t now = time(NULL);
+  char buf[32];
+  sprintf(buf, "%s", ctime(&now));
+  /* Remove trailing newline */
+  buf[strlen(buf)-1] = '\0';
+  write_text_att(cdfid, NC_GLOBAL, "date_created", buf);
+}
+
 void read_grid_atts(parameters_t *params, int fid)
 {
   char buf[MAXSTRLEN];
diff --git a/model/hd/inputs/readparam.c b/model/hd/inputs/readparam.c
index a91041d..aba6de1 100644
--- a/model/hd/inputs/readparam.c
+++ b/model/hd/inputs/readparam.c
@@ -14,7 +14,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: readparam.c 7449 2023-12-13 03:44:20Z her127 $
+ *  $Id: readparam.c 7479 2024-02-01 03:49:06Z riz008 $
  *
  */
 
@@ -1759,7 +1759,6 @@ parameters_t *params_read(FILE *fp)
 	       strcmp(tracer->name, "salt") == 0 || strcmp(tracer->name, "temp") == 0 ) {
 	    params->tdif_v[m] = n;
             m++;
-	    printf("tracer %s %x\n",tracer->name,tracer->type);
 	  }
         } else {
 	  if(!(tracer->type & CLOSURE)) {
@@ -3037,7 +3036,6 @@ parameters_t *auto_params(FILE * fp, int autof)
     for (c = 0; c < params->nvals; c++)
       printf("%d %f\n",c,params->bathy[c]);
   */
-  printf("a %f\n",params->bathy[1]);
   if (!prm_read_double(fp, "BATHYMIN", &params->bmin)) {
     params->bmin = 1e10;
     for (c = 0; c < params->nvals; c++) {
diff --git a/model/lib/ecology/process_library/light_spectral_col.c b/model/lib/ecology/process_library/light_spectral_col.c
index bbf74d7..6c5bf14 100755
--- a/model/lib/ecology/process_library/light_spectral_col.c
+++ b/model/lib/ecology/process_library/light_spectral_col.c
@@ -55,7 +55,8 @@
  *         13. Prevent products being calculated with zero Rrs.
  *         26. Give canopy order in the optical_setup.nc
  *         27. Check out-of-bounds reflectance from benthic substrates - some are zero.
- *         30. Coral skeleton reflectance is hardwired - would need another tracer attribute. 
+ *         30. Coral skeleton reflectance is hardwired - need another tracer attribute.
+ *         31. Suspended macroalgae is hardwired - need another tracer attribute.
  *         39. Colour_of_source_in_water (flu and bio) to add attribute to optical_setup.nc
  *         45. Add potassium decay as light source.
  *         47. Add passive fluorescence for all phytoplankton? Even Symbiodinium?
@@ -76,6 +77,7 @@
  *         72. We don't have a simulated satellite products for Secchi.
  *         73. Light_spectral_col doesn't work for KEYWORD specification of tracers.
  *         74. Likely to be problems with using col->b to identify output columns in fully-coupled version.
+ *         75. Put in ems version into optical_setup.nc file attributes.
  */
 
 #include <stdlib.h>
@@ -1434,14 +1436,7 @@ void light_spectral_col_init(eprocess* p)
 
     write_text_att(ncid1, NC_GLOBAL, "title", "CSIRO Environmental Modelling Suite (EMS) Optical setup file");
     write_text_att(ncid1, NC_GLOBAL, "description", "Optical grid and optical parameter values on the optical grid");
-
-    /* Output time created */
-
-    time_t now = time(NULL);
-    // char buf[32];
-    // sprintf(buf, "%s", ctime(&now));
-    // buf[0] = '\0'; // Remove trailing newline
-    // write_text_att(ncid1, NC_GLOBAL, "date_created", buf);
+    write_date_created(ncid1);
 
     /* Source file and date created */
 
@@ -1488,12 +1483,12 @@ void light_spectral_col_init(eprocess* p)
     nc_put_att_text(ncid1, varid1, "orientation", 35,"First is higher in the water column");
 
     nc_def_var(ncid1,"PARbot",NC_DOUBLE,0,dim_dummy,&varid1);
-    nc_put_att_text(ncid1, varid1, "description", 24,"PAR bot range wavelength");
+    nc_put_att_text(ncid1, varid1, "description", 28,"Lower wavelength edge of PAR");
     nc_put_att_text(ncid1, varid1, "units", 2,"nm");
     nc_put_att_text(ncid1, varid1, "puv_uom",52,"http://vocab.nerc.ac.uk/collection/P06/current/UXNM/");
 
     nc_def_var(ncid1,"PARtop",NC_DOUBLE,0,dim_dummy,&varid1);
-    nc_put_att_text(ncid1, varid1, "description", 24,"PAR top range wavelength");
+    nc_put_att_text(ncid1, varid1, "description", 28,"Upper wavelength edge of PAR");
     nc_put_att_text(ncid1, varid1, "units", 2,"nm");
     nc_put_att_text(ncid1, varid1, "puv_uom",52,"http://vocab.nerc.ac.uk/collection/P06/current/UXNM/");
 
@@ -2201,11 +2196,7 @@ void light_spectral_col_init(eprocess* p)
       write_text_att(ncid, NC_GLOBAL, "title", "CSIRO Environmental Modelling Suite (EMS) optical model output.");
       write_text_att(ncid, NC_GLOBAL, "description", "Spectrally-resolved optical properties in a model column.");
       write_text_att(ncid, NC_GLOBAL, "vertical grid", "Depth is relative to the moving surface, so f(space,time).");
-      time_t now1 = time(NULL); 
-      char buf1[32];
-      // sprintf(buf1, "%s", ctime(&now1));
-      buf1[strlen(buf1)-1] = '\0'; // Remove trailing newline
-      // write_text_att(ncid, NC_GLOBAL, "date_created", buf1);
+      write_date_created(ncid);
       
       nc_def_var(ncid,"i_index",NC_INT,0,0,&varid);
       nc_def_var(ncid,"j_index",NC_INT,0,0,&varid);
diff --git a/model/lib/sediments/sed_init.c b/model/lib/sediments/sed_init.c
index 60e487a..6871ecf 100644
--- a/model/lib/sediments/sed_init.c
+++ b/model/lib/sediments/sed_init.c
@@ -13,7 +13,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: sed_init.c 7186 2022-08-17 05:48:47Z mar644 $
+ *  $Id: sed_init.c 7481 2024-02-01 03:50:21Z riz008 $
  *
  */
 
@@ -572,7 +572,7 @@ static void sed_tracers_init(FILE * prmfd, sediment_t *sediment)
     np = param->ntr + 30;
     //allocate mem
     param->prmpointS = (double ***)p_alloc_2d(param->ncol,np);
-    fprintf(stderr, "sed_init.c: ncol = %d, np = %d \n", param->ncol, np);
+    // fprintf(stderr, "sed_init.c: ncol = %d, np = %d \n", param->ncol, np);
 
     param->prmnameS = (char **)p_alloc_1d(np);
     param->prmindexS = i_alloc_1d(np);
diff --git a/model/lib/waves/waves.c b/model/lib/waves/waves.c
index 4e12ebb..4e6f138 100644
--- a/model/lib/waves/waves.c
+++ b/model/lib/waves/waves.c
@@ -13,7 +13,7 @@
  *  reserved. See the license file for disclaimer and full
  *  use/redistribution conditions.
  *  
- *  $Id: waves.c 7315 2023-04-11 02:03:36Z her127 $
+ *  $Id: waves.c 7482 2024-02-01 03:50:47Z riz008 $
  *
  */
 
@@ -612,20 +612,6 @@ void wave_init(void* model, wave_t *wave, FILE *fp) {
       emstag(LTRACE,"waves:init","Wave tracer %d = %s.\n", n, wave->trname_2d[n]);
   }
 
-  /* Get the grid angles                                            */
-  /* Not used
-  wave->thetau1 = d_alloc_1d(size);
-  wave->thetau2 = d_alloc_1d(size);
-  wave->sinthcell = d_alloc_1d(size);
-  wave->costhcell = d_alloc_1d(size);
-  for (i = 1; i <= wave->cols; i++) {
-    wave->thetau1[i] = i_get_thetau1(model, i);
-    wave->thetau2[i] = i_get_thetau2(model, i);
-    wave->sinthcell[i] = i_get_sinthcell(model, i);
-    wave->sinthcell[i] = i_get_costhcell(model, i);
-  }
-  */
-
   if (!(wave->do_waves & WSWAN)) {
     size = i_get_winsize(model);
     if (wave->Sxyid >= 0 && wave->Syxid >= 0) {
@@ -673,6 +659,21 @@ void wave_init(void* model, wave_t *wave, FILE *fp) {
     wave->do_dir = WWIND;
   }
 
+  /* Get the grid angles                                            */
+  if (wave->do_rs) {
+    wave->thetau1 = d_alloc_1d(size);
+    wave->thetau2 = d_alloc_1d(size);
+    /* Not used */
+    // wave->sinthcell = d_alloc_1d(size);
+    // wave->costhcell = d_alloc_1d(size);
+    for (i = 1; i <= wave->cols; i++) {
+      wave->thetau1[i] = i_get_thetau1(model, i);
+      wave->thetau2[i] = i_get_thetau2(model, i);
+      //wave->sinthcell[i] = i_get_sinthcell(model, i);
+      //wave->sinthcell[i] = i_get_costhcell(model, i);
+    }
+  }
+
   /* Free memory                                                    */
   free((char **)trname_2d);
   free((char **)trname_3d);

From f3de5dc7f6fcb412622335ad483b420204ea0cf1 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Mon, 5 Feb 2024 22:05:10 +1100
Subject: [PATCH 02/12] Make shoc tests use an env var to locate the SHOC
 executable, exit on first failure, not count missing output files as a
 failure

---
 model/tests/hd/test1/run_test1     | 24 +++++-----
 model/tests/hd/test2/run_test2     | 24 +++++-----
 model/tests/hd/test3/run_test3     | 24 +++++-----
 model/tests/hd/test4/run_test4     | 24 +++++-----
 model/tests/hd/test5/run_test5     | 24 +++++-----
 model/tests/hd/test6/run_test6     | 17 +++++---
 model/tests/hd/test7/run_test7     | 70 ++++++++++++++++--------------
 model/tests/hd/test7/test7_sed.prm |  3 +-
 model/tests/hd/test8/run_test8     | 23 +++++-----
 9 files changed, 131 insertions(+), 102 deletions(-)

diff --git a/model/tests/hd/test1/run_test1 b/model/tests/hd/test1/run_test1
index 52d1905..a158f2d 100755
--- a/model/tests/hd/test1/run_test1
+++ b/model/tests/hd/test1/run_test1
@@ -1,23 +1,27 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in1.nc
-rm out1_z.nc
+rm -f in1.nc || true
+rm -f out1_z.nc || true
 
-../shoc -g test1.prm in1.nc
+$SHOC -g test1.prm in1.nc
 
-../shoc -p test1.prm
+$SHOC -p test1.prm
 
 echo "DONE"
 
 echo "Testing sigma model..."
-rm in1.nc
-rm out1_s.nc
+rm -f in1.nc || true
+rm -f out1_s.nc || true
 
-../shoc -g test1_s.prm in1.nc
+$SHOC -g test1_s.prm in1.nc
 
-../shoc -p test1_s.prm
+$SHOC -p test1_s.prm
 
 echo "DONE"
 
-rm in1.nc
+rm -f in1.nc || true
diff --git a/model/tests/hd/test2/run_test2 b/model/tests/hd/test2/run_test2
index 72f64db..438cff9 100755
--- a/model/tests/hd/test2/run_test2
+++ b/model/tests/hd/test2/run_test2
@@ -1,23 +1,27 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in2.nc
-rm out2_z.nc
+rm -f in2.nc || true
+rm -f out2_z.nc || true
 
-../shoc -g test2.prm in2.nc
+$SHOC -g test2.prm in2.nc
 
-../shoc -p test2.prm
+$SHOC -p test2.prm
 
 echo "DONE"
 
 echo "Testing sigma model..."
-rm in2.nc
-rm out2_s.nc
+rm -f in2.nc || true
+rm -f out2_s.nc || true
 
-../shoc -g test2_s.prm in2.nc
+$SHOC -g test2_s.prm in2.nc
 
-../shoc -p test2_s.prm
+$SHOC -p test2_s.prm
 
 echo "DONE"
 
-rm in2.nc
+rm -f in2.nc || true
diff --git a/model/tests/hd/test3/run_test3 b/model/tests/hd/test3/run_test3
index 859d86e..cf78b07 100755
--- a/model/tests/hd/test3/run_test3
+++ b/model/tests/hd/test3/run_test3
@@ -1,23 +1,27 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in3.nc
-rm out3_z.nc
+rm -f in3.nc || true
+rm -f out3_z.nc || true
 
-../shoc -g test3.prm in3.nc
+$SHOC -g test3.prm in3.nc
 
-../shoc -p test3.prm
+$SHOC -p test3.prm
 
 echo "DONE"
 
 echo "Testing sigma model..."
-rm in3.nc
-rm out3_s.nc
+rm -f in3.nc || true
+rm -f out3_s.nc || true
 
-../shoc -g test3_s.prm in3.nc
+$SHOC -g test3_s.prm in3.nc
 
-../shoc -p test3_s.prm
+$SHOC -p test3_s.prm
 
 echo "DONE"
 
-rm in3.nc
+rm -f in3.nc || true
diff --git a/model/tests/hd/test4/run_test4 b/model/tests/hd/test4/run_test4
index 3672f8f..eca447d 100755
--- a/model/tests/hd/test4/run_test4
+++ b/model/tests/hd/test4/run_test4
@@ -1,25 +1,29 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in4.nc
-rm out4_z.nc
+rm -f in4.nc || true
+rm -f out4_z.nc || true
 
-../shoc -g test4.prm in4.nc
+$SHOC -g test4.prm in4.nc
 
 echo "Running, takes ~ 1 minute...."
-../shoc -p test4.prm
+$SHOC -p test4.prm
 
 echo "DONE"
 
 echo "Testing sigma model..."
-rm in4.nc
-rm out4_s.nc
+rm -f in4.nc || true
+rm -f out4_s.nc || true
 
-../shoc -g test4_s.prm in4.nc
+$SHOC -g test4_s.prm in4.nc
 
 echo "Running, takes ~ 1 minute...."
-../shoc -p test4_s.prm
+$SHOC -p test4_s.prm
 
 echo "DONE"
 
-rm in4.nc
+rm -f in4.nc || true
diff --git a/model/tests/hd/test5/run_test5 b/model/tests/hd/test5/run_test5
index 78d8f55..31b3f1d 100755
--- a/model/tests/hd/test5/run_test5
+++ b/model/tests/hd/test5/run_test5
@@ -1,25 +1,29 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in5.nc
-rm out5_z.nc
+rm -f in5.nc || true
+rm -f out5_z.nc || true
 
-../shoc -g test5.prm in5.nc
+$SHOC -g test5.prm in5.nc
 
 echo "Running, takes ~ 30 sec...."
-../shoc -p test5.prm
+$SHOC -p test5.prm
 
 echo "DONE"
 
 echo "Testing sigma model..."
-rm in5.nc
-rm out5_s.nc
+rm -f in5.nc || true
+rm -f out5_s.nc || true
 
-../shoc -g test5_s.prm in5_s.nc
+$SHOC -g test5_s.prm in5_s.nc
 
 echo "Running, takes ~ 30 sec...."
-../shoc -p test5_s.prm
+$SHOC -p test5_s.prm
 
 echo "DONE"
 
-rm in5_s.nc
+rm -f in5_s.nc || true
diff --git a/model/tests/hd/test6/run_test6 b/model/tests/hd/test6/run_test6
index da89730..0efea0f 100755
--- a/model/tests/hd/test6/run_test6
+++ b/model/tests/hd/test6/run_test6
@@ -1,15 +1,18 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in6.nc
-rm out6_z.nc
+rm -f in6.nc || true
+rm -f out6_z.nc || true
 
-../shoc -g test6.prm in6.nc
+$SHOC -g test6.prm in6.nc
 
 echo "Running, takes ~ 2 minutes...."
-../shoc -p test6.prm
+$SHOC -p test6.prm
 
 echo "DONE"
 
-rm in6.nc
-
+rm -f in6.nc || true
diff --git a/model/tests/hd/test7/run_test7 b/model/tests/hd/test7/run_test7
index eab2846..b0c9c19 100755
--- a/model/tests/hd/test7/run_test7
+++ b/model/tests/hd/test7/run_test7
@@ -1,54 +1,58 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in7.nc
-rm out7_z.nc
-rm out_pt.nc
-rm *.eps
+rm -f in7.nc || true
+rm -f out7_z.nc || true
+rm -f out_pt.nc || true
+#rm -f *.eps || true
 
-../shoc -g test7_diag.prm in7.nc
+$SHOC -g test7_diag.prm in7.nc
 
 echo "Running, takes ~ 3 minutes...."
-../shoc -p test7.prm
+$SHOC -p test7.prm
 
 echo "Running diagnostic test, takes ~ 40 minutes...."
-rm out7_d.nc
-rm out_pt.nc
-../shoc -p test7_diag.prm
+rm -f out7_d.nc || true
+rm -f out_pt.nc || true
+$SHOC -p test7_diag.prm
 mv totals.ts totals_diag.ts
 
 echo "Running explicit mapping test, takes ~ 2 minutes...."
-rm in7.nc
-rm out7_e.nc
-../shoc -g test7_em.prm in7.nc
-../shoc -p test7_em.prm
+rm -f in7.nc || true
+rm -f out7_e.nc || true
+$SHOC -g test7_em.prm in7.nc
+$SHOC -p test7_em.prm
 
 echo "Running grid refinement test, takes ~ 3 minutes...."
-rm in7.nc
-rm out7_r.nc
-../shoc -g test7_zoom.prm in7.nc
-../shoc -p test7_zoom.prm
+rm -f in7.nc || true
+rm -f out7_r.nc || true
+$SHOC -g test7_zoom.prm in7.nc
+$SHOC -p test7_zoom.prm
 
 echo "Running sigma model test, takes ~ 3 minutes...."
-rm in7.nc
-rm out7_s.nc
-../shoc -g test7_s.prm in7.nc
-../shoc -p test7_s.prm
+rm -f in7.nc || true
+rm -f out7_s.nc || true
+$SHOC -g test7_s.prm in7.nc
+$SHOC -p test7_s.prm
 
 echo "Running sediment test, takes ~ 3 minutes...."
-rm in7.nc
-rm out7_sed.nc
-../shoc -g test7_sed.prm in7.nc
-../shoc -p test7_sed.prm
+rm -f in7.nc || true
+rm -f out7_sed.nc || true
+$SHOC -g test7_sed.prm in7.nc
+$SHOC -p test7_sed.prm
 
 echo "Running multiple window test, takes ~ 6 minutes...."
-rm in7.nc
-rm out7_1w.nc
-rm out7_4w.nc
-../shoc -g test7_1w.prm in7.nc
-../shoc -p test7_1w.prm
-../shoc -p test7_4w.prm
+rm -f in7.nc || true
+rm -f out7_1w.nc || true
+rm -f out7_4w.nc || true
+$SHOC -g test7_1w.prm in7.nc
+$SHOC -p test7_1w.prm
+$SHOC -p test7_4w.prm
 
 echo "DONE"
 
-rm in7.nc
+rm -f in7.nc || true
diff --git a/model/tests/hd/test7/test7_sed.prm b/model/tests/hd/test7/test7_sed.prm
index 739742e..51308e0 100644
--- a/model/tests/hd/test7/test7_sed.prm
+++ b/model/tests/hd/test7/test7_sed.prm
@@ -1799,4 +1799,5 @@ BATHY    1350
  -99.000
 
 
-ID_CODE SHOC grid|G1.00|H1.00|S0.00|B0.00
+ID_CODE SHOC grid|G1.0|H1.0|S|B
+.00|B0.00
diff --git a/model/tests/hd/test8/run_test8 b/model/tests/hd/test8/run_test8
index c560c1d..83466c6 100755
--- a/model/tests/hd/test8/run_test8
+++ b/model/tests/hd/test8/run_test8
@@ -1,23 +1,24 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
 
 echo "Testing 'z' model..."
-rm in8.nc
-rm out8_z.nc
-rm out8_msl_z.nc
+rm -f in8.nc || true
+rm -f out8_z.nc || true
+rm -f out8_msl_z.nc || true
 
-../shoc -g test8.prm in8.nc
+$SHOC -g test8.prm in8.nc
 
 echo "Running, takes ~ 10 minutes...."
-../shoc -p test8.prm
+$SHOC -p test8.prm
 
 echo "DONE"
 
 echo "Running msl, takes ~ 10 minutes...."
-../shoc -p test8_msl.prm
+$SHOC -p test8_msl.prm
 
 echo "DONE"
 
-rm in8.nc
-
-
-
+rm -f in8.nc || true

From 4a2f3045a6b9342520875fa50d6fb84678f0a528 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Mon, 5 Feb 2024 22:08:09 +1100
Subject: [PATCH 03/12] Ensure SHOC test output files are gitignored

---
 .gitignore | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.gitignore b/.gitignore
index 67a0b70..3a34f29 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,16 @@ libjigsaw64r.so
 # Misc
 sediments-us
 
+# Test output files
+model/tests/**/test*/in*.nc
+model/tests/**/test*/alert.ts
+model/tests/**/test*/alert.txt
+model/tests/**/test*/core*
+model/tests/**/test*/diag.txt
+model/tests/**/test*/flushing.ts
+model/tests/**/test*/loc*.ts
+model/tests/**/test*/out*.nc
+model/tests/**/test*/runlog
+model/tests/**/test*/setup.txt
+model/tests/**/test*/totals*.ts
+model/tests/**/test*/zoom.points

From fb7a199c6d272ca57236963b9599ade21cb5fbb6 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Mon, 5 Feb 2024 22:09:06 +1100
Subject: [PATCH 04/12] Add docker tooling for EMS build and test

---
 .dockerignore            |  5 +++
 Dockerfile               | 71 +++++++++++++++++++++++++++++++++
 docker-compose.yml       | 22 +++++++++++
 model/tests/run-tests.sh | 85 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 183 insertions(+)
 create mode 100644 .dockerignore
 create mode 100644 Dockerfile
 create mode 100644 docker-compose.yml
 create mode 100755 model/tests/run-tests.sh

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..5b2a9d1
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,5 @@
+.env
+.gitlab_ci.yaml
+*.log
+*.md
+docker-compose.yml
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..b1c58ea
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,71 @@
+#------------------------------------------------------------------------------
+# CSIRO EMSr Image Build Script
+#------------------------------------------------------------------------------
+# The recommended base image is one of the onaci/ereefs-netcdf-base variants,
+# as those have been designed to include all the EMS dependencies.
+# Allow it to be overridden in order to choose *which* variant, or even
+# something completely different.
+ARG BASE_IMAGE="onaci/ereefs-netcdf-base:ems"
+FROM ${BASE_IMAGE}
+
+# Record the actual base image used from the FROM command as a label.
+ARG BASE_IMAGE
+LABEL org.opencontainers.image.base.name=${BASE_IMAGE}
+
+# Enable Bash in RUN commands, and ensure that any commands with
+# pipes exit on the first failure.
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Prepare a base directory for all the EMS code
+ARG EMS_BASE="/usr/local/src/ems"
+ENV EMS_BASE="${EMS_BASE}"
+WORKDIR ${EMS_BASE}
+
+# Install all the local EMS Source Code
+COPY ./ ./
+
+# Identify which model executables are to be built, and remove any source directories which should be skipped.
+ARG SHOC=1
+ENV SHOC="${SHOC:+${EMS_BASE}/model/hd/shoc}"
+RUN if [ -z "${SHOC}" ]; then rm -rf "${EMS_BASE}/model/hd"; fi
+
+ARG COMPAS=1
+ENV COMPAS="${COMPAS:+${EMS_BASE}/model/hd-us/compas}"
+RUN if [ -z "${COMPAS}" ]; then rm -rf "${EMS_BASE}/model/hd-us" "${EMS_BASE}/ext/jigsaw"; fi
+
+ARG EMS_RM_DIRS="ext/swan utilities model/boxhd model/lib/da model/lib/exchange"
+RUN if [ -n "${EMS_RM_DIRS}" ]; then for d in $EMS_RM_DIRS; do rm -rf "${EMS_BASE}/${d}"; done; fi
+
+# Compile the EMS components
+# (Keep the build-logs in case they are needed for debugging purposes later)
+ENV EMS_BUILD_LOG="${EMS_BASE}/ems_build.log"
+ARG EMS_CONFIGURE_OPTS="--enable-omp"
+RUN make distclean || true \
+    && conf/configure ${EMS_CONFIGURE_OPTS} 2>&1 | tee -a "${EMS_BUILD_LOG}"
+RUN make clean \
+    && make 2>&1 | tee -a "${EMS_BUILD_LOG}"
+RUN make check install 2>&1 | tee -a "${EMS_BUILD_LOG}"
+
+# Symlink the EMS executable(s) into the default path
+# (This will fail if the executable did not build for some reason)
+RUN if [ -n "${SHOC}" ]; then chmod 0755 "${SHOC}" && ln -s "${SHOC}" /usr/local/bin/shoc; fi; \
+    if [ -n "${COMPAS}" ]; then chmod 0755 "${COMPAS}" && ln -s "${COMPAS}" /usr/local/bin/compas; fi; \
+    ln -s "${EMS_BASE}/model/tests/run-tests.sh" /usr/local/bin/run-ems-tests
+
+# Optionally run all available unit tests
+# (Set this to 1 to fail the build if any tests fail, or any other value to skip running tests)
+ARG EMS_TEST_RUN=1
+ARG EMS_TEST_INCLUDE="${EMS_BASE}"
+ENV EMS_TEST_INCLUDE="${EMS_TEST_INCLUDE}"
+ARG EMS_TEST_EXCLUDE="${EMS_BASE}/model/tests/tracerstats/basic/run_test"
+ENV EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE}"
+RUN if [ $EMS_TEST_RUN -eq 1 ]; then run-ems-tests; fi
+
+# Encode EMS metadata in labels
+LABEL au.csiro.ems.base=${EMS_BASE} \
+      au.csiro.ems.shoc=${SHOC} \
+      au.csiro.ems.compas=${COMPAS}
+
+# Configure the default entrypoint to be the EMS executable
+ENTRYPOINT ["/bin/bash", "-c" ]
+CMD [ "${SHOC:-$COMPAS}", "-v"]
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..9c3ee51
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,22 @@
+---
+#-------------------------------------------------------------------------------
+# Docker development environment composition for ems
+# (Not for production use!)
+#
+# This default compose stack is designed for use on a local development host.
+#-------------------------------------------------------------------------------
+version: "3.8"
+
+services:
+  ems:
+    build:
+      context: .
+      args:
+        SHOC: 1
+        COMPAS: 1
+        EMS_TEST_RUN: 0
+    command: "run-ems-tests"
+    container_name: ems
+    image: onaci/ereefs-ems-shoc:dev
+    volumes:
+      - ./model/tests/:/usr/local/src/ems/model/tests/
diff --git a/model/tests/run-tests.sh b/model/tests/run-tests.sh
new file mode 100755
index 0000000..d80f83c
--- /dev/null
+++ b/model/tests/run-tests.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Runs the EMS test suite(s)
+#
+set +e
+
+SUCCESS_LIST=""
+SUCCESS_COUNT=0
+
+FAIL_LIST=""
+FAIL_COUNT=0
+
+echo "SHOC = '${SHOC}' => '$(which shoc)'"
+echo "COMPAS = '${COMPAS}' => '$(which compas)'"
+echo
+echo "CURL_VERSION = '${CURL_VERSION}'"
+echo "DAP_VERSION = '${DAP_VERSION}'"
+echo "GDAL_VERSION = '${GDAL_VERSION}'"
+echo "HDF5_VERSION = '${HDF5_VERSION}'"
+echo "NETCDF_VERSION = '${NETCDF_VERSION}'"
+echo "NCO_VERSION = '${NCO_VERSION}'"
+echo "PROJ_VERSION = '${PROJ_VERSION}'"
+echo
+
+EMS_TEST_INCLUDE="${EMS_TEST_INCLUDE:-${EMS_BASE}/model/tests}"
+echo "EMS_TEST_INCLUDE = '${EMS_TEST_INCLUDE}'"
+
+EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE:-}"
+if [ -z "${SHOC}" ]; then
+    EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd/*'"
+fi
+if [ -z "${COMPAS}" ]; then
+    EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd-us/*'"
+fi
+echo "EMS_TEST_EXCLUDE = '${EMS_TEST_EXCLUDE}'"
+
+TEST_COUNT=0
+
+for ti in $EMS_TEST_INCLUDE; do
+    FIND_CMD="find ${ti} -type f \( -regex '^.*/run_test[0-9]*$' -o -name 'run*.sh' -o -regex '^.*/run_[^\.]*$' \) ! -iname '.*'"
+    for te in $EMS_TEST_EXCLUDE; do
+        FIND_CMD="${FIND_CMD} ! -path ${te}"
+    done
+    echo "FIND_CMD = '${FIND_CMD}'"
+    TEST_FILES=$(eval "${FIND_CMD}")
+    if [ -z "${TEST_FILES}" ]; then
+        echo "There are no tests to run: Test FAILED"
+        FAIL_LIST="${FAIL_LIST} ${ti}"
+        FAIL_COUNT=$((FAIL_COUNT+1))
+    else
+        echo "Test files to run:"
+        echo "${TEST_FILES}"
+        for tf in $TEST_FILES; do
+            chmod 0755 "${tf}"
+            cd $(dirname "${tf}")
+            echo
+            echo "#-----------------------------------------------------"
+            echo "# Running test file '${tf}' from $(pwd)..."
+            echo "#-----------------------------------------------------"
+            ${tf}
+            TEST_RESULT=$?
+            echo "#......................................................"
+            if [ $TEST_RESULT -eq 0 ]; then
+                echo "exitcode: ${TEST_RESULT} => Test SUCCEEDED"
+                SUCCESS_LIST="${SUCCESS_LIST} ${tf}"
+                SUCCESS_COUNT=$((SUCCESS_COUNT+1))
+            else
+                echo "exitcode: ${TEST_RESULT} => Test FAILED"
+                FAIL_LIST="${FAIL_LIST} ${tf}"
+                FAIL_COUNT=$((FAIL_COUNT+1))
+            fi
+            TEST_COUNT=$((TEST_COUNT+1))
+        done
+    fi
+done
+
+echo
+echo "#-----------------------------------------------------"
+echo
+echo "${SUCCESS_COUNT} of ${TEST_COUNT} tests passed:"
+echo "${SUCCESS_LIST}" | tr ' ' '\n'
+echo
+echo "${FAIL_COUNT} of ${TEST_COUNT} tests failed:"
+echo "${FAIL_LIST}" | tr ' ' '\n'
+exit $FAIL_COUNT

From 09421aec0392f60a14924f6cf631c5ec93e4ebea Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Mon, 5 Feb 2024 22:52:36 +1100
Subject: [PATCH 05/12] Make sediments tests use an env var to locate the SHOC
 executable, exit on first failure, not count missing output files as a
 failure

---
 .../tests/sediments/test1/inputs/test_d.tran  |  2 +
 model/tests/sediments/test1/run.sh            | 28 ++++++------
 model/tests/sediments/test1/run_all.sh        | 45 ++++++++++---------
 .../sediments/test2/inputs/test_flat.tran     |  3 ++
 model/tests/sediments/test2/run.sh            | 30 +++++++------
 model/tests/sediments/test2/run_all.sh        | 23 +++++-----
 .../sediments/test3/inputs/test_advanced.tran |  2 +
 .../sediments/test3/inputs/test_basic.tran    |  2 +
 model/tests/sediments/test3/run.sh            | 28 ++++++------
 model/tests/sediments/test3/run_all.sh        |  8 ++++
 model/tests/sediments/test4/inputs/test.prm   |  2 +
 model/tests/sediments/test4/run_all.sh        | 17 ++++---
 12 files changed, 112 insertions(+), 78 deletions(-)
 create mode 100755 model/tests/sediments/test3/run_all.sh

diff --git a/model/tests/sediments/test1/inputs/test_d.tran b/model/tests/sediments/test1/inputs/test_d.tran
index f678fdb..4c1e8e5 100644
--- a/model/tests/sediments/test1/inputs/test_d.tran
+++ b/model/tests/sediments/test1/inputs/test_d.tran
@@ -313,3 +313,5 @@ TS0.reference        msl
 # Open boundaries
 NBOUNDARIES           0
 
+
+ID_CODE SHOC grid|G1.0|H|S|B
diff --git a/model/tests/sediments/test1/run.sh b/model/tests/sediments/test1/run.sh
index 2f80ad6..80f38f6 100755
--- a/model/tests/sediments/test1/run.sh
+++ b/model/tests/sediments/test1/run.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 
 if [ $# == 0 ] 
 then
@@ -24,26 +24,29 @@ else
    echo "run './run.sh' for info"
 fi
 
+SHOC="${SHOC:-./shoc}"
+mkdir -p outputs tran_outputs
+
 if [ $# == 1 ] 
 then
    echo "Cleaning up"
-   rm outputs/*
-   rm tran_outputs/*
-   rm inputs/in_test.nc
-   rm inputs/test_trans_1990-01.nc
-   rm diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
+   rm -f outputs/*
+   rm -f tran_outputs/*
+   rm -f inputs/in_test.nc
+   rm -f inputs/test_trans_1990-01.nc
+   rm -f diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
    echo "Running $1: hd first and then sediment transport"
-  ./shoc -g inputs/test.prm inputs/in_test.nc
-  ./shoc -p inputs/test.prm
-  ./shoc -t inputs/$1.tran
+   $SHOC -g inputs/test.prm inputs/in_test.nc
+   $SHOC -p inputs/test.prm
+   $SHOC -t inputs/$1.tran
 fi
 
 if [ $# == 2 ] 
 then
    echo "Running $1: only sediment transport"
-   rm tran_outputs/*
-   rm mass.txt sedlog.txt sed_mass_*.txt
-  ./shoc -t inputs/$1.tran
+   rm -f tran_outputs/*
+   rm -f mass.txt sedlog.txt sed_mass_*.txt
+   $SHOC -t inputs/$1.tran
 fi
 
 # post-processing
@@ -51,4 +54,3 @@ cat ./vars sed_mass_end.txt > mass.txt
 rm sed_mass_*.txt
 #python plot.py
 echo "Done"
-
diff --git a/model/tests/sediments/test1/run_all.sh b/model/tests/sediments/test1/run_all.sh
index 8c09273..2695749 100755
--- a/model/tests/sediments/test1/run_all.sh
+++ b/model/tests/sediments/test1/run_all.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 
 if [ $# == 1 ] 
 then
@@ -14,31 +14,34 @@ else
   echo ""
 fi
   
+SHOC="${SHOC:-./shoc}"
+mkdir -p outputs tran_outputs
+
 echo "Cleaning up"
-rm outputs/*
-rm tran_outputs/*
-rm inputs/in_test.nc
-rm inputs/test_trans_1990-01.nc
-rm diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
+rm -f outputs/*
+rm -f tran_outputs/*
+rm -f inputs/in_test.nc
+rm -f inputs/test_trans_1990-01.nc
+rm -f diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
 echo ""
 
 echo "Running hd"
-./shoc -g inputs/test.prm inputs/in_test.nc
-./shoc -p inputs/test.prm
+$SHOC -g inputs/test.prm inputs/in_test.nc
+$SHOC -p inputs/test.prm
 echo "OK: HD forcing files complete"
 echo ""
 
 echo "Running Diffusion test"
-./shoc -t inputs/test_d.tran
+$SHOC -t inputs/test_d.tran
 awk 'BEGIN{a=0;} {if ($NR eq 3) a=$4;} END{if((a-$4)<0.1) \
 print("OK: mass conservation of dissolved tracer"); \
 else print("ERROR: mass conservation of dissolved tracer");}' sed_mass_end.txt
 echo ""
 
 echo "Running Resuspension test"
-rm tran_outputs/*   
-rm sedlog.txt sed_mass_*.txt
-./shoc -t inputs/test_r.tran
+rm -f tran_outputs/*   
+rm -f sedlog.txt sed_mass_*.txt
+$SHOC -t inputs/test_r.tran
 awk 'BEGIN{a=0;} {if ($NR eq 3) a=$5;} END{if((a-$5)<0.1) \
 print("OK: mass conservation of gravel"); \
 else print("ERROR: mass conservation of gravel");}' sed_mass_end.txt
@@ -51,9 +54,9 @@ else print("ERROR: mass conservationof mud");}' sed_mass_end.txt
 echo ""
 
 echo "Running Compaction test"
-rm tran_outputs/*   
-rm sedlog.txt sed_mass_*.txt
-./shoc -t inputs/test_c.tran
+rm -f tran_outputs/*   
+rm -f sedlog.txt sed_mass_*.txt
+$SHOC -t inputs/test_c.tran
 awk 'BEGIN{a=0;} {if ($NR eq 3) a=$4;} END{if((a-$4)<0.1) \
 print("OK: mass conservation of dissolved tracer"); \
 else print("ERROR: mass conservation of dissolved tracer");}' sed_mass_end.txt
@@ -63,18 +66,18 @@ else print("ERROR: mass conservation of mud");}' sed_mass_end.txt
 echo ""
 
 echo "Running Desorption of Pollutant test"
-rm tran_outputs/*   
-rm sedlog.txt sed_mass_*.txt
-./shoc -t inputs/test_p.tran
+rm -f tran_outputs/*   
+rm -f sedlog.txt sed_mass_*.txt
+$SHOC -t inputs/test_p.tran
 awk 'BEGIN{a=0;} {if ($NR eq 3) a=$4+$8;} END{if((a-$4-$8)<0.1) \
 print("OK: mass conservation of pollutant"); \
 else print("ERROR: mass conservation of pollutant");}' sed_mass_end.txt
 echo ""
 
 echo "Running Resuspension, Diffusion and Compaction together"
-rm tran_outputs/*
-rm sedlog.txt sed_mass_*.txt
-./shoc -t inputs/test_rcd.tran
+rm -f tran_outputs/*
+rm -f sedlog.txt sed_mass_*.txt
+$SHOC -t inputs/test_rcd.tran
 awk 'BEGIN{a=0;} {if ($NR eq 3) a=$4;} END{if((a-$4)<0.1) \
 print("OK: mass conservation of dissolved tracer"); \
 else print("ERROR: mass conservation of dissolved tracer");}' sed_mass_end.txt
diff --git a/model/tests/sediments/test2/inputs/test_flat.tran b/model/tests/sediments/test2/inputs/test_flat.tran
index b24c8cf..096f973 100644
--- a/model/tests/sediments/test2/inputs/test_flat.tran
+++ b/model/tests/sediments/test2/inputs/test_flat.tran
@@ -311,3 +311,6 @@ TS0.reference        msl
 # Open boundaries
 NBOUNDARIES           0
 
+
+ID_CODE SHOC grid|G1.0|H|S|B
+
diff --git a/model/tests/sediments/test2/run.sh b/model/tests/sediments/test2/run.sh
index 1ddbafd..44234bc 100755
--- a/model/tests/sediments/test2/run.sh
+++ b/model/tests/sediments/test2/run.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 
 if [ $# == 0 ] 
 then
@@ -16,26 +16,29 @@ else
    echo "run './run.sh' for info"
 fi
 
+SHOC="${SHOC:-./shoc}"
+mkdir -p outputs tran_outputs
+
 if [ $# == 1 ] 
 then
    echo "Cleaning up"
-   rm outputs/*
-   rm tran_outputs/*
-   rm inputs/in_test.nc
-   rm inputs/test_trans_1990-01.nc
-   rm diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
+   rm -f outputs/*
+   rm -f tran_outputs/*
+   rm -f inputs/in_test.nc
+   rm -f inputs/test_trans_1990-01.nc
+   rm -f diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
    echo "Running $1: hd first and then sediment transport"
-  ./shoc -g inputs/test.prm inputs/in_test.nc
-  ./shoc -p inputs/test.prm
-  ./shoc -t inputs/$1.tran
+   $SHOC -g inputs/test.prm inputs/in_test.nc
+   $SHOC -p inputs/test.prm
+   $SHOC -t inputs/$1.tran
 fi
 
 if [ $# == 2 ] 
 then
    echo "Running $1: only sediment transport"
-   rm tran_outputs/*
-   rm mass.txt sedlog.txt sed_mass_*.txt
-  ./shoc -t inputs/$1.tran
+   rm -f tran_outputs/*
+   rm -f mass.txt sedlog.txt sed_mass_*.txt
+   $SHOC -t inputs/$1.tran
 fi
 
 # post-processing
@@ -52,6 +55,5 @@ awk 'BEGIN{a=0;} {if ($NR eq 3) a=$7;} END{if((a-$7)<0.1) \
 print("OK: mass conservation of mud"); \
 else print("ERROR: mass conservation of mud");}' sed_mass_end.txt
 
-rm sed_mass_*.txt
+rm -f sed_mass_*.txt
 echo "Done"
-
diff --git a/model/tests/sediments/test2/run_all.sh b/model/tests/sediments/test2/run_all.sh
index be26607..d662ca3 100755
--- a/model/tests/sediments/test2/run_all.sh
+++ b/model/tests/sediments/test2/run_all.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 
 if [ $# == 1 ] 
 then
@@ -14,23 +14,26 @@ else
   echo ""
 fi
   
+SHOC="${SHOC:-./shoc}"
+mkdir -p outputs tran_outputs
+
 echo "Cleaning up"
-rm outputs/*
-rm tran_outputs/*
-rm inputs/in_test.nc
-rm inputs/test_trans_1990-01.nc
-rm diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
+rm -f outputs/*
+rm -f tran_outputs/*
+rm -f inputs/in_test.nc
+rm -f inputs/test_trans_1990-01.nc
+rm -f diag.txt runlog mass.txt sedlog.txt setup.txt trans.mnc sed_mass_*.txt
 echo ""
 
 echo "Simulating hd in a wind driven, closed basin"
-./shoc -g inputs/test.prm inputs/in_test.nc
-./shoc -p inputs/test.prm
+$SHOC -g inputs/test.prm inputs/in_test.nc
+$SHOC -p inputs/test.prm
 echo "OK: HD forcing files complete"
 echo ""
 
 
 echo "Simulating transport of particulate and dissolver tracers"
-./shoc -t inputs/test_flat.tran
+$SHOC -t inputs/test_flat.tran
 awk 'BEGIN{a=0;} {if ($NR eq 3) a=$4;} END{if((a-$4)<0.1) \
 print("OK: mass conservation of dissolved tracer"); \
 else print("ERROR: mass conservation of dissolved tracer");}' sed_mass_end.txt
@@ -45,5 +48,5 @@ print("OK: mass conservation of mud"); \
 else print("ERROR: mass conservation of mud");}' sed_mass_end.txt
 
 echo ""
-rm  sed_mass_*.txt
+rm -f sed_mass_*.txt
 echo "3D TEST COMPLETE"
diff --git a/model/tests/sediments/test3/inputs/test_advanced.tran b/model/tests/sediments/test3/inputs/test_advanced.tran
index 1402ede..27cf50c 100644
--- a/model/tests/sediments/test3/inputs/test_advanced.tran
+++ b/model/tests/sediments/test3/inputs/test_advanced.tran
@@ -456,3 +456,5 @@ BOUNDARY1.POINTS        3
 1 13
 1 14
 
+
+ID_CODE SHOC grid|G1.0|H|S|B
diff --git a/model/tests/sediments/test3/inputs/test_basic.tran b/model/tests/sediments/test3/inputs/test_basic.tran
index ba1f986..91267c0 100644
--- a/model/tests/sediments/test3/inputs/test_basic.tran
+++ b/model/tests/sediments/test3/inputs/test_basic.tran
@@ -407,3 +407,5 @@ BOUNDARY1.POINTS        3
 1 13
 1 14
 
+
+ID_CODE SHOC grid|G1.0|H|S|B
diff --git a/model/tests/sediments/test3/run.sh b/model/tests/sediments/test3/run.sh
index 3976cd0..6c824ab 100755
--- a/model/tests/sediments/test3/run.sh
+++ b/model/tests/sediments/test3/run.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 
 if [ $# == 0 ] 
 then
@@ -22,27 +22,29 @@ else
    echo "run './run.sh' for info"
 fi
 
+SHOC="${SHOC:-./shoc}"
+mkdir -p outputs tran_outputs
+
 if [ $# == 1 ] 
 then
    echo "Cleaning up"
-   rm outputs/*
-   rm tran_outputs/*
-   rm inputs/in_test.nc
-   rm inputs/test_trans_1990-01.nc
-   rm diag.txt runlog sedlog.txt setup.txt trans.mnc
+   rm -f outputs/*
+   rm -f tran_outputs/*
+   rm -f inputs/in_test.nc
+   rm -f inputs/test_trans_1990-01.nc
+   rm -f diag.txt runlog sedlog.txt setup.txt trans.mnc
    echo "Running $1: hd first and then sediment transport"
-  ./shoc -g inputs/test.prm inputs/in_test.nc
-  ./shoc -p inputs/test.prm
-  ./shoc -t inputs/$1.tran
+  $SHOC -g inputs/test.prm inputs/in_test.nc
+  $SHOC -p inputs/test.prm
+  $SHOC -t inputs/$1.tran
 fi
 
 if [ $# == 2 ] 
 then
    echo "Running $1: only sediment transport"
-   rm tran_outputs/*
-   rm sedlog.txt
-  ./shoc -t inputs/$1.tran
+   rm -f tran_outputs/*
+   rm -f sedlog.txt
+   $SHOC -t inputs/$1.tran
 fi
 
 echo "Done"
-
diff --git a/model/tests/sediments/test3/run_all.sh b/model/tests/sediments/test3/run_all.sh
new file mode 100755
index 0000000..1a5c03e
--- /dev/null
+++ b/model/tests/sediments/test3/run_all.sh
@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+THIS_SCRIPT="${BASH_SOURCE[0]}"
+THIS_DIR=$( cd "$(dirname "${THIS_SCRIPT}")" && pwd)
+
+"${THIS_DIR}/run.sh" "test_basic"
+
+"${THIS_DIR}/run.sh" "test_advanced" "sedi"
diff --git a/model/tests/sediments/test4/inputs/test.prm b/model/tests/sediments/test4/inputs/test.prm
index 6acd496..dabb163 100644
--- a/model/tests/sediments/test4/inputs/test.prm
+++ b/model/tests/sediments/test4/inputs/test.prm
@@ -499,3 +499,5 @@ BATHY    100
  10.000
 
 
+
+ID_CODE SHOC grid|G1.0|H1.0|S|B
diff --git a/model/tests/sediments/test4/run_all.sh b/model/tests/sediments/test4/run_all.sh
index 4afd2a0..ac27756 100755
--- a/model/tests/sediments/test4/run_all.sh
+++ b/model/tests/sediments/test4/run_all.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 
 if [ $# == 1 ] 
 then
@@ -13,16 +13,19 @@ then
 else
   echo ""
 fi
+
+SHOC="${SHOC:-./shoc}"
+mkdir -p outputs
   
 echo "Cleaning up"
-rm outputs/*
-rm inputs/in_test.nc
-rm diag.txt runlog sedlog.txt setup.txt sed_mass_*.txt
+rm -f outputs/*
+rm -f inputs/in_test.nc
+rm -f diag.txt runlog sedlog.txt setup.txt sed_mass_*.txt
 echo ""
 
 echo "Simulating coupled hd and sediment transport in a wind driven, closed basin"
-./shoc -g inputs/test.prm inputs/in_test.nc
-./shoc -p inputs/test.prm
+$SHOC -g inputs/test.prm inputs/in_test.nc
+$SHOC -p inputs/test.prm
 echo "OK: Run complete"
 echo ""
 
@@ -40,5 +43,5 @@ print("OK: mass conservation of mud"); \
 else print("ERROR: mass conservation of mud");}' sed_mass_end.txt
 
 echo ""
-rm  sed_mass_*.txt
+rm  -f sed_mass_*.txt
 echo "3D TEST COMPLETE"

From e1886ea25e3b6a2071332a12b87ce19797e998a3 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Mon, 5 Feb 2024 22:53:21 +1100
Subject: [PATCH 06/12] Make sure the sediments tests are run when run-tests.sh
 is called

---
 .gitignore               | 8 ++++++++
 docker-compose.yml       | 2 ++
 model/tests/run-tests.sh | 8 ++++++--
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 3a34f29..6c4f2c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,11 @@ model/tests/**/test*/runlog
 model/tests/**/test*/setup.txt
 model/tests/**/test*/totals*.ts
 model/tests/**/test*/zoom.points
+model/tests/**/inputs/in_test.nc
+model/tests/**/inputs/test_trans*.nc
+model/tests/**/inputs/test_*.tran
+model/tests/**/outputs
+model/tests/**/tran_outputs
+model/tests/**/sedlog.txt
+model/tests/**/sed_mass*.txt
+model/tests/**/trans.mnc
diff --git a/docker-compose.yml b/docker-compose.yml
index 9c3ee51..17ac7dd 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -17,6 +17,8 @@ services:
         EMS_TEST_RUN: 0
     command: "run-ems-tests"
     container_name: ems
+    environment:
+      EMS_TEST_INCLUDE: '/usr/local/src/ems/model/tests/sediments/test2'
     image: onaci/ereefs-ems-shoc:dev
     volumes:
       - ./model/tests/:/usr/local/src/ems/model/tests/
diff --git a/model/tests/run-tests.sh b/model/tests/run-tests.sh
index d80f83c..c3691ca 100755
--- a/model/tests/run-tests.sh
+++ b/model/tests/run-tests.sh
@@ -25,19 +25,23 @@ echo
 EMS_TEST_INCLUDE="${EMS_TEST_INCLUDE:-${EMS_BASE}/model/tests}"
 echo "EMS_TEST_INCLUDE = '${EMS_TEST_INCLUDE}'"
 
-EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE:-}"
+EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE:-} ${EMS_BASE}/model/tests/run-tests.sh"
 if [ -z "${SHOC}" ]; then
     EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd/*'"
+else
+    EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd/run_tests'"    
 fi
 if [ -z "${COMPAS}" ]; then
     EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd-us/*'"
+else
+    EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd-us/run_tests'"
 fi
 echo "EMS_TEST_EXCLUDE = '${EMS_TEST_EXCLUDE}'"
 
 TEST_COUNT=0
 
 for ti in $EMS_TEST_INCLUDE; do
-    FIND_CMD="find ${ti} -type f \( -regex '^.*/run_test[0-9]*$' -o -name 'run*.sh' -o -regex '^.*/run_[^\.]*$' \) ! -iname '.*'"
+    FIND_CMD="find ${ti} -type f \( -regex '^.*/run_test[0-9]*$' -o -name 'run_all.sh' -o -regex '^.*/run_[^\.]*$' \) ! -iname '.*'"
     for te in $EMS_TEST_EXCLUDE; do
         FIND_CMD="${FIND_CMD} ! -path ${te}"
     done

From 7fc7287ddef1ad747f715b39bad5ae4de6ddf606 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Tue, 6 Feb 2024 02:46:38 +1100
Subject: [PATCH 07/12] Make compas tests use an env var to locate the SHOC and
 COMPAS executables, exit on first failure, not count missing output files as
 a failure

---
 .gitignore                                | 34 ++++++++++++----
 model/tests/hd-us/cetas/run_cetas         | 22 ++++++-----
 model/tests/hd-us/closed/run_closed       | 44 ++++++++++++---------
 model/tests/hd-us/est/run_est             | 45 +++++++++++++---------
 model/tests/hd-us/test2/closed/run_closed | 47 ++++++++++++++---------
 model/tests/hd-us/test2/open/run_open     | 42 +++++++++++---------
 6 files changed, 141 insertions(+), 93 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6c4f2c7..af00e78 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,15 +24,15 @@ sediments-us
 
 # Test output files
 model/tests/**/test*/in*.nc
-model/tests/**/test*/alert.ts
-model/tests/**/test*/alert.txt
-model/tests/**/test*/core*
-model/tests/**/test*/diag.txt
+model/tests/**/alert.ts
+model/tests/**/alert.txt
+model/tests/**/core*
+model/tests/**/diag.txt
 model/tests/**/test*/flushing.ts
-model/tests/**/test*/loc*.ts
-model/tests/**/test*/out*.nc
-model/tests/**/test*/runlog
-model/tests/**/test*/setup.txt
+model/tests/**/loc*.ts
+model/tests/**/out*.nc
+model/tests/**/runlog
+model/tests/**/setup.txt
 model/tests/**/test*/totals*.ts
 model/tests/**/test*/zoom.points
 model/tests/**/inputs/in_test.nc
@@ -43,3 +43,21 @@ model/tests/**/tran_outputs
 model/tests/**/sedlog.txt
 model/tests/**/sed_mass*.txt
 model/tests/**/trans.mnc
+model/tests/hd-us/**/boundary.txt
+model/tests/hd-us/**/obc_spec.txt
+model/tests/hd-us/**/plot*.m
+model/tests/hd-us/**/*_quad.nc
+model/tests/hd-us/**/*_quad*.txt
+model/tests/hd-us/**/*_hex.nc
+model/tests/hd-us/**/*_hex*.txt
+model/tests/hd-us/cetas/auto*.*
+model/tests/hd-us/cetas/cetas*.*
+model/tests/hd-us/cetas/in.*
+model/tests/hd-us/closed/totals.ts
+model/tests/hd-us/est/est*.*
+model/tests/hd-us/est/points.txt
+model/tests/hd-us/est/segment.txt
+model/tests/hd-us/**/in1.nc
+model/tests/hd-us/**/in1*.us
+model/tests/hd-us/**/closed.nc
+model/tests/hd-us/**/closed*.us
diff --git a/model/tests/hd-us/cetas/run_cetas b/model/tests/hd-us/cetas/run_cetas
index 36baa61..bb81eff 100755
--- a/model/tests/hd-us/cetas/run_cetas
+++ b/model/tests/hd-us/cetas/run_cetas
@@ -1,15 +1,17 @@
-#!/bin/csh -f
+#!/bin/csh -ef
 
 echo "Testing COMPAS cetas..."
-rm *.msh
-rm *.txt
-rm *.us
-rm *.jig
-rm *.site
-rm in.nc
+rm -f *.msh
+rm -f *.txt
+rm -f *.us
+rm -f *.jig
+rm -f *.site
+rm -f in.nc
 
-../compas -ag auto.prm
-
-echo "DONE"
+if (! $?COMPAS || "$COMPAS" == "") then
+    set COMPAS='../compas'
+endif
 
+$COMPAS -ag auto.prm
 
+echo "DONE"
diff --git a/model/tests/hd-us/closed/run_closed b/model/tests/hd-us/closed/run_closed
index 2c4f208..26b734a 100755
--- a/model/tests/hd-us/closed/run_closed
+++ b/model/tests/hd-us/closed/run_closed
@@ -1,42 +1,48 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
+
+if (! $?COMPAS || "$COMPAS" == "") then
+    set COMPAS='../compas'
+endif
 
 echo "Testing SHOC..."
-rm closed.nc
-rm out1.nc
+rm -f closed.nc
+rm -f out1.nc
 
-../shoc -g closed.prm closed.nc
+$SHOC -g closed.prm closed.nc
 
-../shoc -p closed.prm
+$SHOC -p closed.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad..."
-rm closed_quad.nc
-rm s_closed_quad.nc
-rm out1_quad.nc
-rm pt.nc
+rm -f closed_quad.nc
+rm -f s_closed_quad.nc
+rm -f out1_quad.nc
+rm -f pt.nc
 
-../compas -g closed_quad.prm closed_quad.nc
+$COMPAS -g closed_quad.prm closed_quad.nc
 
-../compas -p closed_quad.prm
+$COMPAS -p closed_quad.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad 5 window..."
-rm out1_quad5w.nc
+rm -f out1_quad5w.nc
 
-../compas -p closed_quad5w.prm
+$COMPAS -p closed_quad5w.prm
 
 echo "DONE"
 
 echo "Testing COMPAS hex..."
-rm closed_hex.nc
-rm out1_hex.nc
+rm -f closed_hex.nc
+rm -f out1_hex.nc
 
-../compas -g closed_hex.prm closed_hex.nc
+$COMPAS -g closed_hex.prm closed_hex.nc
 
-../compas -p closed_hex.prm
+$COMPAS -p closed_hex.prm
 
 echo "DONE"
-
-
diff --git a/model/tests/hd-us/est/run_est b/model/tests/hd-us/est/run_est
index 00aded8..9825aa7 100755
--- a/model/tests/hd-us/est/run_est
+++ b/model/tests/hd-us/est/run_est
@@ -1,47 +1,54 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
+
+if (! $?COMPAS || "$COMPAS" == "") then
+    set COMPAS='../compas'
+endif
+
 
 echo "Testing SHOC..."
-rm est.nc
-rm out1.nc
+rm -f est.nc
+rm -f out1.nc
 
-../shoc -g est.prm est.nc
+$SHOC -g est.prm est.nc
 
-../shoc -p est.prm
+$SHOC -p est.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad..."
-rm est_quad.nc
-rm out1_quad.nc
+rm -f est_quad.nc
+rm -f out1_quad.nc
 
-../compas -g est_quad.prm est_quad.nc
+$COMPAS -g est_quad.prm est_quad.nc
 
-../compas -p est_quad.prm
+$COMPAS -p est_quad.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad 5 window..."
-rm out1_quad5w.nc
+rm -f out1_quad5w.nc
 
-../compas -p est_quad5w.prm
+$COMPAS -p est_quad5w.prm
 
 echo "DONE"
 
 echo "Testing COMPAS hex..."
-rm est_hex.nc
-rm out1_hex.nc
+rm -f est_hex.nc
+rm -f out1_hex.nc
 
-../compas -g est_hex.prm est_hex.nc
+$COMPAS -g est_hex.prm est_hex.nc
 
-../compas -p est_hex.prm
+$COMPAS -p est_hex.prm
 
 echo "DONE"
 
 echo "Testing COMPAS hex 5 window..."
-rm out1_hex5w.nc
+rm -f out1_hex5w.nc
 
-../compas -p est_hex5w.prm
+$COMPAS -p est_hex5w.prm
 
 echo "DONE"
-
-
diff --git a/model/tests/hd-us/test2/closed/run_closed b/model/tests/hd-us/test2/closed/run_closed
index 96a47e5..f29b010 100755
--- a/model/tests/hd-us/test2/closed/run_closed
+++ b/model/tests/hd-us/test2/closed/run_closed
@@ -1,46 +1,55 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../../shoc'
+endif
+
+if (! $?COMPAS || "$COMPAS" == "") then
+    set COMPAS='../../compas'
+endif
 
 echo "Testing SHOC..."
-rm in1.nc
-rm out1.nc
+rm -f in1.nc
+rm -f out1.nc
 
-../../shoc -g test2.prm in1.nc
+$SHOC -g test2.prm in1.nc
 
-../../shoc -p test2.prm
+$SHOC -p test2.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad..."
-rm in1_quad.nc
-rm s_in1_quad.nc
-rm out1_quad.nc
+rm -f in1_quad.nc
+rm -f s_in1_quad.nc
+rm -f out1_quad.nc
 
-../../compas -g test2_quad.prm in1_quad.nc
+$COMPAS -g test2_quad.prm in1_quad.nc
 
-../../compas -p test2_quad.prm
+$COMPAS -p test2_quad.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad 2 window..."
-rm out1_quad2w.nc
-rm win2.nc
+rm -f out1_quad2w.nc
+rm -f win2.nc
 
-../../compas -p test2_quad2w.prm
+$COMPAS -p test2_quad2w.prm
 
 echo "DONE"
 
 echo "Testing COMPAS hex..."
-rm in1_hex.nc
-rm out1_hex.nc
+rm -f in1_hex.nc
+rm -f out1_hex.nc
 
-../../compas -g test2_hex.prm in1_hex.nc
+$COMPAS -g test2_hex.prm in1_hex.nc
 
-../../compas -p test2_hex.prm
+$COMPAS -p test2_hex.prm
 
 echo "DONE"
 
 echo "Testing COMPAS hex 2 window..."
-rm out1_hex2w.nc
+rm -f out1_hex2w.nc
 
-../../compas -p test2_hex2w.prm
+$COMPAS -p test2_hex2w.prm
 
+echo "DONE"
diff --git a/model/tests/hd-us/test2/open/run_open b/model/tests/hd-us/test2/open/run_open
index ae7847b..52086a9 100755
--- a/model/tests/hd-us/test2/open/run_open
+++ b/model/tests/hd-us/test2/open/run_open
@@ -1,41 +1,47 @@
-#!/bin/csh -f
+#!/bin/csh -ef
+
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../shoc'
+endif
+
+if (! $?COMPAS || "$COMPAS" == "") then
+    set COMPAS='../compas'
+endif
 
 echo "Testing SHOC..."
-rm in1.nc
-rm out1.nc
+rm -f in1.nc
+rm -f out1.nc
 
-../../shoc -g test2.prm in1.nc
+$SHOC -g test2.prm in1.nc
 
-../../shoc -p test2.prm
+$SHOC -p test2.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad..."
-rm in1_quad.nc
-rm s_in1_quad.nc
-rm out1_quad.nc
+rm -f in1_quad.nc
+rm -f s_in1_quad.nc
+rm -f out1_quad.nc
 
-../../compas -g test2_quad.prm in1_quad.nc
+$COMPAS -g test2_quad.prm in1_quad.nc
 
-../../compas -p test2_quad.prm
+$COMPAS -p test2_quad.prm
 
 echo "DONE"
 
 echo "Testing COMPAS quad 2 window..."
-rm out1_quad2w.nc
+rm -f out1_quad2w.nc
 
-../../compas -p test2_quad2w.prm
+$COMPAS -p test2_quad2w.prm
 
 echo "DONE"
 
 echo "Testing COMPAS hex..."
-rm in1_hex.nc
-rm out1_hex.nc
+rm -f in1_hex.nc
+rm -f out1_hex.nc
 
-../../compas -g test2_hex.prm in1_hex.nc
+$COMPAS -g test2_hex.prm in1_hex.nc
 
-../../compas -p test2_hex.prm
+$COMPAS -p test2_hex.prm
 
 echo "DONE"
-
-

From 06cfae5accd4c7ebbdd3231629d7844da513e1b6 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Tue, 6 Feb 2024 14:36:06 +1100
Subject: [PATCH 08/12] Refine handling of tests and test environment

---
 .gitignore                             | 41 +++-------------
 Dockerfile                             | 30 ++++++------
 docker-compose.yml                     |  5 +-
 model/tests/hd-us/.gitignore           | 18 +++++++
 model/tests/hd-us/cetas/.gitignore     | 10 ++++
 model/tests/hd-us/cetas/run_cetas      | 12 +++--
 model/tests/hd-us/est/run_est          |  1 +
 model/tests/hd/.gitignore              |  4 ++
 model/tests/run-tests.sh               | 66 ++++++++++++++++----------
 model/tests/sediments/.gitignore       |  4 ++
 model/tests/tracerstats/basic/run_test | 20 ++++----
 11 files changed, 117 insertions(+), 94 deletions(-)
 create mode 100644 model/tests/hd-us/.gitignore
 create mode 100644 model/tests/hd-us/cetas/.gitignore
 create mode 100644 model/tests/hd/.gitignore
 create mode 100644 model/tests/sediments/.gitignore

diff --git a/.gitignore b/.gitignore
index af00e78..82b6abb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,42 +22,15 @@ libjigsaw64r.so
 # Misc
 sediments-us
 
-# Test output files
-model/tests/**/test*/in*.nc
-model/tests/**/alert.ts
-model/tests/**/alert.txt
-model/tests/**/core*
+# Common test output files
+# (Test directories may also have their own .gitignore files)
+model/tests/ems_test.log
+model/tests/**/alert.*
+model/tests/**/core.*
 model/tests/**/diag.txt
-model/tests/**/test*/flushing.ts
+model/tests/**/in*.nc
 model/tests/**/loc*.ts
+model/tests/**/obc_spec.txt
 model/tests/**/out*.nc
 model/tests/**/runlog
 model/tests/**/setup.txt
-model/tests/**/test*/totals*.ts
-model/tests/**/test*/zoom.points
-model/tests/**/inputs/in_test.nc
-model/tests/**/inputs/test_trans*.nc
-model/tests/**/inputs/test_*.tran
-model/tests/**/outputs
-model/tests/**/tran_outputs
-model/tests/**/sedlog.txt
-model/tests/**/sed_mass*.txt
-model/tests/**/trans.mnc
-model/tests/hd-us/**/boundary.txt
-model/tests/hd-us/**/obc_spec.txt
-model/tests/hd-us/**/plot*.m
-model/tests/hd-us/**/*_quad.nc
-model/tests/hd-us/**/*_quad*.txt
-model/tests/hd-us/**/*_hex.nc
-model/tests/hd-us/**/*_hex*.txt
-model/tests/hd-us/cetas/auto*.*
-model/tests/hd-us/cetas/cetas*.*
-model/tests/hd-us/cetas/in.*
-model/tests/hd-us/closed/totals.ts
-model/tests/hd-us/est/est*.*
-model/tests/hd-us/est/points.txt
-model/tests/hd-us/est/segment.txt
-model/tests/hd-us/**/in1.nc
-model/tests/hd-us/**/in1*.us
-model/tests/hd-us/**/closed.nc
-model/tests/hd-us/**/closed*.us
diff --git a/Dockerfile b/Dockerfile
index b1c58ea..0587297 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,23 +21,18 @@ ARG EMS_BASE="/usr/local/src/ems"
 ENV EMS_BASE="${EMS_BASE}"
 WORKDIR ${EMS_BASE}
 
-# Install all the local EMS Source Code
+# Install all the EMS Source Code
 COPY ./ ./
 
-# Identify which model executables are to be built, and remove any source directories which should be skipped.
-ARG SHOC=1
-ENV SHOC="${SHOC:+${EMS_BASE}/model/hd/shoc}"
-RUN if [ -z "${SHOC}" ]; then rm -rf "${EMS_BASE}/model/hd"; fi
-
-ARG COMPAS=1
-ENV COMPAS="${COMPAS:+${EMS_BASE}/model/hd-us/compas}"
-RUN if [ -z "${COMPAS}" ]; then rm -rf "${EMS_BASE}/model/hd-us" "${EMS_BASE}/ext/jigsaw"; fi
-
-ARG EMS_RM_DIRS="ext/swan utilities model/boxhd model/lib/da model/lib/exchange"
+# Selectively remove some subdirectories that we want to omit
+# (List can be overridden by a build argument)
+ARG EMS_RM_DIRS=""
 RUN if [ -n "${EMS_RM_DIRS}" ]; then for d in $EMS_RM_DIRS; do rm -rf "${EMS_BASE}/${d}"; done; fi
 
 # Compile the EMS components
-# (Keep the build-logs in case they are needed for debugging purposes later)
+# We build with OpenMP support by default for use in a docker container
+# Different options can be specified by overriding the EMS_CONFIGURE_OPTS build argument
+# (The base image *does* include NetCDF and HDF5 with Open MPI support, so that is an option)
 ENV EMS_BUILD_LOG="${EMS_BASE}/ems_build.log"
 ARG EMS_CONFIGURE_OPTS="--enable-omp"
 RUN make distclean || true \
@@ -48,17 +43,20 @@ RUN make check install 2>&1 | tee -a "${EMS_BUILD_LOG}"
 
 # Symlink the EMS executable(s) into the default path
 # (This will fail if the executable did not build for some reason)
+ENV SHOC="${EMS_BASE}/model/hd/shoc" \
+    COMPAS="${EMS_BASE}/model/hd-us/compas"
 RUN if [ -n "${SHOC}" ]; then chmod 0755 "${SHOC}" && ln -s "${SHOC}" /usr/local/bin/shoc; fi; \
     if [ -n "${COMPAS}" ]; then chmod 0755 "${COMPAS}" && ln -s "${COMPAS}" /usr/local/bin/compas; fi; \
     ln -s "${EMS_BASE}/model/tests/run-tests.sh" /usr/local/bin/run-ems-tests
 
 # Optionally run all available unit tests
-# (Set this to 1 to fail the build if any tests fail, or any other value to skip running tests)
+# (Override the EMS_TEST_RUN build argument to have a value of 0 to skip testing at build time)
 ARG EMS_TEST_RUN=1
-ARG EMS_TEST_INCLUDE="${EMS_BASE}"
+ARG EMS_TEST_INCLUDE="${EMS_BASE}/model/tests"
 ENV EMS_TEST_INCLUDE="${EMS_TEST_INCLUDE}"
-ARG EMS_TEST_EXCLUDE="${EMS_BASE}/model/tests/tracerstats/basic/run_test"
+ARG EMS_TEST_EXCLUDE=""
 ENV EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE}"
+ENV EMS_TEST_LOG="${EMS_BASE}/model/tests/ems_test.log"
 RUN if [ $EMS_TEST_RUN -eq 1 ]; then run-ems-tests; fi
 
 # Encode EMS metadata in labels
@@ -66,6 +64,6 @@ LABEL au.csiro.ems.base=${EMS_BASE} \
       au.csiro.ems.shoc=${SHOC} \
       au.csiro.ems.compas=${COMPAS}
 
-# Configure the default entrypoint to be the EMS executable
+# Configure the default entrypoint to be a default EMS executable
 ENTRYPOINT ["/bin/bash", "-c" ]
 CMD [ "${SHOC:-$COMPAS}", "-v"]
diff --git a/docker-compose.yml b/docker-compose.yml
index 17ac7dd..8bdb97c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -12,13 +12,12 @@ services:
     build:
       context: .
       args:
-        SHOC: 1
-        COMPAS: 1
+        #EMS_RM_DIRS: "ext/swan model/lib/da model/lib/exchange"
         EMS_TEST_RUN: 0
     command: "run-ems-tests"
     container_name: ems
     environment:
-      EMS_TEST_INCLUDE: '/usr/local/src/ems/model/tests/sediments/test2'
+      EMS_TEST_INCLUDE: '/usr/local/src/ems/model/tests'
     image: onaci/ereefs-ems-shoc:dev
     volumes:
       - ./model/tests/:/usr/local/src/ems/model/tests/
diff --git a/model/tests/hd-us/.gitignore b/model/tests/hd-us/.gitignore
new file mode 100644
index 0000000..a42fa18
--- /dev/null
+++ b/model/tests/hd-us/.gitignore
@@ -0,0 +1,18 @@
+boundary.txt
+closed*.nc
+closed*.txt
+closed*.us
+est.nc
+est_hex*.txt
+est_hex*.us
+est_quad*.nc
+est_quad*.txt
+plot_*_hex.m
+in1*.txt
+in1*.us
+points.txt
+pt.nc
+s_*_quad.nc
+segment.txt
+test2*.txt
+totals.ts
diff --git a/model/tests/hd-us/cetas/.gitignore b/model/tests/hd-us/cetas/.gitignore
new file mode 100644
index 0000000..367e1dc
--- /dev/null
+++ b/model/tests/hd-us/cetas/.gitignore
@@ -0,0 +1,10 @@
+*.msh
+*.txt
+*.us
+*.jig
+*.site
+in.nc
+in.prm
+outputs/*
+!outputs/.empty
+plot_auto.m
diff --git a/model/tests/hd-us/cetas/run_cetas b/model/tests/hd-us/cetas/run_cetas
index bb81eff..120de9b 100755
--- a/model/tests/hd-us/cetas/run_cetas
+++ b/model/tests/hd-us/cetas/run_cetas
@@ -1,6 +1,12 @@
-#!/bin/csh -ef
+#!/bin/csh -exf
 
 echo "Testing COMPAS cetas..."
+
+if (! $?COMPAS || "$COMPAS" == "") then
+    set COMPAS='../compas'
+endif
+
+set nonomatch
 rm -f *.msh
 rm -f *.txt
 rm -f *.us
@@ -8,10 +14,6 @@ rm -f *.jig
 rm -f *.site
 rm -f in.nc
 
-if (! $?COMPAS || "$COMPAS" == "") then
-    set COMPAS='../compas'
-endif
-
 $COMPAS -ag auto.prm
 
 echo "DONE"
diff --git a/model/tests/hd-us/est/run_est b/model/tests/hd-us/est/run_est
index 9825aa7..c61eeff 100755
--- a/model/tests/hd-us/est/run_est
+++ b/model/tests/hd-us/est/run_est
@@ -22,6 +22,7 @@ echo "DONE"
 echo "Testing COMPAS quad..."
 rm -f est_quad.nc
 rm -f out1_quad.nc
+rm -f s_est_quad.nc
 
 $COMPAS -g est_quad.prm est_quad.nc
 
diff --git a/model/tests/hd/.gitignore b/model/tests/hd/.gitignore
new file mode 100644
index 0000000..03db87d
--- /dev/null
+++ b/model/tests/hd/.gitignore
@@ -0,0 +1,4 @@
+
+flushing.ts
+totals*.ts
+zoom.points
diff --git a/model/tests/run-tests.sh b/model/tests/run-tests.sh
index c3691ca..675e944 100755
--- a/model/tests/run-tests.sh
+++ b/model/tests/run-tests.sh
@@ -2,43 +2,54 @@
 #
 # Runs the EMS test suite(s)
 #
-set +e
 
-SUCCESS_LIST=""
-SUCCESS_COUNT=0
+# Don't exit for errors: we will handle (and count) those.
+set +e
 
-FAIL_LIST=""
-FAIL_COUNT=0
+# Send all output to a logfile as well as stdout / stderr
+EMS_TEST_LOG="${EMS_TEST_LOG:-ems_test.log}"
+if [ -f "${EMS_TEST_LOG}" ]; then
+    truncate -s 0 "${EMS_TEST_LOG}"
+fi
+exec > >(tee -a "${EMS_TEST_LOG}")
+exec 2> >(tee -a "${EMS_TEST_LOG}" >&2)
 
+# Dump some useful information about the test environment
+# (note: some of these vars are set by the base image)
+echo "Testing EMS at $(date --rfc-3339=seconds)"
+echo
 echo "SHOC = '${SHOC}' => '$(which shoc)'"
 echo "COMPAS = '${COMPAS}' => '$(which compas)'"
 echo
-echo "CURL_VERSION = '${CURL_VERSION}'"
-echo "DAP_VERSION = '${DAP_VERSION}'"
-echo "GDAL_VERSION = '${GDAL_VERSION}'"
-echo "HDF5_VERSION = '${HDF5_VERSION}'"
-echo "NETCDF_VERSION = '${NETCDF_VERSION}'"
-echo "NCO_VERSION = '${NCO_VERSION}'"
-echo "PROJ_VERSION = '${PROJ_VERSION}'"
+echo "CURL_VERSION = '${CURL_VERSION:-}', curl binary = '$(which curl)' => $(curl --version | head -n 1)"
+echo "DAP_VERSION = '${DAP_VERSION:-}', getdap4 binary = '$(which getdap4)' => $(getdap4 -V 2>&1)"
+echo "GDAL_VERSION = '${GDAL_VERSION:-}', gdalinfo binary = '$(which gdalinfo)', => $(gdalinfo --version 2>&1)"
+echo "HDF5_VERSION = '${HDF5_VERSION:-}', h5dump binary = '$(which h5dump)' => $(h5dump --version)"
+echo "NETCDF_VERSION = '${NETCDF_VERSION:-}', ncdump binary = '$(which ncdump)' => $(ncdump 2>&1 | tail -n 1)"
+echo "NCO_VERSION = '${NCO_VERSION:-}', ncks binary = '$(which ncks)' => $(ncks --version 2>&1 | xargs)"
+echo "PROJ_VERSION = '${PROJ_VERSION:-}', proj binary = '$(which proj)' => $(proj 2>&1 | head -n 1)"
 echo
 
+# Initialise vars to hold test result statistics
+FAIL_LIST=""
+FAIL_COUNT=0
+SUCCESS_LIST=""
+SUCCESS_COUNT=0
+TEST_COUNT=0
+
+# Parse the include and exclude variables used to select target tests
 EMS_TEST_INCLUDE="${EMS_TEST_INCLUDE:-${EMS_BASE}/model/tests}"
 echo "EMS_TEST_INCLUDE = '${EMS_TEST_INCLUDE}'"
 
-EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE:-} ${EMS_BASE}/model/tests/run-tests.sh"
-if [ -z "${SHOC}" ]; then
-    EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd/*'"
-else
-    EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd/run_tests'"    
-fi
-if [ -z "${COMPAS}" ]; then
+EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE:-} ${EMS_BASE}/model/tests/run-tests.sh '${EMS_BASE}/model/tests/hd/run_tests'"
+if [ ! -f "${COMPAS}" ]; then
+    # Valid scenario: skip the COMPAS tests
     EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd-us/*'"
 else
     EMS_TEST_EXCLUDE="${EMS_TEST_EXCLUDE} '${EMS_BASE}/model/tests/hd-us/run_tests'"
 fi
 echo "EMS_TEST_EXCLUDE = '${EMS_TEST_EXCLUDE}'"
 
-TEST_COUNT=0
 
 for ti in $EMS_TEST_INCLUDE; do
     FIND_CMD="find ${ti} -type f \( -regex '^.*/run_test[0-9]*$' -o -name 'run_all.sh' -o -regex '^.*/run_[^\.]*$' \) ! -iname '.*'"
@@ -48,18 +59,17 @@ for ti in $EMS_TEST_INCLUDE; do
     echo "FIND_CMD = '${FIND_CMD}'"
     TEST_FILES=$(eval "${FIND_CMD}")
     if [ -z "${TEST_FILES}" ]; then
-        echo "There are no tests to run: Test FAILED"
-        FAIL_LIST="${FAIL_LIST} ${ti}"
-        FAIL_COUNT=$((FAIL_COUNT+1))
+        echo "There are no test files to run below include path '${ti}'"
     else
-        echo "Test files to run:"
+        echo "Test files to run below include path '${ti}':"
         echo "${TEST_FILES}"
         for tf in $TEST_FILES; do
             chmod 0755 "${tf}"
             cd $(dirname "${tf}")
             echo
             echo "#-----------------------------------------------------"
-            echo "# Running test file '${tf}' from $(pwd)..."
+            echo "# Running test file '${tf}'"
+            echo "# CWD = '$(pwd)'"
             echo "#-----------------------------------------------------"
             ${tf}
             TEST_RESULT=$?
@@ -86,4 +96,8 @@ echo "${SUCCESS_LIST}" | tr ' ' '\n'
 echo
 echo "${FAIL_COUNT} of ${TEST_COUNT} tests failed:"
 echo "${FAIL_LIST}" | tr ' ' '\n'
-exit $FAIL_COUNT
+if [ $TEST_COUNT -eq 0 ]; then
+    exit -1
+else
+    exit $FAIL_COUNT
+fi
diff --git a/model/tests/sediments/.gitignore b/model/tests/sediments/.gitignore
new file mode 100644
index 0000000..785f762
--- /dev/null
+++ b/model/tests/sediments/.gitignore
@@ -0,0 +1,4 @@
+test_trans*.nc
+sed_mass*.txt
+sedlog.txt
+trans.mnc
diff --git a/model/tests/tracerstats/basic/run_test b/model/tests/tracerstats/basic/run_test
index 53d06e5..b6f535b 100755
--- a/model/tests/tracerstats/basic/run_test
+++ b/model/tests/tracerstats/basic/run_test
@@ -1,23 +1,24 @@
-#!/bin/csh -f
+#!/bin/csh -ef
 
 # Script to run test
 
+if (! $?SHOC || "$SHOC" == "") then
+    set SHOC='../../../hd/shoc'
+endif
+
 # check shoc
-if (! -e ../../../hd/shoc) then
+if (! -e $SHOC) then
     echo "cannot find the shoc executable, please build and try again"
     exit(0)
 endif
 
-# check output dir
-if (! -d out) then
-    mkdir out
-else
-    rm -r out/*
-endif
+# Ensure output dir is empty
+mkdir -p out
+rm -rf out/*
 
 # Kick off model
 echo "Starting model ..."
-../../../hd/shoc -p basic.prm
+$SHOC -p basic.prm
 
 if ($status) then
     echo ".... uh oh!"
@@ -26,4 +27,3 @@ else
     echo " "
     echo "Run out.m in Matlab to see results"
 endif
-

From 88539fa5774bd21c13574944baddc3848891fb22 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Tue, 6 Feb 2024 14:58:14 +1100
Subject: [PATCH 09/12] More tweaks to test runner scripts

---
 .gitignore                             | 2 ++
 Dockerfile                             | 2 +-
 model/tests/hd/.gitignore              | 1 +
 model/tests/tracerstats/basic/run_test | 1 +
 4 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 82b6abb..0906a4f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,5 @@ model/tests/**/obc_spec.txt
 model/tests/**/out*.nc
 model/tests/**/runlog
 model/tests/**/setup.txt
+model/tests/**/out/*
+!model/tests/**/out/.empty
diff --git a/Dockerfile b/Dockerfile
index 0587297..76d2943 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# CSIRO EMSr Image Build Script
+# CSIRO EMS Image Build Script
 #------------------------------------------------------------------------------
 # The recommended base image is one of the onaci/ereefs-netcdf-base variants,
 # as those have been designed to include all the EMS dependencies.
diff --git a/model/tests/hd/.gitignore b/model/tests/hd/.gitignore
index 03db87d..d9494a2 100644
--- a/model/tests/hd/.gitignore
+++ b/model/tests/hd/.gitignore
@@ -1,4 +1,5 @@
 
 flushing.ts
+test7*_sed.prm
 totals*.ts
 zoom.points
diff --git a/model/tests/tracerstats/basic/run_test b/model/tests/tracerstats/basic/run_test
index b6f535b..d04bdac 100755
--- a/model/tests/tracerstats/basic/run_test
+++ b/model/tests/tracerstats/basic/run_test
@@ -13,6 +13,7 @@ if (! -e $SHOC) then
 endif
 
 # Ensure output dir is empty
+set nonomatch
 mkdir -p out
 rm -rf out/*
 

From 11b0d6a830f40fe698e2ae4d6ce5ac5dab427ffa Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Fri, 9 Feb 2024 11:51:36 +1100
Subject: [PATCH 10/12] Seems to run fine on the mainline netcdf-base image
 with all updated libs, so use that by default

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 76d2943..14f9871 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,7 +5,7 @@
 # as those have been designed to include all the EMS dependencies.
 # Allow it to be overridden in order to choose *which* variant, or even
 # something completely different.
-ARG BASE_IMAGE="onaci/ereefs-netcdf-base:ems"
+ARG BASE_IMAGE="onaci/ereefs-netcdf-base:python-3.11-slim-bullseye"
 FROM ${BASE_IMAGE}
 
 # Record the actual base image used from the FROM command as a label.

From d9ddf7f6d4be6fec6271bee6062fad2e7453dad1 Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Fri, 9 Feb 2024 12:05:54 +1100
Subject: [PATCH 11/12] Additional build tooling for Dockerhub repository CI
 build, test and push support

---
 .dockerignore           |  1 +
 docker-compose.test.yml | 22 ++++++++++++++++++
 hooks/README.md         |  7 ++++++
 hooks/build             | 50 +++++++++++++++++++++++++++++++++++++++++
 hooks/post_push         |  8 +++++++
 5 files changed, 88 insertions(+)
 create mode 100644 docker-compose.test.yml
 create mode 100644 hooks/README.md
 create mode 100644 hooks/build
 create mode 100644 hooks/post_push

diff --git a/.dockerignore b/.dockerignore
index 5b2a9d1..f04e101 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -3,3 +3,4 @@
 *.log
 *.md
 docker-compose.yml
+hooks
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
new file mode 100644
index 0000000..a357702
--- /dev/null
+++ b/docker-compose.test.yml
@@ -0,0 +1,22 @@
+---
+#-------------------------------------------------------------------------------
+# Docker *test* environment composition for EMS
+# This exists to support the autotest option for a Dockerhub repository build
+# (See: https://docs.docker.com/docker-hub/builds/automated-testing/)
+#
+# If this repository and rbanch are configured as Dockerhub autotest
+# repositories, this will automatically build and test EMS with all default
+# settings every time a push is made to the repository.
+#
+# The resulting image is not automatically pushed to the DockerHub registry
+# unless the same respository is automatically configured for autobuild.
+#-------------------------------------------------------------------------------
+version: "3.8"
+
+services:
+  ems:
+    build:
+      context: .
+      args:
+        EMS_TEST_RUN: 0
+    command: "run-ems-tests"
diff --git a/hooks/README.md b/hooks/README.md
new file mode 100644
index 0000000..c084349
--- /dev/null
+++ b/hooks/README.md
@@ -0,0 +1,7 @@
+# csiro-coasts/EMS/hooks
+
+The files in this directory are custom build phase hooks for the automated dockerhub builder,
+and are used to set up custom build configurations and docker registry push options for when
+this repository is set up as a DockerHub repository with autobuild enabled.
+
+See <https://docs.docker.com/docker-hub/builds/advanced/#custom-build-phase-hooks> for more information about how they work.
diff --git a/hooks/build b/hooks/build
new file mode 100644
index 0000000..764df0c
--- /dev/null
+++ b/hooks/build
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Source environment variables for this repository branch
+if [ -f .env ]; then
+    source .env
+fi
+
+# Derive some more variables from the dockerhub build environment
+BUILD_TIMESTAMP="$(date --rfc-3339=seconds)"
+DOCKER_TAG="${DOCKER_TAG:-}"
+SAFE_TIMESTAMP="$(echo $BUILD_TIMESTAMP | sed 's/ /T/g' | sed 's/:/-/g' | sed 's/\+.*//')"
+SOURCE_URL="$(git remote get-url origin || true)"
+VERSION_TAG="${SOURCE_BRANCH}_v${SAFE_TIMESTAMP}-${SOURCE_COMMIT}"
+
+
+# Treat most DOCKER_TAG values as overrides for the
+# BASE_IMAGE variable sourced from the .env file
+if [[ -n "${DOCKER_TAG}" ]] && [[ "${DOCKER_TAG}" != "latest" ]] && [[ "${DOCKER_TAG}" != "${SOURCE_BRANCH}" ]]; then
+    # Prepend the non-default docker tag to the versioned docker tag
+    VERSION_TAG="${DOCKER_TAG}_${VERSION_TAG}"
+
+    # Derive the BASE_IMAGE from the DOCKER_TAG
+    if [[ $DOCKER_TAG =~ ^r-base- ]]; then
+        BASE_IMAGE=$(echo "${DOCKER_TAG}" | sed 's/^r-base-/r-base:/')
+    elif [[ $DOCKER_TAG =~ ^rocker-r- ]]; then
+        BASE_IMAGE=$(echo "${DOCKER_TAG}" | sed 's/\-/:/3' | sed 's|\-|/|1')
+    else
+        BASE_IMAGE=$(echo "${DOCKER_TAG}" | sed 's/\-/:/1')
+    fi
+fi
+
+# Build our customised docker image
+docker build --pull \
+    --build-arg "BASE_IMAGE=${BASE_IMAGE:-}" \
+    --build-arg "EMS_RM_DIRS=${EMS_RM_DIRS:-}" \
+    --build-arg "EMS_TEST_RUN=0" \
+    --label "org.opencontainers.image.authors=${BUILD_AUTHORS:-CSIRO Coastal Group}" \
+    --label "org.opencontainers.image.branch=${SOURCE_BRANCH}" \
+    --label "org.opencontainers.image.buildhost=${DOCKER_REPO}" \
+    --label "org.opencontainers.image.created=${BUILD_TIMESTAMP}" \
+    --label "org.opencontainers.image.licenses=https://github.com/csiro-coasts/EMS/blob/master/LICENSE.md" \
+    --label "org.opencontainers.image.revision=${SOURCE_COMMIT}" \
+    --label "org.opencontainers.image.source=${SOURCE_URL}" \
+    --label "org.opencontainers.image.title=${DOCKER_REPO:-csiro-coasts/EMS}" \
+    --label "org.opencontainers.image.url=https://github.com/csiro-coasts/EMS" \
+    --label "org.opencontainers.image.vendor=CSIRO" \
+    --label "org.opencontainers.image.version=${VERSION_TAG}" \
+    -f ${DOCKERFILE_PATH} \
+    -t ${IMAGE_NAME} \
+    .
diff --git a/hooks/post_push b/hooks/post_push
new file mode 100644
index 0000000..4cd9186
--- /dev/null
+++ b/hooks/post_push
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+VERSION_TAG="$(docker image inspect  --format '{{ index .Config.Labels "org.opencontainers.image.version"}}' $IMAGE_NAME || true)"
+if [ -n "${VERSION_TAG}" ]; then
+    IMAGE_TAGGED="${DOCKER_REPO}:${VERSION_TAG}"
+    docker tag $IMAGE_NAME $IMAGE_TAGGED
+    docker push $IMAGE_TAGGED
+fi

From 54b608b6de23d407d8643a3b1dc66d61eb8c2d3b Mon Sep 17 00:00:00 2001
From: Sharon Tickell <Sharon.Tickell@csiro.au>
Date: Mon, 12 Feb 2024 12:07:48 +1100
Subject: [PATCH 12/12] Update tag handling for dockerhub build hook: custom
 tags are used as base-image tags

---
 hooks/build | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/hooks/build b/hooks/build
index 764df0c..16b0d62 100644
--- a/hooks/build
+++ b/hooks/build
@@ -16,17 +16,11 @@ VERSION_TAG="${SOURCE_BRANCH}_v${SAFE_TIMESTAMP}-${SOURCE_COMMIT}"
 # Treat most DOCKER_TAG values as overrides for the
 # BASE_IMAGE variable sourced from the .env file
 if [[ -n "${DOCKER_TAG}" ]] && [[ "${DOCKER_TAG}" != "latest" ]] && [[ "${DOCKER_TAG}" != "${SOURCE_BRANCH}" ]]; then
-    # Prepend the non-default docker tag to the versioned docker tag
-    VERSION_TAG="${DOCKER_TAG}_${VERSION_TAG}"
+    # Assume the custom docker tag is the tag of our desired BASE_IMAGE.
+    BASE_IMAGE="onaci/ereefs-netcdf-base:${DOCKER_TAG}"
 
-    # Derive the BASE_IMAGE from the DOCKER_TAG
-    if [[ $DOCKER_TAG =~ ^r-base- ]]; then
-        BASE_IMAGE=$(echo "${DOCKER_TAG}" | sed 's/^r-base-/r-base:/')
-    elif [[ $DOCKER_TAG =~ ^rocker-r- ]]; then
-        BASE_IMAGE=$(echo "${DOCKER_TAG}" | sed 's/\-/:/3' | sed 's|\-|/|1')
-    else
-        BASE_IMAGE=$(echo "${DOCKER_TAG}" | sed 's/\-/:/1')
-    fi
+    # Override the VERSION_TAG to include it
+    VERSION_TAG="${SOURCE_BRANCH}_${DOCKER_TAG}_v${SAFE_TIMESTAMP}-${SOURCE_COMMIT}"
 fi
 
 # Build our customised docker image