Use aviso triggers to start file conversion

_test/test_cds.py

@@ -29,5 +29,6 @@ def test_get_cds(tmpdir):
                 assert var in fut.variables
                 for att in ["units", "standard_name"]:
                     if att in ref[var].attrs:
+                        print(var, att, ref[var].attrs[att], fut[var].attrs[att])
                         assert att in fut[var].attrs
                         assert ref[var].attrs[att] == fut[var].attrs[att]

aviso/aviso_readme.txt

@@ -0,0 +1,29 @@
+Use of aviso to trigger job scripts on the European Weather Cloud (EWC)
+=======================================================================
+
+
+* The aviso service runs automatically on EWC machines with the data setup
+  It can be activated and deactivated by the command
+    sudo systemctl start aviso.service
+    sudo systemctl stop aviso.service
+  The status can be viewd be
+    sudo systemctl status aviso.service
+
+* For test purposes it my be used to test the behaviour on past trigger
+  events for a given (past) time range. This can be done by the listen
+  command within an env, where aviso is installed, like
+    sudo systemctl stop aviso.service
+    mamba activate avisoenv
+    aviso listen --from 2025-03-05T00:00:00.0Z --to 2025-03-05T08:00:00.0Z
+
+* The file config.yaml in the direactory $HOME/.aviso guides the triggering
+
+* if multiple steps are given in a trigger command, like
+   step: [36,72,144]
+  the programs called by the trigger are executed sequentially, possibly
+  not in the time order of the triggers
+
+* the script get_ecmwf_aviso.sh creates ECMWF grib files and converts
+  them to MSS-convorm NetCDF files. One can speed up the process by
+  parallel download the grib files using "DOWNLOAD_ONLY=yes" in the file
+  settings.config and starting convert_all.sh separately.

aviso/config.yaml.parallel

@@ -0,0 +1,56 @@
+listeners:
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 36
+    triggers:
+      - type: command
+        working_dir: $HOME/data-retrieval/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 72
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 108
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 144
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 228
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out

aviso/config.yaml.sequential

@@ -0,0 +1,12 @@
+listeners:
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: [36, 72, 108, 144, 228]
+    triggers:
+      - type: command
+        working_dir: $HOME/data-retrieval/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out

bin/convert.sh

@@ -58,6 +58,10 @@ rm ${tmpfile}_z
 if [[ x$SFC_PARAMETERS != x"" ]]; then
     echo converting sfc
     cdo -f nc4c -t ecmwf copy grib/${BASE}.sfc.grib $sfcfile
+
+    cdo showatts  $sfcfile
+    echo "ncatted"
+
     ncatted -O \
         -a standard_name,BLH,o,c,atmosphere_boundary_layer_thickness \
         -a standard_name,CI,o,c,sea_ice_area_fraction \
@@ -69,10 +73,23 @@ if [[ x$SFC_PARAMETERS != x"" ]]; then
         -a standard_name,SSTK,o,c,sea_surface_temperature \
         -a standard_name,U10M,o,c,surface_eastward_wind \
         -a standard_name,V10M,o,c,surface_northward_wind \
+        -a standard_name,TCW,o,c,total_column_water \
+        -a standard_name,TCWV,o,c,total_column_cloud_liquid_water \
+        -a standard_name,T2M,o,c,2_meter_temperature \
+        -a standard_name,D2M,o,c,2_meter_dewpoint \
+        -a standard_name,ISHF,o,c,instantaneous_surface_sensible_heat_flux\
+        -a standard_name,IEWS,o,c,instantaneous_x_surface_stress\
+        -a standard_name,INSS,o,c,instantaneous_y_surface_stress\
         -a units,HCC,o,c,dimensionless \
         -a units,LCC,o,c,dimensionless \
         -a units,MCC,o,c,dimensionless \
+        -a units,CI,o,c,dimensionless \
+        -a units,LSM,o,c,dimensionless \
+        -a units,ASN,o,c,dimensionless \
+        -a units,SD,o,c,m \
         $sfcfile
+
+    cdo showatts  $sfcfile
 fi
 # extract lnsp and remove lev dimension.
 grib_copy -w shortName=lnsp grib/${BASE}.ml2.grib ${tmpfile}

bin/convert_all.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+#Copyright (C) 2025 by Forschungszentrum Juelich GmbH
+#Author(s): Jens-Uw Grooss
+
+export MAINDIR=$HOME/data-retrieval
+export BINDIR=$MAINDIR/bin
+
+. ${MAINDIR}/settings.default
+
+if [ ! -f ${MAINDIR}/settings.config ]; then
+    echo Please copy the settings.example to settings.config and configure your setup!
+    exit 1
+fi
+
+. ${MAINDIR}/settings.config
+
+cd $WORKDIR
+
+export YEAR=`date +%Y`
+export MONTH=`date +%m`
+export DAY=`date +%d`
+AMPM=`date +%p`
+if [ $AMPM == AM ]
+then 
+    export HH=00
+else
+    export HH=12
+fi
+
+
+# script should start at 06h/18h to look for 00 12h forecast
+export h_exit=`date --date="+6hours" +%H`
+
+for FCSTEP in $FCSTEPS
+do
+
+# Set path, filenames and variables used later in the script
+    export DATE=${YEAR}-${MONTH}-${DAY}
+    export YMD=${YEAR}${MONTH}${DAY}
+    export TIME=${HH}:00:00
+    export BASE=${DATASET}.${YMD}T${HH}.${FCSTEP}
+    export init_date=${DATE}T${TIME}
+    echo BASE: $BASE
+    export time_units="hours since ${init_date}"
+
+
+    lockfile=grib/${DATASET}.${YMD}T${HH}.${FCSTEP}.ready
+    echo `date` waiting for lockfile $lockfile
+    until  test -e $lockfile
+    do
+	sleep 30
+	h_now=`date +%H`
+	if [[ $h_now == $h_exit ]] 
+	then
+	    echo lockfile $lockfile not found by `date`
+	    echo exiting script
+	    exit 1
+	fi
+    done
+    rm $lockfile
+
+    # Convert grib to netCDF, set init time
+    echo `date`: converting ${FCSTEP}h forecast
+    . $BINDIR/convert.sh
+
+    if [ $ECTRANS_ID == "none" ]
+    then
+	echo "no ectrans transfer -- move data to " $MSSDIR
+	if [ x$TRANSFER_MODEL_LEVELS == x"yes" ]; then
+	    mv $mlfile $MSSDIR
+	fi
+	if [ -f $tlfile ]; then
+	    mv $tlfile $MSSDIR
+	fi
+	if [ -f $plfile ]; then
+	    mv $plfile $MSSDIR
+	fi
+	if [ -f $pvfile ]; then
+	    mv $pvfile $MSSDIR
+	fi
+	if [ -f $alfile ]; then
+	    mv $alfile $MSSDIR
+	fi
+	if [ -f $sfcfile ]; then
+	    mv $sfcfile $MSSDIR
+	fi
+    fi
+done
+echo `date`: converting finished
+
+
+if [[ x$CLEANUP == x"yes" ]]
+then
+  export CYMD=${CLEANUP_YEAR}${CLEANUP_MONTH}${CLEANUP_DAY}
+  export CBASE=${DATASET}.${CYMD}T${HH}.${FCSTEP}
+  echo cleanup $CBASE
+
+  # clean up locally
+  for f in $mlfile $tlfile $plfile $pvfile $alfile $sfcfile grib/${CBASE}*.grib;
+  do
+      if [ -f $f ];
+      then
+          rm $f
+      fi
+  done
+  if [ $ECTRANS_ID == "none" ]
+  then
+    # clean up MSS server dir
+    for f in $MSSDIR/${CBASE}*.nc
+    do
+	if [ -f $f ];
+	then
+            rm $f
+	fi
+    done
+  fi
+fi
+echo `date`: ECMWF data converson finished