diff --git a/_test/mss/2021-01-29T00:00:00.an.sfc.nc b/_test/mss/2021-01-29T00:00:00.an.sfc.nc index a0fb8be..5808da4 100644 Binary files a/_test/mss/2021-01-29T00:00:00.an.sfc.nc and b/_test/mss/2021-01-29T00:00:00.an.sfc.nc differ diff --git a/_test/test_cds.py b/_test/test_cds.py index 7816f12..6e102be 100644 --- a/_test/test_cds.py +++ b/_test/test_cds.py @@ -29,5 +29,6 @@ def test_get_cds(tmpdir): assert var in fut.variables for att in ["units", "standard_name"]: if att in ref[var].attrs: + print(var, att, ref[var].attrs[att], fut[var].attrs[att]) assert att in fut[var].attrs assert ref[var].attrs[att] == fut[var].attrs[att] diff --git a/aviso/aviso_readme.txt b/aviso/aviso_readme.txt new file mode 100644 index 0000000..6f081c9 --- /dev/null +++ b/aviso/aviso_readme.txt @@ -0,0 +1,29 @@ +Use of aviso to trigger job scripts on the European Weather Cloud (EWC) +======================================================================= + + +* The aviso service runs automatically on EWC machines with the data setup + It can be activated and deactivated by the command + sudo systemctl start aviso.service + sudo systemctl stop aviso.service + The status can be viewd be + sudo systemctl status aviso.service + +* For test purposes it my be used to test the behaviour on past trigger + events for a given (past) time range. This can be done by the listen + command within an env, where aviso is installed, like + sudo systemctl stop aviso.service + mamba activate avisoenv + aviso listen --from 2025-03-05T00:00:00.0Z --to 2025-03-05T08:00:00.0Z + +* The file config.yaml in the direactory $HOME/.aviso guides the triggering + +* if multiple steps are given in a trigger command, like + step: [36,72,144] + the programs called by the trigger are executed sequentially, possibly + not in the time order of the triggers + +* the script get_ecmwf_aviso.sh creates ECMWF grib files and converts + them to MSS-convorm NetCDF files. One can speed up the process by + parallel download the grib files using "DOWNLOAD_ONLY=yes" in the file + settings.config and starting convert_all.sh separately. diff --git a/aviso/config.yaml.parallel b/aviso/config.yaml.parallel new file mode 100644 index 0000000..9df4315 --- /dev/null +++ b/aviso/config.yaml.parallel @@ -0,0 +1,56 @@ +listeners: + - event: mars + request: + class: od + expver: 1 + domain: g + stream: oper + step: 36 + triggers: + - type: command + working_dir: $HOME/data-retrieval/aviso + command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out + - event: mars + request: + class: od + expver: 1 + domain: g + stream: oper + step: 72 + triggers: + - type: command + working_dir: $HOME/aviso + command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out + - event: mars + request: + class: od + expver: 1 + domain: g + stream: oper + step: 108 + triggers: + - type: command + working_dir: $HOME/aviso + command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out + - event: mars + request: + class: od + expver: 1 + domain: g + stream: oper + step: 144 + triggers: + - type: command + working_dir: $HOME/aviso + command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out + - event: mars + request: + class: od + expver: 1 + domain: g + stream: oper + step: 228 + triggers: + - type: command + working_dir: $HOME/aviso + command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out diff --git a/aviso/config.yaml.sequential b/aviso/config.yaml.sequential new file mode 100644 index 0000000..95a9821 --- /dev/null +++ b/aviso/config.yaml.sequential @@ -0,0 +1,12 @@ +listeners: + - event: mars + request: + class: od + expver: 1 + domain: g + stream: oper + step: [36, 72, 108, 144, 228] + triggers: + - type: command + working_dir: $HOME/data-retrieval/aviso + command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out diff --git a/bin/convert.sh b/bin/convert.sh index 390f617..d282cd0 100755 --- a/bin/convert.sh +++ b/bin/convert.sh @@ -58,6 +58,10 @@ rm ${tmpfile}_z if [[ x$SFC_PARAMETERS != x"" ]]; then echo converting sfc cdo -f nc4c -t ecmwf copy grib/${BASE}.sfc.grib $sfcfile + + cdo showatts $sfcfile + echo "ncatted" + ncatted -O \ -a standard_name,BLH,o,c,atmosphere_boundary_layer_thickness \ -a standard_name,CI,o,c,sea_ice_area_fraction \ @@ -69,10 +73,23 @@ if [[ x$SFC_PARAMETERS != x"" ]]; then -a standard_name,SSTK,o,c,sea_surface_temperature \ -a standard_name,U10M,o,c,surface_eastward_wind \ -a standard_name,V10M,o,c,surface_northward_wind \ + -a standard_name,TCW,o,c,total_column_water \ + -a standard_name,TCWV,o,c,total_column_cloud_liquid_water \ + -a standard_name,T2M,o,c,2_meter_temperature \ + -a standard_name,D2M,o,c,2_meter_dewpoint \ + -a standard_name,ISHF,o,c,instantaneous_surface_sensible_heat_flux\ + -a standard_name,IEWS,o,c,instantaneous_x_surface_stress\ + -a standard_name,INSS,o,c,instantaneous_y_surface_stress\ -a units,HCC,o,c,dimensionless \ -a units,LCC,o,c,dimensionless \ -a units,MCC,o,c,dimensionless \ + -a units,CI,o,c,dimensionless \ + -a units,LSM,o,c,dimensionless \ + -a units,ASN,o,c,dimensionless \ + -a units,SD,o,c,m \ $sfcfile + + cdo showatts $sfcfile fi # extract lnsp and remove lev dimension. grib_copy -w shortName=lnsp grib/${BASE}.ml2.grib ${tmpfile} diff --git a/bin/convert_all.sh b/bin/convert_all.sh new file mode 100755 index 0000000..fd6c73c --- /dev/null +++ b/bin/convert_all.sh @@ -0,0 +1,118 @@ +#!/bin/bash +#Copyright (C) 2025 by Forschungszentrum Juelich GmbH +#Author(s): Jens-Uw Grooss + +export MAINDIR=$HOME/data-retrieval +export BINDIR=$MAINDIR/bin + +. ${MAINDIR}/settings.default + +if [ ! -f ${MAINDIR}/settings.config ]; then + echo Please copy the settings.example to settings.config and configure your setup! + exit 1 +fi + +. ${MAINDIR}/settings.config + +cd $WORKDIR + +export YEAR=`date +%Y` +export MONTH=`date +%m` +export DAY=`date +%d` +AMPM=`date +%p` +if [ $AMPM == AM ] +then + export HH=00 +else + export HH=12 +fi + + +# script should start at 06h/18h to look for 00 12h forecast +export h_exit=`date --date="+6hours" +%H` + +for FCSTEP in $FCSTEPS +do + +# Set path, filenames and variables used later in the script + export DATE=${YEAR}-${MONTH}-${DAY} + export YMD=${YEAR}${MONTH}${DAY} + export TIME=${HH}:00:00 + export BASE=${DATASET}.${YMD}T${HH}.${FCSTEP} + export init_date=${DATE}T${TIME} + echo BASE: $BASE + export time_units="hours since ${init_date}" + + + lockfile=grib/${DATASET}.${YMD}T${HH}.${FCSTEP}.ready + echo `date` waiting for lockfile $lockfile + until test -e $lockfile + do + sleep 30 + h_now=`date +%H` + if [[ $h_now == $h_exit ]] + then + echo lockfile $lockfile not found by `date` + echo exiting script + exit 1 + fi + done + rm $lockfile + + # Convert grib to netCDF, set init time + echo `date`: converting ${FCSTEP}h forecast + . $BINDIR/convert.sh + + if [ $ECTRANS_ID == "none" ] + then + echo "no ectrans transfer -- move data to " $MSSDIR + if [ x$TRANSFER_MODEL_LEVELS == x"yes" ]; then + mv $mlfile $MSSDIR + fi + if [ -f $tlfile ]; then + mv $tlfile $MSSDIR + fi + if [ -f $plfile ]; then + mv $plfile $MSSDIR + fi + if [ -f $pvfile ]; then + mv $pvfile $MSSDIR + fi + if [ -f $alfile ]; then + mv $alfile $MSSDIR + fi + if [ -f $sfcfile ]; then + mv $sfcfile $MSSDIR + fi + fi +done +echo `date`: converting finished + + +if [[ x$CLEANUP == x"yes" ]] +then + export CYMD=${CLEANUP_YEAR}${CLEANUP_MONTH}${CLEANUP_DAY} + export CBASE=${DATASET}.${CYMD}T${HH}.${FCSTEP} + echo cleanup $CBASE + + # clean up locally + for f in $mlfile $tlfile $plfile $pvfile $alfile $sfcfile grib/${CBASE}*.grib; + do + if [ -f $f ]; + then + rm $f + fi + done + if [ $ECTRANS_ID == "none" ] + then + # clean up MSS server dir + for f in $MSSDIR/${CBASE}*.nc + do + if [ -f $f ]; + then + rm $f + fi + done + fi +fi +echo `date`: ECMWF data converson finished diff --git a/bin/get_ecmwf_aviso.sh b/bin/get_ecmwf_aviso.sh new file mode 100755 index 0000000..e8f7b72 --- /dev/null +++ b/bin/get_ecmwf_aviso.sh @@ -0,0 +1,206 @@ +#!/bin/bash + +#Copyright (C) 2021 by Forschungszentrum Juelich GmbH +#Author(s): Joern Ungermann, May Baer, Jens-Uwe Grooss + +#SBATCH --qos=normal +#SBATCH --job-name=get_ecmwf +#SBATCH --output=get_ecmwf.%j.out +#SBATCH --error=get_ecmwf.%j.out + + +# This script works with the cdo version installed on ECACCESS and +# in an mambaforge environment ncenv that includes cartopy (0.20.1), metpy (1.1.0) +# nco (5.0.4), netcdf4 (1.5.8), scipy (1.7.3) and xarray (0.20.2) + +# Define model domain sector, resolution and id name for ectrans in settings.config + +# defines for performance measurements +N=`date +%s%N` +export PS4='+[$(((`date +%s%N`-$N)/1000000))ms][${BASH_SOURCE}:${LINENO}]: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' +# enable line below for debugging and performance timing +# set -x +export MAINDIR=$HOME/data-retrieval +export BINDIR=$MAINDIR/bin + +. ${MAINDIR}/settings.default + +if [ ! -f ${MAINDIR}/settings.config ]; then + echo Please copy the settings.example to settings.config and configure your setup! + exit 1 +fi + +. ${MAINDIR}/settings.config + +# get forecast date +# If used as a shell script that is run on a event trigger, +# the $MSJ* environment variables contain the corresponding time info. +# This can be done from the web interface or e.g. by the command +# ecaccess-job-submit -ni fc00h036 get_ecmwf.sh +# If these variables are empty, forecast times are defined in settings.config + + +# aviso trigger + +POSITIONAL=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --date) + AVISO_DATE="$2" + shift # past argument + shift # past value + ;; + --stream) + STREAM="$2" + shift # past argument + shift # past value + ;; + --time) + AVISO_HH="$2" + shift # past argument + shift # past value + ;; + --step) + AVISO_STEP="$2" + shift # past argument + shift # past value + ;; +esac +done + +echo `date`: Notification received for stream $STREAM, date $AVISO_DATE, time $AVISO_HH, step $AVISO_STEP +##20250304 00 72 +export YEAR=${AVISO_DATE:0:4} +export MONTH=${AVISO_DATE:4:2} +export DAY=${AVISO_DATE:6:2} +export HH=$AVISO_HH +export FCSTEP=`printf '%03d' $AVISO_STEP` + +case $FCSTEP in + 036) + export STEP=0/to/36/by/3 + ;; + 072) + export STEP=39/to/72/by/3 + ;; + 108) + export STEP=75/to/108/by/3 + ;; + 144) + export STEP=114/to/144/by/6 + ;; + 228) + export STEP=156/to/228/by/12 + ;; + *) +esac + +cd $WORKDIR +mkdir -p mss +mkdir -p grib + +# Set path, filenames and variables used later in the script +export DATE=${YEAR}-${MONTH}-${DAY} +export YMD=${YEAR}${MONTH}${DAY} +export TIME=${HH}:00:00 +export BASE=${DATASET}.${YMD}T${HH}.${FCSTEP} +export init_date=$(date +%Y-%m-%dT%H:%M:%S) +echo BASE: $BASE + +if [[ "$init_date" > "$DATE" ]] +then + export init_date="${DATE}T${TIME}" +fi +export time_units="hours since ${init_date}" + +# Retrieve ml, sfc, pv and pt files +. $BINDIR/download_ecmwf.sh + +if [ $DOWNLOAD_ONLY == "yes" ] +then + lockfile=grib/${BASE}.ready + echo touch $lockfile and exit + touch $lockfile + exit 1 +fi + +# Convert grib to netCDF, set init time +. $BINDIR/convert.sh + +if [ $ECTRANS_ID == "none" ] +then + echo "no ectrans transfer -- move data to " $MSSDIR + if [ x$TRANSFER_MODEL_LEVELS == x"yes" ]; then + mv $mlfile $MSSDIR + fi + if [ -f $tlfile ]; then + mv $tlfile $MSSDIR + fi + if [ -f $plfile ]; then + mv $plfile $MSSDIR + fi + if [ -f $pvfile ]; then + mv $pvfile $MSSDIR + + fi + if [ -f $alfile ]; then + mv $alfile $MSSDIR + fi + if [ -f $sfcfile ]; then + mv $sfcfile $MSSDIR + fi +else + +if ecaccess-association-list | grep -q $ECTRANS_ID; then + echo "Transfering files to "$ECTRANS_ID + if [ x$TRANSFER_MODEL_LEVELS == x"yes" ]; then + ectrans -remote $ECTRANS_ID -source $mlfile -target $mlfile -overwrite -remove + fi + if [ -f $tlfile ]; then + ectrans -remote $ECTRANS_ID -source $tlfile -target $tlfile -overwrite -remove + fi + if [ -f $plfile ]; then + ectrans -remote $ECTRANS_ID -source $plfile -target $plfile -overwrite -remove + fi + if [ -f $pvfile ]; then + ectrans -remote $ECTRANS_ID -source $pvfile -target $pvfile -overwrite -remove + fi + if [ -f $alfile ]; then + ectrans -remote $ECTRANS_ID -source $alfile -target $alfile -overwrite -remove + fi + if [ -f $sfcfile ]; then + ectrans -remote $ECTRANS_ID -source $sfcfile -target $sfcfile -overwrite -remove + fi +fi +fi + +if [[ x$CLEANUP == x"yes" ]] +then + export CYMD=${CLEANUP_YEAR}${CLEANUP_MONTH}${CLEANUP_DAY} + export CBASE=${DATASET}.${CYMD}T${HH}.${FCSTEP} + echo cleanup $CBASE + + # clean up locally + for f in $mlfile $tlfile $plfile $pvfile $alfile $sfcfile grib/${CBASE}*.grib; + do + if [ -f $f ]; + then + rm $f + fi + done + if [ $ECTRANS_ID == "none" ] + then + # clean up MSS server dir + for f in $MSSDIR/${CBASE}*.nc + do + if [ -f $f ]; + then + rm $f + fi + done + fi +fi +echo `date`: ECMWF data converson finished diff --git a/settings.default b/settings.default index 591c261..01d2c27 100644 --- a/settings.default +++ b/settings.default @@ -3,9 +3,20 @@ export PYTHON=python export CLEANUP=no +export CLEANUP_DAY=`date --date=-2day +%d` +export CLEANUP_MONTH=`date --date=-2day +%m` +export CLEANUP_YEAR=`date --date=-2day +%Y` + export MODEL_REDUCTION="-d lev,0,0 -d lev,16,28,4 -d lev,32,124,2" export DATASET=ecmwf +# set ECTRANS_ID to "none" if write to local $MSSDIR +export ECTRANS_ID=none + +# set DOWNLOAD_ONLY to yes if conversion is done by separate script +export DOWNLOAD_ONLY=no + + export TRUNCATION=auto # options: none, auto, TXXX (e.g. T21) export RESOL=auto # options: av (archived), auto, TXXX(e.g. T21); truncation shall replace resol but resol is still needed export ECMWF_TYPE=fc @@ -17,7 +28,7 @@ export ANCILLARY="--pv --theta --tropopause --n2 --pressure" export MODEL_LEVELS=1/to/137 export MODEL_PARAMETERS=T/U/V/W/Q/CC/CLWC/CIWC/D/O3 export MODEL2_PARAMETERS=LNSP/Z -export SFC_PARAMETERS=LSM/LCC/MCC/HCC/MSL/SP/SSTK/TCC/TCW/TCWV/10U/10V/2D/2T/SKT/ASN/CI/BLH/CAPE/SD/IEWS/INSS/ISHF +export SFC_PARAMETERS=LSM/LCC/MCC/HCC/MSL/SP/SSTK/TCC/TCW/TCWV/10U/10V/2D/2T/SKT/ASN/CI/BLH/SD/IEWS/INSS/ISHF export PV_PARAMETERS=Z/PRES/PT/U/V/Q/O3 export PV_LEVELS=2000