add lstm bmi config gen feature with test

.github/workflows/test_datastream_ngiab.yaml

@@ -6,31 +6,23 @@ on:
     branches:
       - main
     paths:
+      - .github/workflow/test_datastream_options.yaml
       - 'docker/**'
-      - 'forcingprocessor/**'
-      - 'scripts/**'  
-      - 'python_tools/**'      
-      - '!docs/**'
-      - '!workflows/**'
-      - '!scripts/README.md'
-      - '!docker/README.md'
-      - '!forcingprocessor/README.md'
-      - '!python_tools/README.md'
-
+      - '!docker/README.md'      
+      - 'scripts/datastream'  
+      - 'src/datastreamcli/**'  
+      - '!src/datastreamcli/README.md'      
+
   pull_request:
     branches:
       - main  
     paths:
+      - .github/workflow/test_datastream_options.yaml
       - 'docker/**'
-      - 'forcingprocessor/**'
-      - 'scripts/**'  
-      - 'python_tools/**'      
-      - '!docs/**'
-      - '!workflows/**'
-      - '!scripts/README.md'
-      - '!docker/README.md'
-      - '!forcingprocessor/README.md'
-      - '!python_tools/README.md'         
+      - '!docker/README.md'      
+      - 'scripts/datastream'  
+      - 'src/datastreamcli/**'  
+      - '!src/datastreamcli/README.md'            
 
 permissions:
   contents: read      
@@ -63,4 +55,4 @@ jobs:
     - name: Base test and NWM_RETRO_V3
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test        
-        ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
+        ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g https://ciroh-community-ngen-datastream.s3.amazonaws.com/v2.2_resources/VPU_09/config/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json

.github/workflows/test_datastream_options.yaml

@@ -6,33 +6,24 @@ on:
     branches:
       - main
     paths:
+      - .github/workflow/test_datastream_options.yaml
       - 'docker/**'
-      - 'forcingprocessor/**'
-      - 'scripts/**'  
-      - 'python_tools/**'      
-      - '!docs/**'
-      - '!workflows/**'
-      - '!scripts/README.md'
-      - '!docker/README.md'
-      - '!forcingprocessor/README.md'
-      - '!python_tools/README.md'
+      - '!docker/README.md'      
+      - 'scripts/datastream'  
+      - 'src/datastreamcli/**'  
+      - '!src/datastreamcli/README.md'      
 
   pull_request:
     branches:
       - main  
     paths:
+      - .github/workflow/test_datastream_options.yaml
       - 'docker/**'
-      - 'forcingprocessor/**'
-      - 'scripts/**'  
-      - 'python_tools/**'      
-      - '!docs/**'
-      - '!workflows/**'
-      - '!scripts/README.md'
-      - '!docker/README.md'
-      - '!forcingprocessor/README.md'
-      - '!python_tools/README.md'            
-
-
+      - '!docker/README.md'      
+      - 'scripts/datastream'  
+      - 'src/datastreamcli/**'  
+      - '!src/datastreamcli/README.md'               
+
 permissions:
   contents: read      
 
@@ -65,12 +56,12 @@ jobs:
 
     - name: Get geopackage 
       run: |
-        curl -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg
+        curl -L -O https://ciroh-community-ngen-datastream.s3.amazonaws.com/v2.2_resources/VPU_09/config/nextgen_VPU_09.gpkg
 
     - name: Base test and NWM_RETRO_V3
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test        
-        ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
+        ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
 
     - name: Cache resource directory and CONF_FILE
       run: |
@@ -90,22 +81,29 @@ jobs:
       if: always()
       run: |
         sudo rm -rf $(pwd)/data/datastream_test
-        ./scripts/datastream -g $(pwd)/palisade.gpkg -R $(pwd)/data/cache/datastream-resources/config/realization_sloth_nom_cfe_pet.json -F $(pwd)/data/cache/datastream-resources/ngen-forcings/1_forcings.nc -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test        
+        forcings_file=$(find "./data/cache/datastream-resources/ngen-forcings/" -name "*.nc")
+        ./scripts/datastream -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/data/cache/datastream-resources/config/realization_sloth_nom_cfe_pet.json -F $forcings_file -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test        
 
     - name: Resource directory test missing all
       if: always()
       run: |
         sudo rm -rf $(pwd)/data/datastream_test
         sudo rm -rf ./data/cache/datastream-resources-missing/ngen-forcings 
         sudo rm -rf ./data/cache/datastream-resources-missing/config/*
-        ./scripts/datastream -r ./data/cache/datastream-resources-missing -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg
+        ./scripts/datastream -r ./data/cache/datastream-resources-missing -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg
 
     - name: Resource directory test
       if: always()
       run: |
         sudo rm -rf $(pwd)/data/datastream_test
         ./scripts/datastream -r ./data/cache/datastream-resources -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test
 
+    - name: Rust LSTM test
+      if: always()
+      run: |
+        sudo rm -rf $(pwd)/data/datastream_test
+        ./scripts/datastream -r ./data/cache/datastream-resources -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -R $(pwd)/configs/ngen/realization_rust_lstm.json
+
     - name: Forcings sources option test NWM_RETRO_V2
       if: always()
       run: |
@@ -129,33 +127,33 @@ jobs:
       if: always()
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test
-        ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -S ciroh-community-ngen-datastream -o git_actions_test
+        ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -S ciroh-community-ngen-datastream -o git_actions_test
         aws s3api wait object-exists --bucket ciroh-community-ngen-datastream --key git_actions_test/ngen-run.tar.gz   
         aws s3 rm s3://ciroh-community-ngen-datastream/git_actions_test --recursive
 
     - name: DAILY short_range 00 today test
       if: always()
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test
-        ./scripts/datastream -s DAILY -C NWM_SHORT_RANGE_00 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
+        ./scripts/datastream -s DAILY -C NWM_SHORT_RANGE_00 -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
 
     - name: DAILY short_range 23 today test
       if: always()
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test
-        ./scripts/datastream -s DAILY -C NWM_SHORT_RANGE_23 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
+        ./scripts/datastream -s DAILY -C NWM_SHORT_RANGE_23 -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
 
     - name: DAILY short_range 00 pick day test
       if: always()
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test
-        ./scripts/datastream -s DAILY -e $(date -d '-15 day' '+%Y%m%d0000') -C NWM_SHORT_RANGE_00 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
+        ./scripts/datastream -s DAILY -e $(date -d '-15 day' '+%Y%m%d0000') -C NWM_SHORT_RANGE_00 -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
 
     - name: DAILY medium_range today test 00 0
       if: always()
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test
-        ./scripts/datastream -s DAILY -C NWM_MEDIUM_RANGE_00_0 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
+        ./scripts/datastream -s DAILY -C NWM_MEDIUM_RANGE_00_0 -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
 
     - name: DAILY medium_range today test 00 3
       if: always()
@@ -168,5 +166,4 @@ jobs:
       if: always()
       run: |        
         sudo rm -rf $(pwd)/data/datastream_test
-        ./scripts/datastream -s DAILY -C NWM_ANALYSIS_ASSIM_EXTEND_16 -e $(date -d '-2 day' '+%Y%m%d0000') -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json      
-
+        ./scripts/datastream -s DAILY -C NWM_ANALYSIS_ASSIM_EXTEND_16 -e $(date -d '-2 day' '+%Y%m%d0000') -d $(pwd)/data/datastream_test -g $(pwd)/nextgen_VPU_09.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json      

configs/ngen/realization_rust_lstm.json

@@ -0,0 +1,44 @@
+{
+    "global": {
+        "formulations": [
+            {
+                "name": "bmi_multi",
+                "params": {
+                    "name": "bmi_multi",
+                    "model_type_name": "lstm",
+                    "forcing_file": "",
+                    "init_config": "",
+                    "allow_exceed_end_time": true,
+                    "main_output_variable": "land_surface_water__runoff_depth",
+                    "modules": [
+                        {
+                        "name": "bmi_c",
+                        "params": {
+                            "name": "bmi_c",
+                            "model_type_name": "bmi_rust",
+                            "init_config": "./config/cat_config/LSTM/{{id}}.yml",
+                            "allow_exceed_end_time": true,
+                            "main_output_variable": "land_surface_water__runoff_depth",
+                            "uses_forcing_file": false,
+                            "registration_function": "register_bmi_lstm",
+                            "library_file": "/dmod/shared_libs/librust_lstm_1025.so"
+                        }
+                    }
+                    ]
+                }
+            }
+        ],
+        "forcing": {
+            "path": "./forcings/forcings.nc",
+            "provider": "NetCDF",
+            "enable_cache": false
+        }
+    },
+    "time": {
+        "start_time": "2010-01-01 00:00:00",
+        "end_time": "2010-01-02 00:00:00",
+        "output_interval": 3600
+    },
+    "remotes_enabled": false,
+    "output_root": "./outputs/ngen"
+}

setup.cfg

@@ -35,6 +35,7 @@ install_requires =
     xarray
     scipy
     requests
+    ruamel.yaml
     ngen_config[all] @ git+https://github.com/ciroh-ua/ngen-cal@master#egg=ngen_config&subdirectory=python/ngen_conf
     ngen_init_config[all] @ git+https://github.com/ciroh-ua/ngen-cal@master#egg=ngen_init_config&subdirectory=python/ngen_init_config
     ngen_config_gen[all] @ git+https://github.com/ciroh-ua/ngen-cal@master#egg=ngen_config_gen&subdirectory=python/ngen_config_gen