Some changes to input format and cleaned up popt structure (resubmitted) (#103)

* update to TrustRegion

* some design changes to TrustRegion

* decoupled GenOpt from Ensemble

* decoupled GenOpt from Ensemble

* Cleaned up code duplication and renamed some stuff

* comments

* improved and cleaned up LineSearch

* added Newton-CG to LineSearch

* improved logging for LineSearch

* dummy message for github check

* empty commit

* udpate BFGS to skip update if negetive curvature

* made input more elegant and cleaned up popt structure

* changed @cache to @lru_cache

---------

Co-authored-by: Mathias Methlie Nilsen <mani@cno-0006.ad.norceresearch.no>
Co-authored-by: Mathias Methlie Nilsen <mani@bgo-1714.ad.norceresearch.no>

Author: 3 people

ensemble/ensemble.py

@@ -3,6 +3,7 @@
 from copy import deepcopy
 import csv
 import datetime as dt
+import pandas as pd
 
 
 class Organize_input():
@@ -109,6 +110,13 @@ def _org_report(self):
                         pred_prim.extend(csv_data)
                 self.keys_fwd['reportpoint'] = pred_prim
 
+            elif isinstance(self.keys_fwd['reportpoint'], dict):
+                self.keys_fwd['reportpoint'] = pd.date_range(**self.keys_fwd['reportpoint']).to_pydatetime().tolist()
+
+            else:
+                pass
+
+
         # Check if assimindex is given as a csv file. If so, we read and make a potential 2D list (if sequential).
         if 'assimindex' in self.keys_pr:
             if isinstance(self.keys_pr['assimindex'], str) and self.keys_pr['assimindex'].endswith('.csv'):

input_output/organize.py

@@ -51,6 +51,12 @@ def ndarray_constructor(loader, node):
         y = yaml.load(fid, Loader=FullLoader)
 
     # Check for dataassim and fwdsim
+    if 'ensemble' in y.keys():
+        keys_en = y['ensemble']
+        check_mand_keywords_en(keys_en)
+    else:
+        keys_en = None
+
     if 'optim' in y.keys():
         keys_pr = y['optim']
         check_mand_keywords_opt(keys_pr)
@@ -59,16 +65,17 @@ def ndarray_constructor(loader, node):
         check_mand_keywords_da(keys_pr)
     else:
         raise KeyError
+    
     if 'fwdsim' in y.keys():
         keys_fwd = y['fwdsim']
     else:
         raise KeyError
 
     # Organize keywords
-    org = Organize_input(keys_pr, keys_fwd)
+    org = Organize_input(keys_pr, keys_fwd, keys_en)
     org.organize()
 
-    return org.get_keys_pr(), org.get_keys_fwd()
+    return org.get_keys_pr(), org.get_keys_fwd(), org.get_keys_en()
 
 
 def convert_txt_to_toml(init_file):

input_output/read_config.py

@@ -301,32 +301,20 @@ def _ext_max_iter(self):
         - ST 7/6-16
         """
         if 'iteration' in self.ensemble.keys_da:
-            # Make sure ITERATION is a list
-            if not isinstance(self.ensemble.keys_da['iteration'][0], list):
-                iter_opts = [self.ensemble.keys_da['iteration']]
-            else:
-                iter_opts = self.ensemble.keys_da['iteration']
-
+            iter_opts = dict(self.ensemble.keys_da['iteration'])
             # Check if 'max_iter' has been given; if not, give error (mandatory in ITERATION)
-            assert 'max_iter' in list(
-                zip(*iter_opts))[0], 'MAX_ITER has not been given in ITERATION!'
-
-            # Extract max. iter
-            max_iter = [item[1] for item in iter_opts if item[0] == 'max_iter'][0]
+            try:
+                max_iter = iter_opts['max_iter']
+            except KeyError:
+                raise AssertionError('MAX_ITER has not been given in ITERATION')
 
         elif 'mda' in self.ensemble.keys_da:
-            # Make sure ITERATION is a list
-            if not isinstance(self.ensemble.keys_da['mda'][0], list):
-                iter_opts = [self.ensemble.keys_da['mda']]
-            else:
-                iter_opts = self.ensemble.keys_da['mda']
-
-            # Check if 'max_iter' has been given; if not, give error (mandatory in ITERATION)
-            assert 'tot_assim_steps' in list(
-                zip(*iter_opts))[0], 'TOT_ASSIM_STEPS has not been given in MDA!'
-
-            # Extract max. iter
-            max_iter = [item[1] for item in iter_opts if item[0] == 'tot_assim_steps'][0]
+            iter_opts = dict(self.ensemble.keys_da['mda'])
+            # Check if 'tot_assim_steps' has been given; if not, raise error (mandatory in MDA)
+            try:
+                max_iter = iter_opts['tot_assim_steps']
+            except KeyError:
+                raise AssertionError('TOT_ASSIM_STEPS has not been given in MDA!')
 
         else:
             max_iter = 1

pipt/loop/assimilation.py

@@ -250,36 +250,22 @@ def _ext_iter_param(self):
         file. These parameters include convergence tolerances and parameters for the damping parameter. Default
         values for these parameters have been given here, if they are not provided in ITERATION.
         """
-
-        # Predefine all the default values
-        self.data_misfit_tol = 0.01
-        self.step_tol = 0.01
-        self.lam = 100
-        self.lam_max = 1e10
-        self.lam_min = 0.01
-        self.gamma = 5
-        self.trunc_energy = 0.95
+        try:
+            options = dict(self.keys_da['iteration'])
+        except:
+            options = dict([self.keys_da['iteration']])
+
+        # unpack options
+        self.data_misfit_tol = options.get('data_misfit_tol', 0.01)
+        self.trunc_energy = options.get('energy', 0.95)
+        self.step_tol  = options.get('step_tol', 0.01)
+        self.lam       = options.get('lambda', 100)
+        self.lam_max   = options.get('lambda_max', 1e10)
+        self.lam_min   = options.get('lambda_min', 0.01)
+        self.gamma     = options.get('lambda_factor', 5)
         self.iteration = 0
 
-        # Loop over options in ITERATION and extract the parameters we want
-        for i, opt in enumerate(list(zip(*self.keys_da['iteration']))[0]):
-            if opt == 'data_misfit_tol':
-                self.data_misfit_tol = self.keys_da['iteration'][i][1]
-            if opt == 'step_tol':
-                self.step_tol = self.keys_da['iteration'][i][1]
-            if opt == 'lambda':
-                self.lam = self.keys_da['iteration'][i][1]
-            if opt == 'lambda_max':
-                self.lam_max = self.keys_da['iteration'][i][1]
-            if opt == 'lambda_min':
-                self.lam_min = self.keys_da['iteration'][i][1]
-            if opt == 'lambda_factor':
-                self.gamma = self.keys_da['iteration'][i][1]
-
-        if 'energy' in self.keys_da:
-            # initial energy (Remember to extract this)
-            self.trunc_energy = self.keys_da['energy']
-            if self.trunc_energy > 1:  # ensure that it is given as percentage
+        if self.trunc_energy > 1:  # ensure that it is given as percentage
                 self.trunc_energy /= 100.
 
 
@@ -593,33 +579,19 @@ def _ext_iter_param(self):
         file. These parameters include convergence tolerances and parameters for the damping parameter. Default
         values for these parameters have been given here, if they are not provided in ITERATION.
         """
-
-        # Predefine all the default values
-        self.data_misfit_tol = 0.01
-        self.step_tol = 0.01
-        self.gamma = 0.2
-        self.gamma_max = 0.5
-        self.gamma_factor = 2.5
-        self.trunc_energy = 0.95
-        self.iteration = 0
-
-        # Loop over options in ITERATION and extract the parameters we want
-        for i, opt in enumerate(list(zip(*self.keys_da['iteration']))[0]):
-            if opt == 'data_misfit_tol':
-                self.data_misfit_tol = self.keys_da['iteration'][i][1]
-            if opt == 'step_tol':
-                self.step_tol = self.keys_da['iteration'][i][1]
-            if opt == 'gamma':
-                self.gamma = self.keys_da['iteration'][i][1]
-            if opt == 'gamma_max':
-                self.gamma_max = self.keys_da['iteration'][i][1]
-            if opt == 'gamma_factor':
-                self.gamma_factor = self.keys_da['iteration'][i][1]
-
-        if 'energy' in self.keys_da:
-            # initial energy (Remember to extract this)
-            self.trunc_energy = self.keys_da['energy']
-            if self.trunc_energy > 1:  # ensure that it is given as percentage
+        try:
+            options = dict(self.keys_da['iteration'])
+        except:
+            options = dict([self.keys_da['iteration']])
+        
+        self.data_misfit_tol = options.get('data_misfit_tol', 0.01)
+        self.trunc_energy = options.get('energy', 0.95)
+        self.step_tol     = options.get('step_tol', 0.01)
+        self.gamma        = options.get('gamma', 0.2)
+        self.gamma_max    = options.get('gamma_max', 0.5)
+        self.gamma_factor = options.get('gamma_factor', 2.5)
+
+        if self.trunc_energy > 1:  # ensure that it is given as percentage
                 self.trunc_energy /= 100.
 
 

pipt/update_schemes/enrml.py

@@ -31,7 +31,7 @@ def __init__(self, keys_da, keys_en, sim):
 
         Parameters
         ----------
-        keys_da['mda'] : list
+        keys_da['mda'] : dict
             - tot_assim_steps: total number of iterations in MDA, e.g., 3
             - inflation_param: covariance inflation factors, e.g., [2, 4, 4]
 
@@ -222,17 +222,16 @@ def _ext_inflation_param(self):
         alpha: list
             Data covariance inflation factor
         """
-        # Make sure MDA is a list
-        if not isinstance(self.keys_da['mda'][0], list):
-            mda_opts = [self.keys_da['mda']]
-        else:
-            mda_opts = self.keys_da['mda']
+        try:
+            mda_opts = dict(self.keys_da['mda'])
+        except:
+            mda_opts = dict([self.keys_da['mda']])
 
         # Check if INFLATION_PARAM has been provided, and if so, extract the value(s). If not, we set alpha to the
         # default value equal to the tot. no. assim. steps
-        if 'inflation_param' in list(zip(*mda_opts))[0]:
+        if 'inflation_param' in mda_opts:
             # Extract value
-            alpha_tmp = [item[1] for item in mda_opts if item[0] == 'inflation_param'][0]
+            alpha_tmp = mda_opts['inflation_param']
 
             # If one value is given, we copy it to all assim. steps. If multiple values are given, we check the
             # number of parameters corresponds to tot. no. assim. steps
@@ -279,22 +278,17 @@ def _ext_assim_steps(self):
         - ST 7/6-16
         - ST 1/3-17: Changed to output list of assim. steps instead of just tot. assim. steps
         """
-        # Make sure MDA is a list
-        if not isinstance(self.keys_da['mda'][0], list):
-            mda_opts = [self.keys_da['mda']]
-        else:
-            mda_opts = self.keys_da['mda']
+        try:
+            mda_opts = dict(self.keys_da['mda'])
+        except:
+            mda_opts = dict([self.keys_da['mda']])
 
+    
         # Check if 'max_iter' has been given; if not, give error (mandatory in ITERATION)
-        assert 'tot_assim_steps' in list(
-            zip(*mda_opts))[0], 'TOT_ASSIM_STEPS has not been given in MDA!'
-
-        # Extract max. iter
-        tot_no_assim = int([item[1]
-                           for item in mda_opts if item[0] == 'tot_assim_steps'][0])
-
-        # Make a list of assim. steps
-        assim_steps = list(range(tot_no_assim))
+        try:
+            assim_steps = list(range(int(mda_opts['tot_assim_steps'])))
+        except KeyError:
+            raise AssertionError('TOT_ASSIM_STEPS has not been given in MDA!')
 
         # If it is a restart run, we remove simulations already done
         if self.restart is True:

pipt/update_schemes/esmda.py

@@ -44,9 +44,7 @@ def ecalc_npv(pred_data, **kwargs):
     report = kwargs.get('true_order', [])
 
     # Economic values
-    npv_const = {}
-    for name, value in keys_opt['npv_const']:
-        npv_const[name] = value
+    npv_const = dict(keys_opt['npv_const'])
 
     # Collect production data
     Qop = []

popt/cost_functions/ecalc_npv.py

@@ -46,9 +46,7 @@ def ecalc_pareto_npv(pred_data, kwargs):
     report = kwargs.get('true_order', [])
 
     # Economic values
-    npv_const = {}
-    for name, value in keys_opt['npv_const']:
-        npv_const[name] = value
+    npv_const = dict(keys_opt['npv_const'])
 
     # Collect production data
     Qop = []

popt/cost_functions/ecalc_pareto_npv.py

@@ -38,9 +38,7 @@ def npv(pred_data, **kwargs):
     report = kwargs.get('true_order', [])
 
     # Economic values
-    npv_const = {}
-    for name, value in keys_opt['npv_const']:
-        npv_const[name] = value
+    npv_const = dict(keys_opt['npv_const'])
 
     values = []
     for i in np.arange(1, len(pred_data)):

popt/cost_functions/npv.py

@@ -32,9 +32,7 @@ def ren_npv(pred_data, kwargs):
     report = kwargs.get('true_order', [])
 
     # Economic values
-    npv_const = {}
-    for name, value in keys_opt['npv_const']:
-        npv_const[name] = value
+    npv_const = dict(keys_opt['npv_const'])
 
     # Loop over timesteps
     values = []