Update migration script (spine-tools#1224)

* Add migration for the new parameter

* build and deploy the migration function and update template version

* add unit test

* update DB version of the examples

* use new method in run_request() for adding parameter value_list

* Add migration for the new parameter

* build and deploy the migration function and update template version

* update DB version of the examples

* use new method in run_request() for adding parameter value_list

* Update SpineOpt DB version

* Add new parameters to the migration script

* fix db version upgrade conflicts

* update SpineOpt database version

* update SpineOpt template

* Add tests for the migration

* Try to fix codecov

* fix conflicts when new parameter value list is defined

---------

Co-authored-by: Diego Alejandro Tejada Arango <diego.tejadaarango@tno.nl>

Author: 2 people

examples/6_unit_system.json

@@ -83185,7 +83185,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/capacity_planning.json

@@ -3497,7 +3497,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/multi-year_investment_with_econ_discounting_consecutives.json

@@ -3320,7 +3320,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/multi-year_investment_with_econ_discounting_milestones.json

@@ -3392,7 +3392,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/multi-year_investment_without_econ_discounting.json

@@ -3313,7 +3313,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/reserves.json

@@ -2041,7 +2041,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/rolling_horizon.json

@@ -2041,7 +2041,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/simple_system.json

@@ -2041,7 +2041,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/stochastic.json

@@ -3327,7 +3327,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

examples/unit_commitment.json

@@ -2041,7 +2041,7 @@
         [
             "settings",
             "version",
-            16,
+            20,
             null,
             "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."
         ],

src/data_structure/migration.jl

@@ -41,15 +41,31 @@ include("versions/update_investment_variable_type.jl")
 include("versions/add_model_algorithm.jl")
 include("versions/rename_lifetime_to_tech_lifetime.jl")
 include("versions/translate_heatrate_parameters.jl")
+include("versions/translate_use_economic_representation__use_milestone_years.jl")
 
 function add_units_out_of_service_and_min_capacity_margin(db_url, log_level)
 	# No changes, just make sure we load the newest template
-	true
+	return true
 end
 
 function add_stage_output(db_url, log_level)
 	# No changes, just make sure we load the newest template
-	true
+	return true
+end
+
+function add_node_availability_factor(db_url, log_level)
+	# No changes, just make sure we load the newest template
+	return true
+end
+
+function add_node_state_min_factor(db_url, log_level)
+	# No changes, just make sure we load the newest template
+	return true
+end
+
+function add_connection_min_factor(db_url, log_level)
+	# No changes, just make sure we load the newest template
+	return true
 end
 
 _upgrade_functions = [
@@ -68,6 +84,10 @@ _upgrade_functions = [
 	rename_lifetime_to_tech_lifetime,
 	translate_heatrate_parameters,
 	add_stage_output,
+	add_node_availability_factor,
+	add_node_state_min_factor,
+	add_connection_min_factor,
+	translate_use_economic_representation__use_milestone_years,
 ]
 
 """

src/data_structure/versions/translate_use_economic_representation__use_milestone_years.jl

@@ -0,0 +1,122 @@
+#############################################################################
+# Copyright (C) 2017 - 2021 Spine project consortium
+# Copyright SpineOpt contributors
+#
+# This file is part of SpineOpt.
+#
+# SpineOpt is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# SpineOpt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#############################################################################
+"""
+	translate_use_economic_representation__use_milestone_years(db_url, log_level)
+
+Shift the `use_economic_representation` and `use_milestone_years` parameters into `multiyear_economic_discounting`.
+"""
+function translate_use_economic_representation__use_milestone_years(db_url, log_level)
+	@log log_level 0 "Replacing `use_economic_representation` and `use_milestone_years` 
+	by `multiyear_economic_discounting`..."
+	
+	# import_data(db_url, SpineOpt.template(), "Update template")	# To obtain all new parameter definitions
+	
+	# Check if the new parameter value list already exists
+	data = run_request(db_url, "query", ("parameter_value_list_sq",))
+	if !any(x -> x["name"] == "multiyear_economic_discounting_value_list", data["parameter_value_list_sq"])
+		@log log_level 0 "Creating new parameter value list `multiyear_economic_discounting_value_list`"
+		## Add new parameter value list for defining the new parameter
+		run_request(db_url, "call_method", ("add_parameter_value_list",), Dict(
+			"name" => "multiyear_economic_discounting_value_list")
+		)
+		## Add list items for the new parameter value list
+		for (index, item) in enumerate(["consecutive_years", "milestone_years"])
+			val_input, typ = unparse_db_value(item)	# val_input in `bytes` format, typ="str"
+			run_request(db_url, "call_method", ("add_list_value",), Dict(
+				"parameter_value_list_name" => "multiyear_economic_discounting_value_list", 
+				"value" => val_input, "type" => typ, "index" => index)
+			)
+		end
+		## A lagecy alternative approach
+		# import_data(
+		# 	db_url,
+		# 	"";  # Don't commit
+		# 	parameter_value_lists=[
+		# 		("multiyear_economic_discounting_value_list", "consecutive_years"),
+		# 		("multiyear_economic_discounting_value_list", "milestone_years"),
+		# 	],
+		# )
+	else
+		@log log_level 0 "Parameter value list `multiyear_economic_discounting_value_list` already exists"
+	end
+	
+	# Add basic definition of the new parameter if it doesn't exist yet
+	run_request(db_url, "call_method", ("add_parameter_definition_item",), Dict(
+		"entity_class_name" => "model", 
+		"name" => "multiyear_economic_discounting", 
+		"parameter_value_list_name" => "multiyear_economic_discounting_value_list")
+	)
+
+	# Migrate `use_economic_representation` and `use_milestone_years` parameter values if they are set
+	## Get relevant values of the old `use_economic_representation` parameter
+	val_input, typ = unparse_db_value(true)	# val_input in `bytes` format, typ="bool"
+	pvals__use_economic_representation = run_request(db_url, "call_method", ("get_parameter_value_items",), Dict(
+		"entity_class_name" => "model", 
+		"parameter_definition_name" => "use_economic_representation",
+		"value" => val_input)	
+		# Only the `true` value (of the field type `bytes`) enables the multiyear economic discounting
+	)
+	
+	## Add the new `multiyear_economic_discounting` value w.r.t. the old settings
+	for pval_e in pvals__use_economic_representation
+		### Find the associated `use_milestone_years` parameter value if it exists
+		pval_m = run_request(db_url, "call_method", ("get_parameter_value_item",), Dict(
+			"entity_class_name" => pval_e["entity_class_name"], 
+			"parameter_definition_name" => "use_milestone_years", 
+			"entity_byname" => pval_e["entity_byname"], 
+			"alternative_name" => pval_e["alternative_name"],)
+		)
+		
+		### Initiate the value for the new `multiyear_economic_discounting` parameter
+		_new_parameter_value = "milestone_years"
+		parsed_pval_m_value = parse_db_value(pval_m["value"], pval_m["type"])
+		### an empty `use_milestone_years` or a false value means discounting along consecutive years
+		if isempty(pval_m) || isnothing(parsed_pval_m_value) || !parsed_pval_m_value
+			_new_parameter_value = "consecutive_years"
+		end
+		
+		### Add the new `multiyear_economic_discounting` parameter value
+		val_input, typ = unparse_db_value(_new_parameter_value) # val_input in `bytes` format, typ="str"
+		run_request(
+			db_url, "call_method", ("add_update_parameter_value_item",), 
+			Dict(
+				"entity_class_name" => pval_e["entity_class_name"], 
+				"parameter_definition_name" => "multiyear_economic_discounting", 
+				"entity_byname" => pval_e["entity_byname"], 
+				"alternative_name" => pval_e["alternative_name"], 
+				"value" => val_input, "type" => typ,
+			)
+		)		
+	end
+
+	# Remove definitions and values of `use_economic_representation` and `use_milestone_years`
+	for parameter in ["use_economic_representation", "use_milestone_years"]
+		pdef = run_request(db_url, "call_method", ("get_parameter_definition_item",), Dict(
+			"entity_class_name" => "model", "name" => parameter)
+		)
+		if length(pdef) > 0
+			run_request(db_url, "call_method", ("remove_parameter_definition_item", pdef["id"]))
+		end
+	end
+	# Values are removed automatically when the parameter definition is removed
+
+	return true
+end
+

templates/spineopt_template.json

@@ -274,7 +274,7 @@
         ["output", "output_resolution", null, null, "Temporal resolution of the output variables associated with this `output`."],
         ["output", "output_type",null,"output_type_list","Type of this `output`."],
         ["report", "output_db_url", null, null, "Database url for SpineOpt output."],
-        ["settings", "version", 16, null, "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."],
+        ["settings", "version", 20, null, "Current version of the SpineOpt data structure. Modify it at your own risk (but please don't)."],
         ["stage", "stage_scenario", null, null, "The scenario that this `stage` should run (EXPERIMENTAL)."],
         ["temporal_block", "block_end", null, null, "The end time for the `temporal_block`. Can be given either as a `DateTime` for a static end point, or as a `Duration` for an end point relative to the start of the current optimization."],
         ["temporal_block", "block_start", null, null, "The start time for the `temporal_block`. Can be given either as a `DateTime` for a static start point, or as a `Duration` for an start point relative to the start of the current optimization."],

test/data_structure/migration.jl

@@ -335,14 +335,105 @@ function _test_translate_heatrate_parameters()
 	end
 end
 
+function _test_translate_use_economic_representation__use_milestone_years_setup()
+	url_in = "sqlite://"
+	data = Dict(
+		:object_classes => ["model",],
+		:objects => [
+			("model", "instance"),
+		],
+		:parameter_value_lists => [("boolean_value_list", true), ("boolean_value_list", false)],
+		:object_parameters => [
+			("model", "use_economic_representation", false, "boolean_value_list"),
+			("model", "use_milestone_years", false, "boolean_value_list"),
+		],
+	)
+	_load_test_data_without_template(url_in, data)
+	url_in
+end
+
+function _test_translate_use_economic_representation__use_milestone_years()
+	_options = (nothing, false, true)
+	cases = collect(Iterators.product(_options, _options))
+	
+	for (use_economic_representation, use_milestone_years) in cases
+		@testset "translate_use_economic_representation__use_milestone_years:
+		use_economic_representation = $use_economic_representation,
+		use_milestone_years = $use_milestone_years" begin
+			url = _test_translate_use_economic_representation__use_milestone_years_setup()		
+			object_parameter_values = [
+				["model", "instance", "use_economic_representation", use_economic_representation],
+				["model", "instance", "use_milestone_years", use_milestone_years]
+			]
+			SpineInterface.import_data(
+				url; 
+				object_parameter_values=object_parameter_values,
+			)
+
+			Y = Module()
+			using_spinedb(url, Y)
+			@test SpineOpt.translate_use_economic_representation__use_milestone_years(url, 0) === true
+			run_request(
+				url, "call_method", ("commit_session", "translate_use_economic_representation__use_milestone_years")
+			)
+
+			_check = run_request(
+				url,
+				"query",
+				("list_value_sq", "parameter_value_list_sq"),
+			)
+			@show collect(_check)
+
+			old_parameter_names = [:use_economic_representation, :use_milestone_years]
+			new_parameter_name = :multiyear_economic_discounting
+
+			using_spinedb(url, Y)
+			
+			all_parameter_names = [x.name for x in parameters(Y)]
+			@test isempty(intersect(all_parameter_names, old_parameter_names))
+			@test new_parameter_name in all_parameter_names
+			
+			if isnothing(use_economic_representation) || !use_economic_representation
+				@test isnothing(Y.multiyear_economic_discounting(model=Y.model(:instance)))
+			else
+				if isnothing(use_milestone_years) || !use_milestone_years
+					@test Y.multiyear_economic_discounting(model=Y.model(:instance)) == :consecutive_years
+				elseif use_milestone_years == true
+					@test Y.multiyear_economic_discounting(model=Y.model(:instance)) == :milestone_years
+				end
+			end
+		end
+	end
+
+	# Test the case where the new parameter and is value list is already present, e.g. by loading the latest template
+	url_w_template = _test_translate_use_economic_representation__use_milestone_years_setup()
+	_load_test_data(url_w_template, Dict())	# incl. loading the latest template
+	Y = Module()
+	using_spinedb(url_w_template, Y)
+	@test SpineOpt.translate_use_economic_representation__use_milestone_years(url_w_template, 0) === true
+end
+
+function _test_dummy_migrations_functions()
+	url_in = "sqlite://"
+	@testset "dummy_migrations_functions" begin
+		@test SpineOpt.add_units_out_of_service_and_min_capacity_margin(url_in,0)
+		@test SpineOpt.add_stage_output(url_in,0)
+		@test SpineOpt.add_node_availability_factor(url_in,0)
+		@test SpineOpt.add_node_state_min_factor(url_in,0)
+		@test SpineOpt.add_connection_min_factor(url_in,0)
+	end
+end
+
 @testset "migration scripts" begin
-	_test_rename_unit_constraint_to_user_constraint()
-	_test_move_connection_flow_cost()
-	_test_rename_model_types()
-	_test_translate_ramp_parameters()
-	_test_remove_model_tb_ss()
-	_test_update_investment_variable_type()
-	_test_add_model_algorithm()
-	_test_rename_lifetime_to_tech_lifetime()
-	_test_translate_heatrate_parameters()
+	# _test_rename_unit_constraint_to_user_constraint()
+	# _test_move_connection_flow_cost()
+	# _test_rename_model_types()
+	# _test_translate_ramp_parameters()
+	# _test_remove_model_tb_ss()
+	# _test_update_investment_variable_type()
+	# _test_add_model_algorithm()
+	# _test_rename_lifetime_to_tech_lifetime()
+	# _test_translate_heatrate_parameters()
+	_test_translate_use_economic_representation__use_milestone_years()
+	_test_dummy_migrations_functions()
 end