Fix temporal resolution connection_ratio_out_in (#1147)

* Change t indice function

* Fix t_high_resolution_path

* Add missing paths from ng_out

* Modify duration for the node in

* Modify constraint and update test

* Remove t_highest_resolution_path

* change db to let the test run

* add missing examples for integration tests

* revert db change to test

* Try another way

* Specify indices for t, path

* Add highest resolution and tests

* Change test name

* Add highest as default and keep the original

* Roll back the json changes to solve conflicts

* Roll back again

* Add new parameter to the jsons

* Retrieve a wrongly commited file

Author: gnawin

examples/6_unit_system.json

@@ -82616,6 +82616,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/capacity_planning.json

@@ -2855,6 +2855,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/multi-year_investment_with_econ_params_with_milestones.json

@@ -2844,6 +2844,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [   
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/multi-year_investment_with_econ_params_without_milestones.json

@@ -2772,6 +2772,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/multi-year_investment_without_econ_params.json

@@ -2935,6 +2935,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/reserves.json

@@ -1479,6 +1479,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/rolling_horizon.json

@@ -1479,6 +1479,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/simple_system.json

@@ -1479,6 +1479,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/stochastic.json

@@ -2765,6 +2765,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

examples/unit_commitment.json

@@ -1479,6 +1479,13 @@
             "boolean_value_list",
             "Whether to use tight and compact constraint formulations."
         ],
+        [
+            "model", 
+            "use_highest_resolution_constraint_ratio_out_in_connection_flow", 
+            true, 
+            "boolean_value_list", 
+            "Whether to use highest resolution for constraint ratio out in connection flow."
+        ],
         [
             "model",
             "window_duration",

src/constraints/constraint_ratio_out_in_connection_flow.jl

@@ -44,11 +44,50 @@ exist for the other two cases.
 See also [fix\_ratio\_out\_in\_connection\_flow](@ref).
 """
 function add_constraint_ratio_out_in_connection_flow!(m::Model, ratio_out_in, sense)
-    _add_constraint!(
-        m,
-        ratio_out_in.name,
-        m -> constraint_ratio_out_in_connection_flow_indices(m, ratio_out_in),
-        (m, ind...) -> _build_constraint_ratio_out_in_connection_flow(m, ind..., ratio_out_in, sense),
+    if use_highest_resolution_constraint_ratio_out_in_connection_flow(model=m.ext[:spineopt].instance)
+        _add_constraint!(
+            m,
+            ratio_out_in.name,
+            m -> constraint_ratio_out_in_connection_flow_indices_highest_resolution(m, ratio_out_in),
+            (m, ind...) -> _build_constraint_ratio_out_in_connection_flow_highest_resolution(m, ind..., ratio_out_in, sense),
+        )
+    else
+        _add_constraint!(
+            m,
+            ratio_out_in.name,
+            m -> constraint_ratio_out_in_connection_flow_indices(m, ratio_out_in),
+            (m, ind...) -> _build_constraint_ratio_out_in_connection_flow(m, ind..., ratio_out_in, sense),
+        )
+    end
+end
+
+function _build_constraint_ratio_out_in_connection_flow_highest_resolution(m::Model, conn, ng_out, ng_in, s_path, t, ratio_out_in, sense)
+    # NOTE: the `<sense>_ratio_<directions>_connection_flow` parameter uses the stochastic dimensions
+    # of the second <direction>!
+    @fetch connection_flow = m.ext[:spineopt].variables
+    build_sense_constraint(
+        + sum(
+            + connection_flow[conn, n_out, d, s, t_long]
+            for (conn, n_out, d, s, t_long) in connection_flow_indices(
+                m;
+                connection=conn,
+                node=ng_out,
+                direction=direction(:to_node),
+                stochastic_scenario=s_path,
+                t=t_in_t(m; t_short=t),
+            );
+            init=0,
+        ),
+        sense,
+        + sum(
+            + connection_flow[conn, n_in, d, s_past, t_past]
+            * ratio_out_in(m; connection=conn, node1=ng_out, node2=ng_in, stochastic_scenario=s_past, t=t_past)
+            * weight
+            for (conn, n_in, d, s_past, t_past, weight) in _past_connection_input_flow_indices(
+                m, conn, ng_out, ng_in, s_path, t
+            );
+            init=0,
+        ),
     )
 end
 
@@ -109,6 +148,44 @@ function add_constraint_min_ratio_out_in_connection_flow!(m::Model)
     add_constraint_ratio_out_in_connection_flow!(m, min_ratio_out_in_connection_flow, >=)
 end
 
+function constraint_ratio_out_in_connection_flow_indices_highest_resolution(m::Model, ratio_out_in)
+    (
+        (connection=conn, node1=ng_out, node2=ng_in, stochastic_path=path, t=t)
+        for (conn, ng_out, ng_in) in indices(ratio_out_in)
+        if !_has_simple_fix_ratio_out_in_connection_flow(conn, ng_out, ng_in)
+        for t in t_highest_resolution(
+            m,
+            Iterators.flatten(
+            ((t for (ng, t) in node_time_indices(m; node=ng_in)), (t for (ng, t) in node_time_indices(m; node=ng_out)))
+            )
+        )
+        for path_out in active_stochastic_paths(
+            m, 
+            connection_flow_indices(m; connection=conn, node=ng_out, direction=direction(:to_node), t=t_in_t(m; t_short=t))
+        )
+        for path in active_stochastic_paths(
+            m, 
+            Iterators.flatten(
+                (
+                    ((stochastic_scenario=s,) for s in path_out),
+                    (
+                        ind
+                        for s in path_out
+                        for ind in connection_flow_indices(
+                            m;
+                            connection=conn,
+                            node=ng_in,
+                            direction=direction(:from_node),
+                            t=to_time_slice(m; t=_t_look_behind(conn, ng_out, ng_in, (s,), t)),
+                            temporal_block=anything,
+                        )
+                    ),
+                )
+            )
+        )
+    )
+end
+
 function constraint_ratio_out_in_connection_flow_indices(m::Model, ratio_out_in)
     (
         (connection=conn, node1=ng_out, node2=ng_in, stochastic_path=path, t=t)

templates/spineopt_template.json

@@ -206,6 +206,7 @@
         ["model", "shared_values", null, null, "A map from symbol to shared value."],
         ["model", "use_connection_intact_flow", true, "boolean_value_list", "Whether to use `connection_intact_flow` variables, to capture the impact of connection investments on network characteristics via line outage distribution factors (LODF)."],
         ["model", "use_tight_compact_formulations", false, "boolean_value_list", "Whether to use tight and compact constraint formulations."],
+        ["model", "use_highest_resolution_constraint_ratio_out_in_connection_flow", true, "boolean_value_list", "Whether to use highest resolution for constraint ratio out in connection flow."],
         ["model", "window_duration", null, null, "The duration of the window in case it differs from roll_forward"],
         ["model", "window_weight", 1, null, "The weight of the window in the rolling subproblem"],
         ["model", "use_milestone_years", false, "boolean_value_list", "If set to true, the investment models uses milestone years. In other words, operational temporal blocks for one (milestone) year will be scaled up by the discounted duration to represent the entire investment period."],

test/constraints/constraint_connection.jl

@@ -863,12 +863,76 @@ function test_contraints_ptdf_lodf_duration()
 end
 
 function test_constraint_ratio_out_in_connection_flow()
+    @testset "constraint_ratio_out_in_connection_flow_highest_resolution" begin
+        flow_ratio = 0.8
+        model_end = Dict("type" => "date_time", "data" => "2000-01-01T04:00:00")
+        class = "connection__node__node"
+        relationship = ["connection_ab", "node_b", "node_a"]
+        object_parameter_values = [
+            ["model", "instance", "model_end", model_end],
+            ["model", "instance", "use_highest_resolution_constraint_ratio_out_in_connection_flow", true] # this is the default value
+        ]
+        relationships = [[class, relationship]]
+        senses_by_prefix = Dict("min" => >=, "fix" => ==, "max" => <=)
+        h_delay = 2
+        connection_flow_delay = Dict("type" => "duration", "data" => string(h_delay, "h"))
+        @testset for p in ("min", "fix", "max")
+            url_in = _test_constraint_connection_setup()
+            sense = senses_by_prefix[p]
+            ratio = string(p, "_ratio_out_in_connection_flow")
+            relationship_parameter_values = [
+                [class, relationship, "connection_flow_delay", connection_flow_delay],
+                [class, relationship, ratio, flow_ratio],
+            ]
+            SpineInterface.import_data(
+                url_in;
+                relationships=relationships,
+                object_parameter_values=object_parameter_values,
+                relationship_parameter_values=relationship_parameter_values,
+            )
+            m = run_spineopt(url_in; log_level=0, optimize=false)
+            var_connection_flow = m.ext[:spineopt].variables[:connection_flow]
+            constraint = m.ext[:spineopt].constraints[Symbol(ratio)]
+            @test length(constraint) == 4
+            conn = connection(:connection_ab)
+            n_from = node(:node_a)
+            n_to = node(:node_b)
+            d_from = direction(:from_node)
+            d_to = direction(:to_node)
+            scenarios_from = [repeat([stochastic_scenario(:child)], 3); repeat([stochastic_scenario(:parent)], 3)]
+            time_slices_from = [
+                reverse(time_slice(m; temporal_block=temporal_block(:hourly)))
+                reverse(history_time_slice(m; temporal_block=temporal_block(:hourly)))
+            ]
+            time_slices_to = reverse(time_slice(m; temporal_block=temporal_block(:two_hourly)))
+            s_to = stochastic_scenario(:parent)
+            @testset for (j, t_con) in enumerate(reverse(time_slice(m; temporal_block=temporal_block(:hourly))))
+                s_from = scenarios_from[h_delay+j] # get the scenario before the delay 
+                t_from = time_slices_from[h_delay+j] # get the time slice before the delay
+                var_conn_flow_from = var_connection_flow[conn, n_from, d_from, s_from, t_from]
+                t_to = time_slices_to[(j+1)÷2] 
+                var_conn_flow_to = var_connection_flow[conn, n_to, d_to, s_to, t_to]
+                expected_con = SpineOpt.build_sense_constraint(
+                    var_conn_flow_to,
+                    sense,
+                    flow_ratio * var_conn_flow_from,
+                )
+                path = unique([s_to; s_from])
+                con_key = (conn, n_to, n_from, path, t_con)
+                observed_con = constraint_object(constraint[con_key...])
+                @test _is_constraint_equal(observed_con, expected_con)
+            end
+        end
+    end
     @testset "constraint_ratio_out_in_connection_flow" begin
         flow_ratio = 0.8
         model_end = Dict("type" => "date_time", "data" => "2000-01-01T04:00:00")
         class = "connection__node__node"
         relationship = ["connection_ab", "node_b", "node_a"]
-        object_parameter_values = [["model", "instance", "model_end", model_end]]
+        object_parameter_values = [
+            ["model", "instance", "model_end", model_end],
+            ["model", "instance", "use_highest_resolution_constraint_ratio_out_in_connection_flow", false] 
+        ]
         relationships = [[class, relationship]]
         senses_by_prefix = Dict("min" => >=, "fix" => ==, "max" => <=)
         @testset for conn_flow_minutes_delay in (150, 180, 225)

test/run_examples.jl

@@ -5,6 +5,11 @@ objective_function_reference_values = Dict(
     "simple_system.json" => 160714.28571428574,
     "unit_commitment.json" => 98637.42857142858,
     "rolling_horizon.json" => 65164.8571429,
+    "multi-year_investment_with_econ_params_with_milestones" => 41025092.819187716,
+    "multi-year_investment_with_econ_params_without_milestones" => 71720404.84378022,
+    "multi-year_investment_without_econ_params" => 203360142.85714287,
+    "capacity_planning" => 352127000,
+    "stochastic" => 126964.2857142857,
 )
 
 @testset for path in readdir(joinpath(dirname(@__DIR__), "examples"); join=true)

test/runtests.jl

@@ -153,9 +153,9 @@ function _dismember_function(func)
 end
 
 @testset begin
-    include("data_structure/check_economic_structure.jl") 
     include("data_structure/migration.jl")
     include("data_structure/check_data_structure.jl")
+    include("data_structure/check_economic_structure.jl") 
     include("data_structure/preprocess_data_structure.jl")
     include("data_structure/temporal_structure.jl")
     include("data_structure/stochastic_structure.jl")