Add enhancements from Markov-Transition

CaseStudy.py

@@ -56,8 +56,9 @@ def __init__(self,
                  power_weightsk_file: str = "Power_WeightsK.xlsx", dPower_WeightsK: pd.DataFrame = None,
                  power_hindex_file: str = "Power_Hindex.xlsx", dPower_Hindex: pd.DataFrame = None,
                  power_impexphubs_file: str = "Power_ImpExpHubs.xlsx", dPower_ImpExpHubs: pd.DataFrame = None,
-                 power_impexpprofiles_file: str = "Power_ImpExpProfiles.xlsx", dPower_ImpExpProfiles: pd.DataFrame = None):
-        self.data_folder = str(data_folder) if str(data_folder).endswith("/") else str(data_folder) + "/"
+                 power_impexpprofiles_file: str = "Power_ImpExpProfiles.xlsx", dPower_ImpExpProfiles: pd.DataFrame = None,
+                 clip_method: str = "none", clip_value: float = 0):
+        self.data_folder = data_folder if data_folder.endswith("/") else data_folder + "/"
         self.do_not_scale_units = do_not_scale_units
         self.do_not_merge_single_node_buses = do_not_merge_single_node_buses
 
@@ -106,11 +107,43 @@ def __init__(self,
             self.power_demand_file = power_demand_file
             self.dPower_Demand = ExcelReader.get_Power_Demand(self.data_folder + self.power_demand_file)
 
+        if dPower_Hindex is not None:
+            self.dPower_Hindex = dPower_Hindex
+        else:
+            self.power_hindex_file = power_hindex_file
+            self.dPower_Hindex = ExcelReader.get_Power_Hindex(self.data_folder + self.power_hindex_file)
+
         if dPower_WeightsRP is not None:
             self.dPower_WeightsRP = dPower_WeightsRP
         else:
             self.power_weightsrp_file = power_weightsrp_file
-            self.dPower_WeightsRP = ExcelReader.get_Power_WeightsRP(self.data_folder + self.power_weightsrp_file)
+            # Calculate dPower_WeightsRP from Hindex
+            dPower_WeightsRPs = []
+            for scenario in self.dPower_Hindex['scenario'].unique().tolist():
+                # Count occurences of each value in column 'rp' of dPower_Hindex
+                dPower_WeightsRP_scenario = pd.DataFrame(self.dPower_Hindex[self.dPower_Hindex['scenario'] == scenario].reset_index()['rp'].value_counts().sort_index())
+                dPower_WeightsRP_scenario = dPower_WeightsRP_scenario.rename(columns={'count': 'pWeight_rp'})
+                dPower_WeightsRP_scenario['scenario'] = scenario  # Add scenario ID
+
+                # Add other columns with default values
+                dPower_WeightsRP_scenario['id'] = np.nan
+                dPower_WeightsRP_scenario['dataPackage'] = np.nan
+                dPower_WeightsRP_scenario['dataSource'] = np.nan
+
+                dPower_WeightsRPs.append(dPower_WeightsRP_scenario)
+
+            dPower_WeightsRP = pd.concat(dPower_WeightsRPs, ignore_index=False)
+
+            if os.path.exists(self.data_folder + self.power_weightsrp_file):  # Compare with given file if it exists
+                self.dPower_WeightsRP = ExcelReader.get_Power_WeightsRP(self.data_folder + self.power_weightsrp_file)
+
+                calculated = dPower_WeightsRP.reset_index().set_index(["rp", "scenario"])
+                fromFile = self.dPower_WeightsRP.reset_index().set_index(["rp", "scenario"])
+                if not (calculated['pWeight_rp'] / calculated['pWeight_rp'].sum()).equals(fromFile['pWeight_rp'] / fromFile['pWeight_rp'].sum()):
+                    printer.warning(f"Values for 'pWeight_rp' in '{self.data_folder + self.power_weightsrp_file}' do not match the calculated values based on '{self.power_hindex_file}'. Please check if this is intended, using the file '{self.data_folder + self.power_weightsrp_file}' instead of the calculated values.")
+            else:  # Use calculated dPower_WeightsRP otherwise
+                printer.warning(f"Executing without 'Power_WeightsRP' (since no file was found at '{self.data_folder + self.power_weightsrp_file}').")
+                self.dPower_WeightsRP = dPower_WeightsRP
 
         if dPower_WeightsK is not None:
             self.dPower_WeightsK = dPower_WeightsK
@@ -124,7 +157,7 @@ def __init__(self,
             self.power_hindex_file = power_hindex_file
             self.dPower_Hindex = ExcelReader.get_Power_Hindex(self.data_folder + self.power_hindex_file)
 
-        self.rpTransitionMatrixAbsolute, self.rpTransitionMatrixRelativeTo, self.rpTransitionMatrixRelativeFrom = self.get_rpTransitionMatrices()
+        self.rpTransitionMatrixAbsolute, self.rpTransitionMatrixRelativeTo, self.rpTransitionMatrixRelativeFrom = self.get_rpTransitionMatrices(clip_method=clip_method, clip_value=clip_value)
 
         if self.dPower_Parameters["pEnableThermalGen"]:
             if dPower_ThermalGen is not None:
@@ -560,7 +593,7 @@ def merge_single_node_buses(self):
             self.dPower_VRESProfiles.sort_index(inplace=True)
 
     # Create transition matrix from Hindex
-    def get_rpTransitionMatrices(self):
+    def get_rpTransitionMatrices(self, clip_method: str = "none", clip_value: float = 0) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
         rps = sorted(self.dPower_Hindex.index.get_level_values('rp').unique().tolist())
         ks = sorted(self.dPower_Hindex.index.get_level_values('k').unique().tolist())
         rpTransitionMatrixAbsolute = pd.DataFrame(0, index=rps, columns=rps)  # Initialize with zeros
@@ -574,6 +607,27 @@ def get_rpTransitionMatrices(self):
             rpTransitionMatrixAbsolute.at[previous_rp, rp] += 1
             previous_rp = rp
 
+        # Clip according to selected method
+        match clip_method:
+            case "none":
+                pass
+            case "absolute_count":  # Get 'clip_value' highest values of each row of the transition matrix, set all others to 0
+                if int(clip_value) != clip_value or clip_value < 0:
+                    raise ValueError(f"For 'absolute_count', clip_value must be a non-negative integer, not {clip_value}.")
+                for rp in rps:
+                    threshold = rpTransitionMatrixAbsolute.loc[rp].nlargest(int(clip_value)).min()
+                    if (rpTransitionMatrixAbsolute.loc[rp] == threshold).sum() > 1:
+                        printer.warning(f"For rp {rp}, there are multiple values with the same value as the threshold ({threshold}). This means that more than {clip_value} values are kept.")
+                    rpTransitionMatrixAbsolute.loc[rp, rpTransitionMatrixAbsolute.loc[rp] < threshold] = 0
+            case "relative_to_highest":  # Get all values that are at least 'clip_value' * 100 % of the highest value of each row of the transition matrix, set all others to 0
+                if clip_value < 0 or clip_value > 1:
+                    raise ValueError(f"For 'relative_to_highest', clip_value must be between 0 and 1, not {clip_value}.")
+                for rp in rps:
+                    threshold = rpTransitionMatrixAbsolute.loc[rp].max() * clip_value
+                    rpTransitionMatrixAbsolute.loc[rp][rpTransitionMatrixAbsolute.loc[rp] < threshold] = 0
+            case _:
+                raise ValueError(f"clip_method must be either 'none', 'absolute_count' or 'relative_to_highest', not {clip_method}.")
+
         # Calculate relative transition matrix (nerd info: for the sum, the axis is irrelevant, as there are the same number of transitions to an rp as there are transitions from an rp away. For the division however, the axis matters)
         rpTransitionMatrixRelativeTo = rpTransitionMatrixAbsolute.div(rpTransitionMatrixAbsolute.sum(axis=1), axis=0)  # Sum of probabilities is 1 for r -> all others
         rpTransitionMatrixRelativeFrom = rpTransitionMatrixAbsolute.div(rpTransitionMatrixAbsolute.sum(axis=0), axis=1)  # Sum of probabilities is 1 for all others -> r
@@ -591,43 +645,58 @@ def to_full_hourly_model(self, inplace: bool) -> Optional['CaseStudy']:
         """
         caseStudy = self.copy() if not inplace else self
 
+        # First Adjustment of Hindex (important if the case study was filtered before, to get a coherent p-index)
+        caseStudy.dPower_Hindex = caseStudy.dPower_Hindex.reset_index()
+        for i in caseStudy.dPower_Hindex.index:
+            caseStudy.dPower_Hindex.loc[i, "p"] = f"h{i + 1:0>4}"
+        caseStudy.dPower_Hindex = caseStudy.dPower_Hindex.set_index(["p", "rp", "k"])
+
         # Adjust Demand
         adjusted_demand = []
-        for i, _ in caseStudy.dPower_BusInfo.iterrows():
-            for h, row in caseStudy.dPower_Hindex.iterrows():
-                adjusted_demand.append(["rp01", h[0].replace("h", "k"), i, caseStudy.dPower_Demand.loc[(h[1], h[2], i), "Demand"]])
+        for i in caseStudy.dPower_BusInfo.index:
+            for h in caseStudy.dPower_Hindex.index:
+                adjusted_demand.append(["rp01", h[0].replace("h", "k"), i, caseStudy.dPower_Demand.loc[(h[1], h[2], i), "value"], "ScenarioA", None, None, None])
 
-        caseStudy.dPower_Demand = pd.DataFrame(adjusted_demand, columns=["rp", "k", "i", "Demand"])
+        caseStudy.dPower_Demand = pd.DataFrame(adjusted_demand, columns=["rp", "k", "i", "value", "scenario", "id", "dataPackage", "dataSource"])
         caseStudy.dPower_Demand = caseStudy.dPower_Demand.set_index(["rp", "k", "i"])
 
         # Adjust VRESProfiles
         if hasattr(caseStudy, "dPower_VRESProfiles"):
             adjusted_vresprofiles = []
             caseStudy.dPower_VRESProfiles.sort_index(inplace=True)
             for g in caseStudy.dPower_VRESProfiles.index.get_level_values('g').unique().tolist():
-                if len(caseStudy.dPower_VRESProfiles.loc[:, :, g]) > 0:  # Check if VRESProfiles has entries for g
-                    for h, row in caseStudy.dPower_Hindex.iterrows():
-                        adjusted_vresprofiles.append(["rp01", h[0].replace("h", "k"), g, caseStudy.dPower_VRESProfiles.loc[(h[1], h[2], g), "Capacity"]])
+                for h in caseStudy.dPower_Hindex.index:
+                    adjusted_vresprofiles.append(["rp01", h[0].replace("h", "k"), g, caseStudy.dPower_VRESProfiles.loc[(h[1], h[2], g), "value"], "ScenarioA", None, None, None])
 
-            caseStudy.dPower_VRESProfiles = pd.DataFrame(adjusted_vresprofiles, columns=["rp", "k", "g", "Capacity"])
+            caseStudy.dPower_VRESProfiles = pd.DataFrame(adjusted_vresprofiles, columns=["rp", "k", "g", "value", "scenario", "id", "dataPackage", "dataSource"])
             caseStudy.dPower_VRESProfiles = caseStudy.dPower_VRESProfiles.set_index(["rp", "k", "g"])
 
+        # Adjust Inflows
+        if hasattr(caseStudy, "dPower_Inflows"):
+            adjusted_inflows = []
+            caseStudy.dPower_Inflows.sort_index(inplace=True)
+            for g in caseStudy.dPower_Inflows.index.get_level_values('g').unique().tolist():
+                for h in caseStudy.dPower_Hindex.index:
+                    adjusted_inflows.append(["rp01", h[0].replace("h", "k"), g, caseStudy.dPower_Inflows.loc[(h[1], h[2], g), "value"], "ScenarioA", None, None, None])
+            caseStudy.dPower_Inflows = pd.DataFrame(adjusted_inflows, columns=["rp", "k", "g", "value", "scenario", "id", "dataPackage", "dataSource"])
+            caseStudy.dPower_Inflows = caseStudy.dPower_Inflows.set_index(["rp", "k", "g"])
+
         # Adjust Hindex
         caseStudy.dPower_Hindex = caseStudy.dPower_Hindex.reset_index()
-        for i, row in caseStudy.dPower_Hindex.iterrows():
-            caseStudy.dPower_Hindex.loc[i] = f"h{i + 1:0>4}", f"rp01", f"k{i + 1:0>4}", None, None, None
+        for i in caseStudy.dPower_Hindex.index:
+            caseStudy.dPower_Hindex.loc[i] = f"h{i + 1:0>4}", f"rp01", f"k{i + 1:0>4}", None, None, None, "ScenarioA"
         caseStudy.dPower_Hindex = caseStudy.dPower_Hindex.set_index(["p", "rp", "k"])
 
         # Adjust WeightsK
         caseStudy.dPower_WeightsK = caseStudy.dPower_WeightsK.reset_index()
         caseStudy.dPower_WeightsK = caseStudy.dPower_WeightsK.drop(caseStudy.dPower_WeightsK.index)
         for i in range(len(caseStudy.dPower_Hindex)):
-            caseStudy.dPower_WeightsK.loc[i] = f"k{i + 1:0>4}", None, 1, None, None
+            caseStudy.dPower_WeightsK.loc[i] = f"{caseStudy.dPower_Hindex.index[i][2]}", None, 1, None, None, "ScenarioA"
         caseStudy.dPower_WeightsK = caseStudy.dPower_WeightsK.set_index("k")
 
         # Adjust WeightsRP
         caseStudy.dPower_WeightsRP = caseStudy.dPower_WeightsRP.drop(caseStudy.dPower_WeightsRP.index)
-        caseStudy.dPower_WeightsRP.loc["rp01"] = 1
+        caseStudy.dPower_WeightsRP.loc["rp01"] = None, 1, None, None, "ScenarioA"
 
         if not inplace:
             return caseStudy
@@ -669,7 +738,7 @@ def filter_timesteps(self, start: str, end: str, inplace: bool = False) -> Optio
         case_study = self if inplace else self.copy()
 
         for df_name in CaseStudy.k_dependent_dataframes:
-            if hasattr(case_study, df_name):
+            if hasattr(case_study, df_name) and getattr(case_study, df_name) is not None:
                 df = getattr(case_study, df_name)
 
                 index = df.index.names
@@ -706,3 +775,43 @@ def filter_representative_periods(self, rp: str, inplace: bool = False) -> Optio
                 setattr(case_study, df_name, filtered_df)
 
         return None if inplace else case_study
+
+    def shift_ks(self, shift: int, inplace: bool = False) -> Optional[Self]:
+        """
+        Shifts all k indices by the given amount, i.e., if shift is 4, then the first 4
+        timesteps are moved to the back of the time series.
+
+        :param shift: The amount to shift the k indices by.
+        :param inplace: If True, modifies the current instance. If False, returns a new instance.
+        :return: None if inplace is True, otherwise a new CaseStudy instance.
+        """
+        case_study = self if inplace else self.copy()
+
+        for df_name in CaseStudy.k_dependent_dataframes:
+            if df_name in ["dPower_WeightsK", "dPower_Hindex"]:
+                continue  # These dataframes are not shifted, as they are not time series
+
+            if hasattr(case_study, df_name):
+                df = getattr(case_study, df_name)
+                if df is None or df.empty:
+                    continue
+
+                index = df.index.names
+                df = df.reset_index()
+
+                df["k_int"] = df["k"].str.replace("k", "").astype(int)
+                k_int_max = df["k_int"].max()
+                k_int_min = df["k_int"].min()
+
+                df["k_int_new"] = ((df["k_int"] - k_int_min + shift) % (k_int_max - k_int_min + 1)) + k_int_min
+
+                df["k"] = "k" + df["k_int_new"].astype(str).str.zfill(4)
+                df = df.drop(columns=["k_int", "k_int_new"])
+                df = df.set_index(index)
+
+                # Sort by index to ensure that the order of the indices is correct after shifting
+                df = df.sort_index()
+
+                setattr(case_study, df_name, df)
+
+        return None if inplace else case_study