From b00c26b7650945e2993e4776acd42bfcdc8b4230 Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Tue, 26 Aug 2025 13:38:22 +0200 Subject: [PATCH 01/10] Implement inflows to capacity factor --- Utilities.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Utilities.py diff --git a/Utilities.py b/Utilities.py new file mode 100644 index 0000000..0bf6b40 --- /dev/null +++ b/Utilities.py @@ -0,0 +1,41 @@ +import numpy as np +import pandas as pd + + +def inflowsToCapacityFactor(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vresProfiles_df: pd.DataFrame) -> pd.DataFrame: + """ + Convert inflows to capacity factors and concat them to vresProfiles_df. + + - inflows_df: inflow data with inflows per generator (g) and representative period (rp). + - vres_df: contains generator technical data, including 'MaxProd'. + - vresProfiles_df: existing VRES profiles (indexed by rp, k, g). + """ + df = inflows_df.reset_index() + + # Prepare vres_df with ['g','MaxProd'] + vres_tmp = vres_df.reset_index()[['g', 'MaxProd']] + maxprod = ( + vres_tmp.drop_duplicates('g') + .set_index('g')['MaxProd'] + .astype(float) + ) + + # Merge MaxProd into inflows + df = df.merge(maxprod, on='g', how='left') + + # Divide inflow value by MaxProd + df['value'] = df['value'] / df['MaxProd'].replace(0, np.nan) + + # Drop helper column + df = df.drop(columns=['MaxProd']) + + # Ensure required metadata columns + meta_cols = ['dataPackage', 'dataSource', 'id', 'scenario'] + for col in meta_cols: + if col not in df.columns: + df[col] = vresProfiles_df[col].iloc[0] + + # Restore index structure (rp, k, g) + df = df.set_index(['rp', 'k', 'g']) + + return pd.concat([vresProfiles_df, df], axis=0).sort_index(level="k") From 23356e068d4b043b9895ac6e8e2c9826652d2a72 Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Tue, 26 Aug 2025 13:38:56 +0200 Subject: [PATCH 02/10] Implement capacity factor to inflows --- Utilities.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Utilities.py b/Utilities.py index 0bf6b40..8adad33 100644 --- a/Utilities.py +++ b/Utilities.py @@ -39,3 +39,39 @@ def inflowsToCapacityFactor(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vre df = df.set_index(['rp', 'k', 'g']) return pd.concat([vresProfiles_df, df], axis=0).sort_index(level="k") + + +def capacityFactorToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFrame, inflows_df: pd.DataFrame) -> pd.DataFrame: + """ + Convert capacity factors in vresProfiles_df back to inflows. + + - vresProfiles_df: DataFrame with capacity factors (indexed by rp, k, g). + - vres_df: DataFrame containing generator technical data, including 'MaxProd'. + - inflows_df: template inflows DataFrame (used to filter only those generators that are inflow-based). + """ + df = vresProfiles_df.reset_index() + + # Get list of inflow generators + inflow_generators = inflows_df.reset_index()['g'].unique() + + # Prepare vres_df with ['g','MaxProd'] + vres_tmp = vres_df.reset_index()[['g', 'MaxProd']] + maxprod = ( + vres_tmp.drop_duplicates('g') + .set_index('g')['MaxProd'] + .astype(float) + ) + + # Keep only inflow generators + df = df[df['g'].isin(inflow_generators)] + + # Merge MaxProd + df = df.merge(maxprod, on='g', how='left') + + # Multiply capacity factor by MaxProd + df['value'] = df['value'] * df['MaxProd'] + + # Drop helper column + df = df.drop(columns=['MaxProd']) + + return df.set_index(['rp', 'k', 'g']).sort_index(level="k") From 0d472087bdbbe996dce5bfef63160e6575691c69 Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Tue, 26 Aug 2025 16:19:23 +0200 Subject: [PATCH 03/10] Rename functions to enhance coherence --- Utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utilities.py b/Utilities.py index 8adad33..1419ade 100644 --- a/Utilities.py +++ b/Utilities.py @@ -2,7 +2,7 @@ import pandas as pd -def inflowsToCapacityFactor(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vresProfiles_df: pd.DataFrame) -> pd.DataFrame: +def inflowsToCapacityFactors(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vresProfiles_df: pd.DataFrame) -> pd.DataFrame: """ Convert inflows to capacity factors and concat them to vresProfiles_df. @@ -41,7 +41,7 @@ def inflowsToCapacityFactor(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vre return pd.concat([vresProfiles_df, df], axis=0).sort_index(level="k") -def capacityFactorToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFrame, inflows_df: pd.DataFrame) -> pd.DataFrame: +def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFrame, inflows_df: pd.DataFrame) -> pd.DataFrame: """ Convert capacity factors in vresProfiles_df back to inflows. From 38648b93bbd6c6a0296762a7a284dc6a3e78095d Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Tue, 26 Aug 2025 16:19:50 +0200 Subject: [PATCH 04/10] Assume all columns are in inflows --- Utilities.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Utilities.py b/Utilities.py index 1419ade..fa7d3e0 100644 --- a/Utilities.py +++ b/Utilities.py @@ -29,12 +29,6 @@ def inflowsToCapacityFactors(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vr # Drop helper column df = df.drop(columns=['MaxProd']) - # Ensure required metadata columns - meta_cols = ['dataPackage', 'dataSource', 'id', 'scenario'] - for col in meta_cols: - if col not in df.columns: - df[col] = vresProfiles_df[col].iloc[0] - # Restore index structure (rp, k, g) df = df.set_index(['rp', 'k', 'g']) From 1b894b94b9e63f339a80898f9dc4ddc0590761a6 Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Tue, 26 Aug 2025 16:23:58 +0200 Subject: [PATCH 05/10] Raise an error if duplicated generators are found --- Utilities.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Utilities.py b/Utilities.py index fa7d3e0..0489efc 100644 --- a/Utilities.py +++ b/Utilities.py @@ -14,14 +14,14 @@ def inflowsToCapacityFactors(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vr # Prepare vres_df with ['g','MaxProd'] vres_tmp = vres_df.reset_index()[['g', 'MaxProd']] - maxprod = ( - vres_tmp.drop_duplicates('g') - .set_index('g')['MaxProd'] - .astype(float) - ) + + if vres_tmp['g'].duplicated().any(): + raise ValueError("Duplicated generator found in Power_VRES.") + + maxProd = vres_tmp.set_index('g')['MaxProd'].astype(float) # Merge MaxProd into inflows - df = df.merge(maxprod, on='g', how='left') + df = df.merge(maxProd, on='g', how='left') # Divide inflow value by MaxProd df['value'] = df['value'] / df['MaxProd'].replace(0, np.nan) @@ -50,17 +50,17 @@ def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFram # Prepare vres_df with ['g','MaxProd'] vres_tmp = vres_df.reset_index()[['g', 'MaxProd']] - maxprod = ( - vres_tmp.drop_duplicates('g') - .set_index('g')['MaxProd'] - .astype(float) - ) + + if vres_tmp['g'].duplicated().any(): + raise ValueError("Duplicated generator found in Power_VRES.") + + maxProd = vres_tmp.set_index('g')['MaxProd'].astype(float) # Keep only inflow generators df = df[df['g'].isin(inflow_generators)] # Merge MaxProd - df = df.merge(maxprod, on='g', how='left') + df = df.merge(maxProd, on='g', how='left') # Multiply capacity factor by MaxProd df['value'] = df['value'] * df['MaxProd'] From 7a7b33f3b65c480ad3dc66fdf473e19a2b1d2f0b Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Tue, 26 Aug 2025 17:31:34 +0200 Subject: [PATCH 06/10] Not reset indexes for inflows dataframe --- Utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utilities.py b/Utilities.py index 0489efc..f64456d 100644 --- a/Utilities.py +++ b/Utilities.py @@ -10,7 +10,7 @@ def inflowsToCapacityFactors(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vr - vres_df: contains generator technical data, including 'MaxProd'. - vresProfiles_df: existing VRES profiles (indexed by rp, k, g). """ - df = inflows_df.reset_index() + df = inflows_df.copy() # Prepare vres_df with ['g','MaxProd'] vres_tmp = vres_df.reset_index()[['g', 'MaxProd']] From 1393e9799543de76bfa736ae378ff2cfc059c299 Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Tue, 26 Aug 2025 17:32:06 +0200 Subject: [PATCH 07/10] Raise an error if Max Production is null for any generator --- Utilities.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Utilities.py b/Utilities.py index f64456d..69d4029 100644 --- a/Utilities.py +++ b/Utilities.py @@ -1,4 +1,3 @@ -import numpy as np import pandas as pd @@ -20,18 +19,18 @@ def inflowsToCapacityFactors(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vr maxProd = vres_tmp.set_index('g')['MaxProd'].astype(float) - # Merge MaxProd into inflows - df = df.merge(maxProd, on='g', how='left') + # Join MaxProd into inflows + df = df.join(maxProd, on='g', how='left') + + if df['MaxProd'].isna().any() or (df['MaxProd'] == 0).any(): + raise ValueError("MaxProd is missing or zero for some generators in inflows.") # Divide inflow value by MaxProd - df['value'] = df['value'] / df['MaxProd'].replace(0, np.nan) + df['value'] = df['value'] / df['MaxProd'] # Drop helper column df = df.drop(columns=['MaxProd']) - # Restore index structure (rp, k, g) - df = df.set_index(['rp', 'k', 'g']) - return pd.concat([vresProfiles_df, df], axis=0).sort_index(level="k") @@ -59,8 +58,11 @@ def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFram # Keep only inflow generators df = df[df['g'].isin(inflow_generators)] - # Merge MaxProd - df = df.merge(maxProd, on='g', how='left') + # Join MaxProd + df = df.join(maxProd, on='g', how='left') + + if df['MaxProd'].isna().any() or (df['MaxProd'] == 0).any(): + raise ValueError("MaxProd is missing or zero for some generators in inflows.") # Multiply capacity factor by MaxProd df['value'] = df['value'] * df['MaxProd'] From 5253b562974c2c6d3dd8f2f44548580bb09c9bed Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Wed, 27 Aug 2025 10:47:54 +0200 Subject: [PATCH 08/10] Add values of Inflows at the end of VRESProfiles --- Utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utilities.py b/Utilities.py index 69d4029..fbb929d 100644 --- a/Utilities.py +++ b/Utilities.py @@ -31,7 +31,7 @@ def inflowsToCapacityFactors(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vr # Drop helper column df = df.drop(columns=['MaxProd']) - return pd.concat([vresProfiles_df, df], axis=0).sort_index(level="k") + return pd.concat([vresProfiles_df, df], axis=0) def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFrame, inflows_df: pd.DataFrame) -> pd.DataFrame: From ac0395789f682d1e487f4132b8fc15bee016ba6b Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Wed, 27 Aug 2025 10:49:26 +0200 Subject: [PATCH 09/10] Add bool to remove Inflows from the original VRESProfiles --- Utilities.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Utilities.py b/Utilities.py index fbb929d..a795d32 100644 --- a/Utilities.py +++ b/Utilities.py @@ -34,13 +34,14 @@ def inflowsToCapacityFactors(inflows_df: pd.DataFrame, vres_df: pd.DataFrame, vr return pd.concat([vresProfiles_df, df], axis=0) -def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFrame, inflows_df: pd.DataFrame) -> pd.DataFrame: +def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFrame, inflows_df: pd.DataFrame, remove_Inflows_from_VRESProfiles_inplace: bool = False) -> pd.DataFrame: """ Convert capacity factors in vresProfiles_df back to inflows. - vresProfiles_df: DataFrame with capacity factors (indexed by rp, k, g). - vres_df: DataFrame containing generator technical data, including 'MaxProd'. - inflows_df: template inflows DataFrame (used to filter only those generators that are inflow-based). + - remove_Inflows_from_VRESProfiles_inplace: if True, remove inflow generators from the original vresProfiles_df. """ df = vresProfiles_df.reset_index() @@ -70,4 +71,9 @@ def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFram # Drop helper column df = df.drop(columns=['MaxProd']) + # Remove inflow generators from vresProfiles_df after calculation if requested + if remove_Inflows_from_VRESProfiles_inplace: + mask = ~vresProfiles_df.index.get_level_values('g').isin(inflow_generators) + vresProfiles_df.drop(vresProfiles_df.index[~mask], inplace=True) + return df.set_index(['rp', 'k', 'g']).sort_index(level="k") From 5db060392afcf68aa47bc9cd49f39282b93e90ad Mon Sep 17 00:00:00 2001 From: Marco Anarmo Date: Wed, 27 Aug 2025 11:14:27 +0200 Subject: [PATCH 10/10] Delete reverse operator in mask definition and usage --- Utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utilities.py b/Utilities.py index a795d32..ab12cf6 100644 --- a/Utilities.py +++ b/Utilities.py @@ -73,7 +73,7 @@ def capacityFactorsToInflows(vresProfiles_df: pd.DataFrame, vres_df: pd.DataFram # Remove inflow generators from vresProfiles_df after calculation if requested if remove_Inflows_from_VRESProfiles_inplace: - mask = ~vresProfiles_df.index.get_level_values('g').isin(inflow_generators) - vresProfiles_df.drop(vresProfiles_df.index[~mask], inplace=True) + mask = vresProfiles_df.index.get_level_values('g').isin(inflow_generators) + vresProfiles_df.drop(vresProfiles_df.index[mask], inplace=True) return df.set_index(['rp', 'k', 'g']).sort_index(level="k")