diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..812dd86 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,30 @@ +name: CI + +on: + push: + pull_request: + +jobs: + run_tests: + name: Test on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -el {0} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + steps: + - name: Checkout repository + uses: actions/checkout@v4.2.2 + + - name: Set up Conda environment + uses: conda-incubator/setup-miniconda@v3.2.0 + with: + activate-environment: InOutModule_env + environment-file: environment.yml + auto-activate-base: false + + - name: Run tests + run: python -m pytest \ No newline at end of file diff --git a/.gitignore b/.gitignore index c4d5a07..53c0264 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ __pycache__/ examples/output # Temporary files -~$*.xlsx \ No newline at end of file +~$*.xlsx + +# Development environment files +.idea/ \ No newline at end of file diff --git a/CaseStudy.py b/CaseStudy.py index 8162dde..aac2af7 100644 --- a/CaseStudy.py +++ b/CaseStudy.py @@ -5,12 +5,12 @@ import numpy as np import pandas as pd -from InOutModule import ExcelReader +import ExcelReader class CaseStudy: - def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = False, + def __init__(self, data_folder: str, do_not_merge_single_node_buses: bool = False, global_parameters_file: str = "Global_Parameters.xlsx", dGlobal_Parameters: pd.DataFrame = None, power_parameters_file: str = "Power_Parameters.xlsx", dPower_Parameters: pd.DataFrame = None, power_businfo_file: str = "Power_BusInfo.xlsx", dPower_BusInfo: pd.DataFrame = None, @@ -27,7 +27,7 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F power_hindex_file: str = "Power_Hindex.xlsx", dPower_Hindex: pd.DataFrame = None, power_impexphubs_file: str = "Power_ImpExpHubs.xlsx", dPower_ImpExpHubs: pd.DataFrame = None, power_impexpprofiles_file: str = "Power_ImpExpProfiles.xlsx", dPower_ImpExpProfiles: pd.DataFrame = None): - self.example_folder = example_folder if example_folder.endswith("/") else example_folder + "/" + self.data_folder = data_folder if data_folder.endswith("/") else data_folder + "/" self.do_not_merge_single_node_buses = do_not_merge_single_node_buses if dGlobal_Parameters is not None: @@ -46,37 +46,37 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F self.dPower_BusInfo = dPower_BusInfo else: self.power_businfo_file = power_businfo_file - self.dPower_BusInfo = ExcelReader.get_dPower_BusInfo(self.example_folder + self.power_businfo_file) + self.dPower_BusInfo = ExcelReader.get_dPower_BusInfo(self.data_folder + self.power_businfo_file) if dPower_Network is not None: self.dPower_Network = dPower_Network else: self.power_network_file = power_network_file - self.dPower_Network = ExcelReader.get_dPower_Network(self.example_folder + self.power_network_file) + self.dPower_Network = ExcelReader.get_dPower_Network(self.data_folder + self.power_network_file) if dPower_Demand is not None: self.dPower_Demand = dPower_Demand else: self.power_demand_file = power_demand_file - self.dPower_Demand = ExcelReader.get_dPower_Demand(self.example_folder + self.power_demand_file) + self.dPower_Demand = ExcelReader.get_dPower_Demand(self.data_folder + self.power_demand_file) if dPower_WeightsRP is not None: self.dPower_WeightsRP = dPower_WeightsRP else: self.power_weightsrp_file = power_weightsrp_file - self.dPower_WeightsRP = ExcelReader.get_dPower_WeightsRP(self.example_folder + self.power_weightsrp_file) + self.dPower_WeightsRP = ExcelReader.get_dPower_WeightsRP(self.data_folder + self.power_weightsrp_file) if dPower_WeightsK is not None: self.dPower_WeightsK = dPower_WeightsK else: self.power_weightsk_file = power_weightsk_file - self.dPower_WeightsK = ExcelReader.get_dPower_WeightsK(self.example_folder + self.power_weightsk_file) + self.dPower_WeightsK = ExcelReader.get_dPower_WeightsK(self.data_folder + self.power_weightsk_file) if dPower_Hindex is not None: self.dPower_Hindex = dPower_Hindex else: self.power_hindex_file = power_hindex_file - self.dPower_Hindex = ExcelReader.get_dPower_Hindex(self.example_folder + self.power_hindex_file) + self.dPower_Hindex = ExcelReader.get_dPower_Hindex(self.data_folder + self.power_hindex_file) self.rpTransitionMatrixAbsolute, self.rpTransitionMatrixRelativeTo, self.rpTransitionMatrixRelativeFrom = self.get_rpTransitionMatrices() @@ -85,7 +85,7 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F self.dPower_ThermalGen = dPower_ThermalGen else: self.power_thermalgen_file = power_thermalgen_file - self.dPower_ThermalGen = ExcelReader.get_dPower_ThermalGen(self.example_folder + self.power_thermalgen_file) + self.dPower_ThermalGen = ExcelReader.get_dPower_ThermalGen(self.data_folder + self.power_thermalgen_file) if self.dPower_Parameters["pEnableRoR"]: if dPower_RoR is not None: @@ -105,13 +105,13 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F self.dPower_VRES = dPower_VRES else: self.power_vres_file = power_vres_file - self.dPower_VRES = ExcelReader.get_dPower_VRES(self.example_folder + self.power_vres_file) + self.dPower_VRES = ExcelReader.get_dPower_VRES(self.data_folder + self.power_vres_file) if dPower_VRESProfiles is not None: self.dPower_VRESProfiles = dPower_VRESProfiles else: self.power_vresprofiles_file = power_vresprofiles_file - self.dPower_VRESProfiles = ExcelReader.get_dPower_VRESProfiles(self.example_folder + self.power_vresprofiles_file) + self.dPower_VRESProfiles = ExcelReader.get_dPower_VRESProfiles(self.data_folder + self.power_vresprofiles_file) if self.dPower_Parameters["pEnableStorage"]: if dPower_Storage is not None: @@ -143,7 +143,7 @@ def copy(self): return copy.deepcopy(self) def get_dGlobal_Parameters(self): - dGlobal_Parameters = pd.read_excel(self.example_folder + self.global_parameters_file, skiprows=[0, 1]) + dGlobal_Parameters = pd.read_excel(self.data_folder + self.global_parameters_file, skiprows=[0, 1]) dGlobal_Parameters = dGlobal_Parameters.drop(dGlobal_Parameters.columns[0], axis=1) dGlobal_Parameters = dGlobal_Parameters.set_index('Sectors') @@ -154,7 +154,7 @@ def get_dGlobal_Parameters(self): return dGlobal_Parameters def get_dPower_Parameters(self): - dPower_Parameters = pd.read_excel(self.example_folder + self.power_parameters_file, skiprows=[0, 1]) + dPower_Parameters = pd.read_excel(self.data_folder + self.power_parameters_file, skiprows=[0, 1]) dPower_Parameters = dPower_Parameters.drop(dPower_Parameters.columns[0], axis=1) dPower_Parameters = dPower_Parameters.dropna(how="all") dPower_Parameters = dPower_Parameters.set_index('General') @@ -186,14 +186,14 @@ def yesNo_to_bool(df: pd.DataFrame, columns_to_be_changed: list[str]): return df def get_dPower_RoR(self): - dPower_RoR = self.read_generator_data(self.example_folder + self.power_ror_file) + dPower_RoR = self.read_generator_data(self.data_folder + self.power_ror_file) dPower_RoR['InvestCostEUR'] = dPower_RoR['MaxProd'] * 1e-3 * (dPower_RoR['InvestCostPerMW'] * 1e-3 + dPower_RoR['InvestCostPerMWh'] * 1e-3 * dPower_RoR['Ene2PowRatio']) dPower_RoR['MaxProd'] *= 1e-3 return dPower_RoR def get_dPower_Storage(self): - dPower_Storage = self.read_generator_data(self.example_folder + self.power_storage_file) + dPower_Storage = self.read_generator_data(self.data_folder + self.power_storage_file) dPower_Storage['pOMVarCostEUR'] = dPower_Storage['OMVarCost'] * 1e-3 dPower_Storage['IniReserve'] = dPower_Storage['IniReserve'].fillna(0) dPower_Storage['MinReserve'] = dPower_Storage['MinReserve'].fillna(0) @@ -204,7 +204,7 @@ def get_dPower_Storage(self): return dPower_Storage def get_dPower_Inflows(self): - dPower_Inflows = pd.read_excel(self.example_folder + self.power_inflows_file, skiprows=[0, 1, 3, 4, 5]) + dPower_Inflows = pd.read_excel(self.data_folder + self.power_inflows_file, skiprows=[0, 1, 3, 4, 5]) dPower_Inflows = dPower_Inflows.drop(dPower_Inflows.columns[0], axis=1) dPower_Inflows = dPower_Inflows.rename(columns={dPower_Inflows.columns[0]: "rp", dPower_Inflows.columns[1]: "g"}) dPower_Inflows = dPower_Inflows.melt(id_vars=['rp', 'g'], var_name='k', value_name='Inflow') @@ -212,7 +212,7 @@ def get_dPower_Inflows(self): return dPower_Inflows def get_dPower_ImpExpHubs(self): - dPower_ImpExpHubs = pd.read_excel(self.example_folder + self.power_impexphubs_file, skiprows=[0, 1, 3, 4, 5]) + dPower_ImpExpHubs = pd.read_excel(self.data_folder + self.power_impexphubs_file, skiprows=[0, 1, 3, 4, 5]) dPower_ImpExpHubs = dPower_ImpExpHubs.drop(dPower_ImpExpHubs.columns[0], axis=1) dPower_ImpExpHubs = dPower_ImpExpHubs.set_index(['hub', 'i']) @@ -238,7 +238,7 @@ def get_dPower_ImpExpHubs(self): def get_dPower_ImpExpProfiles(self): with warnings.catch_warnings(action="ignore", category=UserWarning): # Otherwise there is a warning regarding data validation in the Excel-File (see https://stackoverflow.com/questions/53965596/python-3-openpyxl-userwarning-data-validation-extension-not-supported) - dPower_ImpExpProfiles = pd.read_excel(self.example_folder + self.power_impexpprofiles_file, skiprows=[0, 1, 3, 4, 5], sheet_name='Power ImpExpProfiles') + dPower_ImpExpProfiles = pd.read_excel(self.data_folder + self.power_impexpprofiles_file, skiprows=[0, 1, 3, 4, 5], sheet_name='Power ImpExpProfiles') dPower_ImpExpProfiles = dPower_ImpExpProfiles.drop(dPower_ImpExpProfiles.columns[0], axis=1) dPower_ImpExpProfiles = dPower_ImpExpProfiles.melt(id_vars=['hub', 'rp', 'Type'], var_name='k', value_name='Value') diff --git a/ExcelReader.py b/ExcelReader.py index 70f7e73..7b025e5 100644 --- a/ExcelReader.py +++ b/ExcelReader.py @@ -5,7 +5,7 @@ import pandas as pd from openpyxl import load_workbook -from InOutModule.printer import Printer +from printer import Printer printer = Printer.getInstance() diff --git a/ExcelWriter.py b/ExcelWriter.py index 0d5e38f..4d99aec 100644 --- a/ExcelWriter.py +++ b/ExcelWriter.py @@ -8,10 +8,10 @@ import pandas as pd from openpyxl.utils.dataframe import dataframe_to_rows -import InOutModule.TableDefinition -from InOutModule import ExcelReader, TableDefinition -from InOutModule.TableDefinition import CellStyle, Alignment, Font, Color, Text, Column, NumberFormat, TableDefinition -from InOutModule.printer import Printer +import ExcelReader +import TableDefinition +from TableDefinition import CellStyle, Alignment, Font, Color, Text, Column, NumberFormat, TableDefinition +from printer import Printer package_directory_ExcelWriter = os.path.dirname(os.path.abspath(__file__)) @@ -40,7 +40,7 @@ def __init__(self, excel_definitions_path: str = "TableDefinitions.xml"): pass @staticmethod - def __setCellStyle(cell_style: InOutModule.TableDefinition.CellStyle, target_cell: openpyxl.cell.cell): + def __setCellStyle(cell_style: CellStyle, target_cell: openpyxl.cell.cell): """ Set the cell style of a target cell based on the given cell style. diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..a76f37c --- /dev/null +++ b/environment.yml @@ -0,0 +1,14 @@ +name: InOutModule_env +channels: + - defaults + - conda-forge +dependencies: + - openpyxl=3.1.5 + - pandas=2.2.3 + - pip=24.0 + - pytest=8.4.0 + - python=3.12.2 + - rich=13.7.1 + - rich-argparse=1.6.0 + - pip: + - pulp==2.9.0 diff --git a/printer.py b/printer.py index 04b2d1f..c753835 100644 --- a/printer.py +++ b/printer.py @@ -1,6 +1,5 @@ import datetime -from pyomo import environ as pyo from rich.console import Console @@ -217,6 +216,7 @@ def pprint_zoi_var(var, zoi, index_positions: list = None, decimals: int = 2): :param decimals: The number of decimal places to display for the variable's value. :return: None """ + from pyomo import environ as pyo # Import Pyomo environment for variable handling if index_positions is None: index_positions = [0] diff --git a/templates/Power_VRESProfiles-template.xlsx b/templates/Power_VRESProfiles-template.xlsx deleted file mode 100644 index 2668639..0000000 Binary files a/templates/Power_VRESProfiles-template.xlsx and /dev/null differ diff --git a/tests/test_ExcelReaderWriter.py b/tests/test_ExcelReaderWriter.py new file mode 100644 index 0000000..e2b4a17 --- /dev/null +++ b/tests/test_ExcelReaderWriter.py @@ -0,0 +1,36 @@ +import pytest + +import ExcelReader as ExcelReader +from ExcelWriter import ExcelWriter +from printer import Printer + +printer = Printer.getInstance() + +case_study_folder = "examples/" +ew = ExcelWriter() +combinations = [ + ("Power_Hindex", f"{case_study_folder}Power_Hindex.xlsx", ExcelReader.get_dPower_Hindex, ew.write_dPower_Hindex), + ("Power_WeightsRP", f"{case_study_folder}Power_WeightsRP.xlsx", ExcelReader.get_dPower_WeightsRP, ew.write_dPower_WeightsRP), + ("Power_WeightsK", f"{case_study_folder}Power_WeightsK.xlsx", ExcelReader.get_dPower_WeightsK, ew.write_dPower_WeightsK), + ("Power_BusInfo", f"{case_study_folder}Power_BusInfo.xlsx", ExcelReader.get_dPower_BusInfo, ew.write_dPower_BusInfo), + ("Power_Network", f"{case_study_folder}Power_Network.xlsx", ExcelReader.get_dPower_Network, ew.write_dPower_Network), + ("Power_Demand", f"{case_study_folder}Power_Demand.xlsx", ExcelReader.get_dPower_Demand, ew.write_dPower_Demand), + ("Power_ThermalGen", f"{case_study_folder}Power_ThermalGen.xlsx", ExcelReader.get_dPower_ThermalGen, ew.write_dPower_ThermalGen), + ("Power_VRES", f"{case_study_folder}Power_VRES.xlsx", ExcelReader.get_dPower_VRES, ew.write_VRES), + ("Power_VRESProfiles", f"{case_study_folder}Power_VRESProfiles.xlsx", ExcelReader.get_dPower_VRESProfiles, ew.write_VRESProfiles), + ("Data_Sources", f"{case_study_folder}Data_Sources.xlsx", ExcelReader.get_dData_Sources, ew.write_dData_Sources), + ("Data_Packages", f"{case_study_folder}Data_Packages.xlsx", ExcelReader.get_dData_Packages, ew.write_dData_Packages), +] + + +@pytest.mark.parametrize("excel_definition_id, file_path, read, write", combinations) +def test_read_and_write(excel_definition_id, file_path, read, write, tmp_path): + printer.information(f"Writing '{excel_definition_id}', read from '{file_path}'") + + data = read(file_path, True, True) + write(data, str(tmp_path)) + + printer.information(f"Comparing '{tmp_path}/{excel_definition_id}.xlsx' against source file '{file_path}'") + filesEqual = ExcelReader.compare_Excels(file_path, f"{tmp_path}/{excel_definition_id}.xlsx", dont_check_formatting=False) + + assert filesEqual, f"Read and/or Write for {excel_definition_id} are faulty - please check!"