Add automatic testing to InOutModule

.github/workflows/ci.yaml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  run_tests:
+    name: Test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4.2.2
+
+      - name: Set up Conda environment
+        uses: conda-incubator/setup-miniconda@v3.2.0
+        with:
+          activate-environment: InOutModule_env
+          environment-file: environment.yml
+          auto-activate-base: false
+
+      - name: Run tests
+        run: python -m pytest

.gitignore

@@ -5,4 +5,7 @@ __pycache__/
 examples/output
 
 # Temporary files
-~$*.xlsx
+~$*.xlsx
+
+# Development environment files
+.idea/

CaseStudy.py

@@ -5,12 +5,12 @@
 import numpy as np
 import pandas as pd
 
-from InOutModule import ExcelReader
+import ExcelReader
 
 
 class CaseStudy:
 
-    def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = False,
+    def __init__(self, data_folder: str, do_not_merge_single_node_buses: bool = False,
                  global_parameters_file: str = "Global_Parameters.xlsx", dGlobal_Parameters: pd.DataFrame = None,
                  power_parameters_file: str = "Power_Parameters.xlsx", dPower_Parameters: pd.DataFrame = None,
                  power_businfo_file: str = "Power_BusInfo.xlsx", dPower_BusInfo: pd.DataFrame = None,
@@ -27,7 +27,7 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F
                  power_hindex_file: str = "Power_Hindex.xlsx", dPower_Hindex: pd.DataFrame = None,
                  power_impexphubs_file: str = "Power_ImpExpHubs.xlsx", dPower_ImpExpHubs: pd.DataFrame = None,
                  power_impexpprofiles_file: str = "Power_ImpExpProfiles.xlsx", dPower_ImpExpProfiles: pd.DataFrame = None):
-        self.example_folder = example_folder if example_folder.endswith("/") else example_folder + "/"
+        self.data_folder = data_folder if data_folder.endswith("/") else data_folder + "/"
         self.do_not_merge_single_node_buses = do_not_merge_single_node_buses
 
         if dGlobal_Parameters is not None:
@@ -46,37 +46,37 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F
             self.dPower_BusInfo = dPower_BusInfo
         else:
             self.power_businfo_file = power_businfo_file
-            self.dPower_BusInfo = ExcelReader.get_dPower_BusInfo(self.example_folder + self.power_businfo_file)
+            self.dPower_BusInfo = ExcelReader.get_dPower_BusInfo(self.data_folder + self.power_businfo_file)
 
         if dPower_Network is not None:
             self.dPower_Network = dPower_Network
         else:
             self.power_network_file = power_network_file
-            self.dPower_Network = ExcelReader.get_dPower_Network(self.example_folder + self.power_network_file)
+            self.dPower_Network = ExcelReader.get_dPower_Network(self.data_folder + self.power_network_file)
 
         if dPower_Demand is not None:
             self.dPower_Demand = dPower_Demand
         else:
             self.power_demand_file = power_demand_file
-            self.dPower_Demand = ExcelReader.get_dPower_Demand(self.example_folder + self.power_demand_file)
+            self.dPower_Demand = ExcelReader.get_dPower_Demand(self.data_folder + self.power_demand_file)
 
         if dPower_WeightsRP is not None:
             self.dPower_WeightsRP = dPower_WeightsRP
         else:
             self.power_weightsrp_file = power_weightsrp_file
-            self.dPower_WeightsRP = ExcelReader.get_dPower_WeightsRP(self.example_folder + self.power_weightsrp_file)
+            self.dPower_WeightsRP = ExcelReader.get_dPower_WeightsRP(self.data_folder + self.power_weightsrp_file)
 
         if dPower_WeightsK is not None:
             self.dPower_WeightsK = dPower_WeightsK
         else:
             self.power_weightsk_file = power_weightsk_file
-            self.dPower_WeightsK = ExcelReader.get_dPower_WeightsK(self.example_folder + self.power_weightsk_file)
+            self.dPower_WeightsK = ExcelReader.get_dPower_WeightsK(self.data_folder + self.power_weightsk_file)
 
         if dPower_Hindex is not None:
             self.dPower_Hindex = dPower_Hindex
         else:
             self.power_hindex_file = power_hindex_file
-            self.dPower_Hindex = ExcelReader.get_dPower_Hindex(self.example_folder + self.power_hindex_file)
+            self.dPower_Hindex = ExcelReader.get_dPower_Hindex(self.data_folder + self.power_hindex_file)
 
         self.rpTransitionMatrixAbsolute, self.rpTransitionMatrixRelativeTo, self.rpTransitionMatrixRelativeFrom = self.get_rpTransitionMatrices()
 
@@ -85,7 +85,7 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F
                 self.dPower_ThermalGen = dPower_ThermalGen
             else:
                 self.power_thermalgen_file = power_thermalgen_file
-                self.dPower_ThermalGen = ExcelReader.get_dPower_ThermalGen(self.example_folder + self.power_thermalgen_file)
+                self.dPower_ThermalGen = ExcelReader.get_dPower_ThermalGen(self.data_folder + self.power_thermalgen_file)
 
         if self.dPower_Parameters["pEnableRoR"]:
             if dPower_RoR is not None:
@@ -105,13 +105,13 @@ def __init__(self, example_folder: str, do_not_merge_single_node_buses: bool = F
                 self.dPower_VRES = dPower_VRES
             else:
                 self.power_vres_file = power_vres_file
-                self.dPower_VRES = ExcelReader.get_dPower_VRES(self.example_folder + self.power_vres_file)
+                self.dPower_VRES = ExcelReader.get_dPower_VRES(self.data_folder + self.power_vres_file)
 
             if dPower_VRESProfiles is not None:
                 self.dPower_VRESProfiles = dPower_VRESProfiles
             else:
                 self.power_vresprofiles_file = power_vresprofiles_file
-                self.dPower_VRESProfiles = ExcelReader.get_dPower_VRESProfiles(self.example_folder + self.power_vresprofiles_file)
+                self.dPower_VRESProfiles = ExcelReader.get_dPower_VRESProfiles(self.data_folder + self.power_vresprofiles_file)
 
         if self.dPower_Parameters["pEnableStorage"]:
             if dPower_Storage is not None:
@@ -143,7 +143,7 @@ def copy(self):
         return copy.deepcopy(self)
 
     def get_dGlobal_Parameters(self):
-        dGlobal_Parameters = pd.read_excel(self.example_folder + self.global_parameters_file, skiprows=[0, 1])
+        dGlobal_Parameters = pd.read_excel(self.data_folder + self.global_parameters_file, skiprows=[0, 1])
         dGlobal_Parameters = dGlobal_Parameters.drop(dGlobal_Parameters.columns[0], axis=1)
         dGlobal_Parameters = dGlobal_Parameters.set_index('Sectors')
 
@@ -154,7 +154,7 @@ def get_dGlobal_Parameters(self):
         return dGlobal_Parameters
 
     def get_dPower_Parameters(self):
-        dPower_Parameters = pd.read_excel(self.example_folder + self.power_parameters_file, skiprows=[0, 1])
+        dPower_Parameters = pd.read_excel(self.data_folder + self.power_parameters_file, skiprows=[0, 1])
         dPower_Parameters = dPower_Parameters.drop(dPower_Parameters.columns[0], axis=1)
         dPower_Parameters = dPower_Parameters.dropna(how="all")
         dPower_Parameters = dPower_Parameters.set_index('General')
@@ -186,14 +186,14 @@ def yesNo_to_bool(df: pd.DataFrame, columns_to_be_changed: list[str]):
         return df
 
     def get_dPower_RoR(self):
-        dPower_RoR = self.read_generator_data(self.example_folder + self.power_ror_file)
+        dPower_RoR = self.read_generator_data(self.data_folder + self.power_ror_file)
 
         dPower_RoR['InvestCostEUR'] = dPower_RoR['MaxProd'] * 1e-3 * (dPower_RoR['InvestCostPerMW'] * 1e-3 + dPower_RoR['InvestCostPerMWh'] * 1e-3 * dPower_RoR['Ene2PowRatio'])
         dPower_RoR['MaxProd'] *= 1e-3
         return dPower_RoR
 
     def get_dPower_Storage(self):
-        dPower_Storage = self.read_generator_data(self.example_folder + self.power_storage_file)
+        dPower_Storage = self.read_generator_data(self.data_folder + self.power_storage_file)
         dPower_Storage['pOMVarCostEUR'] = dPower_Storage['OMVarCost'] * 1e-3
         dPower_Storage['IniReserve'] = dPower_Storage['IniReserve'].fillna(0)
         dPower_Storage['MinReserve'] = dPower_Storage['MinReserve'].fillna(0)
@@ -204,15 +204,15 @@ def get_dPower_Storage(self):
         return dPower_Storage
 
     def get_dPower_Inflows(self):
-        dPower_Inflows = pd.read_excel(self.example_folder + self.power_inflows_file, skiprows=[0, 1, 3, 4, 5])
+        dPower_Inflows = pd.read_excel(self.data_folder + self.power_inflows_file, skiprows=[0, 1, 3, 4, 5])
         dPower_Inflows = dPower_Inflows.drop(dPower_Inflows.columns[0], axis=1)
         dPower_Inflows = dPower_Inflows.rename(columns={dPower_Inflows.columns[0]: "rp", dPower_Inflows.columns[1]: "g"})
         dPower_Inflows = dPower_Inflows.melt(id_vars=['rp', 'g'], var_name='k', value_name='Inflow')
         dPower_Inflows = dPower_Inflows.set_index(['rp', 'g', 'k'])
         return dPower_Inflows
 
     def get_dPower_ImpExpHubs(self):
-        dPower_ImpExpHubs = pd.read_excel(self.example_folder + self.power_impexphubs_file, skiprows=[0, 1, 3, 4, 5])
+        dPower_ImpExpHubs = pd.read_excel(self.data_folder + self.power_impexphubs_file, skiprows=[0, 1, 3, 4, 5])
         dPower_ImpExpHubs = dPower_ImpExpHubs.drop(dPower_ImpExpHubs.columns[0], axis=1)
         dPower_ImpExpHubs = dPower_ImpExpHubs.set_index(['hub', 'i'])
 
@@ -238,7 +238,7 @@ def get_dPower_ImpExpHubs(self):
 
     def get_dPower_ImpExpProfiles(self):
         with warnings.catch_warnings(action="ignore", category=UserWarning):  # Otherwise there is a warning regarding data validation in the Excel-File (see https://stackoverflow.com/questions/53965596/python-3-openpyxl-userwarning-data-validation-extension-not-supported)
-            dPower_ImpExpProfiles = pd.read_excel(self.example_folder + self.power_impexpprofiles_file, skiprows=[0, 1, 3, 4, 5], sheet_name='Power ImpExpProfiles')
+            dPower_ImpExpProfiles = pd.read_excel(self.data_folder + self.power_impexpprofiles_file, skiprows=[0, 1, 3, 4, 5], sheet_name='Power ImpExpProfiles')
         dPower_ImpExpProfiles = dPower_ImpExpProfiles.drop(dPower_ImpExpProfiles.columns[0], axis=1)
         dPower_ImpExpProfiles = dPower_ImpExpProfiles.melt(id_vars=['hub', 'rp', 'Type'], var_name='k', value_name='Value')
 

ExcelReader.py

@@ -5,7 +5,7 @@
 import pandas as pd
 from openpyxl import load_workbook
 
-from InOutModule.printer import Printer
+from printer import Printer
 
 printer = Printer.getInstance()
 

ExcelWriter.py

@@ -8,10 +8,10 @@
 import pandas as pd
 from openpyxl.utils.dataframe import dataframe_to_rows
 
-import InOutModule.TableDefinition
-from InOutModule import ExcelReader, TableDefinition
-from InOutModule.TableDefinition import CellStyle, Alignment, Font, Color, Text, Column, NumberFormat, TableDefinition
-from InOutModule.printer import Printer
+import ExcelReader
+import TableDefinition
+from TableDefinition import CellStyle, Alignment, Font, Color, Text, Column, NumberFormat, TableDefinition
+from printer import Printer
 
 package_directory_ExcelWriter = os.path.dirname(os.path.abspath(__file__))
 
@@ -40,7 +40,7 @@ def __init__(self, excel_definitions_path: str = "TableDefinitions.xml"):
         pass
 
     @staticmethod
-    def __setCellStyle(cell_style: InOutModule.TableDefinition.CellStyle, target_cell: openpyxl.cell.cell):
+    def __setCellStyle(cell_style: CellStyle, target_cell: openpyxl.cell.cell):
         """
         Set the cell style of a target cell based on the given cell style.
 

environment.yml

@@ -0,0 +1,14 @@
+name: InOutModule_env
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - openpyxl=3.1.5
+  - pandas=2.2.3
+  - pip=24.0
+  - pytest=8.4.0
+  - python=3.12.2
+  - rich=13.7.1
+  - rich-argparse=1.6.0
+  - pip:
+      - pulp==2.9.0

printer.py

@@ -1,6 +1,5 @@
 import datetime
 
-from pyomo import environ as pyo
 from rich.console import Console
 
 
@@ -217,6 +216,7 @@ def pprint_zoi_var(var, zoi, index_positions: list = None, decimals: int = 2):
     :param decimals: The number of decimal places to display for the variable's value.
     :return: None
     """
+    from pyomo import environ as pyo  # Import Pyomo environment for variable handling
 
     if index_positions is None:
         index_positions = [0]

tests/test_ExcelReaderWriter.py

@@ -0,0 +1,36 @@
+import pytest
+
+import ExcelReader as ExcelReader
+from ExcelWriter import ExcelWriter
+from printer import Printer
+
+printer = Printer.getInstance()
+
+case_study_folder = "examples/"
+ew = ExcelWriter()
+combinations = [
+    ("Power_Hindex", f"{case_study_folder}Power_Hindex.xlsx", ExcelReader.get_dPower_Hindex, ew.write_dPower_Hindex),
+    ("Power_WeightsRP", f"{case_study_folder}Power_WeightsRP.xlsx", ExcelReader.get_dPower_WeightsRP, ew.write_dPower_WeightsRP),
+    ("Power_WeightsK", f"{case_study_folder}Power_WeightsK.xlsx", ExcelReader.get_dPower_WeightsK, ew.write_dPower_WeightsK),
+    ("Power_BusInfo", f"{case_study_folder}Power_BusInfo.xlsx", ExcelReader.get_dPower_BusInfo, ew.write_dPower_BusInfo),
+    ("Power_Network", f"{case_study_folder}Power_Network.xlsx", ExcelReader.get_dPower_Network, ew.write_dPower_Network),
+    ("Power_Demand", f"{case_study_folder}Power_Demand.xlsx", ExcelReader.get_dPower_Demand, ew.write_dPower_Demand),
+    ("Power_ThermalGen", f"{case_study_folder}Power_ThermalGen.xlsx", ExcelReader.get_dPower_ThermalGen, ew.write_dPower_ThermalGen),
+    ("Power_VRES", f"{case_study_folder}Power_VRES.xlsx", ExcelReader.get_dPower_VRES, ew.write_VRES),
+    ("Power_VRESProfiles", f"{case_study_folder}Power_VRESProfiles.xlsx", ExcelReader.get_dPower_VRESProfiles, ew.write_VRESProfiles),
+    ("Data_Sources", f"{case_study_folder}Data_Sources.xlsx", ExcelReader.get_dData_Sources, ew.write_dData_Sources),
+    ("Data_Packages", f"{case_study_folder}Data_Packages.xlsx", ExcelReader.get_dData_Packages, ew.write_dData_Packages),
+]
+
+
+@pytest.mark.parametrize("excel_definition_id, file_path, read, write", combinations)
+def test_read_and_write(excel_definition_id, file_path, read, write, tmp_path):
+    printer.information(f"Writing '{excel_definition_id}', read from '{file_path}'")
+
+    data = read(file_path, True, True)
+    write(data, str(tmp_path))
+
+    printer.information(f"Comparing '{tmp_path}/{excel_definition_id}.xlsx' against source file '{file_path}'")
+    filesEqual = ExcelReader.compare_Excels(file_path, f"{tmp_path}/{excel_definition_id}.xlsx", dont_check_formatting=False)
+
+    assert filesEqual, f"Read and/or Write for {excel_definition_id} are faulty - please check!"