Added main project file

Author: dames

BatchDataLoader.py …l_data_loader_project/BatchDataLoader.pyBatchDataLoader.py renamed to relational_data_loader_project/BatchDataLoader.py BatchDataLoader.py renamed to relational_data_loader_project/BatchDataLoader.py

@@ -1,8 +1,8 @@
 import logging
-from BatchDataLoader import BatchDataLoader
-from DestinationTableManager import DestinationTableManager
-from DataLoadTracker import DataLoadTracker
-from SourceTableManager import SourceTableManager
+from relational_data_loader_project.BatchDataLoader import BatchDataLoader
+from relational_data_loader_project.DestinationTableManager import DestinationTableManager
+from relational_data_loader_project.DataLoadTracker import DataLoadTracker
+from relational_data_loader_project.SourceTableManager import SourceTableManager
 import json
 
 
@@ -21,7 +21,8 @@ def start_import(self, source_engine, target_engine, configuration_name, full_lo
         self.logger.debug("Execute Starting")
         destination_table_manager = DestinationTableManager()
 
-        columns = self.remove_invalid_columns(pipeline_configuration['source_table'], pipeline_configuration['columns'], source_engine)
+        columns = self.remove_invalid_columns(pipeline_configuration['source_table'], pipeline_configuration['columns'],
+                                              source_engine)
 
         if full_load:
             self.logger.info("Full-load is set. Recreating the staging table.")

DataLoadManager.py …l_data_loader_project/DataLoadManager.pyDataLoadManager.py renamed to relational_data_loader_project/DataLoadManager.py DataLoadManager.py renamed to relational_data_loader_project/DataLoadManager.py

@@ -0,0 +1,45 @@
+import logging
+from relational_data_loader_project.DataLoadManager import DataLoadManager
+from sqlalchemy import create_engine
+import argparse
+
+
+def main(args):
+    configure_logging()
+    source_engine = create_engine(args['source-engine'])
+    destination_engine = create_engine(args['destination-engine'])
+
+    data_load_manager = DataLoadManager(args['configuration-folder'])
+    data_load_manager.start_import(source_engine, destination_engine, "provider", True)
+
+
+def configure_logging():
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    log = logging.getLogger()
+    console_stream_handler = logging.StreamHandler()
+    console_stream_handler.setFormatter(formatter)
+    log.addHandler(console_stream_handler)
+    log.setLevel(logging.DEBUG)
+    return
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description='Relational Data Loader')
+
+    parser.add_argument('source-engine', metavar='source-engine',
+                        help='The source engine. Eg: mssql+pyodbc://dwsource')
+
+    parser.add_argument('destination-engine', metavar='destination-engine',
+                        help='The destination engine. Eg: postgresql+psycopg2://postgres:xxxx@localhost/dest_dw')
+
+    parser.add_argument('configuration-folder', metavar='configuration-folder',
+                        help='The configration folder. Eg C:\\_dev\\oscars-misc\\el-pipeline-spike\\configuraton\\')
+
+    return vars(parser.parse_args())
+
+
+if __name__ == "__main__":
+    args = get_arguments()
+    main(args)
+
+
+