[OSC-1241] improve logging for data frame to csv to psql process

Author: Sean Budd

README.md

@@ -81,6 +81,29 @@ Use the following vscode settings by either:
 
 ### Testing
 
+### Postgres debugging
+
+Ensure the database you are using is in utf8 mode. You cannot change encoding once the database is created.
+
+```sql
+
+CREATE DATABASE "my_database"
+    WITH OWNER "postgres"
+    ENCODING 'UTF8'
+    TEMPLATE template0;
+
+```
+
+Also ensure that the database has the CITEXT extension by logging into the DB and adding it
+
+```sql
+
+>>>psql my_database
+
+CREATE EXTENSION CITEXT;
+
+```
+
 #### Integration
 
 The test batch files assume there is a user by the name of `postgres` on the system.

modules/BatchDataLoader.py

@@ -58,6 +58,11 @@ def write_data_frame_to_table(self, data_frame):
         raw = self.target_db.raw_connection()
         curs = raw.cursor()
 
+        # log CSV on debug
+        if self.logger.getEffectiveLevel() == logging.DEBUG:
+            with open(f'{qualified_target_table}.csv', 'w', encoding='utf-8') as f:
+                f.write(data.getvalue())
+
         column_array = list(
             map(lambda source_colum_name: self.get_destination_column_name(source_colum_name), data_frame.columns))
         column_list = ','.join(map(str, column_array))