Improved Null handling. Nullable dates/ints now work.

Author: dames

configuraton/provider.json

@@ -82,6 +82,23 @@
         "type": "sqlalchemy.DateTime",
         "nullable": true
       }
+    },
+     {
+      "source_name": "LongString",
+      "destination": {
+        "name": "long_string",
+        "type": "citext.CIText",
+        "nullable": true
+      }
+    },
+     {
+      "source_name": "UnicodeString",
+      "destination": {
+        "name": "unicode_string",
+        "type": "citext.CIText",
+        "nullable": true
+      }
     }
+
   ]
 }

relational_data_loader_project/BatchDataLoader.py

@@ -61,14 +61,14 @@ def write_data_frame_to_table(self, data_frame, table_configuration, target_engi
         destination_table = "{0}.{1}".format(table_configuration['schema'], table_configuration['name'])
         self.logger.info("Starting write to table {0}".format(destination_table))
         data = StringIO()
-        data_frame.to_csv(data, header=False, index=False)
+        data_frame.to_csv(data, header=False, index=False, na_rep='')
         data.seek(0)
         raw = target_engine.raw_connection()
         curs = raw.cursor()
 
         column_array = list(map(lambda cfg: cfg['destination']['name'], self.columns))
 
-        curs.copy_from(data, destination_table, sep=',', columns=column_array)
+        curs.copy_from(data, destination_table, sep=',', columns=column_array, null='')
         self.logger.info("Completed write to table {0}".format(destination_table))
 
         curs.connection.commit()