Merge pull request #1 from PageUpPeopleOrg/OSC-907-CI

Relational data loader integration tests

Author: DavidAmesPup

.gitignore

@@ -11,13 +11,13 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+lib/
 build/
 develop-eggs/
 dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

README.md

@@ -1,2 +1,39 @@
 # RelationalDataLoader
-A utility for taking data from MS-SQL and loading it into PostgeSQL
+## About
+A utility for taking data from MS-SQL and loading it into PostgreSQL
+
+
+## Usage
+Execute  `py rdl.py SOURCE DESTINATION CONFIGURATION-FOLDER [log-level] [full-refresh]`
+
+Where `SOURCE` takes the following formats
+**CSV:**  `csv://.\test_data\full_refresh`
+**MSSQL:**  `mssql+pyodbc://dwsource`
+
+In the above example, dwsource is a 64bit ODBC system dsn
+
+
+`DESTINATION` takes the following format
+**PostgreSQL:**  `postgresql+psycopg2://postgres:xxxx@localhost/dest_dw`
+
+
+### Examples
+#### CSV Source
+
+`py rdl.py csv://.\test_data\full_refresh postgresql+psycopg2://postgres:xxxx@localhost/dest_dw .\configuration\ --log-level INFO --full-refresh yes`
+`py rdl.py csv://.\test_data\incremental_refresh postgresql+psycopg2://postgres:xxxx@localhost/dest_dw .\configuration\ --log-level INFO --full-refresh no`
+
+
+#### MSSQL Source
+
+
+
+### Troubleshooting
+Run with  `--log-level DEBUG` on the command line.
+
+
+##Other Notes
+###Testing
+The test batch files assume there is a user by the name of `postgres` on the system.
+It also sends through a nonense password - it is assumed that the target system is running in 'trust' mode.
+See https://www.postgresql.org/docs/9.1/static/auth-pg-hba-conf.html for details on trust mode

appveyor.yml

@@ -0,0 +1,54 @@
+version: 1.0.{build}
+build: off
+
+environment:
+  PGUSER: "postgres"
+  PGPASSWORD: "Password12!"
+  PGPORT: 5432
+  PGHOST: "localhost"
+  DBNAME: relational_data_loader_integration_tests
+  matrix:
+    - PYTHON: "C:\\Python36-x64" #This needs to be a double slash
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
+
+services:
+  - postgresql101
+  - mssql2016
+
+init:
+  - set PATH=C:\program Files\PostgreSQL\10\bin\;%PATH%
+  - ps: Set-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   ::1/128        trust"
+  - ps: Add-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   127.0.0.1/32   trust"
+  #Enable this line to enable RDP for the build.
+  #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
+install:
+
+  #Setup Python
+  - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% #Swap path to python to latest version (as per matrix above)
+  - python --version
+  - python -m venv c:\projects\relational-data-loader\venv\
+
+build_script:
+  #Setup the source MSSQL database
+  - sqlcmd -b -E -S "(local)\SQL2016" -i .\integration_tests\mssql_source\source_database_setup\create_database.sql
+  - sqlcmd -b -E -f 65001 -S "(local)\SQL2016" -d RelationalDataLoaderIntegrationTestSource -i .\integration_tests\mssql_source\source_database_setup\create_large_table.sql
+
+  #Setup the target PostgreSQL database
+  - psql -c "SELECT VERSION()"
+  - createdb %DBNAME%
+  - psql -d %DBNAME% -c "CREATE EXTENSION IF NOT EXISTS citext"
+
+  #Install the dependencies for rdl.
+  - pip install -r requirements.txt
+
+
+test_script:
+  - test_full_refresh_from_csv.cmd
+  - test_incremental_refresh_from_csv.cmd
+  - test_full_refresh_from_mssql.cmd
+
+on_finish:
+  #Enable this line to make the build pause after completion for RDP troubleshooting.
+  #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql

@@ -0,0 +1,35 @@
+SET client_encoding TO 'UTF8';
+
+DROP TABLE IF EXISTS results;
+
+CREATE TEMPORARY TABLE results AS
+  WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1) AS (
+    SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String'
+    UNION ALL
+    SELECT 2, NULL, NULL, NULL, NULL, NULL
+    UNION ALL
+    SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave'
+    UNION ALL
+    SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ'
+    UNION ALL
+    SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test'
+  ),
+
+  actual AS (
+    SELECT  id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1
+    FROM    rdl_integration_tests.load_source_data
+  )
+
+  SELECT * FROM expected
+  EXCEPT
+  SELECT * FROM actual;
+
+DO $$
+BEGIN
+    PERFORM * FROM results;
+    IF FOUND THEN RAISE EXCEPTION '[FULL REFRESH TEST] FAIL: The actual data did not match the expected data for the CSV refresh';
+    ELSE
+    RAISE NOTICE '[FULL REFRESH TEST] PASS';
+    END IF;
+END $$;
+

integration_tests/csv_source/assertions/column_test_incremental_refresh_assertions.sql

@@ -0,0 +1,38 @@
+SET client_encoding TO 'UTF8';
+DROP TABLE IF EXISTS results;
+
+CREATE TEMPORARY TABLE results AS
+  WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1) AS (
+    SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String'
+    UNION ALL
+    SELECT 2, NULL, NULL, NULL, NULL, NULL
+    UNION ALL
+    SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave'
+    UNION ALL
+    SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ'
+    UNION ALL
+    SELECT 5, NULL, NULL, NULL, NULL, 'This row WAS updated in the incremental review test'
+    UNION ALL
+    SELECT 6, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String'
+    UNION ALL
+    SELECT 7, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'Another Basic String'
+  ),
+
+  actual AS (
+    SELECT  id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1
+    FROM    rdl_integration_tests.load_source_data
+  )
+
+  SELECT * FROM expected
+  EXCEPT
+  SELECT * FROM actual;
+
+DO $$
+BEGIN
+    PERFORM * FROM results;
+    IF FOUND THEN RAISE EXCEPTION '[INCREMENTAL REFRESH TEST] FAIL: The actual data did not match the expected data for the CSV refresh';
+    ELSE
+    RAISE NOTICE '[INCREMENTAL REFRESH TEST] PASS';
+    END IF;
+END $$;
+

configuration/ColumnTest.json …_tests/csv_source/config/ColumnTest.jsonconfiguration/ColumnTest.json renamed to integration_tests/csv_source/config/ColumnTest.json configuration/ColumnTest.json renamed to integration_tests/csv_source/config/ColumnTest.json

@@ -26,7 +26,7 @@
       "source_name": "IntColumn1",
       "destination": {
         "name": "int_column_1",
-        "type": "citext.CIText",
+        "type": "sqlalchemy.Numeric",
         "nullable": true
       }
     },

test_data/full_refresh/ColumnTest.csv …_source/full_refresh_data/ColumnTest.csvtest_data/full_refresh/ColumnTest.csv renamed to integration_tests/csv_source/full_refresh_data/ColumnTest.csv test_data/full_refresh/ColumnTest.csv renamed to integration_tests/csv_source/full_refresh_data/ColumnTest.csv

@@ -2,5 +2,5 @@ id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1
 1,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am
 2,,,,,
 3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,
-4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ ",,,,
+4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,
 5,"This row will be updated in the incremental review test"

…_data/incremental_refresh/ColumnTest.csv …/incremental_refresh_data/ColumnTest.csvtest_data/incremental_refresh/ColumnTest.csv renamed to integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv test_data/incremental_refresh/ColumnTest.csv renamed to integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv

@@ -0,0 +1,13 @@
+SET client_encoding TO 'UTF8';
+
+
+DO $$
+  BEGIN
+  IF (SELECT COUNT(*) FROM rdl_integration_tests.load_large_data ) = 1000000 THEN
+    RAISE NOTICE '[LARGE MSSQL IMPORT TEST] PASS';
+  ELSE
+    RAISE EXCEPTION '[LARGE MSSQL IMPORT TEST] FAIL: Did not find the required 1,000,000 rows.';
+  END IF;
+END $$;
+
+

integration_tests/mssql_source/assertions/large_table_test_full_refresh_assertions.sql

@@ -0,0 +1,75 @@
+{
+
+  "source_table": {
+    "name": "LargeTable",
+    "schema": "dbo",
+    "primary_key": "Id"
+  },
+  "target_schema": "rdl_integration_tests",
+  "stage_table": "stage_large_data",
+  "load_table": "load_large_data",
+
+  "batch": {
+    "size": 100000
+  },
+  "columns": [
+    {
+      "source_name": "Id",
+      "destination": {
+        "name": "id",
+        "type": "sqlalchemy.Integer",
+        "nullable": false,
+        "primary_key": true
+      }
+    },
+    {
+      "source_name": "DateColumn1",
+      "destination": {
+        "name": "date_column_1",
+        "type": "sqlalchemy.DateTime",
+        "nullable": true
+      }
+    },
+    {
+      "source_name": "IntColumn1",
+      "destination": {
+        "name": "int_column_1",
+        "type": "sqlalchemy.Numeric",
+        "nullable": true
+      }
+    },
+    {
+      "source_name": "DateColumn2",
+      "destination": {
+        "name": "date_column_2",
+        "type": "sqlalchemy.DateTime",
+        "nullable": true
+      }
+    },
+     {
+      "source_name": "StringColumn1",
+      "destination": {
+        "name": "string_column_1",
+        "type": "citext.CIText",
+        "nullable": true
+      }
+    },
+    {
+      "source_name": "StringColumn2",
+      "destination": {
+        "name": "string_column_2",
+        "type": "citext.CIText",
+        "nullable": true
+      }
+    },
+    {
+      "source_name": "GuidColumn",
+      "destination": {
+        "name": "guid_column",
+        "type": "citext.CIText",
+        "nullable": true
+      }
+    }
+
+  ]
+}