Merge pull request #16 from PageUpPeopleOrg/task/shave-that-yak

[OSC-1236] - Minor useful logic changes and yak shaves

Author: ChintanRaval

README.md

@@ -52,6 +52,33 @@ See `test_*.cmd` scripts for usage samples.
 
 ## Development
 
+### Linting
+
+Use autopep8 before pushing commits (include the "." for the folder)
+
+`>>>autopep8 --in-place --aggressive --aggressive --recursive --max-line-length=120 .`
+
+Use the following vscode settings by either:
+
+- System wide: Ctrl+Shift+P, "open settings (json)"
+- For this project only: add the following to ./vscode/settings.json
+
+```json
+{
+    "python.pythonPath": (your python venv here),
+    "files.trimFinalNewlines": true,
+    "files.trimTrailingWhitespace": true,
+    "files.insertFinalNewline": true,
+    "files.eol": "\n",
+    "editor.renderWhitespace": "all",
+    "editor.tabCompletion": "on",
+    "editor.trimAutoWhitespace": true,
+    "editor.insertSpaces": true,
+    "editor.rulers": [80, 120],
+    "editor.formatOnSave": true
+}
+```
+
 ### Testing
 
 #### Integration

__init__.py

@@ -7,6 +7,7 @@ environment:
   PGPORT: 5432
   PGHOST: "localhost"
   DBNAME: relational_data_loader_integration_tests
+  MSSQLSTR: "{'username': '','password': '','server_string': '(local)\\SQL2016'}"
   matrix:
     - PYTHON: "C:\\Python37-x64" #This needs to be a double slash
       PYTHON_VERSION: "3.7.x"
@@ -24,7 +25,6 @@ init:
   #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 
 install:
-
   #Setup Python
   - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% #Swap path to python to latest version (as per matrix above)
   - python --version
@@ -44,15 +44,27 @@ build_script:
   #Install the dependencies for rdl.
   - pip install -r requirements.txt
 
-
 test_script:
+  # unit tests
+  - test_run_unit_tests.cmd
+
+  # integration tests
   - test_full_refresh_from_csv.cmd
   - test_full_refresh_from_mssql.cmd
   - test_incremental_refresh_from_csv.cmd
   - test_incremental_refresh_from_mssql.cmd
+
   - test_full_refresh_from_mssql.cmd
   - test_incremental_refresh_from_mssql.cmd
 
+  - sqlcmd -b -E -f 65001 -S "(local)\SQL2016" -d RelationalDataLoaderIntegrationTestSource -i .\integration_tests\mssql_source\source_database_setup\change_compound_pk.sql
+
+  - test_incremental_refresh_from_mssql.cmd
+
+  - sqlcmd -b -E -f 65001 -S "(local)\SQL2016" -d RelationalDataLoaderIntegrationTestSource -i .\integration_tests\mssql_source\source_database_setup\change_large_table.sql
+
+  - test_full_refresh_from_mssql.cmd
+
 on_finish:
   #Enable this line to make the build pause after completion for RDP troubleshooting.
   #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

appveyor.yml

@@ -4,31 +4,36 @@ DROP TABLE IF EXISTS results;
 
 CREATE TEMPORARY TABLE results AS
 WITH
-  expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1, big_int_column_1, bool_column_1)
-  AS
-  (
-    SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212, True
+    expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1, big_int_column_1, bool_column_1)
+    AS
+    (
+        SELECT
+        1, 111.0, '1976-12-01'::DATE,
+        12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP,
+        'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212,
+        True
     UNION ALL
-  SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
-UNION ALL
-  SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True
-UNION ALL
-  SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
-UNION ALL
-  SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL, NULL
-),
+        SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
+    UNION ALL
+        SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True
+    UNION ALL
+        SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
+    UNION ALL
+        SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL, NULL
+    ),
 
-  actual AS
-(
+    actual
+    AS
+    (
     SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1, big_int_column_1, bool_column_1
-FROM rdl_integration_tests.load_source_data
-  )
+    FROM rdl_integration_tests.load_source_data
+    )
 
-  SELECT *
-  FROM expected
-EXCEPT
-  SELECT *
-  FROM actual;
+    SELECT *
+    FROM expected
+    EXCEPT
+    SELECT *
+    FROM actual;
 
 DO $$
 BEGIN

integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql

@@ -1,5 +1,4 @@
 {
-
   "source_table": {
     "name": "ColumnTest",
     "schema": "dbo",
@@ -54,42 +53,37 @@
         "nullable": true
       }
     },
-     {
+    {
       "source_name": "StringColumn1",
       "destination": {
         "name": "string_column_1",
         "type": "string",
         "nullable": true
       }
     },
-     {
+    {
       "source_name": "GuidColumn1",
       "destination": {
         "name": "guid_column_1",
         "type": "guid",
         "nullable": true
       }
     },
-     {
+    {
       "source_name": "BigIntColumn1",
       "destination": {
         "name": "big_int_column_1",
         "type": "bigint",
         "nullable": true
       }
     },
-     {
+    {
       "source_name": "BoolColumn1",
       "destination": {
         "name": "bool_column_1",
         "type": "boolean",
         "nullable": true
       }
     }
-
-
-
-
-
   ]
 }

integration_tests/csv_source/config/ColumnTest.json

@@ -3,4 +3,4 @@ id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColum
 2,,,,,,,,
 3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,,,,1
 4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,,AABC8093-FE4C-477A-BBD7-FB5C02055A7E,,0
-5,"This row will be updated in the incremental review test",,
+5,"This row will be updated in the incremental review test",,

integration_tests/csv_source/full_refresh_data/ColumnTest.csv

@@ -1,4 +1,4 @@
 id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1,BoolColumn1
 5,"This row WAS updated in the incremental review test",,,,,,,1
 6,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,,0
-7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,,,1
+7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,,,1

integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv

@@ -1,13 +1,16 @@
-SET client_encoding TO 'UTF8';
-
+SET client_encoding
+TO 'UTF8';
 
 DO $$
-  BEGIN
-  IF (SELECT COUNT(*) FROM rdl_integration_tests.load_large_data ) = 1000000 THEN
+BEGIN
+  IF ((SELECT COUNT(*)
+    FROM rdl_integration_tests.load_large_data ) = 1000000)
+    OR
+    ((SELECT COUNT(*)
+    FROM rdl_integration_tests.load_large_data ) = 1000001) THEN
     RAISE NOTICE '[LARGE MSSQL IMPORT TEST] PASS';
-  ELSE
+ELSE
     RAISE EXCEPTION '[LARGE MSSQL IMPORT TEST] FAIL: Did not find the required 1,000,000 rows.';
-  END IF;
+END
+IF;
 END $$;
-
-

integration_tests/mssql_source/assertions/large_table_test_full_refresh_assertions.sql

@@ -2,7 +2,7 @@
   "source_table": {
     "name": "CompoundPk",
     "schema": "dbo",
-    "primary_keys": ["Id1","Id2"]
+    "primary_keys": ["Id1", "Id2"]
   },
   "target_schema": "rdl_integration_tests",
   "stage_table": "stage_compound_pk",
@@ -21,7 +21,7 @@
         "primary_key": true
       }
     },
-      {
+    {
       "source_name": "Id2",
       "destination": {
         "name": "id_2",
@@ -30,6 +30,5 @@
         "primary_key": true
       }
     }
-
   ]
 }

integration_tests/mssql_source/config/CompoundPkTest.json

@@ -45,7 +45,7 @@
         "nullable": true
       }
     },
-     {
+    {
       "source_name": "StringColumn1",
       "destination": {
         "name": "string_column_1",
@@ -69,15 +69,13 @@
         "nullable": true
       }
     },
-     {
+    {
       "source_name": "BoolColumn",
       "destination": {
         "name": "bool_column_1",
         "type": "boolean",
         "nullable": true
       }
     }
-
-
   ]
 }