Skip to content

Commit c3668e3

Browse files
rfdearbornrfdearborn
committed
Initialize tests
1 parent 357b478 commit c3668e3

File tree

8 files changed

+350
-8
lines changed

8 files changed

+350
-8
lines changed

‎.github/workflows/test.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Run Tests
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- uses: actions/checkout@v2
15+
- name: Set up Python 3.7
16+
uses: actions/setup-python@v2
17+
with:
18+
python-version: '3.7'
19+
- name: Install dependencies
20+
run: |
21+
pip install requests
22+
pip install mock
23+
- name: Run unit tests
24+
run: |
25+
python -m unittest tests/test_unit.py
26+
- name: Run integration tests
27+
run: |
28+
python -m unittest tests/test_integration.py

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1-
.vscode
1+
.vscode
2+
__pycache__
3+
/tests/__pycache__

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,5 +69,4 @@ jobs:
6969
7070
## Todo
7171
72-
- Tests
7372
- Visualize models graph

‎dbt_docs_to_notion.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,10 @@ def get_owner(data, catalog_nodes, model_name):
5858
return get_paths_or_empty(catalog_nodes, [[model_name, 'metadata', 'owner']], '')
5959

6060

61-
def main():
62-
model_records_to_write = sys.argv[1:] # 'all' or list of model names
61+
def main(argv=None):
62+
if argv is None:
63+
argv = sys.argv
64+
model_records_to_write = argv[1:] # 'all' or list of model names
6365
print(f'Model records to write: {model_records_to_write}')
6466

6567
###### load nodes from dbt docs ######
@@ -313,15 +315,15 @@ def main():
313315
"children": columns_table_children_obj
314316
}
315317
},
316-
# Raw SQL
318+
# Raw Code
317319
{
318320
"object": "block",
319321
"type": "heading_1",
320322
"heading_1": {
321323
"rich_text": [
322324
{
323325
"type": "text",
324-
"text": { "content": "Raw SQL" }
326+
"text": { "content": "Raw Code" }
325327
}
326328
]
327329
}
@@ -341,15 +343,15 @@ def main():
341343
"language": "sql"
342344
}
343345
},
344-
# Compiled SQL
346+
# Compiled Code
345347
{
346348
"object": "block",
347349
"type": "heading_1",
348350
"heading_1": {
349351
"rich_text": [
350352
{
351353
"type": "text",
352-
"text": { "content": "Compiled SQL" }
354+
"text": { "content": "Compiled Code" }
353355
}
354356
]
355357
}
@@ -509,5 +511,6 @@ def main():
509511
json=record_obj
510512
)
511513

514+
512515
if __name__ == '__main__':
513516
main()

‎tests/__init__.py‎

Whitespace-only changes.

‎tests/mock_data.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Mock Data for dbt and Notion API
2+
3+
# Mock dbt Data
4+
DBT_MOCK_CATALOG = {
5+
"nodes": {
6+
"model.test.model_1": {
7+
"columns": {
8+
"column_1": {
9+
"type": "TEXT"
10+
},
11+
"column_2": {
12+
"type": "TEXT"
13+
},
14+
},
15+
"metadata": {
16+
"owner": "owner@example.com"
17+
},
18+
"stats": {
19+
"row_count": {
20+
"value": 1,
21+
},
22+
"bytes": {
23+
"value": 1000000,
24+
},
25+
},
26+
},
27+
},
28+
}
29+
30+
DBT_MOCK_MANIFEST = {
31+
"nodes": {
32+
"model.test.model_1": {
33+
"resource_type": "model",
34+
"columns": {
35+
"column_1": {
36+
"description": "Description for column 1"
37+
},
38+
"column_2": {
39+
"description": "Description for column 2"
40+
},
41+
},
42+
"raw_code": "SELECT 1",
43+
"compiled_code": "SELECT 1",
44+
"name": "model_1",
45+
"description": "Description for model 1",
46+
"relation_name": "model.test.model_1",
47+
"depends_on": ["model.test.model_2"],
48+
"tags": ["tag1", "tag2"],
49+
},
50+
},
51+
}
52+
53+
# Mock Notion API Responses
54+
NOTION_MOCK_EXISTENT_CHILD_PAGE_QUERY = {
55+
"results": [
56+
{
57+
"id": "mock_child_id",
58+
"child_database": {
59+
"title": "dbt Models",
60+
},
61+
},
62+
],
63+
}
64+
65+
NOTION_MOCK_EXISTENT_DATABASE_RECORDS_QUERY = {
66+
"results": [
67+
{
68+
"id": "mock_record_id",
69+
},
70+
],
71+
}
72+
73+
NOTION_MOCK_NONEXISTENT_QUERY = {
74+
"results": [],
75+
}
76+
77+
NOTION_MOCK_DATABASE_CREATE = {
78+
"id": "mock_database_id",
79+
}
80+
81+
NOTION_MOCK_RECORD_CREATE = {
82+
"id": "mock_record_id",
83+
}

‎tests/test_integration.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
import json
2+
import os
3+
import unittest
4+
from unittest.mock import patch, Mock
5+
6+
from dbt_docs_to_notion import main
7+
from tests.mock_data import (
8+
DBT_MOCK_MANIFEST,
9+
DBT_MOCK_CATALOG,
10+
NOTION_MOCK_EXISTENT_CHILD_PAGE_QUERY,
11+
NOTION_MOCK_EXISTENT_DATABASE_RECORDS_QUERY,
12+
NOTION_MOCK_NONEXISTENT_QUERY,
13+
NOTION_MOCK_DATABASE_CREATE,
14+
NOTION_MOCK_RECORD_CREATE,
15+
)
16+
17+
18+
class TestDbtDocsToNotionIntegration(unittest.TestCase):
19+
20+
def setUp(self):
21+
patch('dbt_docs_to_notion.json.load').start().side_effect = [DBT_MOCK_MANIFEST, DBT_MOCK_CATALOG]
22+
patch('dbt_docs_to_notion.open', new_callable=unittest.mock.mock_open, read_data="data").start()
23+
self.comparison_catalog = DBT_MOCK_CATALOG['nodes']['model.test.model_1']
24+
self.comparison_manifest = DBT_MOCK_MANIFEST['nodes']['model.test.model_1']
25+
self.recorded_requests = []
26+
27+
def tearDown(self):
28+
patch.stopall()
29+
30+
def _verify_database_obj(self, database_obj):
31+
title = database_obj['title'][0]
32+
self.assertEqual(title['type'], 'text')
33+
self.assertEqual(title['text']['content'], os.environ['DATABASE_NAME'])
34+
parent = database_obj['parent']
35+
self.assertEqual(parent['type'], 'page_id')
36+
self.assertIsInstance(parent['page_id'], str)
37+
properties = database_obj['properties']
38+
self.assertEqual(properties['Name'], {'title': {}})
39+
self.assertEqual(properties['Description'], {'rich_text': {}})
40+
self.assertEqual(properties['Owner'], {'rich_text': {}})
41+
self.assertEqual(properties['Relation'], {'rich_text': {}})
42+
self.assertEqual(
43+
properties['Approx Rows'],
44+
{'number': {'format': 'number_with_commas'}}
45+
)
46+
self.assertEqual(
47+
properties['Approx GB'],
48+
{'number': {'format': 'number_with_commas'}}
49+
)
50+
self.assertEqual(properties['Depends On'], {'rich_text': {}})
51+
self.assertEqual(properties['Tags'], {'rich_text': {}})
52+
53+
def _verify_record_obj(self, record_obj):
54+
parent = record_obj['parent']
55+
self.assertEqual(parent['database_id'], NOTION_MOCK_DATABASE_CREATE['id'])
56+
properties = record_obj['properties']
57+
self.assertEqual(properties['Name']['title'][0]['text']['content'], self.comparison_manifest['name'])
58+
self.assertEqual(properties['Description']['rich_text'][0]['text']['content'], self.comparison_manifest['description'])
59+
self.assertEqual(properties['Owner']['rich_text'][0]['text']['content'], self.comparison_catalog['metadata']['owner'])
60+
self.assertEqual(properties['Relation']['rich_text'][0]['text']['content'], self.comparison_manifest['relation_name'])
61+
self.assertEqual(properties['Approx Rows']['number'], self.comparison_catalog['stats']['row_count']['value'])
62+
self.assertEqual(properties['Approx GB']['number'], self.comparison_catalog['stats']['bytes']['value']/1e9)
63+
self.assertEqual(properties['Depends On']['rich_text'][0]['text']['content'], json.dumps(self.comparison_manifest['depends_on']))
64+
self.assertEqual(properties['Tags']['rich_text'][0]['text']['content'], json.dumps(self.comparison_manifest['tags']))
65+
66+
def _verify_record_children_obj(self, record_children_obj):
67+
toc_child_block = record_children_obj[0]
68+
self.assertEqual(toc_child_block['object'], 'block')
69+
self.assertEqual(toc_child_block['type'], 'table_of_contents')
70+
columns_header_child_block = record_children_obj[1]
71+
self.assertEqual(columns_header_child_block['object'], 'block')
72+
self.assertEqual(columns_header_child_block['type'], 'heading_1')
73+
self.assertEqual(columns_header_child_block['heading_1']['rich_text'][0]['text']['content'], 'Columns')
74+
columns_child_block = record_children_obj[2]
75+
self.assertEqual(columns_child_block['object'], 'block')
76+
self.assertEqual(columns_child_block['type'], 'table')
77+
self.assertEqual(columns_child_block['table']['table_width'], 3)
78+
self.assertEqual(columns_child_block['table']['has_column_header'], True)
79+
self.assertEqual(columns_child_block['table']['has_row_header'], False)
80+
columns_table_children_obj = columns_child_block['table']['children']
81+
columns_table_header_row = columns_table_children_obj[0]
82+
self.assertEqual(columns_table_header_row['type'], 'table_row')
83+
self.assertEqual(columns_table_header_row['table_row']['cells'][0][0]['plain_text'], 'Column')
84+
self.assertEqual(columns_table_header_row['table_row']['cells'][1][0]['plain_text'], 'Type')
85+
self.assertEqual(columns_table_header_row['table_row']['cells'][2][0]['plain_text'], 'Description')
86+
columns_table_row = columns_table_children_obj[1]
87+
self.assertEqual(columns_table_row['type'], 'table_row')
88+
self.assertEqual(columns_table_row['table_row']['cells'][0][0]['plain_text'], list(self.comparison_catalog['columns'].keys())[0])
89+
self.assertEqual(columns_table_row['table_row']['cells'][1][0]['plain_text'], list(self.comparison_catalog['columns'].values())[0]['type'])
90+
self.assertEqual(columns_table_row['table_row']['cells'][2][0]['plain_text'], list(self.comparison_manifest['columns'].values())[0]['description'])
91+
raw_code_header_child_block = record_children_obj[3]
92+
self.assertEqual(raw_code_header_child_block['object'], 'block')
93+
self.assertEqual(raw_code_header_child_block['type'], 'heading_1')
94+
self.assertEqual(raw_code_header_child_block['heading_1']['rich_text'][0]['text']['content'], 'Raw Code')
95+
raw_code_child_block = record_children_obj[4]
96+
self.assertEqual(raw_code_child_block['object'], 'block')
97+
self.assertEqual(raw_code_child_block['type'], 'code')
98+
self.assertEqual(raw_code_child_block['code']['language'], 'sql')
99+
self.assertEqual(raw_code_child_block['code']['rich_text'][0]['text']['content'], self.comparison_manifest['raw_code'])
100+
compiled_code_header_child_block = record_children_obj[5]
101+
self.assertEqual(compiled_code_header_child_block['object'], 'block')
102+
self.assertEqual(compiled_code_header_child_block['type'], 'heading_1')
103+
self.assertEqual(compiled_code_header_child_block['heading_1']['rich_text'][0]['text']['content'], 'Compiled Code')
104+
compiled_code_child_block = record_children_obj[6]
105+
self.assertEqual(compiled_code_child_block['object'], 'block')
106+
self.assertEqual(compiled_code_child_block['type'], 'code')
107+
self.assertEqual(compiled_code_child_block['code']['language'], 'sql')
108+
self.assertEqual(compiled_code_child_block['code']['rich_text'][0]['text']['content'], self.comparison_manifest['compiled_code'])
109+
110+
@patch('dbt_docs_to_notion.make_request')
111+
def test_create_new_database(self, mock_make_request):
112+
def _mocked_make_request(endpoint, querystring, method, **request_kwargs):
113+
self.recorded_requests.append((endpoint, method))
114+
if endpoint == 'blocks/' and method == 'GET':
115+
return NOTION_MOCK_NONEXISTENT_QUERY
116+
elif endpoint == 'databases/' and querystring == '' and method == 'POST':
117+
database_obj = request_kwargs['json']
118+
self._verify_database_obj(database_obj)
119+
return NOTION_MOCK_DATABASE_CREATE
120+
elif endpoint == 'databases/' and '/query' in querystring and method == 'POST':
121+
return NOTION_MOCK_NONEXISTENT_QUERY
122+
elif endpoint == 'pages/' and method == 'POST':
123+
record_obj = request_kwargs['json']
124+
self._verify_record_obj(record_obj)
125+
record_children_obj = request_kwargs['json']['children']
126+
self._verify_record_children_obj(record_children_obj)
127+
return NOTION_MOCK_RECORD_CREATE
128+
mock_make_request.side_effect = _mocked_make_request
129+
130+
main(argv=[None, 'all'])
131+
132+
self.assertEqual(
133+
self.recorded_requests,
134+
[
135+
('blocks/', 'GET'),
136+
('databases/', 'POST'),
137+
('databases/', 'POST'),
138+
('pages/', 'POST'),
139+
]
140+
)
141+
142+
@patch('dbt_docs_to_notion.make_request')
143+
def test_update_existing_database(self, mock_make_request):
144+
def _mocked_make_request(endpoint, querystring, method, **request_kwargs):
145+
self.recorded_requests.append((endpoint, method))
146+
if endpoint == 'blocks/' and method == 'GET':
147+
return NOTION_MOCK_EXISTENT_CHILD_PAGE_QUERY
148+
elif endpoint == 'databases/' and '/query' in querystring and method == 'POST':
149+
return NOTION_MOCK_EXISTENT_DATABASE_RECORDS_QUERY
150+
elif endpoint == 'pages/' and method == 'PATCH':
151+
record_obj = request_kwargs['json']
152+
self._verify_record_obj(record_obj)
153+
return {} # response is thrown away
154+
elif endpoint == 'blocks/' and method == 'DELETE':
155+
return {} # response is thrown away
156+
elif endpoint == 'blocks/' and method == 'PATCH':
157+
record_children_obj = request_kwargs['json']['children']
158+
self._verify_record_children_obj(record_children_obj)
159+
return {} # response is thrown away
160+
mock_make_request.side_effect = _mocked_make_request
161+
162+
main(argv=[None, 'all'])
163+
164+
self.assertEqual(
165+
self.recorded_requests,
166+
[
167+
('blocks/', 'GET'),
168+
('databases/', 'POST'),
169+
('pages/mock_record_id', 'PATCH'),
170+
('blocks/', 'GET'),
171+
('blocks/', 'DELETE'),
172+
('blocks/', 'PATCH'),
173+
]
174+
)
175+
176+
177+
if __name__ == '__main__':
178+
unittest.main()

0 commit comments

Comments
 (0)