diff --git a/dtran/main.py b/dtran/main.py index 1fd0a54..44a3511 100644 --- a/dtran/main.py +++ b/dtran/main.py @@ -12,9 +12,13 @@ def cli(): ignore_unknown_options=True, allow_extra_args=True, )) -@click.option("--config", help="full path to config") +@click.option("--config", help="Full path to the config.") +@click.option( + "--dryrun", is_flag=True, + help="Only check parsed inputs without actual execution." +) @click.pass_context -def create_pipeline(ctx, config=None): +def create_pipeline(ctx, config=None, dryrun=False): """ Creates a pipeline and execute it based on given config and input(optional). To specify the input to pipeline, use (listed in ascending priority): @@ -36,6 +40,10 @@ def create_pipeline(ctx, config=None): parser = ConfigParser(user_inputs) parsed_pipeline, parsed_inputs = parser.parse(config) + if dryrun: + print(parsed_inputs) + return + # Execute the pipeline parsed_pipeline.exec(parsed_inputs) diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..e5e6a48 --- /dev/null +++ b/test/README.md @@ -0,0 +1,6 @@ +## Prereq +Install Pytest in the current container/env + +## Run +Run the following scripts in /ws: +`python -m pytest test/cli_unit_test.py` \ No newline at end of file diff --git a/test/cli_unit_test.py b/test/cli_unit_test.py new file mode 100644 index 0000000..ee7e488 --- /dev/null +++ b/test/cli_unit_test.py @@ -0,0 +1,103 @@ +import pytest +from click.testing import CliRunner +from unittest.mock import patch, call, MagicMock + +from dtran.main import cli + + +# https://click.palletsprojects.com/en/7.x/testing/ +@pytest.mark.parametrize( + "func_name, attr_name, arg_value", + [ + ("", "", ""), + ("substitute", "attr", "some value"), + ] +) +@patch('dtran.main.ConfigParser', autospec=True) +def test_cli_valid(parser_mock, func_name, attr_name, arg_value): + """ + This function tests 3 valid scenarios: + 1) User does not specify any inputs + 2) User updates existing inputs + 3) User inserts new valid inputs + """ + pipeline_mock = MagicMock() + mock_parsed_inputs = { + "keep_attr": "keep this value", + "substitute_attr": "substitute this value" + } + + parser_mock.return_value.parse.return_value = ( + pipeline_mock, + mock_parsed_inputs + ) + + mock_user_inputs = {} + + runner = CliRunner() + if not func_name or not attr_name: + result = runner.invoke(cli, [ + 'create_pipeline', '--config', 'config/path' + ]) + else: + mock_user_inputs = { + (func_name, attr_name): arg_value + } + result = runner.invoke(cli, [ + 'create_pipeline', '--config', 'config/path', + f'--{func_name}.{attr_name}={arg_value}', + ]) + + assert result.exit_code == 0 + assert parser_mock.mock_calls == [call(mock_user_inputs), call().parse('config/path')] + assert pipeline_mock.exec.mock_calls[-1] == call(mock_parsed_inputs) + + +@pytest.mark.parametrize( + "func_name, attr_name, arg_value", + [ + ("substitute", "attr", "some value"), + ] +) +def test_cli_invalid(func_name, attr_name, arg_value): + """ + This function tests invalid cli input scenario. + """ + runner = CliRunner() + arg = f'--{func_name}//{attr_name}={arg_value}' + result = runner.invoke(cli, [ + 'create_pipeline', '--config', 'config/path', arg + ]) + + assert result.exit_code == 0 + assert f"user input: '{arg}' should have format '--FuncName.Attr=value'" in result.output + + +@pytest.mark.parametrize( + "func_name, attr_name, arg_value", + [ + ("substitute", "attr", "some value"), + ] +) +@patch('dtran.main.ConfigParser', autospec=True) +def test_cli_dryrun(parser_mock, func_name, attr_name, arg_value): + pipeline_mock = MagicMock() + mock_parsed_inputs = { + "keep_attr": "keep this value", + "substitute_attr": "substitute this value" + } + + parser_mock.return_value.parse.return_value = ( + pipeline_mock, + mock_parsed_inputs + ) + + runner = CliRunner() + result = runner.invoke(cli, [ + 'create_pipeline', '--config', 'config/path', + f'--{func_name}.{attr_name}={arg_value}', '--dryrun' + ]) + + for parsed_key, parsed_value in mock_parsed_inputs.items(): + assert parsed_key in result.output + assert parsed_value in result.output diff --git a/test/parser_unit_test.py b/test/parser_unit_test.py new file mode 100644 index 0000000..b5206a2 --- /dev/null +++ b/test/parser_unit_test.py @@ -0,0 +1,37 @@ +import pytest +from dtran.config_parser import ConfigParser +from unittest.mock import patch, call, MagicMock +from funcs import ReadFunc, UnitTransFunc, GraphWriteFunc + + +@pytest.mark.parametrize("config_path", [ + "test/sample_config.json", "test/sample_config.yml" +]) +@patch('dtran.config_parser.Pipeline') +def test_config_parser_no_user_inputs(pipeline_mock, config_path): + pipeline_mock.return_value = MagicMock() + + parser = ConfigParser({}) + parsed_pipeline, parsed_inputs = parser.parse(config_path) + assert "$/liter" in parsed_inputs.values() + assert pipeline_mock.mock_calls[-1] == call( + [ReadFunc, UnitTransFunc, GraphWriteFunc], + [(['unit_trans', 1, 'graph'], ['read_func', 1, 'data']), (['graph_write_func', 1, 'graph'], ['unit_trans', 1, 'graph'])] + ) + + +@pytest.mark.parametrize("config_path", [ + "test/sample_config.json", "test/sample_config.yml" +]) +@patch('dtran.config_parser.Pipeline') +def test_config_parser_with_user_inputs(pipeline_mock, config_path): + pipeline_mock.return_value = MagicMock() + + parser = ConfigParser({("MyCustomName2", "unit_desired"): "$/oz"}) + parsed_pipeline, parsed_inputs = parser.parse(config_path) + assert "$/oz" in parsed_inputs.values() + assert pipeline_mock.mock_calls[-1] == call( + [ReadFunc, UnitTransFunc, GraphWriteFunc], + [(['unit_trans', 1, 'graph'], ['read_func', 1, 'data']), + (['graph_write_func', 1, 'graph'], ['unit_trans', 1, 'graph'])] + ) diff --git a/test/sample_config.json b/test/sample_config.json new file mode 100644 index 0000000..0165142 --- /dev/null +++ b/test/sample_config.json @@ -0,0 +1,33 @@ +{ + "version": "1", + "adapters": { + "MyCustomName1": { + "comment": "My First Adapter", + "adapter": "funcs.ReadFunc", + "inputs": { + "repr_file": "./examples/demo/s01_ethiopia_commodity_price.yml", + "resources": "./examples/demo/s01_ethiopia_commodity_price.csv" + } + }, + "MyCustomName2": { + "comment": "My Second Adapter", + "adapter": "funcs.UnitTransFunc", + "inputs": { + "graph": "$.MyCustomName1.data", + "unit_value": "rdf:value", + "unit_label": "eg:unit", + "unit_desired": "$/liter" + } + }, + "MyCustomName3": { + "comment": "My Third Adapter", + "adapter": "funcs.GraphWriteFunc", + "inputs": { + "graph": "$.MyCustomName2.graph", + "main_class": "qb:Observation", + "output_file": "./examples/demo/s01_ethiopia_commodity_price_write.csv", + "mapped_columns": {} + } + } + } +} \ No newline at end of file diff --git a/test/sample_config.yml b/test/sample_config.yml new file mode 100644 index 0000000..7ddcfb2 --- /dev/null +++ b/test/sample_config.yml @@ -0,0 +1,24 @@ +version: "1" +adapters: + MyCustomName1: + comment: My First Adapter + adapter: funcs.ReadFunc + inputs: + repr_file: ./examples/demo/s01_ethiopia_commodity_price.yml + resources: ./examples/demo/s01_ethiopia_commodity_price.csv + MyCustomName2: + comment: My Second Adapter + adapter: funcs.UnitTransFunc + inputs: + graph: $.MyCustomName1.data + unit_value: rdf:value + unit_label: eg:unit + unit_desired: $/liter + MyCustomName3: + comment: My Third Adapter + adapter: funcs.GraphWriteFunc + inputs: + graph: $.MyCustomName2.graph + main_class: qb:Observation + output_file: ./examples/demo/s01_ethiopia_commodity_price_write.csv + mapped_columns: {} \ No newline at end of file