diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4087c1a..d0ce079 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 1.0.0 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P(rc|dev))(?P\d+))? @@ -18,3 +18,5 @@ values = first_value = 1 [bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 38e50c7..10ae6c9 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -1,5 +1,4 @@ name: Coverage - on: push: branches: @@ -9,20 +8,23 @@ on: pull_request: branches: - main - + - dev jobs: coverage: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.9 uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Install poetry uses: abatilo/actions-poetry@v2 - name: Setup a local virtual environment run: | poetry config virtualenvs.create true --local poetry config virtualenvs.in-project true --local + poetry lock - uses: actions/cache@v3 name: Define a cache for the virtual environment based on the dependencies lock file with: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2e1efc9..4de7ea0 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,21 +1,26 @@ name: Lint - on: push: - + pull_request: + branches: + - main + - dev jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.9 uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Install poetry uses: abatilo/actions-poetry@v2 - name: Setup a local virtual environment run: | poetry config virtualenvs.create true --local poetry config virtualenvs.in-project true --local + poetry lock - uses: actions/cache@v3 name: Define a cache for the virtual environment based on the dependencies lock file with: diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7d2bf90..624cb99 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,7 +5,6 @@ on: - main - dev - issue/** - jobs: validate_focus: runs-on: ubuntu-latest @@ -14,8 +13,10 @@ jobs: steps: - name: Check out repository code uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.9 uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Install poetry uses: abatilo/actions-poetry@v2 - name: Setup a local virtual environment diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 4f27f59..41c5cb3 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -4,7 +4,6 @@ on: tags: - 'v\d\.\d\.\d' - 'v\d\.\d\.\d-(dev|rc)\d' - jobs: publish: permissions: @@ -12,14 +11,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.8 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.11 - name: Install poetry uses: abatilo/actions-poetry@v2 - name: Install dependencies run: | - poetry build + find -type l -exec bash -c 'ln -f "$(readlink -m "$0")" "$0"' {} \; + poetry build --format=sdist - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 840ab9a..ac2bab2 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -1,5 +1,4 @@ name: Unittest - on: push: branches: @@ -9,15 +8,14 @@ on: pull_request: branches: - main - + - dev jobs: test: - runs-on: ubuntu-latest - + runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [ "3.8", "3.9", "3.10", "3.11" ] - + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + os: [ windows-latest, ubuntu-latest, macos-latest ] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -35,7 +33,7 @@ jobs: name: Define a cache for the virtual environment based on the dependencies lock file with: path: ./.venv - key: venv-${{ hashFiles('poetry.lock') }} + key: venv-${{ hashFiles('poetry.lock') }}-${{ matrix.os }}-${{ matrix.python-version }} - name: Install dependencies run: | poetry install diff --git a/README.md b/README.md index 0cfde3f..e3a5897 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ tbd ### Prerequisites -- Python 3.8+ +- Python 3.9+ - Poetry (Package & Dependency Manager) ### Installation @@ -40,6 +40,22 @@ Using Poetry, you can install the project's dependencies with: ```bash poetry install ``` +#### 4. Downgrade Multimethod + +By default Multimethod 2.0 is installed, focus_validator works with 1.9.1 + +```bash +poetry add "multimethod@1.9.1" +``` + +#### 5. Install Virtual Environment Shell + +Shell is not installed by default, you need to install it. + +```bash +poetry self add poetry-plugin-shell +``` + ## Usage @@ -67,6 +83,14 @@ poetry run pytest Ensure you have `pytest` defined as a development dependency in your `pyproject.toml`. +If running on legacy CPUs and the tests crash on the polars library, run the following locally only: + +```bash +poetry add polars-lts-cpu +``` + +This will align the polars execution with your system hardware. It should NOT be committed back into the repository. + ## License This project is licensed under the MIT License - see the `LICENSE` file for details. diff --git a/build.py b/build.py new file mode 100644 index 0000000..7959370 --- /dev/null +++ b/build.py @@ -0,0 +1,21 @@ +import os +import pathlib +import shutil +import yaml + +def copy_rules(basedir): + with open(os.path.join(basedir, 'version_sets.yaml'), 'r') as file: + version_sets = yaml.safe_load(file) + + for version, base_files in version_sets.items(): + dest = os.path.join(basedir, 'version_sets', version) + if os.path.exists(dest): + shutil.rmtree(dest) + pathlib.Path(dest).mkdir(parents=True) + for f in base_files: + src_file = os.path.join(basedir, 'base_rule_definitions', f) + dest_file = os.path.join(dest, f) + shutil.copyfile(src_file, dest_file) + +if __name__ == "__main__": + copy_rules(basedir='focus_validator/rules') diff --git a/focus_validator/config_objects/common.py b/focus_validator/config_objects/common.py index 6b65705..04bf49c 100644 --- a/focus_validator/config_objects/common.py +++ b/focus_validator/config_objects/common.py @@ -1,7 +1,8 @@ from enum import Enum from typing import List, Literal -from pydantic import BaseModel +import sqlglot +from pydantic import BaseModel, field_validator class AllowNullsCheck(BaseModel): @@ -12,6 +13,22 @@ class ValueInCheck(BaseModel): value_in: List[str] +class SQLQueryCheck(BaseModel): + sql_query: str + + @field_validator("sql_query") + def check_sql_query(cls, sql_query): + returned_columns = [ + column.alias + for column in sqlglot.parse_one(sql_query).find_all(sqlglot.exp.Alias) + ] + + assert returned_columns == [ + "check_output" + ], "SQL query must only return a column called 'check_output'" + return sql_query + + SIMPLE_CHECKS = Literal["check_unique", "column_required"] @@ -20,6 +37,7 @@ class DataTypes(Enum): DECIMAL = "decimal" DATETIME = "datetime" CURRENCY_CODE = "currency-code" + STRINGIFIED_JSON_OBJECT = "stringified-json-object" class DataTypeCheck(BaseModel): @@ -50,3 +68,8 @@ def generate_check_friendly_name(check, column_id): return f"{column_id} does not allow null values." elif isinstance(check, DataTypeCheck): return f"{column_id} requires values of type {check.data_type.value}." + elif isinstance(check, SQLQueryCheck): + sql_query = " ".join([word.strip() for word in check.sql_query.split()]) + return f"{column_id} requires values that return true when evaluated by the following SQL query: {sql_query}" + else: + raise NotImplementedError(f"Check {check} not implemented.") diff --git a/focus_validator/config_objects/focus_to_pandera_schema_converter.py b/focus_validator/config_objects/focus_to_pandera_schema_converter.py index 826836c..d21fac4 100644 --- a/focus_validator/config_objects/focus_to_pandera_schema_converter.py +++ b/focus_validator/config_objects/focus_to_pandera_schema_converter.py @@ -1,7 +1,10 @@ +import os from itertools import groupby from typing import Dict, List, Optional, Set, Union +import pandas as pd import pandera as pa +import sqlglot from pandera.api.pandas.types import PandasDtypeInputTypes from focus_validator.config_objects import ChecklistObject, InvalidRule, Rule @@ -10,11 +13,25 @@ ChecklistObjectStatus, DataTypeCheck, DataTypes, + SQLQueryCheck, ValueInCheck, ) from focus_validator.config_objects.override import Override from focus_validator.exceptions import FocusNotImplementedError +# group index column adds a column to the dataframe which is used to group the dataframe, otherwise the default +# groupby function does not carry forward all rows in the dataframe causing it to not have row numbers +GROUP_INDEX_COLUMN = "group_index_column" + + +def __groupby_fnc__(df: pd.DataFrame, column_alias: List[str]): + """ + Custom groupby function to be used with pandera check_sql_query, allowing null values + Default groupby function does not allow null values + """ + df[GROUP_INDEX_COLUMN] = range(0, len(df)) + return df.groupby(column_alias + [GROUP_INDEX_COLUMN], dropna=False) + class FocusToPanderaSchemaConverter: @staticmethod @@ -40,9 +57,22 @@ def __generate_pandera_check__(rule: Rule, check_id): return pa.Check.check_value_in( allowed_values=check.value_in, error=error_string ) + elif isinstance(check, SQLQueryCheck): + column_alias = [ + column.alias_or_name + for column in sqlglot.parse_one(check.sql_query).find_all( + sqlglot.exp.Column + ) + ] + return pa.Check.check_sql_query( + sql_query=check.sql_query, + error=error_string, + column_alias=column_alias, + groupby=lambda df: __groupby_fnc__(df=df, column_alias=column_alias), + ) elif isinstance(check, AllowNullsCheck): return pa.Check.check_not_null( - error=error_string, ignore_na=False, allow_nulls=check.allow_nulls + error=error_string, ignore_na=check.allow_nulls ) else: raise FocusNotImplementedError( @@ -77,6 +107,14 @@ def __generate_column_definition__( error=f"{rule.check_id}:::Ensures that column is of {data_type.value} type.", ) ) + elif data_type == DataTypes.STRINGIFIED_JSON_OBJECT: + pandera_type = None + column_checks.append( + pa.Check.check_stringified_json_object_dtype( + ignore_na=True, + error=f"{rule.check_id}:::Ensures that column is of {data_type.value} type.", + ) + ) else: pandera_type = pa.String @@ -151,7 +189,7 @@ def generate_pandera_schema( for rule in rules: if isinstance(rule, InvalidRule): checklist[rule.rule_path] = ChecklistObject( - check_name=rule.rule_path, + check_name=os.path.splitext(os.path.basename(rule.rule_path))[0], column_id="Unknown", error=f"{rule.error_type}: {rule.error}", status=ChecklistObjectStatus.ERRORED, @@ -180,4 +218,7 @@ def generate_pandera_schema( overrides=overrides, schema_dict=schema_dict, ) - return pa.DataFrameSchema(schema_dict, strict=False), checklist + return ( + pa.DataFrameSchema(schema_dict, strict=False), + checklist, + ) diff --git a/focus_validator/config_objects/override.py b/focus_validator/config_objects/override.py index d52104c..25147d5 100644 --- a/focus_validator/config_objects/override.py +++ b/focus_validator/config_objects/override.py @@ -1,7 +1,7 @@ from typing import List import yaml -from pydantic import BaseModel +from pydantic.v1 import BaseModel class Override(BaseModel): diff --git a/focus_validator/config_objects/rule.py b/focus_validator/config_objects/rule.py index aa8defe..aa4bc53 100644 --- a/focus_validator/config_objects/rule.py +++ b/focus_validator/config_objects/rule.py @@ -1,13 +1,16 @@ -from typing import Optional, Union +import os +from typing import Annotated, Optional, Union import yaml -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic_core.core_schema import ValidationInfo from focus_validator.config_objects.common import ( SIMPLE_CHECKS, AllowNullsCheck, ChecklistObjectStatus, DataTypeCheck, + SQLQueryCheck, ValueInCheck, generate_check_friendly_name, ) @@ -27,48 +30,58 @@ class Rule(BaseModel): check_id: str column_id: str - check: Union[SIMPLE_CHECKS, AllowNullsCheck, ValueInCheck, DataTypeCheck] + check: Union[ + SIMPLE_CHECKS, AllowNullsCheck, ValueInCheck, DataTypeCheck, SQLQueryCheck + ] - check_friendly_name: Optional[ - str + check_friendly_name: Annotated[ + Optional[str], Field(validate_default=True) ] = None # auto generated or else can be overwritten - check_type_friendly_name: Optional[str] = None - - class Config: - extra = "forbid" # prevents config from containing any undesirable keys - frozen = ( - True # prevents any modification to any attribute onces loaded from config - ) - - @root_validator - def root_val(cls, values): - """ - Root validator that checks for all options passed in the config and generate missing options. - """ - - check = values.get("check") - check_friendly_name = values.get("check_friendly_name") - column_id = values.get("column_id") - if check is not None: + check_type_friendly_name: Annotated[ + Optional[str], Field(validate_default=True) + ] = None + + model_config = ConfigDict( + extra="forbid", # prevents config from containing any undesirable keys + frozen=True, # prevents any modification to any attribute onces loaded from config + ) + + @field_validator("check_friendly_name") + def validate_or_generate_check_friendly_name( + cls, check_friendly_name, validation_info: ValidationInfo + ): + values = validation_info.data + if ( + check_friendly_name is None + and values.get("check") is not None + and values.get("column_id") is not None + ): + check_friendly_name = generate_check_friendly_name( + check=values["check"], column_id=values["column_id"] + ) + return check_friendly_name + + @field_validator("check_type_friendly_name") + def validate_or_generate_check_type_friendly_name( + cls, check_type_friendly_name, validation_info: ValidationInfo + ): + values = validation_info.data + if values.get("check") is not None and values.get("column_id") is not None: + check = values.get("check") if isinstance(check, str): check_type_friendly_name = "".join( [word.title() for word in check.split("_")] ) else: check_type_friendly_name = check.__class__.__name__ - values["check_type_friendly_name"] = check_type_friendly_name - - if check_friendly_name is None and column_id is not None: - values["check_friendly_name"] = generate_check_friendly_name( - check=check, column_id=column_id - ) - - return values + return check_type_friendly_name @staticmethod def load_yaml( rule_path, column_namespace: Optional[str] = None ) -> Union["Rule", InvalidRule]: + rule_path_basename = os.path.splitext(os.path.basename(rule_path))[0] + try: with open(rule_path, "r") as f: rule_obj = yaml.safe_load(f) @@ -80,10 +93,15 @@ def load_yaml( ): rule_obj["column"] = f"{column_namespace}:{rule_obj['column']}" - return Rule.parse_obj(rule_obj) + if isinstance(rule_obj, dict) and "check_id" not in rule_obj: + rule_obj["check_id"] = rule_path_basename + + return Rule.model_validate(rule_obj) except Exception as e: return InvalidRule( - rule_path=rule_path, error=str(e), error_type=e.__class__.__name__ + rule_path=rule_path_basename, + error=str(e), + error_type=e.__class__.__name__, ) diff --git a/focus_validator/data_loaders/csv_data_loader.py b/focus_validator/data_loaders/csv_data_loader.py index 28a82cd..2a75a3b 100644 --- a/focus_validator/data_loaders/csv_data_loader.py +++ b/focus_validator/data_loaders/csv_data_loader.py @@ -6,4 +6,4 @@ def __init__(self, data_filename): self.data_filename = data_filename def load(self): - return pd.read_csv(self.data_filename, keep_default_na=False) + return pd.read_csv(self.data_filename) diff --git a/focus_validator/data_loaders/data_loader.py b/focus_validator/data_loaders/data_loader.py index 5232192..b52d243 100644 --- a/focus_validator/data_loaders/data_loader.py +++ b/focus_validator/data_loaders/data_loader.py @@ -1,15 +1,8 @@ -import magic - from focus_validator.data_loaders.csv_data_loader import CSVDataLoader from focus_validator.data_loaders.parquet_data_loader import ParquetDataLoader from focus_validator.exceptions import FocusNotImplementedError -def get_file_mime_type(filename): - f = magic.Magic(uncompress=True) - return f.from_file(filename=filename) - - class DataLoader: def __init__(self, data_filename): self.data_filename = data_filename @@ -17,16 +10,12 @@ def __init__(self, data_filename): self.data_loader = self.data_loader_class(self.data_filename) def find_data_loader(self): - file_mime_type = get_file_mime_type(self.data_filename) - - if file_mime_type in ["ASCII text", "CSV text"]: + if self.data_filename.endswith(".csv"): return CSVDataLoader - elif file_mime_type == "Apache Parquet": + elif self.data_filename.endswith(".parquet"): return ParquetDataLoader else: - raise FocusNotImplementedError( - msg=f"Validator for file_type {file_mime_type} not implemented yet." - ) + raise FocusNotImplementedError("File type not implemented yet.") def load(self): return self.data_loader.load() diff --git a/focus_validator/main.py b/focus_validator/main.py index 0377313..7550266 100644 --- a/focus_validator/main.py +++ b/focus_validator/main.py @@ -1,8 +1,7 @@ import argparse -import os import sys -from focus_validator.validator import Validator +from focus_validator.validator import DEFAULT_VERSION_SETS_PATH, Validator def main(): @@ -33,11 +32,11 @@ def main(): help="Allow transitional rules in validation", ) parser.add_argument( - "--validate-version", default="0.5", help="Version of FOCUS to validate against" + "--validate-version", default="1.0", help="Version of FOCUS to validate against" ) parser.add_argument( "--rule-set-path", - default=os.path.join("focus_validator", "rules", "version_sets"), + default=DEFAULT_VERSION_SETS_PATH, help="Path to rules definitions", ) parser.add_argument( diff --git a/focus_validator/outputter/outputter_console.py b/focus_validator/outputter/outputter_console.py index 289a142..a0b4471 100644 --- a/focus_validator/outputter/outputter_console.py +++ b/focus_validator/outputter/outputter_console.py @@ -1,5 +1,6 @@ +import math + import pandas as pd -from tabulate import tabulate from focus_validator.config_objects import Rule from focus_validator.rules.spec_rules import ValidationResult @@ -19,7 +20,7 @@ def __restructure_check_list__(result_set: ValidationResult): else: check_type = "ERRORED" - row_obj = value.dict() + row_obj = value.model_dump() row_obj.update( { "check_type": check_type, @@ -27,6 +28,7 @@ def __restructure_check_list__(result_set: ValidationResult): } ) rows.append(row_obj) + df = pd.DataFrame(rows) df.rename( columns={ @@ -53,17 +55,51 @@ def __restructure_check_list__(result_set: ValidationResult): def write(self, result_set: ValidationResult): self.result_set = result_set - - checklist = self.__restructure_check_list__(result_set) - print("Checklist:") - print(tabulate(checklist, headers="keys", tablefmt="psql")) - if result_set.failure_cases is not None: - print("Checks summary:") - print( - tabulate( - tabular_data=result_set.failure_cases, # type: ignore - headers="keys", - tablefmt="psql", + aggregated_failures = result_set.failure_cases.groupby( + by=["Check Name", "Description"], as_index=False + ).aggregate(lambda x: collapse_occurrence_range(x.unique().tolist())) + + print("Errors encountered:") + for _, fail in aggregated_failures.iterrows(): + print( + f'{fail["Check Name"]} failed:\n\tDescription: {fail["Description"]}\n\tRows: {fail["Row #"] if fail["Row #"] else "(whole file)"}\n\tExample values: {fail["Values"] if fail["Values"] else "(none)"}\n' ) - ) + print("Validation failed!") + else: + print("Validation succeeded.") + + +def collapse_occurrence_range(occurrence_range: list): + start = None + i = None + collapsed = [] + + # Edge case + if len(occurrence_range) == 1: + if isinstance(occurrence_range[0], float) and math.isnan(occurrence_range[0]): + return "" + if occurrence_range[0] is None: + return "" + + for n in sorted(occurrence_range): + if not isinstance(n, int) and not (isinstance(n, float) and not math.isnan(n)): + return ",".join([str(x) for x in occurrence_range]) + elif i is None: + start = i = int(n) + elif n == i + 1: + i = int(n) + elif i: + if i == start: + collapsed.append(f"{start}") + else: + collapsed.append(f"{start}-{i}") + start = i = int(n) + + if start is not None: + if i == start: + collapsed.append(f"{start}") + else: + collapsed.append(f"{start}-{i}") + + return ",".join(collapsed) diff --git a/focus_validator/outputter/outputter_unittest.py b/focus_validator/outputter/outputter_unittest.py index 274e682..8b4f5b5 100644 --- a/focus_validator/outputter/outputter_unittest.py +++ b/focus_validator/outputter/outputter_unittest.py @@ -1,5 +1,4 @@ import logging -import re import sys import xml.etree.cElementTree as ET from datetime import datetime, timezone @@ -121,7 +120,7 @@ def write(self, result_set): ) # format the results for processing - rows = [v.dict() for v in result_set.checklist.values()] + rows = [v.model_dump() for v in result_set.checklist.values()] # Setup a Formatter and initiate with result totals formatter = UnittestFormatter( @@ -146,9 +145,9 @@ def write(self, result_set): # Add the testcases to the testsuites added_testsuites = {} - for testcase in [ - r for r in rows if re.match(r"^FV-[D,M][0-9]{3}-[0-9]{4}$", r["check_name"]) - ]: + for testcase in rows: + if testcase["status"].value == "errored": + continue test_suite_id = testcase["check_name"].rsplit("-", 1)[0] if test_suite_id not in added_testsuites: formatter.add_testsuite( diff --git a/focus_validator/rules/.gitignore b/focus_validator/rules/.gitignore new file mode 100644 index 0000000..0605180 --- /dev/null +++ b/focus_validator/rules/.gitignore @@ -0,0 +1,2 @@ +# Generated by build.py at wheel-time +version_sets/ diff --git a/focus_validator/rules/base_rule_definitions/FV-M002-0001.yaml b/focus_validator/rules/base_rule_definitions/AmortizedCost_IsDecimal.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-M002-0001.yaml rename to focus_validator/rules/base_rule_definitions/AmortizedCost_IsDecimal.yaml index b05e342..0d70c5c 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M002-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/AmortizedCost_IsDecimal.yaml @@ -1,4 +1,3 @@ -check_id: FV-M002-0001 column_id: AmortizedCost check: data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/FV-M002-0002.yaml b/focus_validator/rules/base_rule_definitions/AmortizedCost_NotNull.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-M002-0002.yaml rename to focus_validator/rules/base_rule_definitions/AmortizedCost_NotNull.yaml index 274fe38..dc45435 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M002-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/AmortizedCost_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-M002-0002 column_id: AmortizedCost check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-M002-0003.yaml b/focus_validator/rules/base_rule_definitions/AmortizedCost_Required.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-M002-0003.yaml rename to focus_validator/rules/base_rule_definitions/AmortizedCost_Required.yaml index 6eadde2..1225820 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M002-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/AmortizedCost_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-M002-0003 column_id: AmortizedCost check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D014-0001.yaml b/focus_validator/rules/base_rule_definitions/AvailabilityZone_IsString.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D014-0001.yaml rename to focus_validator/rules/base_rule_definitions/AvailabilityZone_IsString.yaml index 571abbe..05a737a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D014-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/AvailabilityZone_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D014-0001 column_id: AvailabilityZone check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D014-0002.yaml b/focus_validator/rules/base_rule_definitions/AvailabilityZone_Nullable.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D014-0002.yaml rename to focus_validator/rules/base_rule_definitions/AvailabilityZone_Nullable.yaml index 5c3522c..326126e 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D014-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/AvailabilityZone_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D014-0002 column_id: AvailabilityZone check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/FV-M001-0001.yaml b/focus_validator/rules/base_rule_definitions/BilledCost_IsDecimal.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-M001-0001.yaml rename to focus_validator/rules/base_rule_definitions/BilledCost_IsDecimal.yaml index 554cac5..ff3039a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M001-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCost_IsDecimal.yaml @@ -1,4 +1,3 @@ -check_id: FV-M001-0001 column_id: BilledCost check: data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/FV-M001-0002.yaml b/focus_validator/rules/base_rule_definitions/BilledCost_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-M001-0002.yaml rename to focus_validator/rules/base_rule_definitions/BilledCost_NotNull.yaml index fcac9d7..d18817f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M001-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCost_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-M001-0002 column_id: BilledCost check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-M001-0003.yaml b/focus_validator/rules/base_rule_definitions/BilledCost_Required.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-M001-0003.yaml rename to focus_validator/rules/base_rule_definitions/BilledCost_Required.yaml index 35a9787..7407c46 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M001-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCost_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-M001-0003 column_id: BilledCost check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D010-0001.yaml b/focus_validator/rules/base_rule_definitions/BilledCurrency_IsCurrencyCode.yaml similarity index 72% rename from focus_validator/rules/base_rule_definitions/FV-D010-0001.yaml rename to focus_validator/rules/base_rule_definitions/BilledCurrency_IsCurrencyCode.yaml index 49dca29..5a26704 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D010-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCurrency_IsCurrencyCode.yaml @@ -1,4 +1,3 @@ -check_id: FV-D010-0001 column_id: BilledCurrency check: data_type: currency-code diff --git a/focus_validator/rules/base_rule_definitions/FV-D010-0002.yaml b/focus_validator/rules/base_rule_definitions/BilledCurrency_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D010-0002.yaml rename to focus_validator/rules/base_rule_definitions/BilledCurrency_NotNull.yaml index ddae15e..1094457 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D010-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCurrency_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D010-0002 column_id: BilledCurrency check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D010-0003.yaml b/focus_validator/rules/base_rule_definitions/BilledCurrency_Required.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D010-0003.yaml rename to focus_validator/rules/base_rule_definitions/BilledCurrency_Required.yaml index 9dcd117..76a1828 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D010-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCurrency_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D010-0003 column_id: BilledCurrency check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D006-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountId_IsString.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D006-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountId_IsString.yaml index d8effe8..ce3124a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D006-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountId_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D006-0001 column_id: BillingAccountId check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D006-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountId_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D006-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountId_NotNull.yaml index f8eebe0..2fbb907 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D006-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountId_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D006-0002 column_id: BillingAccountId check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D006-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountId_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D006-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountId_Required.yaml index d08c2e0..3e61188 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D006-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountId_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D006-0003 column_id: BillingAccountId check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D005-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountName_IsString.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D005-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountName_IsString.yaml index 46e7872..8408869 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D005-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountName_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D005-0001 column_id: BillingAccountName check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D005-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountName_Nullable.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D005-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountName_Nullable.yaml index ac2c851..9115bbf 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D005-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountName_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D005-0002 column_id: BillingAccountName check: allow_nulls: True diff --git a/focus_validator/rules/base_rule_definitions/FV-D005-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountName_Required.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D005-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountName_Required.yaml index 8d091b3..672716f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D005-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountName_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D005-0003 column_id: BillingAccountName check: column_required diff --git a/focus_validator/rules/base_rule_definitions/BillingCurrency_IsCurrencyCode.yaml b/focus_validator/rules/base_rule_definitions/BillingCurrency_IsCurrencyCode.yaml new file mode 100644 index 0000000..6c37675 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/BillingCurrency_IsCurrencyCode.yaml @@ -0,0 +1,3 @@ +column_id: BillingCurrency +check: + data_type: currency-code diff --git a/focus_validator/rules/base_rule_definitions/BillingCurrency_NotNull.yaml b/focus_validator/rules/base_rule_definitions/BillingCurrency_NotNull.yaml new file mode 100644 index 0000000..f1fcc1f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/BillingCurrency_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: BillingCurrency +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/BillingCurrency_Required.yaml b/focus_validator/rules/base_rule_definitions/BillingCurrency_Required.yaml new file mode 100644 index 0000000..5fae986 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/BillingCurrency_Required.yaml @@ -0,0 +1,2 @@ +column_id: BillingCurrency +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D011-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_IsDateTime.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D011-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodEnd_IsDateTime.yaml index d97835b..dbe136a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D011-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D011-0001 column_id: BillingPeriodEnd check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D011-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D011-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodEnd_NotNull.yaml index 1210ee0..f8a50cf 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D011-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D011-0002 column_id: BillingPeriodEnd check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D011-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D011-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodEnd_Required.yaml index 75224a1..b864b76 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D011-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D011-0003 column_id: BillingPeriodEnd check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D012-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_IsDateTime.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D012-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodStart_IsDateTime.yaml index f6d695f..58e567e 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D012-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D012-0001 column_id: BillingPeriodStart check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D012-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_NotNull.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D012-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodStart_NotNull.yaml index b7d93bf..7c9b272 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D012-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D012-0002 column_id: BillingPeriodStart check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D012-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_Required.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D012-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodStart_Required.yaml index 049e524..d2ae0c2 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D012-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D012-0003 column_id: BillingPeriodStart check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeCategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeCategory_IsString.yaml new file mode 100644 index 0000000..7d40985 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeCategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeCategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeCategory_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeCategory_NotNull.yaml new file mode 100644 index 0000000..cbad900 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeCategory_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeCategory +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeCategory_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeCategory_Required.yaml new file mode 100644 index 0000000..ad38523 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeCategory_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeCategory +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeClass_Enum.yaml b/focus_validator/rules/base_rule_definitions/ChargeClass_Enum.yaml new file mode 100644 index 0000000..819096c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeClass_Enum.yaml @@ -0,0 +1,4 @@ +column_id: ChargeClass +check: + value_in: + - "Correction" diff --git a/focus_validator/rules/base_rule_definitions/ChargeClass_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeClass_IsString.yaml new file mode 100644 index 0000000..d22958b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeClass_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeClass +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeClass_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ChargeClass_Nullable.yaml new file mode 100644 index 0000000..92b55bd --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeClass_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ChargeClass +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ChargeClass_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeClass_Required.yaml new file mode 100644 index 0000000..8a72a22 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeClass_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeClass +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml new file mode 100644 index 0000000..920a124 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeDescription +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml new file mode 100644 index 0000000..01dd7c9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeDescription +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_Nullable.yaml new file mode 100644 index 0000000..1d790cf --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ChargeDescription +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml new file mode 100644 index 0000000..f08f045 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeDescription +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml new file mode 100644 index 0000000..021af67 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml @@ -0,0 +1,6 @@ +column_id: ChargeFrequency +check: + value_in: + - "One-Time" + - "Recurring" + - "Usage-Based" \ No newline at end of file diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml new file mode 100644 index 0000000..ea4a54a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeFrequency +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml new file mode 100644 index 0000000..49eaedf --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeFrequency +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml new file mode 100644 index 0000000..97ad0df --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeFrequency +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D017-0001.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_IsDateTime.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D017-0001.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodEnd_IsDateTime.yaml index 78bcef0..2e5f049 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D017-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D017-0001 column_id: ChargePeriodEnd check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D017-0002.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D017-0002.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodEnd_NotNull.yaml index 15b43c4..5157bdd 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D017-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D017-0002 column_id: ChargePeriodEnd check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D017-0003.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D017-0003.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodEnd_Required.yaml index 08e058e..b91ee77 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D017-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D017-0003 column_id: ChargePeriodEnd check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D016-0001.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_IsDateTime.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D016-0001.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodStart_IsDateTime.yaml index 4ea40e2..4b9ad12 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D016-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D016-0001 column_id: ChargePeriodStart check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D016-0002.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_NotNull.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D016-0002.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodStart_NotNull.yaml index bed8364..d40abd4 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D016-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D016-0002 column_id: ChargePeriodStart check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D016-0003.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_Required.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D016-0003.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodStart_Required.yaml index d62ca89..aca0c01 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D016-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D016-0003 column_id: ChargePeriodStart check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml new file mode 100644 index 0000000..6dec960 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml @@ -0,0 +1,13 @@ +column_id: ChargeSubcategory +check: + value_in: + # Allowed when ChargeType=Usage + - "On-Demand" + - "Used Commitment" + - "Unused Commitment" + - "Usage" + # Allowed when ChargeType=Adjustment + - "Refund" + - "Credit" + - "Rounding Error" + - "General Adjustment" diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml new file mode 100644 index 0000000..cd7545e --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeSubcategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml new file mode 100644 index 0000000..9ca2483 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeSubcategory +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml new file mode 100644 index 0000000..1deb197 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeSubcategory +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0003.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_Enum.yaml similarity index 81% rename from focus_validator/rules/base_rule_definitions/FV-D001-0003.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_Enum.yaml index 825ad29..694456f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_Enum.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0003 column_id: ChargeType check: value_in: diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0001.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D001-0001.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_IsString.yaml index 48277c2..8369e69 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0001 column_id: ChargeType check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0004.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D001-0004.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_NotNull.yaml index 0a37e9e..e3a8e80 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0004.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0004 column_id: ChargeType check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0002.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_Required.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D001-0002.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_Required.yaml index 794bb8b..de31444 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_Required.yaml @@ -1,3 +1,2 @@ -check_id: FV-D001-0002 column_id: ChargeType check: column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml new file mode 100644 index 0000000..2ad823d --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml @@ -0,0 +1,5 @@ +column_id: CommitmentDiscountCategory +check: + value_in: + - "Spend" + - "Usage" diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml new file mode 100644 index 0000000..afe06ed --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml new file mode 100644 index 0000000..4b4d889 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Nullable.yaml new file mode 100644 index 0000000..d4f8eee --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml new file mode 100644 index 0000000..49b7f23 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml new file mode 100644 index 0000000..650305b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml new file mode 100644 index 0000000..1b7e48c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml new file mode 100644 index 0000000..212e16b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml new file mode 100644 index 0000000..ef740d6 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountName +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml new file mode 100644 index 0000000..99c567a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountName +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml new file mode 100644 index 0000000..d279fb9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml @@ -0,0 +1,2 @@ +column_id: CommitmentDiscountName +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Enum.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Enum.yaml new file mode 100644 index 0000000..52c5115 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Enum.yaml @@ -0,0 +1,5 @@ +column_id: CommitmentDiscountStatus +check: + value_in: + - "Used" + - "Unused" diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_IsString.yaml new file mode 100644 index 0000000..df5b2a4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountStatus +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Nullable.yaml new file mode 100644 index 0000000..f2acf4b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountStatus +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Required.yaml new file mode 100644 index 0000000..50ce075 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountStatus_Required.yaml @@ -0,0 +1,2 @@ +column_id: CommitmentDiscountStatus +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml new file mode 100644 index 0000000..9daa993 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountType +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml new file mode 100644 index 0000000..2197a0b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountType +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Required.yaml new file mode 100644 index 0000000..c8a79a0 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Required.yaml @@ -0,0 +1,2 @@ +column_id: CommitmentDiscountType +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ConsumedQuantity_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/ConsumedQuantity_IsDecimal.yaml new file mode 100644 index 0000000..4634dba --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ConsumedQuantity_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: ConsumedQuantity +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/ConsumedQuantity_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ConsumedQuantity_Nullable.yaml new file mode 100644 index 0000000..6dd2e7b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ConsumedQuantity_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ConsumedQuantity +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ConsumedQuantity_Required.yaml b/focus_validator/rules/base_rule_definitions/ConsumedQuantity_Required.yaml new file mode 100644 index 0000000..839d051 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ConsumedQuantity_Required.yaml @@ -0,0 +1,2 @@ +column_id: ConsumedQuantity +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ConsumedUnit_IsString.yaml b/focus_validator/rules/base_rule_definitions/ConsumedUnit_IsString.yaml new file mode 100644 index 0000000..d2b77c7 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ConsumedUnit_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ConsumedUnit +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ConsumedUnit_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ConsumedUnit_Nullable.yaml new file mode 100644 index 0000000..ebcd3a7 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ConsumedUnit_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ConsumedUnit +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ConsumedUnit_Required.yaml b/focus_validator/rules/base_rule_definitions/ConsumedUnit_Required.yaml new file mode 100644 index 0000000..340d306 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ConsumedUnit_Required.yaml @@ -0,0 +1,2 @@ +column_id: ConsumedUnit +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ContractedCost_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/ContractedCost_IsDecimal.yaml new file mode 100644 index 0000000..5fca515 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ContractedCost_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: ContractedCost +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/ContractedCost_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ContractedCost_NotNull.yaml new file mode 100644 index 0000000..a7f975e --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ContractedCost_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ContractedCost +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ContractedCost_Required.yaml b/focus_validator/rules/base_rule_definitions/ContractedCost_Required.yaml new file mode 100644 index 0000000..9576cef --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ContractedCost_Required.yaml @@ -0,0 +1,2 @@ +column_id: ContractedCost +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_IsDecimal.yaml new file mode 100644 index 0000000..26212ab --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: ContractedUnitPrice +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_Nullable.yaml new file mode 100644 index 0000000..16baa8f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ContractedUnitPrice +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_Required.yaml b/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_Required.yaml new file mode 100644 index 0000000..09f2516 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ContractedUnitPrice_Required.yaml @@ -0,0 +1,2 @@ +column_id: ContractedUnitPrice +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/EffectiveCost_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/EffectiveCost_IsDecimal.yaml new file mode 100644 index 0000000..55adf42 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/EffectiveCost_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: EffectiveCost +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/EffectiveCost_NotNull.yaml b/focus_validator/rules/base_rule_definitions/EffectiveCost_NotNull.yaml new file mode 100644 index 0000000..6120158 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/EffectiveCost_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: EffectiveCost +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/EffectiveCost_Required.yaml b/focus_validator/rules/base_rule_definitions/EffectiveCost_Required.yaml new file mode 100644 index 0000000..a78d71c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/EffectiveCost_Required.yaml @@ -0,0 +1,3 @@ +column_id: EffectiveCost +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D018-0001.yaml b/focus_validator/rules/base_rule_definitions/FV-D018-0001.yaml deleted file mode 100644 index 136be02..0000000 --- a/focus_validator/rules/base_rule_definitions/FV-D018-0001.yaml +++ /dev/null @@ -1,4 +0,0 @@ -check_id: FV-D018-0001 -column_id: SubAccountName -check: - data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D018-0003.yaml b/focus_validator/rules/base_rule_definitions/FV-D018-0003.yaml deleted file mode 100644 index 25ab012..0000000 --- a/focus_validator/rules/base_rule_definitions/FV-D018-0003.yaml +++ /dev/null @@ -1,4 +0,0 @@ -check_id: FV-D018-0003 -column_id: SubAccountName -check: - column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D003-0001.yaml b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_IsString.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D003-0001.yaml rename to focus_validator/rules/base_rule_definitions/InvoiceIssuer_IsString.yaml index 15848b9..a98271b 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D003-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D003-0001 column_id: InvoiceIssuer check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D003-0002.yaml b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_NotNull.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D003-0002.yaml rename to focus_validator/rules/base_rule_definitions/InvoiceIssuer_NotNull.yaml index 5347a76..5e4060a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D003-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D003-0002 column_id: InvoiceIssuer check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/InvoiceIssuer_Required.yaml b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_Required.yaml new file mode 100644 index 0000000..607ec84 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_Required.yaml @@ -0,0 +1,2 @@ +column_id: InvoiceIssuer +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ListCost_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/ListCost_IsDecimal.yaml new file mode 100644 index 0000000..02b7f93 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListCost_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: ListCost +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/ListCost_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ListCost_NotNull.yaml new file mode 100644 index 0000000..3259446 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListCost_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ListCost +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ListCost_Required.yaml b/focus_validator/rules/base_rule_definitions/ListCost_Required.yaml new file mode 100644 index 0000000..f0749c4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListCost_Required.yaml @@ -0,0 +1,3 @@ +column_id: ListCost +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/ListUnitPrice_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/ListUnitPrice_IsDecimal.yaml new file mode 100644 index 0000000..6bbd392 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListUnitPrice_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: ListUnitPrice +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/ListUnitPrice_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Nullable.yaml new file mode 100644 index 0000000..e93fb24 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ListUnitPrice +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ListUnitPrice_Required.yaml b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Required.yaml new file mode 100644 index 0000000..f0beb93 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Required.yaml @@ -0,0 +1,3 @@ +column_id: ListUnitPrice +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml new file mode 100644 index 0000000..2e42fa9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml @@ -0,0 +1,7 @@ +column_id: PricingCategory +check: + value_in: + - "On-Demand" + - "Dynamic" + - "Commitment-Based" + - "Other" diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Enum_1.0.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum_1.0.yaml new file mode 100644 index 0000000..0300b55 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum_1.0.yaml @@ -0,0 +1,7 @@ +column_id: PricingCategory +check: + value_in: + - "Standard" + - "Dynamic" + - "Committed" + - "Other" diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml new file mode 100644 index 0000000..2065111 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: PricingCategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml new file mode 100644 index 0000000..49c257f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingCategory +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml new file mode 100644 index 0000000..4904b04 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml @@ -0,0 +1,2 @@ +column_id: PricingCategory +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/PricingQuantity_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/PricingQuantity_IsDecimal.yaml new file mode 100644 index 0000000..c2f22d2 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingQuantity_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: PricingQuantity +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/PricingQuantity_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingQuantity_Nullable.yaml new file mode 100644 index 0000000..2cda91c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingQuantity_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingQuantity +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingQuantity_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingQuantity_Required.yaml new file mode 100644 index 0000000..6dfdc20 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingQuantity_Required.yaml @@ -0,0 +1,3 @@ +column_id: PricingQuantity +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml new file mode 100644 index 0000000..f138fa4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml @@ -0,0 +1,3 @@ +column_id: PricingUnit +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml new file mode 100644 index 0000000..34e841a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingUnit +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml new file mode 100644 index 0000000..e2c935e --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml @@ -0,0 +1,2 @@ +column_id: PricingUnit +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D004-0001.yaml b/focus_validator/rules/base_rule_definitions/Provider_IsString.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D004-0001.yaml rename to focus_validator/rules/base_rule_definitions/Provider_IsString.yaml index 5b79316..6b5e61f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D004-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/Provider_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D004-0001 column_id: Provider check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D004-0002.yaml b/focus_validator/rules/base_rule_definitions/Provider_NotNull.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D004-0002.yaml rename to focus_validator/rules/base_rule_definitions/Provider_NotNull.yaml index 52e2f9e..dfdea5f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D004-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/Provider_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D004-0002 column_id: Provider check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D004-0003.yaml b/focus_validator/rules/base_rule_definitions/Provider_Required.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D004-0003.yaml rename to focus_validator/rules/base_rule_definitions/Provider_Required.yaml index 17e18d1..88906a7 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D004-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/Provider_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D004-0003 column_id: Provider check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D007-0001.yaml b/focus_validator/rules/base_rule_definitions/Publisher_IsString.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D007-0001.yaml rename to focus_validator/rules/base_rule_definitions/Publisher_IsString.yaml index 4bba250..8502a38 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D007-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/Publisher_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D007-0001 column_id: Publisher check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D007-0002.yaml b/focus_validator/rules/base_rule_definitions/Publisher_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D007-0002.yaml rename to focus_validator/rules/base_rule_definitions/Publisher_NotNull.yaml index 4f82b78..122fa87 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D007-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/Publisher_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D007-0002 column_id: Publisher check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D007-0003.yaml b/focus_validator/rules/base_rule_definitions/Publisher_Required.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D007-0003.yaml rename to focus_validator/rules/base_rule_definitions/Publisher_Required.yaml index 94b52d4..9d4dc1e 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D007-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/Publisher_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D007-0003 column_id: Publisher check: column_required diff --git a/focus_validator/rules/base_rule_definitions/RegionId_IsString.yaml b/focus_validator/rules/base_rule_definitions/RegionId_IsString.yaml new file mode 100644 index 0000000..3af92bb --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/RegionId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: RegionId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/RegionId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/RegionId_Nullable.yaml new file mode 100644 index 0000000..fa2ef8a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/RegionId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: RegionId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/RegionId_Required.yaml b/focus_validator/rules/base_rule_definitions/RegionId_Required.yaml new file mode 100644 index 0000000..2ed5aa9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/RegionId_Required.yaml @@ -0,0 +1,2 @@ +column_id: RegionId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/RegionName_IsString.yaml b/focus_validator/rules/base_rule_definitions/RegionName_IsString.yaml new file mode 100644 index 0000000..a04cac3 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/RegionName_IsString.yaml @@ -0,0 +1,3 @@ +column_id: RegionName +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/RegionName_Nullable.yaml b/focus_validator/rules/base_rule_definitions/RegionName_Nullable.yaml new file mode 100644 index 0000000..3f470f2 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/RegionName_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: RegionName +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/RegionName_Required.yaml b/focus_validator/rules/base_rule_definitions/RegionName_Required.yaml new file mode 100644 index 0000000..c7f2f2b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/RegionName_Required.yaml @@ -0,0 +1,2 @@ +column_id: RegionName +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D013-0001.yaml b/focus_validator/rules/base_rule_definitions/Region_IsString.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D013-0001.yaml rename to focus_validator/rules/base_rule_definitions/Region_IsString.yaml index fc77602..8733163 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D013-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/Region_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D013-0001 column_id: Region check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D013-0002.yaml b/focus_validator/rules/base_rule_definitions/Region_NotNull.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D013-0002.yaml rename to focus_validator/rules/base_rule_definitions/Region_NotNull.yaml index 19ba00c..fcc15a5 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D013-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/Region_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D013-0002 column_id: Region check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D013-0003.yaml b/focus_validator/rules/base_rule_definitions/Region_Required.yaml similarity index 65% rename from focus_validator/rules/base_rule_definitions/FV-D013-0003.yaml rename to focus_validator/rules/base_rule_definitions/Region_Required.yaml index e27245d..60968e1 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D013-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/Region_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D013-0003 column_id: Region check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D002-0001.yaml b/focus_validator/rules/base_rule_definitions/ResourceID_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D002-0001.yaml rename to focus_validator/rules/base_rule_definitions/ResourceID_IsString.yaml index fa8f571..42913bb 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D002-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceID_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D002-0001 column_id: ResourceID check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D002-0002.yaml b/focus_validator/rules/base_rule_definitions/ResourceID_Nullable.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D002-0002.yaml rename to focus_validator/rules/base_rule_definitions/ResourceID_Nullable.yaml index 862411d..f855723 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D002-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceID_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D002-0002 column_id: ResourceID check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ResourceID_Required.yaml b/focus_validator/rules/base_rule_definitions/ResourceID_Required.yaml new file mode 100644 index 0000000..0bec5db --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceID_Required.yaml @@ -0,0 +1,2 @@ +column_id: ResourceID +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D008-0001.yaml b/focus_validator/rules/base_rule_definitions/ResourceName_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D008-0001.yaml rename to focus_validator/rules/base_rule_definitions/ResourceName_IsString.yaml index 6aafb9c..06bc202 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D008-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceName_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D008-0001 column_id: ResourceName check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D008-0002.yaml b/focus_validator/rules/base_rule_definitions/ResourceName_Nullable.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D008-0002.yaml rename to focus_validator/rules/base_rule_definitions/ResourceName_Nullable.yaml index fd9878e..bcd02f8 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D008-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceName_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D008-0002 column_id: ResourceName check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/FV-D008-0003.yaml b/focus_validator/rules/base_rule_definitions/ResourceName_Required.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D008-0003.yaml rename to focus_validator/rules/base_rule_definitions/ResourceName_Required.yaml index 1821b6b..b713ea3 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D008-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceName_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D008-0003 column_id: ResourceName check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml new file mode 100644 index 0000000..68a5230 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ResourceType +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml new file mode 100644 index 0000000..99cf430 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ResourceType +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml new file mode 100644 index 0000000..5df814f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml @@ -0,0 +1,2 @@ +column_id: ResourceType +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0004.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_Enum.yaml similarity index 95% rename from focus_validator/rules/base_rule_definitions/FV-D015-0004.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_Enum.yaml index fb68aae..85f90f3 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0004.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_Enum.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0004 column_id: ServiceCategory check_friendly_name: "ServiceCategory must have a value defined in spec." check: diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0001.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_IsString.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D015-0001.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_IsString.yaml index 629223b..f359331 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0001 column_id: ServiceCategory check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0002.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D015-0002.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_NotNull.yaml index a5f600b..e2825f6 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0002 column_id: ServiceCategory check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0003.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D015-0003.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_Required.yaml index 7290024..ea364b3 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0003 column_id: ServiceCategory check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D009-0001.yaml b/focus_validator/rules/base_rule_definitions/ServiceName_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D009-0001.yaml rename to focus_validator/rules/base_rule_definitions/ServiceName_IsString.yaml index b5956f1..7429b14 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D009-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceName_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D009-0001 column_id: ServiceName check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D009-0002.yaml b/focus_validator/rules/base_rule_definitions/ServiceName_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D009-0002.yaml rename to focus_validator/rules/base_rule_definitions/ServiceName_NotNull.yaml index 8d0b787..2c62718 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D009-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceName_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D009-0002 column_id: ServiceName check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D009-0003.yaml b/focus_validator/rules/base_rule_definitions/ServiceName_Required.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D009-0003.yaml rename to focus_validator/rules/base_rule_definitions/ServiceName_Required.yaml index fdede46..5c07502 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D009-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceName_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D009-0003 column_id: ServiceName check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml new file mode 100644 index 0000000..4b43197 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SkuId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml new file mode 100644 index 0000000..d8656e9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: SkuId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml b/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml new file mode 100644 index 0000000..3fdb39d --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SkuId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml new file mode 100644 index 0000000..76460dd --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SkuPriceId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml new file mode 100644 index 0000000..76d2fc1 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml @@ -0,0 +1,9 @@ +column_id: SkuPriceId +check_friendly_name: SkuPriceId must be set for certain values of ChargeType +check: + sql_query: | + SELECT CASE + WHEN ChargeType IN ('Purchase', 'Usage', 'Refund') AND SkuPriceId IS NULL THEN FALSE + ELSE TRUE + END AS check_output + FROM df; diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml new file mode 100644 index 0000000..5352d19 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SkuPriceId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml new file mode 100644 index 0000000..aa49628 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SubAccountId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml new file mode 100644 index 0000000..03f41eb --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: SubAccountId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml new file mode 100644 index 0000000..d492ff7 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SubAccountId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SubAccountName_IsString.yaml b/focus_validator/rules/base_rule_definitions/SubAccountName_IsString.yaml new file mode 100644 index 0000000..5847d66 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountName_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SubAccountName +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D018-0002.yaml b/focus_validator/rules/base_rule_definitions/SubAccountName_Nullable.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D018-0002.yaml rename to focus_validator/rules/base_rule_definitions/SubAccountName_Nullable.yaml index 8520687..d861749 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D018-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/SubAccountName_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D018-0002 column_id: SubAccountName check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml b/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml new file mode 100644 index 0000000..635ab28 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml @@ -0,0 +1,2 @@ +column_id: SubAccountName +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml b/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml new file mode 100644 index 0000000..2832cea --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + data_type: stringified-json-object diff --git a/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml b/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml new file mode 100644 index 0000000..b06ee4c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/Tags_Required.yaml b/focus_validator/rules/base_rule_definitions/Tags_Required.yaml new file mode 100644 index 0000000..54ae7c2 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_Required.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml new file mode 100644 index 0000000..e7071b4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml new file mode 100644 index 0000000..05d4bfb --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml new file mode 100644 index 0000000..3a21150 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml new file mode 100644 index 0000000..89fa9e0 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml @@ -0,0 +1,3 @@ +column_id: UsageUnit +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml new file mode 100644 index 0000000..b5fb4b5 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: UsageUnit +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml new file mode 100644 index 0000000..a23616c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml @@ -0,0 +1,2 @@ +column_id: UsageUnit +check: column_required diff --git a/focus_validator/rules/checks.py b/focus_validator/rules/checks.py index 06d5723..0cf7932 100644 --- a/focus_validator/rules/checks.py +++ b/focus_validator/rules/checks.py @@ -1,8 +1,13 @@ -import re +import json from datetime import datetime +from typing import Union +import numpy as np import pandas as pd +import pandasql +import pandera as pa from pandera import extensions +from pandera.errors import SchemaError from focus_validator.utils.download_currency_codes import get_currency_codes @@ -16,11 +21,8 @@ def is_camel_case(column_name): @extensions.register_check_method() -def check_not_null(pandas_obj: pd.Series, allow_nulls: bool): - # TODO: works for string type, need to verify for other data types - check_values = pandas_obj.isnull() | (pandas_obj == "") - if not allow_nulls: - check_values = check_values | (pandas_obj == "NULL") +def check_not_null(pandas_obj: pd.Series): + check_values = pandas_obj.isnull() return ~check_values @@ -34,19 +36,61 @@ def check_value_in(pandas_obj: pd.Series, allowed_values): return pandas_obj.isin(allowed_values) -@extensions.register_check_method() -def check_datetime_dtype(pandas_obj: pd.Series): - pattern = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") +@extensions.register_check_method(check_type="groupby") +def check_sql_query(df_groups, sql_query, column_alias): + grouped_elements = [] + for values in list(df_groups): + row_obj = {} + for column_name, value in zip(column_alias + ["index"], values): + row_obj[column_name] = value + grouped_elements.append(row_obj) - def __validate_date_obj__(value: str): - if not (isinstance(value, str) and re.match(pattern, value)): - return False + df = pd.DataFrame(grouped_elements) + check_output = pandasql.sqldf(sql_query, locals())["check_output"] + + # Getting the index of rows where the series values are False + false_indexes = [i for i, val in enumerate(check_output) if not val] + if false_indexes: + # Extracting those rows from the dataframe + extracted_rows = df.loc[false_indexes].to_dict("records")[:3] + false_indexes = [i for i, val in enumerate(check_output) if not val] + + # for the given indexes in false_indexes list, we are extracting the rows from the dataframe and + # add column_alias value to failure_case column and index to index column + failure_cases = df[df.index.isin(false_indexes)].copy() + failure_cases.loc[:, "failure_case"] = [ + ",".join([f"{column}:{row[column]}" for column in column_alias]) + for _, row in failure_cases.iterrows() + ] + + raise SchemaError( + schema=pa.DataFrameSchema(), + data=None, + message="", + failure_cases=failure_cases, + ) + + return True - try: - datetime.strptime(value[:-1], "%Y-%m-%dT%H:%M:%S") - return True - except ValueError: - return False + +@extensions.register_check_method() +def check_datetime_dtype(pandas_obj: pd.Series): + def __validate_date_obj__(value: Union[str, datetime]): + if isinstance(value, str): + # fix of python 3.10 and lower, strings ending with Z are not parsed automatically + if value.endswith("Z"): + value = value.replace("Z", "+00:00") + + try: + value = datetime.fromisoformat(value) + except ValueError: + # failed to parse iso string + return False + + if isinstance(value, datetime): + return value.tzname() == "UTC" + else: + return isinstance(value, np.datetime64) return pd.Series(map(__validate_date_obj__, pandas_obj.values)) @@ -57,3 +101,15 @@ def check_currency_code_dtype(pandas_obj: pd.Series): return pd.Series( map(lambda v: isinstance(v, str) and v in currency_codes, pandas_obj.values) ) + + +@extensions.register_check_method() +def check_stringified_json_object_dtype(pandas_obj: pd.Series): + def __validate_stringified_json_object__(value: str): + try: + parsed = json.loads(value) + return isinstance(parsed, dict) + except Exception: + return False + + return pd.Series(map(__validate_stringified_json_object__, pandas_obj.values)) diff --git a/focus_validator/rules/version_sets.yaml b/focus_validator/rules/version_sets.yaml new file mode 100644 index 0000000..1948d5b --- /dev/null +++ b/focus_validator/rules/version_sets.yaml @@ -0,0 +1,323 @@ +"1.0": + - AvailabilityZone_IsString.yaml + - AvailabilityZone_Nullable.yaml + - BilledCost_IsDecimal.yaml + - BilledCost_NotNull.yaml + - BilledCost_Required.yaml + - BillingAccountId_Required.yaml + - BillingAccountId_IsString.yaml + - BillingAccountId_NotNull.yaml + - BillingAccountName_Required.yaml + - BillingAccountName_IsString.yaml + - BillingAccountName_Nullable.yaml + - BillingCurrency_Required.yaml + - BillingCurrency_IsCurrencyCode.yaml + - BillingCurrency_NotNull.yaml + - BillingPeriodEnd_Required.yaml + - BillingPeriodEnd_IsDateTime.yaml + - BillingPeriodEnd_NotNull.yaml + - BillingPeriodStart_Required.yaml + - BillingPeriodStart_IsDateTime.yaml + - BillingPeriodStart_NotNull.yaml + - ChargeCategory_Required.yaml + - ChargeCategory_NotNull.yaml + - ChargeCategory_IsString.yaml + - ChargeClass_Required.yaml + - ChargeClass_Nullable.yaml + - ChargeClass_IsString.yaml + - ChargeClass_Enum.yaml + - ChargeDescription_Required.yaml + - ChargeDescription_IsString.yaml + - ChargeDescription_Nullable.yaml + - ChargeFrequency_Enum.yaml + - ChargeFrequency_IsString.yaml + - ChargeFrequency_NotNull.yaml + - ChargePeriodEnd_Required.yaml + - ChargePeriodEnd_IsDateTime.yaml + - ChargePeriodEnd_NotNull.yaml + - ChargePeriodStart_Required.yaml + - ChargePeriodStart_IsDateTime.yaml + - ChargePeriodStart_NotNull.yaml + - CommitmentDiscountCategory_Required.yaml + - CommitmentDiscountCategory_Enum.yaml + - CommitmentDiscountCategory_IsString.yaml + - CommitmentDiscountCategory_Nullable.yaml + - CommitmentDiscountId_Required.yaml + - CommitmentDiscountId_IsString.yaml + - CommitmentDiscountName_Required.yaml + - CommitmentDiscountId_Nullable.yaml + - CommitmentDiscountName_IsString.yaml + - CommitmentDiscountName_Nullable.yaml + - CommitmentDiscountType_Required.yaml + - CommitmentDiscountType_IsString.yaml + - CommitmentDiscountType_Nullable.yaml + - CommitmentDiscountStatus_Required.yaml + - CommitmentDiscountStatus_IsString.yaml + - CommitmentDiscountStatus_Nullable.yaml + - CommitmentDiscountStatus_Enum.yaml + - ConsumedQuantity_Required.yaml + - ConsumedQuantity_IsDecimal.yaml + - ConsumedQuantity_Nullable.yaml + - ConsumedUnit_Required.yaml + - ConsumedUnit_IsString.yaml + - ConsumedUnit_Nullable.yaml + - ContractedCost_Required.yaml + - ContractedCost_IsDecimal.yaml + - ContractedCost_NotNull.yaml + - ContractedUnitPrice_Required.yaml + - ContractedUnitPrice_IsDecimal.yaml + - ContractedUnitPrice_Nullable.yaml + - EffectiveCost_Required.yaml + - EffectiveCost_IsDecimal.yaml + - EffectiveCost_NotNull.yaml + - InvoiceIssuer_Required.yaml + - InvoiceIssuer_IsString.yaml + - InvoiceIssuer_NotNull.yaml + - ListCost_Required.yaml + - ListCost_IsDecimal.yaml + - ListCost_NotNull.yaml + - ListUnitPrice_Required.yaml + - ListUnitPrice_IsDecimal.yaml + - ListUnitPrice_Nullable.yaml + - PricingCategory_Required.yaml + - PricingCategory_Enum_1.0.yaml + - PricingCategory_IsString.yaml + - PricingCategory_Nullable.yaml + - PricingQuantity_Required.yaml + - PricingQuantity_IsDecimal.yaml + - PricingQuantity_Nullable.yaml + - PricingUnit_Required.yaml + - PricingUnit_IsString.yaml + - PricingUnit_Nullable.yaml + - Provider_Required.yaml + - Provider_IsString.yaml + - Provider_NotNull.yaml + - Publisher_Required.yaml + - Publisher_IsString.yaml + - Publisher_NotNull.yaml + - RegionId_Required.yaml + - RegionId_Nullable.yaml + - RegionId_IsString.yaml + - RegionName_Required.yaml + - RegionName_Nullable.yaml + - RegionName_IsString.yaml + - ResourceID_Required.yaml + - ResourceID_IsString.yaml + - ResourceID_Nullable.yaml + - ResourceName_Required.yaml + - ResourceName_IsString.yaml + - ResourceName_Nullable.yaml + - ResourceType_Required.yaml + - ResourceType_IsString.yaml + - ResourceType_Nullable.yaml + - ServiceCategory_Required.yaml + - ServiceCategory_Enum.yaml + - ServiceCategory_IsString.yaml + - ServiceCategory_NotNull.yaml + - ServiceName_Required.yaml + - ServiceName_IsString.yaml + - ServiceName_NotNull.yaml + - SkuId_Required.yaml + - SkuId_IsString.yaml + - SkuId_Nullable.yaml + - SkuPriceId_Required.yaml + - SkuPriceId_IsString.yaml + - SkuPriceId_Nullable.yaml + - SubAccountId_Required.yaml + - SubAccountId_IsString.yaml + - SubAccountId_Nullable.yaml + - SubAccountName_Required.yaml + - SubAccountName_IsString.yaml + - SubAccountName_Nullable.yaml + - Tags_Required.yaml + - Tags_IsJSONObject.yaml + - Tags_Nullable.yaml + +"1.0-preview": + - ChargeDescription_IsString.yaml + - ChargeDescription_Required.yaml + - ChargeDescription_NotNull.yaml + - ChargeFrequency_Enum.yaml + - ChargeFrequency_IsString.yaml + - ChargeFrequency_NotNull.yaml + - ChargeFrequency_Required.yaml + - ChargeSubcategory_Enum.yaml + - ChargeSubcategory_IsString.yaml + - ChargeSubcategory_NotNull.yaml + - ChargeSubcategory_Required.yaml + - CommitmentDiscountCategory_Enum.yaml + - CommitmentDiscountCategory_IsString.yaml + - CommitmentDiscountCategory_NotNull.yaml + - CommitmentDiscountCategory_Required.yaml + - CommitmentDiscountId_IsString.yaml + - CommitmentDiscountId_Nullable.yaml + - CommitmentDiscountId_Required.yaml + - CommitmentDiscountName_IsString.yaml + - CommitmentDiscountName_Nullable.yaml + - CommitmentDiscountName_Required.yaml + - CommitmentDiscountType_IsString.yaml + - CommitmentDiscountType_Nullable.yaml + - EffectiveCost_IsDecimal.yaml + - EffectiveCost_NotNull.yaml + - EffectiveCost_Required.yaml + - ListCost_IsDecimal.yaml + - ListCost_NotNull.yaml + - ListCost_Required.yaml + - ListUnitPrice_IsDecimal.yaml + - ListUnitPrice_Nullable.yaml + - ListUnitPrice_Required.yaml + - PricingCategory_Enum.yaml + - PricingCategory_IsString.yaml + - PricingCategory_Nullable.yaml + - PricingCategory_Required.yaml + - PricingQuantity_IsDecimal.yaml + - PricingQuantity_Nullable.yaml + - PricingQuantity_Required.yaml + - PricingUnit_IsString.yaml + - PricingUnit_Nullable.yaml + - PricingUnit_Required.yaml + - ResourceType_IsString.yaml + - ResourceType_Nullable.yaml + - ResourceType_Required.yaml + - SkuId_IsString.yaml + - SkuId_Nullable.yaml + - SkuId_Required.yaml + - SkuPriceId_IsString.yaml + - SkuPriceId_Nullable.yaml + - SkuPriceId_Required.yaml + - Tags_IsJSONObject.yaml + - Tags_Nullable.yaml + - Tags_Required.yaml + - UsageQuantity_IsDecimal.yaml + - UsageQuantity_Nullable.yaml + - UsageQuantity_Required.yaml + - UsageUnit_IsString.yaml + - UsageUnit_Nullable.yaml + - UsageUnit_Required.yaml + + ### Removed 0.5 content is left here in commented form for reference + # - AmortizedCost_IsDecimal.yaml + # - AmortizedCost_NotNull.yaml + # - AmortizedCost_Required.yaml + + ### Unmodified 0.5 content follows + - AvailabilityZone_IsString.yaml + - AvailabilityZone_Nullable.yaml + - BilledCost_IsDecimal.yaml + - BilledCost_NotNull.yaml + - BilledCost_Required.yaml + - BilledCurrency_IsCurrencyCode.yaml + - BilledCurrency_NotNull.yaml + - BilledCurrency_Required.yaml + - BillingAccountId_IsString.yaml + - BillingAccountId_NotNull.yaml + - BillingAccountId_Required.yaml + - BillingAccountName_IsString.yaml + - BillingAccountName_Nullable.yaml + - BillingAccountName_Required.yaml + - BillingPeriodEnd_IsDateTime.yaml + - BillingPeriodEnd_NotNull.yaml + - BillingPeriodEnd_Required.yaml + - BillingPeriodStart_IsDateTime.yaml + - BillingPeriodStart_NotNull.yaml + - BillingPeriodStart_Required.yaml + - ChargePeriodEnd_IsDateTime.yaml + - ChargePeriodEnd_NotNull.yaml + - ChargePeriodEnd_Required.yaml + - ChargePeriodStart_IsDateTime.yaml + - ChargePeriodStart_NotNull.yaml + - ChargePeriodStart_Required.yaml + - ChargeType_Enum.yaml + - ChargeType_IsString.yaml + - ChargeType_NotNull.yaml + - ChargeType_Required.yaml + - InvoiceIssuer_IsString.yaml + - InvoiceIssuer_NotNull.yaml + - Provider_IsString.yaml + - Provider_NotNull.yaml + - Provider_Required.yaml + - Publisher_IsString.yaml + - Publisher_NotNull.yaml + - Publisher_Required.yaml + - Region_IsString.yaml + - Region_NotNull.yaml + - Region_Required.yaml + - ResourceID_IsString.yaml + - ResourceID_Nullable.yaml + - ResourceName_IsString.yaml + - ResourceName_Nullable.yaml + - ResourceName_Required.yaml + - ServiceCategory_Enum.yaml + - ServiceCategory_IsString.yaml + - ServiceCategory_NotNull.yaml + - ServiceCategory_Required.yaml + - ServiceName_IsString.yaml + - ServiceName_NotNull.yaml + - ServiceName_Required.yaml + - SubAccountName_IsString.yaml + - SubAccountName_Nullable.yaml + - SubAccountName_Required.yaml + +"0.5": + - AmortizedCost_IsDecimal.yaml + - AmortizedCost_NotNull.yaml + - AmortizedCost_Required.yaml + - AvailabilityZone_IsString.yaml + - AvailabilityZone_Nullable.yaml + - BilledCost_IsDecimal.yaml + - BilledCost_NotNull.yaml + - BilledCost_Required.yaml + - BilledCurrency_IsCurrencyCode.yaml + - BilledCurrency_NotNull.yaml + - BilledCurrency_Required.yaml + - BillingAccountId_IsString.yaml + - BillingAccountId_NotNull.yaml + - BillingAccountId_Required.yaml + - BillingAccountName_IsString.yaml + - BillingAccountName_Nullable.yaml + - BillingAccountName_Required.yaml + - BillingPeriodEnd_IsDateTime.yaml + - BillingPeriodEnd_NotNull.yaml + - BillingPeriodEnd_Required.yaml + - BillingPeriodStart_IsDateTime.yaml + - BillingPeriodStart_NotNull.yaml + - BillingPeriodStart_Required.yaml + - ChargePeriodEnd_IsDateTime.yaml + - ChargePeriodEnd_NotNull.yaml + - ChargePeriodEnd_Required.yaml + - ChargePeriodStart_IsDateTime.yaml + - ChargePeriodStart_NotNull.yaml + - ChargePeriodStart_Required.yaml + - ChargeType_Enum.yaml + - ChargeType_IsString.yaml + - ChargeType_NotNull.yaml + - ChargeType_Required.yaml + - InvoiceIssuer_IsString.yaml + - InvoiceIssuer_NotNull.yaml + - Provider_IsString.yaml + - Provider_NotNull.yaml + - Provider_Required.yaml + - Publisher_IsString.yaml + - Publisher_NotNull.yaml + - Publisher_Required.yaml + - Region_IsString.yaml + - Region_NotNull.yaml + - Region_Required.yaml + - ResourceID_IsString.yaml + - ResourceID_Nullable.yaml + - ResourceName_IsString.yaml + - ResourceName_Nullable.yaml + - ResourceName_Required.yaml + - ServiceCategory_Enum.yaml + - ServiceCategory_IsString.yaml + - ServiceCategory_NotNull.yaml + - ServiceCategory_Required.yaml + - ServiceName_IsString.yaml + - ServiceName_NotNull.yaml + - ServiceName_Required.yaml + - SubAccountId_IsString.yaml + - SubAccountId_Nullable.yaml + - SubAccountId_Required.yaml + - SubAccountName_IsString.yaml + - SubAccountName_Nullable.yaml + - SubAccountName_Required.yaml diff --git a/focus_validator/rules/version_sets/0.5/FV-D001-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D001-0001.yaml deleted file mode 120000 index 4bcf5b5..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D001-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D001-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D001-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D001-0002.yaml deleted file mode 120000 index 2c7e15e..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D001-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D001-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D001-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D001-0003.yaml deleted file mode 120000 index ec90fc1..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D001-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D001-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D001-0004.yaml b/focus_validator/rules/version_sets/0.5/FV-D001-0004.yaml deleted file mode 120000 index ed757eb..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D001-0004.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D001-0004.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D002-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D002-0001.yaml deleted file mode 120000 index 02086c6..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D002-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D002-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D002-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D002-0002.yaml deleted file mode 120000 index 8ef80bc..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D002-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D002-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D003-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D003-0001.yaml deleted file mode 120000 index e77d2c1..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D003-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D003-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D003-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D003-0002.yaml deleted file mode 120000 index 29c5e03..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D003-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D003-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D004-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D004-0001.yaml deleted file mode 120000 index a93f4b7..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D004-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D004-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D004-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D004-0002.yaml deleted file mode 120000 index b0602a8..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D004-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D004-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D004-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D004-0003.yaml deleted file mode 120000 index 389277a..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D004-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D004-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D005-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D005-0001.yaml deleted file mode 120000 index 6d5f94a..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D005-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D005-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D005-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D005-0002.yaml deleted file mode 120000 index 32f5465..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D005-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D005-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D005-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D005-0003.yaml deleted file mode 120000 index bbee430..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D005-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D005-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D006-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D006-0001.yaml deleted file mode 120000 index 8e16a1c..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D006-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D006-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D006-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D006-0002.yaml deleted file mode 120000 index 88e4cbc..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D006-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D006-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D006-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D006-0003.yaml deleted file mode 120000 index b9a757c..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D006-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D006-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D007-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D007-0001.yaml deleted file mode 120000 index 4f78ee5..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D007-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D007-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D007-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D007-0002.yaml deleted file mode 120000 index 9c3db63..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D007-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D007-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D007-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D007-0003.yaml deleted file mode 120000 index ba6f50a..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D007-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D007-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D008-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D008-0001.yaml deleted file mode 120000 index 1640433..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D008-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D008-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D008-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D008-0002.yaml deleted file mode 120000 index c02783b..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D008-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D008-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D008-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D008-0003.yaml deleted file mode 120000 index 8f00221..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D008-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D008-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D009-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D009-0001.yaml deleted file mode 120000 index 368d590..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D009-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D009-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D009-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D009-0002.yaml deleted file mode 120000 index 2405772..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D009-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D009-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D009-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D009-0003.yaml deleted file mode 120000 index a523bc8..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D009-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D009-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D010-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D010-0001.yaml deleted file mode 120000 index 1357b5d..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D010-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D010-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D010-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D010-0002.yaml deleted file mode 120000 index cd76c95..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D010-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D010-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D010-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D010-0003.yaml deleted file mode 120000 index b7d100f..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D010-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D010-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D011-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D011-0001.yaml deleted file mode 120000 index 75e3aad..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D011-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D011-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D011-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D011-0002.yaml deleted file mode 120000 index 7b382bc..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D011-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D011-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D011-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D011-0003.yaml deleted file mode 120000 index 61f6fd4..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D011-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D011-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D012-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D012-0001.yaml deleted file mode 120000 index 1e9e641..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D012-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D012-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D012-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D012-0002.yaml deleted file mode 120000 index be582ec..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D012-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D012-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D012-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D012-0003.yaml deleted file mode 120000 index 4cfb13b..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D012-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D012-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D013-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D013-0001.yaml deleted file mode 120000 index 25188de..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D013-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D013-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D013-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D013-0002.yaml deleted file mode 120000 index b32d2a3..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D013-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D013-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D013-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D013-0003.yaml deleted file mode 120000 index bf23ef9..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D013-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D013-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D014-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D014-0001.yaml deleted file mode 120000 index 049b62b..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D014-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D014-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D014-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D014-0002.yaml deleted file mode 120000 index c7dd9f1..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D014-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D014-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D015-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D015-0001.yaml deleted file mode 120000 index a24ec44..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D015-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D015-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D015-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D015-0002.yaml deleted file mode 120000 index 79c9fb8..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D015-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D015-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D015-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D015-0003.yaml deleted file mode 120000 index 7d24efb..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D015-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D015-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D015-0004.yaml b/focus_validator/rules/version_sets/0.5/FV-D015-0004.yaml deleted file mode 120000 index c0181be..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D015-0004.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D015-0004.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D016-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D016-0001.yaml deleted file mode 120000 index 2be25df..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D016-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D016-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D016-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D016-0002.yaml deleted file mode 120000 index 45979de..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D016-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D016-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D016-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D016-0003.yaml deleted file mode 120000 index 7775e96..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D016-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D016-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D017-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D017-0001.yaml deleted file mode 120000 index e2d4dba..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D017-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D017-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D017-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D017-0002.yaml deleted file mode 120000 index ba541bd..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D017-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D017-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D017-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D017-0003.yaml deleted file mode 120000 index 58d0afe..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D017-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D017-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D018-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-D018-0001.yaml deleted file mode 120000 index bc4ec53..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D018-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D018-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D018-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-D018-0002.yaml deleted file mode 120000 index 2877388..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D018-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D018-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-D018-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-D018-0003.yaml deleted file mode 120000 index 6274407..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-D018-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-D018-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-M001-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-M001-0001.yaml deleted file mode 120000 index 7770868..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-M001-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-M001-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-M001-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-M001-0002.yaml deleted file mode 120000 index ed859f5..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-M001-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-M001-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-M001-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-M001-0003.yaml deleted file mode 120000 index dbb95d4..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-M001-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-M001-0003.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-M002-0001.yaml b/focus_validator/rules/version_sets/0.5/FV-M002-0001.yaml deleted file mode 120000 index 8ba19c4..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-M002-0001.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-M002-0001.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-M002-0002.yaml b/focus_validator/rules/version_sets/0.5/FV-M002-0002.yaml deleted file mode 120000 index 21902b4..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-M002-0002.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-M002-0002.yaml \ No newline at end of file diff --git a/focus_validator/rules/version_sets/0.5/FV-M002-0003.yaml b/focus_validator/rules/version_sets/0.5/FV-M002-0003.yaml deleted file mode 120000 index f31f6da..0000000 --- a/focus_validator/rules/version_sets/0.5/FV-M002-0003.yaml +++ /dev/null @@ -1 +0,0 @@ -../../base_rule_definitions/FV-M002-0003.yaml \ No newline at end of file diff --git a/focus_validator/utils/download_currency_codes.py b/focus_validator/utils/download_currency_codes.py index a748556..53e0d08 100644 --- a/focus_validator/utils/download_currency_codes.py +++ b/focus_validator/utils/download_currency_codes.py @@ -7,7 +7,7 @@ CURRENCY_CODE_CSV_PATH = "focus_validator/utils/currency_codes.csv" -def download_currency_codes(): +def download_currency_codes(): # pragma: no cover r = requests.get(DATAHUB_URL) root = ET.fromstring(r.content.decode()) @@ -25,5 +25,5 @@ def get_currency_codes(): return set(df["currency_codes"].values) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover download_currency_codes() diff --git a/focus_validator/validator.py b/focus_validator/validator.py index 9d8ec35..18cacf1 100644 --- a/focus_validator/validator.py +++ b/focus_validator/validator.py @@ -1,10 +1,20 @@ -from pkg_resources import resource_filename +import importlib.resources from focus_validator.data_loaders import data_loader from focus_validator.outputter.outputter import Outputter from focus_validator.rules.spec_rules import SpecRules -DEFAULT_VERSION_SETS_PATH = resource_filename("focus_validator.rules", "version_sets") +try: + DEFAULT_VERSION_SETS_PATH = str( + importlib.resources.files("focus_validator.rules").joinpath("version_sets") + ) +except AttributeError: + # for compatibility with python 3.8, which does not support files api in importlib + from pkg_resources import resource_filename + + DEFAULT_VERSION_SETS_PATH = resource_filename( + "focus_validator.rules", "version_sets" + ) class Validator: diff --git a/pyproject.toml b/pyproject.toml index 34a360d..beaf50f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,24 +1,42 @@ [tool.poetry] -name = "focus-spec-validator" -version = "0.5.0" +name = "focus_validator" +version = "1.0.0" description = "FOCUS spec validator." authors = [] readme = "README.md" -packages = [{include = "focus_validator"}] +packages = [{ include = "focus_validator" }] +include = [ + { path = "focus_validator/rules/version_sets/0.5/*" }, + { path = "focus_validator/rules/version_sets/1.0-preview/*" }, + { path = "focus_validator/rules/version_sets/1.0/*" } +] +# TODO: For some reason, this doesn't exclude anything +exclude = [ + { path = "focus_validator/rules/base_rule_definitions/*" } +] + +[tool.poetry.build] +generate-setup-file = false +script = "build.py" [tool.poetry.dependencies] -python = "^3.8.1" -pandas = "^1" +python = "^3.9" +pandas = "^2" tabulate = "*" pyarrow = "*" -pydantic = "^1" -python-magic = "*" +pydantic = "^2" pyyaml = "*" requests = "*" -pandera = "^0.16" +pandera = { version = "^0.17.2" } +sqlglot = "^18.7.0" +numpy = { version = "^1.26"} +pytz = "^2023.3.post1" +pandasql = "^0.7.3" +polars = "^0.20.3" +ddt = "^1.7.1" [tool.poetry.group.dev.dependencies] -black = {extras = ["d"], version = "^23.7.0"} +black = { extras = ["d"], version = "^23.7.0" } polyfactory = "^2.7.0" pytest = "^7.4.0" pytest-cov = "^4.1.0" @@ -28,14 +46,14 @@ types-tabulate = "^0.9.0.3" pandas-stubs = "^2.0.2.230605" types-pyyaml = "^6.0.12.11" types-requests = "^2.31.0.2" -pandera = {extras = ["mypy"], version = "^0.16.1"} +pandera = { version = "^0.17.2", extras = ["mypy"] } isort = "^5.12.0" flake8 = "^6.1.0" pre-commit = "^3.3.3" bump2version = "^1.0.1" [build-system] -requires = ["poetry-core"] +requires = ["poetry-core", "pyyaml"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] diff --git a/setup.cfg b/setup.cfg index 9d6fa07..146ca93 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,3 +13,9 @@ line_length=88 [mypy] plugins = pandera.mypy + +[coverage:run] +omit = focus_validator/utils/*.py + +[mypy-pandasql.*] +ignore_missing_imports = True diff --git a/tests/attributes/test_attribute_json.py b/tests/attributes/test_attribute_json.py new file mode 100644 index 0000000..e1ed79b --- /dev/null +++ b/tests/attributes/test_attribute_json.py @@ -0,0 +1,78 @@ +from unittest import TestCase +from uuid import uuid4 + +import pandas as pd +from pandera.errors import SchemaErrors + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import ( + ChecklistObjectStatus, + DataTypeCheck, + DataTypes, +) +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + + +# noinspection DuplicatedCode +class TestAttributeJSONObject(TestCase): + def __eval_function__(self, sample_value, should_fail): + random_column_id = str(uuid4()) + random_check_id = str(uuid4()) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=[ + Rule( + check_id=random_check_id, + column_id=random_column_id, + check=DataTypeCheck(data_type=DataTypes.STRINGIFIED_JSON_OBJECT), + ) + ] + ) + + sample_data = pd.DataFrame([{random_column_id: sample_value}]) + + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases = e.failure_cases + + validation_result = ValidationResult( + failure_cases=failure_cases, checklist=checklist + ) + validation_result.process_result() + + if should_fail: + self.assertIsNotNone(validation_result.failure_cases) + records = validation_result.failure_cases.to_dict(orient="records") + self.assertEqual(len(records), 1) + collected_values = [record["Values"] for record in records] + self.assertEqual(collected_values, [sample_value]) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.FAILED, + ) + else: + self.assertIsNone(validation_result.failure_cases) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.PASSED, + ) + + def test_valid_json(self): + self.__eval_function__('{"my-cool-tag": "focus", "my-cool-tag-2": "whahoo"}', False) + + def test_valid_json_empty(self): + self.__eval_function__('{}', False) + + def test_valid_json_bad_data_type(self): + self.__eval_function__(0, True) + + def test_valid_json_null_value(self): + self.__eval_function__(None, False) + + def test_valid_json_empty_string(self): + self.__eval_function__("", True) diff --git a/tests/attributes/test_datetime_column_load_from_csv.py b/tests/attributes/test_datetime_column_load_from_csv.py new file mode 100644 index 0000000..0a1c661 --- /dev/null +++ b/tests/attributes/test_datetime_column_load_from_csv.py @@ -0,0 +1,167 @@ +import tempfile +from datetime import datetime +from unittest import TestCase +from uuid import uuid4 + +import pytz +import pandas as pd +import pytz +from pandera.errors import SchemaErrors + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import ( + DataTypeCheck, + DataTypes, + ChecklistObjectStatus, +) +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + + +# noinspection DuplicatedCode +class TestDatetimeColumnLoadFromCSV(TestCase): + def __assert_values__( + self, random_column_id, should_fail, sample_value, sample_data + ): + random_check_id = str(uuid4()) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=[ + Rule( + check_id=str(uuid4()), + column_id=random_column_id, + check="column_required", + check_friendly_name="Column required.", + ), + Rule( + check_id=random_check_id, + column_id=random_column_id, + check=DataTypeCheck(data_type=DataTypes.DATETIME), + ), + ] + ) + + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases = e.failure_cases + + validation_result = ValidationResult( + failure_cases=failure_cases, checklist=checklist + ) + validation_result.process_result() + + if should_fail: + self.assertIsNotNone(validation_result.failure_cases) + records = validation_result.failure_cases.to_dict(orient="records") + self.assertEqual(len(records), 1) + collected_values = [record["Values"] for record in records] + self.assertEqual(collected_values, [sample_value]) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.FAILED, + ) + else: + self.assertIsNone(validation_result.failure_cases) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.PASSED, + ) + + def test_load_column_with_valid_datetime_utc(self): + """ + Test case ensuring UTC datetime field passes test case. + """ + + random_column_id = str(uuid4()) + utc_datetime = datetime.now(tz=pytz.UTC) + + sample_df = pd.DataFrame([{random_column_id: utc_datetime}]) + + with tempfile.NamedTemporaryFile(suffix=".csv", mode="r+") as temp_file: + sample_df.to_csv(temp_file) + temp_file.seek(0) + read_df = pd.read_csv(temp_file) + + self.__assert_values__( + random_column_id=random_column_id, + should_fail=False, + sample_value=str(utc_datetime), + sample_data=read_df, + ) + + def test_load_column_with_valid_datetime_naive(self): + """ + Test case ensuring naive datetime field fails test case. + """ + + random_column_id = str(uuid4()) + naive_datetime = datetime.now(tz=None) + + sample_df = pd.DataFrame([{random_column_id: naive_datetime}]) + + with tempfile.NamedTemporaryFile(suffix=".csv", mode="r+") as temp_file: + sample_df.to_csv(temp_file) + temp_file.seek(0) + read_df = pd.read_csv(temp_file) + + self.__assert_values__( + random_column_id=random_column_id, + should_fail=True, + sample_value=str(naive_datetime), + sample_data=read_df, + ) + + def test_load_column_with_valid_datetime_not_utc(self): + """ + Test case ensures non UTC datetime value fails validation. + """ + + random_column_id = str(uuid4()) + + local_timezone = pytz.timezone("America/Los_Angeles") + aware_datetime = local_timezone.localize(datetime.now()) + + # generate random dataframe + sample_df = pd.DataFrame([{random_column_id: aware_datetime}]) + + with tempfile.NamedTemporaryFile(suffix=".csv", mode="r+") as temp_file: + # write csv to temporary location and read to simulate df read + sample_df.to_csv(temp_file) + temp_file.seek(0) + read_df = pd.read_csv(temp_file) + + self.__assert_values__( + random_column_id=random_column_id, + should_fail=True, + sample_value=str(aware_datetime), + sample_data=read_df, + ) + + def test_load_column_with_invalid_datetime(self): + """ + Test case ensures invalid date value fails validation. + """ + + random_column_id = str(uuid4()) + + bad_value = str(uuid4()) + + # generate random dataframe + sample_df = pd.DataFrame([{random_column_id: bad_value}]) + + with tempfile.NamedTemporaryFile(suffix=".csv", mode="r+") as temp_file: + # write csv to temporary location and read to simulate df read + sample_df.to_csv(temp_file) + temp_file.seek(0) + read_df = pd.read_csv(temp_file) + + self.__assert_values__( + random_column_id=random_column_id, + should_fail=True, + sample_value=bad_value, + sample_data=read_df, + ) diff --git a/tests/checks/test_null_value_check.py b/tests/checks/test_null_value_check.py index 0a71879..a2e1f45 100644 --- a/tests/checks/test_null_value_check.py +++ b/tests/checks/test_null_value_check.py @@ -83,7 +83,7 @@ def test_null_value_not_allowed_invalid_case(self): allow_nulls=False, data_type=DataTypes.STRING ) sample_data = pd.DataFrame( - [{"test_dimension": "NULL"}, {"test_dimension": "val2"}] + [{"test_dimension": None}, {"test_dimension": "val2"}] ) schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( rules=rules, override_config=None @@ -104,12 +104,14 @@ def test_null_value_not_allowed_invalid_case(self): "Column": "test_dimension", "Check Name": "allow_null", "Description": " test_dimension does not allow null values.", - "Values": "NULL", + "Values": None, "Row #": 1, }, ) - def test_null_value_allowed_invalid_case_with_empty_strings(self): + def test_null_value_allowed_valid_case_with_empty_strings(self): + # ensure that check does not treat empty strings as null values + rules = self.__generate_sample_rule_type_string__( allow_nulls=True, data_type=DataTypes.STRING ) @@ -123,23 +125,11 @@ def test_null_value_allowed_invalid_case_with_empty_strings(self): ) self.assertEqual( validation_result.checklist["allow_null"].status, - ChecklistObjectStatus.FAILED, - ) - self.assertIsNotNone(validation_result.failure_cases) - failure_cases_dict = validation_result.failure_cases.to_dict(orient="records") - self.assertEqual(len(failure_cases_dict), 1) - self.assertEqual( - failure_cases_dict[0], - { - "Column": "test_dimension", - "Check Name": "allow_null", - "Description": " test_dimension allows null values.", - "Values": "", - "Row #": 2, - }, + ChecklistObjectStatus.PASSED, ) + self.assertIsNone(validation_result.failure_cases) - def test_null_value_allowed_invalid_case_with_nan_values(self): + def test_null_value_allowed_case_with_explicit_null_values(self): rules = self.__generate_sample_rule_type_string__( allow_nulls=True, data_type=DataTypes.STRING ) @@ -155,18 +145,6 @@ def test_null_value_allowed_invalid_case_with_nan_values(self): ) self.assertEqual( validation_result.checklist["allow_null"].status, - ChecklistObjectStatus.FAILED, - ) - self.assertIsNotNone(validation_result.failure_cases) - failure_cases_dict = validation_result.failure_cases.to_dict(orient="records") - self.assertEqual(len(failure_cases_dict), 1) - self.assertEqual( - failure_cases_dict[0], - { - "Column": "test_dimension", - "Check Name": "allow_null", - "Description": " test_dimension allows null values.", - "Values": None, - "Row #": 2, - }, + ChecklistObjectStatus.PASSED, ) + self.assertIsNone(validation_result.failure_cases) diff --git a/tests/checks/test_sql_query_check.py b/tests/checks/test_sql_query_check.py new file mode 100644 index 0000000..8698df3 --- /dev/null +++ b/tests/checks/test_sql_query_check.py @@ -0,0 +1,142 @@ +import json +from unittest import TestCase + +import pandas as pd +from pandera.errors import SchemaErrors +from pydantic import ValidationError + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import ( + SQLQueryCheck, + DataTypes, + DataTypeCheck, +) +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + + +# noinspection SqlNoDataSourceInspection,SqlDialectInspection +class TestSQLQueryCheck(TestCase): + @staticmethod + def __generate_sample_rule_type_string__(allow_nulls: bool, data_type: DataTypes): + return [ + Rule( + check_id="sql_check_for_multiple_columns", + column_id="test_dimension", + check=SQLQueryCheck( + sql_query=""" + SELECT + test_dimension, + CASE WHEN test_dimension = 'some-value' THEN true ELSE false END AS check_output + FROM df; + """ + ), + ), + Rule( + check_id="test_dimension", + column_id="test_dimension", + check=DataTypeCheck(data_type=data_type), + ), + ] + + @staticmethod + def __validate_helper__(schema, checklist, sample_data): + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases: pd.DataFrame = e.failure_cases + + validation_result = ValidationResult( + checklist=checklist, failure_cases=failure_cases + ) + validation_result.process_result() + return validation_result + + def test_sql_check_for_multiple_columns(self): + test_sql_query = "SELECT * FROM table" + + with self.assertRaises(ValidationError) as cm: + SQLQueryCheck(sql_query=test_sql_query) + self.assertIn( + "Assertion failed, SQL query must only return a column called 'check_output'", + str(cm.exception), + ) + + def test_sql_check_with_invalid_sql(self): + """ + Check for sql query that do not return column called check + """ + + # noinspection SqlDialectInspection,SqlNoDataSourceInspection + test_sql_query = """SELECT + product_id, + (CASE + WHEN product_id = 'a' THEN TRUE + ELSE FALSE + END) AS check_output + FROM Products;""" + + # this query should be valid + SQLQueryCheck(sql_query=test_sql_query) + + def test_null_value_allowed_valid_case(self): + rules = self.__generate_sample_rule_type_string__( + allow_nulls=True, data_type=DataTypes.STRING + ) + sample_data = pd.DataFrame( + [ + {"test_dimension": "NULL", "column_2": "some-value"}, + {"test_dimension": "some-value", "column_2": "some-value"}, + {"test_dimension": "some-value", "column_2": "some-value"}, + {"test_dimension": "NULL", "column_2": "some-value"}, + ] + ) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=rules, override_config=None + ) + validation_result = self.__validate_helper__( + schema=schema, checklist=checklist, sample_data=sample_data + ) + + failure_cases_dict = validation_result.failure_cases.to_dict(orient="records") + + self.assertEqual(len(failure_cases_dict), 2) + self.assertEqual( + failure_cases_dict, + [ + { + "Column": "test_dimension", + "Check Name": "sql_check_for_multiple_columns", + "Description": " test_dimension requires values that return true when evaluated by the following SQL query: SELECT test_dimension, CASE WHEN test_dimension = 'some-value' THEN true ELSE false END AS check_output FROM df;", + "Values": "test_dimension:NULL,test_dimension:NULL", + "Row #": 1, + }, + { + "Column": "test_dimension", + "Check Name": "sql_check_for_multiple_columns", + "Description": " test_dimension requires values that return true when evaluated by the following SQL query: SELECT test_dimension, CASE WHEN test_dimension = 'some-value' THEN true ELSE false END AS check_output FROM df;", + "Values": "test_dimension:NULL,test_dimension:NULL", + "Row #": 4, + }, + ], + ) + + def test_pass_case(self): + rules = self.__generate_sample_rule_type_string__( + allow_nulls=True, data_type=DataTypes.STRING + ) + sample_data = pd.DataFrame( + [{"test_dimension": "some-value"}, {"test_dimension": "some-value"}] + ) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=rules, override_config=None + ) + validation_result = self.__validate_helper__( + schema=schema, checklist=checklist, sample_data=sample_data + ) + self.assertIsNone(validation_result.failure_cases) diff --git a/tests/checks/test_sql_query_check_from_config.py b/tests/checks/test_sql_query_check_from_config.py new file mode 100644 index 0000000..bdbb022 --- /dev/null +++ b/tests/checks/test_sql_query_check_from_config.py @@ -0,0 +1,119 @@ +import os +import tempfile +from unittest import TestCase + +import pandas as pd +from pandera.errors import SchemaErrors + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import DataTypeCheck, DataTypes +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + +YAML_CONFIG = """ +check_id: SkuPriceId +column_id: SkuPriceId +check_friendly_name: SkuPriceId must be set for certain values of ChargeType +check: + sql_query: | + SELECT CASE + WHEN ChargeType IN ('Purchase', 'Usage', 'Refund') AND SkuPriceId IS NULL THEN FALSE + ELSE TRUE + END AS check_output + FROM df; +""" + + +class TestSQLQueryCheckConfig(TestCase): + def test_config_from_yaml(self): + with tempfile.TemporaryDirectory() as temp_dir: + sample_file_path = os.path.join(temp_dir, "D001_S001.yaml") + + with open(sample_file_path, "w") as fd: + fd.write(YAML_CONFIG) + + rule = Rule.load_yaml(sample_file_path) + + dimension_checks = [ + Rule( + check_id="SkuPriceId", + column_id="SkuPriceId", + check=DataTypeCheck(data_type=DataTypes.STRING), + ), + Rule( + check_id="test_dimension", + column_id="test_dimension", + check=DataTypeCheck(data_type=DataTypes.STRING), + ), + Rule( + check_id="ChargeType", + column_id="ChargeType", + check=DataTypeCheck(data_type=DataTypes.STRING), + ), + ] + + sample_data = pd.DataFrame( + [ + { + "test_dimension": "some-value", + "SkuPriceId": "some-value", + "ChargeType": "Purchase", + }, + { + "test_dimension": "some-value", + "SkuPriceId": None, + "ChargeType": "Purchase", + }, + { + "test_dimension": "some-value", + "SkuPriceId": "random-value", + "ChargeType": "Purchase", + }, + { + "test_dimension": "some-value", + "SkuPriceId": None, + "ChargeType": "Purchase", + }, + ] + ) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=dimension_checks + [rule], override_config=None + ) + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases = e.failure_cases + + validation_result = ValidationResult( + checklist=checklist, failure_cases=failure_cases + ) + validation_result.process_result() + + failure_cases_dict = validation_result.failure_cases.to_dict(orient="records") + self.assertEqual(len(failure_cases_dict), 2) + + # row # does not match nan, need to fix thsi + # failure_cases_dict[0] + self.assertEqual( + failure_cases_dict, + [ + { + "Column": "SkuPriceId", + "Check Name": "SkuPriceId", + "Description": " SkuPriceId must be set for certain values of ChargeType", + "Values": "ChargeType:Purchase,SkuPriceId:nan", + "Row #": 2, + }, + { + "Column": "SkuPriceId", + "Check Name": "SkuPriceId", + "Description": " SkuPriceId must be set for certain values of ChargeType", + "Values": "ChargeType:Purchase,SkuPriceId:nan", + "Row #": 4, + }, + ], + ) diff --git a/tests/config_objects/test_check_friendly_name.py b/tests/config_objects/test_check_friendly_name.py index 1208753..c526b67 100644 --- a/tests/config_objects/test_check_friendly_name.py +++ b/tests/config_objects/test_check_friendly_name.py @@ -2,12 +2,14 @@ from uuid import uuid4 from polyfactory.factories.pydantic_factory import ModelFactory +from pydantic import ValidationError from focus_validator.config_objects import Rule from focus_validator.config_objects.common import ( AllowNullsCheck, DataTypeCheck, ValueInCheck, + SQLQueryCheck, ) @@ -20,7 +22,15 @@ def test_default_friendly_name_is_generated(self): ) for _ in range(1000): # there is no way to generate all values for a field type - random_model = model_factory.build() + try: + random_model = model_factory.build() + except ValidationError as e: + if "SQLQueryCheck" in str(e): + # SQLQueryCheck is not supported by ModelFactory + continue + else: + raise e + if random_model.check == "column_required": self.assertEqual( random_model.check_friendly_name, @@ -53,6 +63,8 @@ def test_default_friendly_name_is_generated(self): random_model.check_friendly_name, f"{random_column_name} must have a value from the list: {options}.", ) + elif isinstance(random_model.check, SQLQueryCheck): + pass else: raise NotImplementedError( f"check_type: {random_model.check} not implemented" diff --git a/tests/config_objects/test_check_type_friendly_name.py b/tests/config_objects/test_check_type_friendly_name.py index 392ad89..1395eec 100644 --- a/tests/config_objects/test_check_type_friendly_name.py +++ b/tests/config_objects/test_check_type_friendly_name.py @@ -18,7 +18,15 @@ def test_generate_name_for_check_types(self): model_factory = ModelFactory.create_factory(model=Rule) for _ in range(1000): # there is no way to generate all values for a field type - random_model = model_factory.build() + try: + random_model = model_factory.build() + except ValidationError as e: + if "SQLQueryCheck" in str(e): + # SQLQueryCheck is not supported by ModelFactory + continue + else: + raise e + self.assertIn( random_model.check_type_friendly_name, [ @@ -41,21 +49,33 @@ def test_random_value_is_ignored(self): self.assertEqual(sample.check_type_friendly_name, "CheckUnique") def test_data_type_config(self): + # Ensures that the check_type_friendly_name is generated correctly for DataTypeCheck + model_factory = ModelFactory.create_factory(model=Rule) + # generate random rule object sample_data_type = model_factory.build( **{"check": DataTypeCheck(data_type=DataTypes.STRING)} ) + self.assertEqual(sample_data_type.check_type_friendly_name, "DataTypeCheck") def test_check_type_config_deny_update(self): model_factory = ModelFactory.create_factory(model=Rule) - sample_data_type = model_factory.build() - with self.assertRaises(TypeError) as cm: + try: + sample_data_type = model_factory.build() + except ValidationError as e: + if "SQLQueryCheck" in str(e): + # SQLQueryCheck is not supported by ModelFactory + return + else: + raise e + + with self.assertRaises(ValidationError) as cm: sample_data_type.check_type_friendly_name = "new_value" self.assertIn( - '"Rule" is immutable and does not support item assignment', + "Instance is frozen", str(cm.exception), ) @@ -69,5 +89,6 @@ def test_assign_bad_type(self): ) self.assertEqual(len(cm.exception.errors()), 1) self.assertIn( - "value is not a valid enumeration member; permitted:", str(cm.exception) + "Input should be 'string', 'decimal', 'datetime', 'currency-code' or 'stringified-json-object'", + str(cm.exception), ) diff --git a/tests/config_objects/test_load_bad_rule_config_file.py b/tests/config_objects/test_load_bad_rule_config_file.py index 66b5f19..afd89d5 100644 --- a/tests/config_objects/test_load_bad_rule_config_file.py +++ b/tests/config_objects/test_load_bad_rule_config_file.py @@ -48,30 +48,38 @@ def test_load_schema(self): _, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( rules=rules, override_config=None ) + self.assertEqual( - checklist["FV-D001-0001"].status, ChecklistObjectStatus.PENDING + checklist["bad_rule_config_empty_file"].status, + ChecklistObjectStatus.ERRORED, ) - self.assertIsNone(checklist["FV-D001-0001"].error) - self.assertIsNotNone(checklist["FV-D001-0001"].friendly_name) - self.assertEqual(checklist["FV-D001"].column_id, "ChargeType") - self.assertEqual(checklist["FV-D001"].status, ChecklistObjectStatus.PENDING) - self.assertIsNone(checklist["FV-D001"].error) - self.assertIsNotNone(checklist["FV-D001"].friendly_name) self.assertEqual( - checklist["FV-D001"].friendly_name, "Ensures that column is of string type." + checklist["valid_rule_config_column_metadata"].column_id, "ChargeType" + ) + self.assertEqual( + checklist["valid_rule_config_column_metadata"].status, + ChecklistObjectStatus.PENDING, + ) + self.assertIsNone(checklist["valid_rule_config_column_metadata"].error) + self.assertIsNotNone( + checklist["valid_rule_config_column_metadata"].friendly_name + ) + self.assertEqual( + checklist["valid_rule_config_column_metadata"].friendly_name, + "Ensures that column is of string type.", ) - for errored_file_paths in [ - "tests/samples/rule_configs/bad_rule_config_empty_file.yaml", - "tests/samples/rule_configs/bad_rule_config_missing_check.yaml", + for errored_checks in [ + "bad_rule_config_empty_file", + "bad_rule_config_missing_check", ]: self.assertEqual( - checklist[errored_file_paths].status, ChecklistObjectStatus.ERRORED + checklist[errored_checks].status, ChecklistObjectStatus.ERRORED ) - self.assertIsNotNone(checklist[errored_file_paths].error) - self.assertIsNone(checklist[errored_file_paths].friendly_name) - self.assertEqual(checklist[errored_file_paths].column_id, "Unknown") + self.assertIsNotNone(checklist[errored_checks].error) + self.assertIsNone(checklist[errored_checks].friendly_name) + self.assertEqual(checklist[errored_checks].column_id, "Unknown") def test_load_schema_without_valid_column_metadata(self): rules = [ @@ -88,11 +96,13 @@ def test_load_schema_without_valid_column_metadata(self): rules=rules, override_config=None ) self.assertEqual( - checklist["FV-D001-0001"].status, ChecklistObjectStatus.ERRORED + checklist["bad_rule_config_missing_check"].status, + ChecklistObjectStatus.ERRORED, ) - self.assertEqual( - checklist["FV-D001-0001"].error, - "ConfigurationError: No configuration found for column.", + print(checklist["bad_rule_config_missing_check"].error) + self.assertRegex( + checklist["bad_rule_config_missing_check"].error, + "ValidationError:.*", ) - self.assertIsNotNone(checklist["FV-D001-0001"].friendly_name) - self.assertEqual(checklist["FV-D001-0001"].column_id, "ChargeType") + self.assertIsNotNone(checklist["valid_rule_config"].friendly_name) + self.assertEqual(checklist["valid_rule_config"].column_id, "ChargeType") diff --git a/tests/data_loaders/test_null_value_loader.py b/tests/data_loaders/test_null_value_loader.py index b45c9a7..5bf77e3 100644 --- a/tests/data_loaders/test_null_value_loader.py +++ b/tests/data_loaders/test_null_value_loader.py @@ -12,7 +12,7 @@ def test_null_value_from_csv(self): sample_data = pd.DataFrame([{"value": "NULL"}]) buffer = io.BytesIO() - sample_data.to_csv(buffer, index=False) + sample_data.to_csv(buffer, index=False, lineterminator="\n") buffer.seek(0) self.assertEqual(buffer.read(), b"value\nNULL\n") @@ -21,13 +21,13 @@ def test_null_value_from_csv(self): loader = CSVDataLoader(buffer) data = loader.load() - self.assertEqual(data.to_dict(orient="records")[0], {"value": "NULL"}) + self.assertTrue(pd.isnull(data.to_dict(orient="records")[0]["value"])) def test_null_value_from_csv_with_missing_value(self): sample_data = pd.DataFrame([{"value": None}]) buffer = io.BytesIO() - sample_data.to_csv(buffer, index=False) + sample_data.to_csv(buffer, index=False, lineterminator="\n") buffer.seek(0) self.assertEqual(buffer.read(), b'value\n""\n') @@ -36,7 +36,7 @@ def test_null_value_from_csv_with_missing_value(self): loader = CSVDataLoader(buffer) data = loader.load() - self.assertEqual(data.to_dict(orient="records")[0], {"value": ""}) + self.assertTrue(pd.isnull(data.to_dict(orient="records")[0]["value"])) def test_null_value_from_parquet(self): sample_data = pd.DataFrame([{"value": "NULL"}]) diff --git a/tests/outputter/test_outputter_console.py b/tests/outputter/test_outputter_console.py index a947a39..bef733d 100644 --- a/tests/outputter/test_outputter_console.py +++ b/tests/outputter/test_outputter_console.py @@ -4,7 +4,7 @@ from focus_validator.config_objects.focus_to_pandera_schema_converter import ( FocusToPanderaSchemaConverter, ) -from focus_validator.outputter.outputter_console import ConsoleOutputter +from focus_validator.outputter.outputter_console import ConsoleOutputter, collapse_occurrence_range from focus_validator.rules.spec_rules import ValidationResult from focus_validator.validator import Validator @@ -34,6 +34,16 @@ def test_failure_output(self): ], ) + def test_collapse_range(self): + self.assertEqual( + collapse_occurrence_range([1, 5, 6, 7, 23.0]), + '1,5-7,23' + ) + self.assertEqual( + collapse_occurrence_range(['category', 'category2', 'category3']), + 'category,category2,category3' + ) + def test_output_with_bad_configs_loaded(self): schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( rules=[ @@ -50,6 +60,7 @@ def test_output_with_bad_configs_loaded(self): outputter = ConsoleOutputter(output_destination=None) checklist = outputter.__restructure_check_list__(result_set=validation_result) + outputter.write(validation_result) self.assertEqual( checklist.to_dict(orient="records"), [ diff --git a/tests/samples/all_pass.csv b/tests/samples/all_pass_0.5.csv similarity index 64% rename from tests/samples/all_pass.csv rename to tests/samples/all_pass_0.5.csv index 4123811..e94f7c0 100644 --- a/tests/samples/all_pass.csv +++ b/tests/samples/all_pass_0.5.csv @@ -1,3 +1,3 @@ -InvoiceIssuer,ResourceID,ChargeType,Provider,BillingAccountName,BillingAccountId,Publisher,ResourceName,ServiceName,BilledCurrency,BillingPeriodEnd,BillingPeriodStart,Region,ServiceCategory,ChargePeriodStart,ChargePeriodEnd,BilledCost,AmortizedCost -test-entity-1,r1,Purchase,A,Account-1,a1,Publisher1,Test Resource 1,Sample-service-1,AUD,2023-06-01T00:00:00Z,2023-06-30T00:00:00Z,A,Compute,2023-06-01T00:00:00Z,2023-06-01T00:01:00Z,0.1,0.01 -test-entity-2,r2,Adjustment,B,NULL,a2,Publisher2,Test Resource 2,Sample-service-1,USD,2023-06-01T00:00:00Z,2023-06-30T00:00:00Z,B,Media,2023-06-01T00:00:00Z,2023-06-01T00:01:00Z,1,1 +InvoiceIssuer,ResourceID,ChargeType,Provider,BillingAccountName,BillingAccountId,Publisher,ResourceName,ServiceName,BilledCurrency,BillingPeriodEnd,BillingPeriodStart,Region,ServiceCategory,ChargePeriodStart,ChargePeriodEnd,BilledCost,AmortizedCost,SubAccountName,SubAccountId +test-entity-1,r1,Purchase,A,Account-1,a1,Publisher1,Test Resource 1,Sample-service-1,AUD,2023-06-01T00:00:00Z,2023-06-30T00:00:00Z,A,Compute,2023-06-01T00:00:00Z,2023-06-01T00:01:00Z,0.1,0.01,IT,a09876543212 +test-entity-2,r2,Adjustment,B,NULL,a2,Publisher2,Test Resource 2,Sample-service-1,USD,2023-06-01T00:00:00Z,2023-06-30T00:00:00Z,B,Media,2023-06-01T00:00:00Z,2023-06-01T00:01:00Z,1,1,Production,a123456789098 diff --git a/tests/samples/csv_random_data_generate_at_scale.py b/tests/samples/csv_random_data_generate_at_scale.py new file mode 100644 index 0000000..3e8015d --- /dev/null +++ b/tests/samples/csv_random_data_generate_at_scale.py @@ -0,0 +1,80 @@ +import functools +import polars as pl +from faker import Faker +import random +from datetime import datetime, timedelta +import pytz +import logging +import time + +fake = Faker() + +def get_aws_invoice_issuer(num_records): + aws_entities = [ + 'AWS Inc.', 'Amazon Web Services', 'AWS Marketplace', + 'Amazon Data Services', 'AWS CloudFront', 'Amazon S3 Billing', + 'Amazon EC2 Billing', 'AWS Lambda Billing' + ] + return [random.choice(aws_entities) for _ in range(num_records)] + +# ... similar functions for other non-date attributes ... + +def get_random_datetimes(num_records, start_date, end_date): + return [fake.date_time_between(start_date=start_date, end_date=end_date, tzinfo=pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ') for _ in range(num_records)] + +def log_execution_time(func): + """Decorator to log the execution time of a function.""" + @functools.wraps(func) + def wrapper(*args, **kwargs): + start_time = time.time() + result = func(*args, **kwargs) + end_time = time.time() + logging.info(f"{func.__name__} executed in {end_time - start_time:.2f} seconds") + return result + return wrapper + +@log_execution_time +def generate_and_write_fake_focuses(csv_filename, num_records): + now = datetime.now(pytz.utc) + thirty_days_ago = now - timedelta(days=30) + + df = pl.DataFrame({ + 'InvoiceIssuer': [random.choice([ 'AWS Inc.', 'Amazon Web Services', 'AWS Marketplace', 'Amazon Data Services', + 'AWS CloudFront', 'Amazon S3 Billing', 'Amazon EC2 Billing', 'AWS Lambda Billing']) for _ in range(num_records)], + 'ResourceID': [fake.uuid4() for _ in range(num_records)], + 'ChargeType': [random.choice(['Adjustment', 'Purchase', 'Tax', 'Usage']) for _ in range(num_records)], + 'Provider': [fake.company() for _ in range(num_records)], + 'BillingAccountName': [fake.company() for _ in range(num_records)], + 'SubAccountName': get_random_datetimes(num_records, thirty_days_ago, now), + 'BillingAccountId': [fake.uuid4() for _ in range(num_records)], + 'Publisher': [f"{fake.company()} {random.choice(['Software', 'Service', 'Platform'])} {random.choice(['Inc.', 'LLC', 'Ltd.', 'Group', 'Technologies', 'Solutions'])}" for _ in range(num_records)], + 'ResourceName': [f"{random.choice(['i-', 'vol-', 'snap-', 'ami-', 'bucket-', 'db-'])}{fake.hexify(text='^^^^^^^^', upper=False)}" for _ in range(num_records)], + 'ServiceName': [random.choice([ + 'Amazon EC2', 'Amazon S3', 'AWS Lambda', 'Amazon RDS', + 'Amazon DynamoDB', 'Amazon VPC', 'Amazon Route 53', + 'Amazon CloudFront', 'AWS Elastic Beanstalk', 'Amazon SNS', + 'Amazon SQS', 'Amazon Redshift', 'AWS CloudFormation', + 'AWS IAM', 'Amazon EBS', 'Amazon ECS', 'Amazon EKS', + 'Amazon ElastiCache', 'AWS Fargate', 'AWS Glue' + ]) for _ in range(num_records)], + 'BilledCurrency': ['USD' for _ in range(num_records)], + 'BillingPeriodEnd': get_random_datetimes(num_records, thirty_days_ago, now), + 'BillingPeriodStart': get_random_datetimes(num_records, thirty_days_ago, now), + 'Region': [random.choice([ + 'us-east-1', 'us-west-1', 'us-west-2', 'eu-west-1', 'eu-central-1', + 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'ap-northeast-2', + 'ap-south-1', 'sa-east-1', 'ca-central-1', 'eu-north-1', 'eu-west-2', + 'eu-west-3', 'ap-east-1', 'me-south-1', 'af-south-1', 'eu-south-1' + ]) for _ in range(num_records)], + 'ServiceCategory': [random.choice([ + 'AI and Machine Learning', 'Analytics', 'Business Applications', 'Compute', 'Databases', 'Developer Tools', 'Multicloud', + 'Identity', 'Integration', 'Internet of Things', 'Management and Governance', 'Media', 'Migration', 'Mobile', 'Networking', + 'Security', 'Storage', 'Web', 'Other' + ]) for _ in range(num_records)], + 'ChargePeriodStart': get_random_datetimes(num_records, thirty_days_ago, now), + 'ChargePeriodEnd': get_random_datetimes(num_records, thirty_days_ago, now), + 'BilledCost': [fake.pyfloat(left_digits=3, right_digits=2, positive=True) for _ in range(num_records)], + 'AmortizedCost': [fake.pyfloat(left_digits=3, right_digits=2, positive=True) for _ in range(num_records)] + }) + + df.write_csv(csv_filename) \ No newline at end of file diff --git a/tests/samples/rule_configs/bad_rule_config_empty_file.yaml b/tests/samples/rule_configs/bad_rule_config_empty_file.yaml index 8179223..e69de29 100644 --- a/tests/samples/rule_configs/bad_rule_config_empty_file.yaml +++ b/tests/samples/rule_configs/bad_rule_config_empty_file.yaml @@ -1,4 +0,0 @@ -check_id: FV-D001-0001 -column: ChargeType -validation_config: - check_friendly_name: "ChargeType must have a value from the list: {values}." diff --git a/tests/samples/rule_configs/bad_rule_config_missing_check.yaml b/tests/samples/rule_configs/bad_rule_config_missing_check.yaml index e69de29..8242360 100644 --- a/tests/samples/rule_configs/bad_rule_config_missing_check.yaml +++ b/tests/samples/rule_configs/bad_rule_config_missing_check.yaml @@ -0,0 +1,3 @@ +column_id: ChargeType +validation_config: + check_friendly_name: "ChargeType must have a value from the list: {values}." diff --git a/tests/samples/rule_configs/valid_rule_config.yaml b/tests/samples/rule_configs/valid_rule_config.yaml index 7d0c2bd..0c20356 100644 --- a/tests/samples/rule_configs/valid_rule_config.yaml +++ b/tests/samples/rule_configs/valid_rule_config.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0001 column_id: ChargeType check_friendly_name: "ChargeType must have a value from the list: {values}." check: diff --git a/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml b/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml index a89f1e8..8369e69 100644 --- a/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml +++ b/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001 column_id: ChargeType check: data_type: string diff --git a/tests/test_match_check_id_rule_config_file.py b/tests/test_match_check_id_rule_config_file.py index d011b02..18620c7 100644 --- a/tests/test_match_check_id_rule_config_file.py +++ b/tests/test_match_check_id_rule_config_file.py @@ -15,16 +15,3 @@ def test_match_check_id_in_base_definitions(self): rule = Rule.load_yaml(rule_path=rule_path) self.assertIsInstance(rule, Rule) self.assertEqual(rule.check_id, Path(name).stem) - - def test_version_sets(self): - for root, dirs, files in os.walk( - "focus_validator/rules/version_sets", topdown=False - ): - for name in files: - rule_path = os.path.join(root, name) - self.assertTrue( - os.path.islink(rule_path), f"path not a sym link, {rule_path}" - ) - self.assertTrue( - Path(rule_path).exists(), f"invalid sym link, {rule_path}" - ) diff --git a/tests/test_performance_profiler.py b/tests/test_performance_profiler.py new file mode 100644 index 0000000..bb2b568 --- /dev/null +++ b/tests/test_performance_profiler.py @@ -0,0 +1,107 @@ +import cProfile +import csv +import io +import logging +import os +import pstats +import time +import unittest +from ddt import ddt, data, unpack + +from tests.samples.csv_random_data_generate_at_scale import generate_and_write_fake_focuses +from focus_validator.validator import Validator + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - [%(funcName)s] - %(message)s') + +@ddt +class TestPerformanceProfiler(unittest.TestCase): + + def profile_to_csv(self, profiling_result, csv_file): + with open(csv_file, 'w', newline='') as f: + w = csv.writer(f) + # Write the headers + headers = ['ncalls', 'tottime', 'percall', 'cumtime', 'percall', 'filename:lineno(function)'] + w.writerow(headers) + + # Write each row + for row in profiling_result.stats.items(): + func_name, (cc, nc, tt, ct, callers) = row + w.writerow([nc, tt, tt/nc, ct, ct/cc, func_name]) + + def execute_profiler(self, file_name, performance_threshold): + # Set the environment variable for logging level + env = os.environ.copy() + env["LOG_LEVEL"] = "INFO" + + # Get the current directory of this test file + test_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + version_set_path=os.path.join(base_dir, "focus_validator", "rules", "version_sets") + validator = Validator( + data_filename=os.path.join(test_dir, '../' + file_name), + override_filename=None, + rule_set_path=version_set_path, + rules_version="0.5", + output_type="console", + output_destination=None, + column_namespace=None, + ) + + # Set up the profiler + profiler = cProfile.Profile() + profiler.enable() + + # The original performance testing code + start_time = time.time() + validator.validate() + end_time = time.time() + duration = end_time - start_time + logging.info(f"File: {file_name} Duration: {duration} seconds") + + # Stop the profiler + profiler.disable() + + # Save profiling data to a file + profiling_result = pstats.Stats(profiler) + profile_file_name = "profiling_data_" + file_name + self.profile_to_csv(profiling_result, profile_file_name) + + # Optionally print out profiling report to the console + s = io.StringIO() + sortby = 'cumulative' # Can be changed to 'time', 'calls', etc. + ps = pstats.Stats(profiler, stream=s).sort_stats(sortby) + ps.print_stats(10) + logging.info(s.getvalue()) + + #Execution time check + self.assertLess(duration, performance_threshold, f"Performance test exceeded threshold. Duration: {duration} seconds") + + @data( + # ("fake_focuses500000.csv", 60.0, 500000, "validate_500000_records"), + # ("fake_focuses250000.csv", 60.0, 250000, "validate_250000_records"), + # ("fake_focuses100000.csv", 30.0, 100000, "validate_100000_records"), + # ("fake_focuses50000.csv", 15.0, 50000, "validate_50000_records"), + # ("fake_focuses10000.csv", 7.0, 10000, "validate_10000_records"), + # ("fake_focuses5000.csv", 3.0, 5000, "validate_5000_records"), + ("fake_focuses2000.csv", 3.0, 2000, "validate_2000_records"), + ("fake_focuses2000.csv", 3.0, 1000, "validate_1000_records") + ) + @unpack + def test_param_validator_performance(self, file_name, performance_threshold, number_of_records, case_id): + with self.subTest(case_id=case_id): + # Set the environment variable for logging level + env = os.environ.copy() + env["LOG_LEVEL"] = "INFO" + + logging.info("Generating file with {number_of_records} records.") + generate_and_write_fake_focuses(file_name, number_of_records) + self.execute_profiler(str(file_name), performance_threshold) + + logging.info("Cleaning up test file.") + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + if os.path.exists(os.path.join(base_dir, file_name)): + os.remove(os.path.join(base_dir, file_name)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_progressive_performance.py b/tests/test_progressive_performance.py new file mode 100644 index 0000000..02e9799 --- /dev/null +++ b/tests/test_progressive_performance.py @@ -0,0 +1,126 @@ +import logging +import os +import subprocess +import time +import unittest + +from tests.samples.csv_random_data_generate_at_scale import generate_and_write_fake_focuses + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - [%(funcName)s] - %(message)s') + + +class TestProgressivePerformance(unittest.TestCase): + @classmethod + def setUpClass(cls): + #Generate 1000 fake focuses to a CSV file + cls.csv_filename_1000 = 'fake_focuses1000.csv' + cls.csv_filename_10000 = 'fake_focuses10000.csv' + cls.csv_filename_50000 = 'fake_focuses50000.csv' + cls.csv_filename_100000 = 'fake_focuses100000.csv' + cls.csv_filename_250000 = 'fake_focuses250000.csv' + cls.csv_filename_500000 = 'fake_focuses500000.csv' + + logging.info("Generating file with 1,000 records") + cls.generate_test_file(str(cls.csv_filename_1000), 1000) + + # logging.info("Generating file with 10,0000 records") + # cls.generate_test_file(str(cls.csv_filename_10000), 10000) + + # logging.info("Generating file with 50,0000 records") + # cls.generate_test_file(str(cls.csv_filename_50000), 50000) + + # logging.info("Generating file with 100,0000 records") + # cls.generate_test_file(str(cls.csv_filename_100000), 100000) + + # logging.info("Generating file with 250,0000 records") + # cls.generate_test_file(str(cls.csv_filename_250000), 250000) + + # logging.info("Generating file with 500,0000 records") + # cls.generate_test_file(str(cls.csv_filename_500000), 500000) + + @classmethod + def tearDownClass(cls): + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + if os.path.exists(os.path.join(base_dir, 'fake_focuses.csv')): + os.remove(os.path.join(base_dir, 'fake_focuses.csv')) + + if os.path.exists(os.path.join(base_dir, str(cls.csv_filename_1000))): + os.remove(os.path.join(base_dir, str(cls.csv_filename_1000))) + + if os.path.exists(os.path.join(base_dir, str(cls.csv_filename_10000))): + os.remove(os.path.join(base_dir, str(cls.csv_filename_10000))) + + if os.path.exists(os.path.join(base_dir, str(cls.csv_filename_50000))): + os.remove(os.path.join(base_dir, str(cls.csv_filename_50000))) + + if os.path.exists(os.path.join(base_dir, str(cls.csv_filename_100000))): + os.remove(os.path.join(base_dir, str(cls.csv_filename_100000))) + + if os.path.exists(os.path.join(base_dir, str(cls.csv_filename_250000))): + os.remove(os.path.join(base_dir, str(cls.csv_filename_250000))) + + if os.path.exists(os.path.join(base_dir, str(cls.csv_filename_500000))): + os.remove(os.path.join(base_dir, str(cls.csv_filename_500000))) + + @classmethod + def generate_test_file(cls, csv_filename, number_of_records): + #Generate fake focuses to a CSV file + # fake_focuses = generate_fake_focus(number_of_records) + + # write_fake_focuses_to_csv(fake_focuses, csv_filename) + generate_and_write_fake_focuses(csv_filename, number_of_records) + + + def run_validator(self, args): + # Get the current directory of this test file + test_dir = os.path.dirname(os.path.abspath(__file__)) + + # Construct the path to the application directory + app_dir = os.path.join(test_dir, '../focus_validator') + # Set the environment variable for logging level + env = os.environ.copy() + env["LOG_LEVEL"] = "INFO" + + command = ['poetry', 'run', 'python', os.path.join(app_dir, 'main.py')] + args + return subprocess.run(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True) + + def test_1000_record_csv_performance(self): + self.execute_performance(str(self.csv_filename_1000), 25.0) + + # def test_10000_record_csv_performance(self): + # self.execute_performance(str(self.csv_filename_10000), 25.0) + + # def test_50000_record_csv_performance(self): + # self.execute_performance(str(self.csv_filename_50000), 150.0) + + # def test_100000_record_csv_performance(self): + # self.execute_performance(str(self.csv_filename_100000), 300.0) + + # def test_250000_record_csv_performance(self): + # self.execute_performance(str(self.csv_filename_250000), 300.0) + + # def test_500000_record_csv_performance(self): + # self.execute_performance(str(self.csv_filename_500000), 300.0) + + def execute_performance(self, file_name, performance_threshold): + # Get the current directory of this test file + test_dir = os.path.dirname(os.path.abspath(__file__)) + + start_time = time.time() + + # Command to execute the focus_validator tool + result = self.run_validator(['--data-file', os.path.join(test_dir, '../' + file_name)]) + print(result.stdout) + + end_time = time.time() + duration = end_time - start_time + logging.info(f"File: {file_name} Duration: {duration} seconds") + + self.assertLess(duration, performance_threshold, f"Performance test exceeded threshold. Duration: {duration} seconds") + self.assertEqual(result.returncode, 0, "Focus Validator did not exit cleanly.") + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_rule_config_load.py b/tests/test_rule_config_load.py new file mode 100644 index 0000000..2e3f2b9 --- /dev/null +++ b/tests/test_rule_config_load.py @@ -0,0 +1,29 @@ +import os + +import pytest + +from focus_validator.config_objects import Rule +from focus_validator.rules.spec_rules import SpecRules +from focus_validator.validator import DEFAULT_VERSION_SETS_PATH + + +def rules_version(): + return sorted([x for x in os.walk(DEFAULT_VERSION_SETS_PATH)][0][1]) + + +@pytest.mark.parametrize("focus_spec_version", rules_version()) +def test_rules_load_with_no_errors(focus_spec_version): + """ + Test loading of rules with no errors + """ + spec_rules = SpecRules( + override_filename=None, + rule_set_path=DEFAULT_VERSION_SETS_PATH, + rules_version=focus_spec_version, + column_namespace=None, + ) + spec_rules.load() + + for rule in spec_rules.rules: + # ensures that the rule is a Rule object and not InvalidRule + assert isinstance(rule, Rule) diff --git a/tests/test_validate_default_configs.py b/tests/test_validate_default_configs.py index 17d79d8..c994ab7 100644 --- a/tests/test_validate_default_configs.py +++ b/tests/test_validate_default_configs.py @@ -29,59 +29,3 @@ def test_version_sets_have_valid_config(self): self.assertIsNot( result.checklist[check_id].status, ChecklistObjectStatus.ERRORED ) - - def test_default_rules_with_sample_data(self): - check_id_pattern = re.compile(r"FV-[D,M]\d{3}-\d{4}$") - - for root, dirs, files in os.walk( - "focus_validator/rules/version_sets", topdown=False - ): - column_test_suites = [] - for file_path in files: - rule_path = os.path.join(root, file_path) - rule = Rule.load_yaml(rule_path=rule_path) - self.assertIsInstance(rule, Rule) - - column_id = rule.column_id - self.assertIsNotNone(re.match(check_id_pattern, rule.check_id)) - - check_column_id = rule.check_id.split("-")[1] - local_check_id = rule.check_id.split("-")[2] - column_test_suites.append((column_id, check_column_id, local_check_id)) - - # sort column test suites to allow grouping by column - column_test_suites = sorted(column_test_suites, key=lambda item: item[0]) - for _, test_suites in groupby(column_test_suites, key=lambda item: item[0]): - test_suites = list(test_suites) - self.assertEqual( - len(set([test_suite[1] for test_suite in test_suites])), 1 - ) - local_check_ids = [int(test_suite[2]) for test_suite in test_suites] - # check all ids are in order - self.assertEqual( - sorted(local_check_ids), list(range(1, len(local_check_ids) + 1)) - ) - - def test_metric_file_format_metric_vs_dimension(self): - metric_check_id_pattern = re.compile(r"FV-M\d{3}-\d{4}$") - dimension_check_id_pattern = re.compile(r"FV-D\d{3}-\d{4}$") - - for root, dirs, files in os.walk( - "focus_validator/rules/version_sets", topdown=False - ): - for file_path in files: - rule_path = os.path.join(root, file_path) - rule = Rule.load_yaml(rule_path=rule_path) - self.assertIsInstance(rule, Rule) - - if isinstance(rule.check, DataTypeCheck): - if rule.check.data_type == DataTypes.DECIMAL: - self.assertIsNotNone( - re.match(metric_check_id_pattern, rule.check_id), - "For metric column type check_id format should be FV-MYYY-YYYY", - ) - else: - self.assertIsNotNone( - re.match(dimension_check_id_pattern, rule.check_id), - "For metric column type check_id format should be FV-DYYY-YYYY", - )