From a3586926daaf66c9bac38cc067aae4438e8488e2 Mon Sep 17 00:00:00 2001 From: catsona Date: Thu, 25 Sep 2025 16:29:28 +0300 Subject: [PATCH 01/10] feat(connectors): BI-5971 Upgrade YDB SDK dependencies --- lib/dl_connector_ydb/pyproject.toml | 1 + lib/dl_type_transformer/dl_type_transformer/type_transformer.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/dl_connector_ydb/pyproject.toml b/lib/dl_connector_ydb/pyproject.toml index 2aee44c9ef..475d110b32 100644 --- a/lib/dl_connector_ydb/pyproject.toml +++ b/lib/dl_connector_ydb/pyproject.toml @@ -41,6 +41,7 @@ dl-core-testing = {path = "../dl_core_testing"} dl-formula-testing = {path = "../dl_formula_testing"} dl-testing = {path = "../dl_testing"} frozendict = "*" +dl-sqlalchemy-ydb = {path = "../../lib/dl_sqlalchemy_ydb"} pytest = "*" requests = "*" diff --git a/lib/dl_type_transformer/dl_type_transformer/type_transformer.py b/lib/dl_type_transformer/dl_type_transformer/type_transformer.py index c23118f90e..dcf9701dde 100644 --- a/lib/dl_type_transformer/dl_type_transformer/type_transformer.py +++ b/lib/dl_type_transformer/dl_type_transformer/type_transformer.py @@ -72,6 +72,8 @@ def make_datetime(value: Any) -> Optional[datetime.datetime]: def make_int(value: Any) -> Optional[int]: if value is None: return None + # if isinstance(value, datetime.timedelta): + # return int(value.total_seconds()) if isinstance(value, float) and (math.isinf(value) or math.isnan(value)): return None return int(value) From 19bcc48a89ec2aeb940c9cdca931188b84361c63 Mon Sep 17 00:00:00 2001 From: catsona Date: Fri, 26 Sep 2025 11:55:34 +0300 Subject: [PATCH 02/10] cleanup --- lib/dl_type_transformer/dl_type_transformer/type_transformer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/dl_type_transformer/dl_type_transformer/type_transformer.py b/lib/dl_type_transformer/dl_type_transformer/type_transformer.py index dcf9701dde..c23118f90e 100644 --- a/lib/dl_type_transformer/dl_type_transformer/type_transformer.py +++ b/lib/dl_type_transformer/dl_type_transformer/type_transformer.py @@ -72,8 +72,6 @@ def make_datetime(value: Any) -> Optional[datetime.datetime]: def make_int(value: Any) -> Optional[int]: if value is None: return None - # if isinstance(value, datetime.timedelta): - # return int(value.total_seconds()) if isinstance(value, float) and (math.isinf(value) or math.isnan(value)): return None return int(value) From 24269668736be19c15db03b128d4f2fad6d987d8 Mon Sep 17 00:00:00 2001 From: catsona Date: Mon, 29 Sep 2025 19:17:49 +0300 Subject: [PATCH 03/10] feat(connectors): BI-6611 DB_CAST for YDB --- .../formula/definitions/functions_type.py | 458 ++++++++++++++++++ .../db/formula/test_functions_array.py | 33 ++ .../formula/test_functions_type_conversion.py | 200 ++++++++ 3 files changed, 691 insertions(+) create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py index 7402a16b8c..138f185455 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py @@ -1,13 +1,467 @@ import sqlalchemy as sa +import ydb_sqlalchemy as ydb_sa +from dl_formula.core.datatype import DataType from dl_formula.definitions.base import TranslationVariant import dl_formula.definitions.functions_type as base +import dl_sqlalchemy_ydb.dialect as ydb_dialect from dl_connector_ydb.formula.constants import YqlDialect as D V = TranslationVariant.make +TYPES_SPEC = { + wlts.name: wlts + for wlts in [ + base.WhitelistTypeSpec(name="Bool", sa_type=sa.BOOLEAN), + base.WhitelistTypeSpec(name="Int8", sa_type=ydb_sa.types.Int8), + base.WhitelistTypeSpec(name="Int16", sa_type=ydb_sa.types.Int16), + base.WhitelistTypeSpec(name="Int32", sa_type=ydb_sa.types.Int32), + base.WhitelistTypeSpec(name="Int64", sa_type=ydb_sa.types.Int64), + base.WhitelistTypeSpec(name="UInt8", sa_type=ydb_sa.types.UInt8), + base.WhitelistTypeSpec(name="UInt16", sa_type=ydb_sa.types.UInt16), + base.WhitelistTypeSpec(name="UInt32", sa_type=ydb_sa.types.UInt32), + base.WhitelistTypeSpec(name="UInt64", sa_type=ydb_sa.types.UInt64), + base.WhitelistTypeSpec(name="Double", sa_type=sa.types.FLOAT), + base.WhitelistTypeSpec(name="Float", sa_type=sa.types.FLOAT), + base.WhitelistTypeSpec(name="Decimal", sa_type=sa.DECIMAL, arg_types=base.DECIMAL_CAST_ARG_T), + base.WhitelistTypeSpec(name="Utf8", sa_type=sa.types.TEXT), + base.WhitelistTypeSpec(name="String", sa_type=sa.types.TEXT), + base.WhitelistTypeSpec(name="Date", sa_type=sa.types.DATE), + base.WhitelistTypeSpec(name="Datetime", sa_type=ydb_dialect.YqlDateTime), + base.WhitelistTypeSpec(name="Timestamp", sa_type=ydb_dialect.YqlTimestamp), + base.WhitelistTypeSpec(name="Uuid", sa_type=sa.types.TEXT), + ] +} + +BOOL_TYPES_SPEC = [ + TYPES_SPEC["Bool"], +] + +INT_TYPES_SPEC = [ + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], +] + +DECIMAL_TYPES_SPEC = [ + TYPES_SPEC["Decimal"], +] + +STRING_TYPES_SPEC = [ + TYPES_SPEC["Utf8"], + TYPES_SPEC["String"], +] + +FLOAT_TYPES_SPEC = [ + TYPES_SPEC["Double"], + TYPES_SPEC["Float"], +] + + +class FuncDbCastYQLBase(base.FuncDbCastBase): + # For numeric types see: https://ydb.tech/docs/en/yql/reference/types/primitive#casting-to-numeric-types + # Type cast tables date: 2025-09-29. + # + # Type Bool Int8 Int16 Int32 Int64 Uint8 Uint16 Uint32 Uint64 Float Double Decimal <- (Target Type) + # Bool — Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] No + # Int8 Yes2 — Yes Yes Yes Yes[3] Yes[3] Yes[3] Yes[3] Yes Yes Yes + # Int16 Yes2 Yes[4] — Yes Yes Yes[3,4] Yes[3] Yes[3] Yes[3] Yes Yes Yes + # Int32 Yes2 Yes[4] Yes[4] — Yes Yes[3,4] Yes[3,4] Yes[3] Yes[3] Yes Yes Yes + # Int64 Yes2 Yes[4] Yes[4] Yes[4] — Yes[3,4] Yes[3,4] Yes[3,4] Yes[3] Yes Yes Yes + # Uint8 Yes2 Yes[4] Yes Yes Yes — Yes Yes Yes Yes Yes Yes + # Uint16 Yes2 Yes[4] Yes[4] Yes Yes Yes[4] — Yes Yes Yes Yes Yes + # Uint32 Yes2 Yes[4] Yes[4] Yes[4] Yes Yes[4] Yes[4] — Yes Yes Yes Yes + # Uint64 Yes2 Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] — Yes Yes Yes + # Float Yes2 Yes[4] Yes[4] Yes[4] Yes[4] Yes[3,4] Yes[3,4] Yes[3,4] Yes[3,4] — Yes No + # Double Yes2 Yes[4] Yes[4] Yes[4] Yes[4] Yes[3,4] Yes[3,4] Yes[3,4] Yes[3,4] Yes — No + # Decimal No Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes — + # String Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes + # Utf8 Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes + # Json No No No No No No No No No No No No + # Yson Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] No + # Uuid No No No No No No No No No No No No + # Date No Yes[4] Yes[4] Yes Yes Yes[4] Yes Yes Yes Yes Yes No + # Datetime No Yes[4] Yes[4] Yes[4] Yes Yes[4] Yes[4] Yes Yes Yes Yes No + # Timestamp No Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes Yes Yes No + # Interval No Yes[4] Yes[4] Yes[4] Yes Yes[3,4] Yes[3,4] Yes[3,4] Yes[3] Yes Yes No + # ^ + # | + # (Source Type) + # + # Type String Utf8 Json Yson Uuid + # Bool Yes No No No No + # INT Yes No No No No + # Uint Yes No No No No + # Float Yes No No No No + # Double Yes No No No No + # Decimal Yes No No No No + # String — Yes Yes Yes Yes + # Utf8 Yes — No No No + # Json Yes Yes — No No + # Yson Yes[4] No No No No + # Uuid Yes Yes No No — + # Date Yes Yes No No No + # Datetime Yes Yes No No No + # Timestamp Yes Yes No No No + # Interval Yes Yes No No No + # + # Type Date Datetime Timestamp Interval + # Bool No No No No + # INT Yes Yes Yes Yes + # Uint Yes Yes Yes Yes + # Float No No No No + # Double No No No No + # Decimal No No No No + # String Yes Yes Yes Yes + # Utf8 Yes Yes Yes Yes + # Json No No No No + # Yson No No No No + # Uuid No No No No + # Date — Yes Yes No + # Datetime Yes — Yes No + # Timestamp Yes Yes — No + # Interval No No No — + # + # [1] - True is converted to 1 and False to 0. + # [2] - Any value other than 0 is converted to True, 0 is converted to False. + # [3] - Possible only in case of a non-negative value. + # [4] - Possible only within the valid range. + # [5] - Using the built-in function Yson::ConvertTo. + + WHITELISTS = { + yql_dialect: { + # TODO: Decimal + # TODO: DyNumber + # TODO: Json + # TODO: JsonDocument + # TODO: Yson + # TODO: Cast Integer to Interval + # Commented - not supported + DataType.BOOLEAN: [ + # > Bool + TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + # TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + # TYPES_SPEC["Utf8"], + # > Date + # TYPES_SPEC["Date"], + # > Datetime + # TYPES_SPEC["Datetime"], + # > Timestamp + # TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.INTEGER: [ + # Interval can not be casted to Bool + # Interval can not be casted to Decimal + # > Bool + TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + # TYPES_SPEC["Utf8"], + # > Date + TYPES_SPEC["Date"], + # > Datetime + TYPES_SPEC["Datetime"], + # > Timestamp + TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.FLOAT: [ + # > Bool + TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + # TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + # TYPES_SPEC["Utf8"], + # > Date + # TYPES_SPEC["Date"], + # > Datetime + # TYPES_SPEC["Datetime"], + # > Timestamp + # TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.STRING: [ + # Utf8 can not be casted to Uuid + # > Bool + TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + TYPES_SPEC["Utf8"], + # > Date + TYPES_SPEC["Date"], + # > Datetime + TYPES_SPEC["Datetime"], + # > Timestamp + TYPES_SPEC["Timestamp"], + # > UUID + TYPES_SPEC["Uuid"], + ], + DataType.DATE: [ + # > Bool + # TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + # TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + TYPES_SPEC["Utf8"], + # > Date + TYPES_SPEC["Date"], + # > Datetime + TYPES_SPEC["Datetime"], + # > Timestamp + TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.DATETIME: [ + # > Bool + # TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + # TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + TYPES_SPEC["Utf8"], + # > Date + TYPES_SPEC["Date"], + # > Datetime + TYPES_SPEC["Datetime"], + # > Timestamp + TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.DATETIMETZ: [ + # > Bool + # TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + # TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + TYPES_SPEC["Utf8"], + # > Date + TYPES_SPEC["Date"], + # > Datetime + TYPES_SPEC["Datetime"], + # > Timestamp + TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.GENERICDATETIME: [ + # > Bool + # TYPES_SPEC["Bool"], + # > INT + TYPES_SPEC["Int8"], + TYPES_SPEC["Int16"], + TYPES_SPEC["Int32"], + TYPES_SPEC["Int64"], + # > UINT + TYPES_SPEC["UInt8"], + TYPES_SPEC["UInt16"], + TYPES_SPEC["UInt32"], + TYPES_SPEC["UInt64"], + # > Float + TYPES_SPEC["Float"], + # > Double + TYPES_SPEC["Double"], + # > Decimal + # TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + TYPES_SPEC["Utf8"], + # > Date + TYPES_SPEC["Date"], + # > Datetime + TYPES_SPEC["Datetime"], + # > Timestamp + TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.UUID: [ + # > Bool + # TYPES_SPEC["Bool"], + # > INT + # TYPES_SPEC["Int8"], + # TYPES_SPEC["Int16"], + # TYPES_SPEC["Int32"], + # TYPES_SPEC["Int64"], + # > UINT + # TYPES_SPEC["UInt8"], + # TYPES_SPEC["UInt16"], + # TYPES_SPEC["UInt32"], + # TYPES_SPEC["UInt64"], + # > Float + # TYPES_SPEC["Float"], + # > Double + # TYPES_SPEC["Double"], + # > Decimal + # TYPES_SPEC["Decimal"], + # > String + TYPES_SPEC["String"], + # > Utf8 + TYPES_SPEC["Utf8"], + # > Date + # TYPES_SPEC["Date"], + # > Datetime + # TYPES_SPEC["Datetime"], + # > Timestamp + # TYPES_SPEC["Timestamp"], + # > UUID + # TYPES_SPEC["Uuid"], + ], + DataType.ARRAY_STR: [], + DataType.ARRAY_INT: [], + DataType.ARRAY_FLOAT: [], + # DataType.GEOPOINT: [ + # ], + # DataType.GEOPOLYGON: [ + # ], + # DataType.MARKUP: [ + # ], + # DataType.TREE_STR: [ + # ], + } + for yql_dialect in (D.YQL,) + } + + +class FuncDbCastYQL2(FuncDbCastYQLBase, base.FuncDbCast2): + pass + + +class FuncDbCastYQL3(FuncDbCastYQLBase, base.FuncDbCast3): + pass + + +class FuncDbCastYQL4(FuncDbCastYQLBase, base.FuncDbCast4): + pass + DEFINITIONS_TYPE = [ # bool @@ -44,6 +498,10 @@ ), # datetimetz base.FuncDatetimeTZConst.for_dialect(D.YQL), + # db_cast + FuncDbCastYQL2(), + FuncDbCastYQL3(), + FuncDbCastYQL4(), # float base.FuncFloatNumber( variants=[ diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py new file mode 100644 index 0000000000..c2056a6ae3 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py @@ -0,0 +1,33 @@ +import sqlalchemy as sa + +from dl_formula_testing.evaluator import DbEvaluator +from dl_formula_testing.testcases.functions_array import DefaultArrayFunctionFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class ArrayFunctionYDBTestSuite(DefaultArrayFunctionFormulaConnectorTestSuite): + make_decimal_cast = None + make_float_cast = "Double?" + make_float_array_cast = "List?" + make_str_array_cast = "List?" + + def test_startswith_string_array(self, dbe: DbEvaluator, data_table: sa.Table) -> None: + assert dbe.eval("STARTSWITH([arr_str_value], [arr_str_value])", from_=data_table) + assert not dbe.eval( + 'STARTSWITH([arr_str_value], DB_CAST(ARRAY("", "cde", NULL), "varchar[]"))', from_=data_table + ) + + def test_array_contains_all_string_array(self, dbe: DbEvaluator, data_table: sa.Table) -> None: + assert dbe.eval('CONTAINS_ALL([arr_str_value], DB_CAST(ARRAY("cde"), "varchar[]"))', from_=data_table) + assert dbe.eval('CONTAINS_ALL([arr_str_value], DB_CAST(ARRAY("cde", NULL), "varchar[]"))', from_=data_table) + assert not dbe.eval('CONTAINS_ALL(DB_CAST(ARRAY("cde"), "varchar[]"), [arr_str_value])', from_=data_table) + + def test_array_contains_any_string_array(self, dbe: DbEvaluator, data_table: sa.Table) -> None: + assert dbe.eval('CONTAINS_ANY([arr_str_value], DB_CAST(ARRAY("cde"), "varchar[]"))', from_=data_table) + assert dbe.eval('CONTAINS_ANY([arr_str_value], DB_CAST(ARRAY("123", NULL), "varchar[]"))', from_=data_table) + assert dbe.eval('CONTAINS_ANY(DB_CAST(ARRAY("cde"), "varchar[]"), [arr_str_value])', from_=data_table) + + +class TestArrayFunctionYDB(YQLTestBase, ArrayFunctionYDBTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py index 7b7360f5ed..cb6e81de42 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py @@ -1,7 +1,20 @@ +import contextlib +import datetime +from typing import ( + Generator, + Optional, +) + +import pytest +import sqlalchemy as sa + +from dl_formula.core.datatype import DataType +import dl_formula.core.exc as exc from dl_formula_testing.evaluator import DbEvaluator from dl_formula_testing.testcases.functions_type_conversion import ( DefaultBoolTypeFunctionFormulaConnectorTestSuite, DefaultDateTypeFunctionFormulaConnectorTestSuite, + DefaultDbCastTypeFunctionFormulaConnectorTestSuite, DefaultFloatTypeFunctionFormulaConnectorTestSuite, DefaultGenericDatetimeTypeFunctionFormulaConnectorTestSuite, DefaultGeopointTypeFunctionFormulaConnectorTestSuite, @@ -75,3 +88,190 @@ class TestGeopointTypeFunctionYQL(YQLTestBase, DefaultGeopointTypeFunctionFormul class TestGeopolygonTypeFunctionYQL(YQLTestBase, DefaultGeopolygonTypeFunctionFormulaConnectorTestSuite): pass + + +# DB_CAST + + +class DbCastTypeFunctionYQLTestSuite( + DefaultDbCastTypeFunctionFormulaConnectorTestSuite, +): + def test_db_cast_ydb(self, dbe: DbEvaluator, data_table: sa.Table) -> None: + # Valid cast + value = dbe.eval("[int_value]", from_=data_table) + assert dbe.eval('DB_CAST(FLOAT([int_value]), "Double")', from_=data_table) == pytest.approx(float(value)) + + # # Test that it works with bool + dbe.eval('DB_CAST(BOOL([int_value]), "Double")', from_=data_table) + # Test that it works with int + dbe.eval('DB_CAST(INT([int_value]), "Int64")', from_=data_table) + # Test that it works with float + dbe.eval('DB_CAST(FLOAT([int_value]), "Double")', from_=data_table) + # Test that it works with string + dbe.eval('DB_CAST(STR([int_value]), "Utf8")', from_=data_table) + + # Cast to decimal with correct arguments + assert dbe.eval('DB_CAST([int_value], "Decimal", 5, 0)', from_=data_table) == value + + # Invalid number of arguments for Decimal + with pytest.raises(exc.TranslationError): + dbe.eval('DB_CAST([int_value], "Decimal", 5)', from_=data_table) + + with pytest.raises(exc.TranslationError): + dbe.eval('DB_CAST([int_value], "Decimal", "5", "3")', from_=data_table) + + # Invalid cast from Integer to Uuid + with pytest.raises(exc.TranslationError): + dbe.eval('DB_CAST([int_value], "Uuid")', from_=data_table) + + # Cast into itself + assert dbe.eval('DB_CAST(DB_CAST([int_value], "Int64"), "Int64")', from_=data_table) == value + + # Cast and cast back + assert dbe.eval('DB_CAST(DB_CAST(DB_CAST([int_value], "Int64"), "UInt64"), "Int64")', from_=data_table) == value + + # Castn't + with pytest.raises(exc.TranslationError): + assert dbe.eval('DB_CAST([int_value], "meow")', from_=data_table) == value + + @contextlib.contextmanager + def make_ydb_type_test_data_table( + self, dbe: DbEvaluator, table_schema_name: Optional[str] + ) -> Generator[sa.Table, None, None]: + db = dbe.db + table_spec = self.generate_table_spec(table_name_prefix="ydb_type_test_table") + + columns = [ + sa.Column("bool_value", sa.Boolean()), + sa.Column("int64_value", sa.Integer(), primary_key=True), + sa.Column("float_value", sa.Float()), + sa.Column("string_value", sa.Text()), + sa.Column("date_value", sa.Date()), + sa.Column("datetime_value", sa.DateTime()), + ] + + table = self.lowlevel_make_sa_table( + db=db, table_spec=table_spec, table_schema_name=table_schema_name, columns=columns + ) + + db.create_table(table) + + table_data = [ + { + "bool_value": True, + "int64_value": 42, + "float_value": 0.1 + 0.2, + "string_value": "lobster", + "date_value": datetime.date(2000, 1, 2), + "datetime_value": datetime.datetime(2000, 1, 2, 4, 5, 6, 7), + }, + ] + + db.insert_into_table(table, table_data) + + try: + yield table + finally: + dbe.db.drop_table(table) + + @pytest.fixture(scope="class") + def ydb_type_test_data_table( + self, dbe: DbEvaluator, table_schema_name: Optional[str] + ) -> Generator[sa.Table, None, None]: + with self.make_ydb_type_test_data_table(dbe=dbe, table_schema_name=table_schema_name) as table: + yield table + + # YDB-specific field types for formula testing + YDB_TYPE_FIELD_TYPES = { + "bool_value": DataType.BOOLEAN, + "int64_value": DataType.INTEGER, + "float_value": DataType.FLOAT, + "string_value": DataType.STRING, + "timestamp_value": DataType.DATETIME, # YDB TIMESTAMP maps to DATETIME in formula system + "date_value": DataType.DATE, + "datetime_value": DataType.DATETIME, + } + + @pytest.fixture(scope="function") + def ydb_data_test_table_field_types_patch(self, monkeypatch) -> None: + ydb_field_types = {**self.YDB_TYPE_FIELD_TYPES} + + monkeypatch.setattr("dl_formula_testing.evaluator.FIELD_TYPES", ydb_field_types) + + return ydb_field_types + + def _test_db_cast_ydb_bool( + self, + dbe: DbEvaluator, + ydb_type_test_data_table: sa.Table, + target: str, + cast_args: tuple[int, int] | None, + ok: bool, + ydb_data_test_table_field_types_patch, + source_column: str, + ) -> None: + if cast_args: + cast_args_str = ", ".join(cast_args) + query_string = f'DB_CAST([{source_column}], "{target}", {cast_args_str})' + else: + query_string = f'DB_CAST([{source_column}], "{target}")' + + if ok: + dbe.eval(query_string, from_=ydb_type_test_data_table) + else: + with pytest.raises(exc.TranslationError): + dbe.eval(query_string, from_=ydb_type_test_data_table) + + @pytest.mark.parametrize( + "target,cast_args,ok", + [ + # Bool + ("Bool", None, True), + # Int + ("Int8", None, True), + ("Int16", None, True), + ("Int32", None, True), + ("Int64", None, True), + ("UInt8", None, True), + ("UInt16", None, True), + ("UInt32", None, True), + # Float + ("Float", None, True), + ("Double", None, True), + # String + ("String", None, True), + ("Utf8", None, True), + # Date + ("Date", None, True), + ("Datetime", None, True), + ("Timestamp", None, True), + # Uuid + ("Uuid", None, True), + ], + ) + def test_db_cast_ydb_bool( + self, + dbe: DbEvaluator, + # data_table: sa.Table, + ydb_type_test_data_table: sa.Table, + target: str, + cast_args: tuple[int, int] | None, + ok: bool, + ydb_data_test_table_field_types_patch, + ) -> None: + self._test_db_cast_ydb_bool( + dbe=dbe, + ydb_type_test_data_table=ydb_type_test_data_table, + target=target, + cast_args=cast_args, + ok=ok, + ydb_data_test_table_field_types_patch=ydb_data_test_table_field_types_patch, + source_column="bool_value", + ) + + +class TestDbCastTypeFunctionYQL( + YQLTestBase, + DbCastTypeFunctionYQLTestSuite, +): + pass From 7da76f5033e19f30a187e8fbc48c40025d0c667c Mon Sep 17 00:00:00 2001 From: catsona Date: Mon, 29 Sep 2025 19:21:38 +0300 Subject: [PATCH 04/10] reorder classes --- .../formula/test_functions_type_conversion.py | 136 +++++++++--------- 1 file changed, 69 insertions(+), 67 deletions(-) diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py index cb6e81de42..4577ddb8ca 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py @@ -134,72 +134,6 @@ def test_db_cast_ydb(self, dbe: DbEvaluator, data_table: sa.Table) -> None: with pytest.raises(exc.TranslationError): assert dbe.eval('DB_CAST([int_value], "meow")', from_=data_table) == value - @contextlib.contextmanager - def make_ydb_type_test_data_table( - self, dbe: DbEvaluator, table_schema_name: Optional[str] - ) -> Generator[sa.Table, None, None]: - db = dbe.db - table_spec = self.generate_table_spec(table_name_prefix="ydb_type_test_table") - - columns = [ - sa.Column("bool_value", sa.Boolean()), - sa.Column("int64_value", sa.Integer(), primary_key=True), - sa.Column("float_value", sa.Float()), - sa.Column("string_value", sa.Text()), - sa.Column("date_value", sa.Date()), - sa.Column("datetime_value", sa.DateTime()), - ] - - table = self.lowlevel_make_sa_table( - db=db, table_spec=table_spec, table_schema_name=table_schema_name, columns=columns - ) - - db.create_table(table) - - table_data = [ - { - "bool_value": True, - "int64_value": 42, - "float_value": 0.1 + 0.2, - "string_value": "lobster", - "date_value": datetime.date(2000, 1, 2), - "datetime_value": datetime.datetime(2000, 1, 2, 4, 5, 6, 7), - }, - ] - - db.insert_into_table(table, table_data) - - try: - yield table - finally: - dbe.db.drop_table(table) - - @pytest.fixture(scope="class") - def ydb_type_test_data_table( - self, dbe: DbEvaluator, table_schema_name: Optional[str] - ) -> Generator[sa.Table, None, None]: - with self.make_ydb_type_test_data_table(dbe=dbe, table_schema_name=table_schema_name) as table: - yield table - - # YDB-specific field types for formula testing - YDB_TYPE_FIELD_TYPES = { - "bool_value": DataType.BOOLEAN, - "int64_value": DataType.INTEGER, - "float_value": DataType.FLOAT, - "string_value": DataType.STRING, - "timestamp_value": DataType.DATETIME, # YDB TIMESTAMP maps to DATETIME in formula system - "date_value": DataType.DATE, - "datetime_value": DataType.DATETIME, - } - - @pytest.fixture(scope="function") - def ydb_data_test_table_field_types_patch(self, monkeypatch) -> None: - ydb_field_types = {**self.YDB_TYPE_FIELD_TYPES} - - monkeypatch.setattr("dl_formula_testing.evaluator.FIELD_TYPES", ydb_field_types) - - return ydb_field_types - def _test_db_cast_ydb_bool( self, dbe: DbEvaluator, @@ -270,8 +204,76 @@ def test_db_cast_ydb_bool( ) +class DbCastYQLTestSuiteBase(YQLTestBase): + @contextlib.contextmanager + def make_ydb_type_test_data_table( + self, dbe: DbEvaluator, table_schema_name: Optional[str] + ) -> Generator[sa.Table, None, None]: + db = dbe.db + table_spec = self.generate_table_spec(table_name_prefix="ydb_type_test_table") + + columns = [ + sa.Column("bool_value", sa.Boolean()), + sa.Column("int64_value", sa.Integer(), primary_key=True), + sa.Column("float_value", sa.Float()), + sa.Column("string_value", sa.Text()), + sa.Column("date_value", sa.Date()), + sa.Column("datetime_value", sa.DateTime()), + ] + + table = self.lowlevel_make_sa_table( + db=db, table_spec=table_spec, table_schema_name=table_schema_name, columns=columns + ) + + db.create_table(table) + + table_data = [ + { + "bool_value": True, + "int64_value": 42, + "float_value": 0.1 + 0.2, + "string_value": "lobster", + "date_value": datetime.date(2000, 1, 2), + "datetime_value": datetime.datetime(2000, 1, 2, 4, 5, 6, 7), + }, + ] + + db.insert_into_table(table, table_data) + + try: + yield table + finally: + dbe.db.drop_table(table) + + @pytest.fixture(scope="class") + def ydb_type_test_data_table( + self, dbe: DbEvaluator, table_schema_name: Optional[str] + ) -> Generator[sa.Table, None, None]: + with self.make_ydb_type_test_data_table(dbe=dbe, table_schema_name=table_schema_name) as table: + yield table + + # YDB-specific field types for formula testing + YDB_TYPE_FIELD_TYPES = { + "bool_value": DataType.BOOLEAN, + "int64_value": DataType.INTEGER, + "float_value": DataType.FLOAT, + "string_value": DataType.STRING, + "timestamp_value": DataType.DATETIME, # YDB TIMESTAMP maps to DATETIME in formula system + "date_value": DataType.DATE, + "datetime_value": DataType.DATETIME, + } + + @pytest.fixture(scope="function") + def ydb_data_test_table_field_types_patch(self, monkeypatch) -> None: + ydb_field_types = {**self.YDB_TYPE_FIELD_TYPES} + + monkeypatch.setattr("dl_formula_testing.evaluator.FIELD_TYPES", ydb_field_types) + + return ydb_field_types + + class TestDbCastTypeFunctionYQL( - YQLTestBase, + DbCastYQLTestSuiteBase, DbCastTypeFunctionYQLTestSuite, ): pass From 1bdcc0948f8aa6e8f0317f266c4945fc0f123901 Mon Sep 17 00:00:00 2001 From: catsona Date: Mon, 29 Sep 2025 20:14:40 +0300 Subject: [PATCH 05/10] more tests --- .../formula/definitions/functions_type.py | 162 +++----------- .../formula/test_functions_type_conversion.py | 198 +++++++++++++++++- 2 files changed, 217 insertions(+), 143 deletions(-) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py index 138f185455..698b816527 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py @@ -65,6 +65,26 @@ ] +class YQLDbCastArgTypes(base.DbCastArgTypes): + def __init__(self) -> None: + # See DbCastArgTypes.__init__ + super(base.DbCastArgTypes, self).__init__( + arg_types=[ + { + DataType.BOOLEAN, + DataType.INTEGER, + DataType.FLOAT, + DataType.STRING, + DataType.DATE, + DataType.ARRAY_INT, + DataType.ARRAY_FLOAT, + DataType.ARRAY_STR, + }, + DataType.CONST_STRING, + ] + ) + + class FuncDbCastYQLBase(base.FuncDbCastBase): # For numeric types see: https://ydb.tech/docs/en/yql/reference/types/primitive#casting-to-numeric-types # Type cast tables date: 2025-09-29. @@ -135,6 +155,10 @@ class FuncDbCastYQLBase(base.FuncDbCastBase): # [4] - Possible only within the valid range. # [5] - Using the built-in function Yson::ConvertTo. + argument_types = [ + YQLDbCastArgTypes(), + ] + WHITELISTS = { yql_dialect: { # TODO: Decimal @@ -307,147 +331,11 @@ class FuncDbCastYQLBase(base.FuncDbCastBase): # > UUID # TYPES_SPEC["Uuid"], ], - DataType.DATETIME: [ - # > Bool - # TYPES_SPEC["Bool"], - # > INT - TYPES_SPEC["Int8"], - TYPES_SPEC["Int16"], - TYPES_SPEC["Int32"], - TYPES_SPEC["Int64"], - # > UINT - TYPES_SPEC["UInt8"], - TYPES_SPEC["UInt16"], - TYPES_SPEC["UInt32"], - TYPES_SPEC["UInt64"], - # > Float - TYPES_SPEC["Float"], - # > Double - TYPES_SPEC["Double"], - # > Decimal - # TYPES_SPEC["Decimal"], - # > String - TYPES_SPEC["String"], - # > Utf8 - TYPES_SPEC["Utf8"], - # > Date - TYPES_SPEC["Date"], - # > Datetime - TYPES_SPEC["Datetime"], - # > Timestamp - TYPES_SPEC["Timestamp"], - # > UUID - # TYPES_SPEC["Uuid"], - ], - DataType.DATETIMETZ: [ - # > Bool - # TYPES_SPEC["Bool"], - # > INT - TYPES_SPEC["Int8"], - TYPES_SPEC["Int16"], - TYPES_SPEC["Int32"], - TYPES_SPEC["Int64"], - # > UINT - TYPES_SPEC["UInt8"], - TYPES_SPEC["UInt16"], - TYPES_SPEC["UInt32"], - TYPES_SPEC["UInt64"], - # > Float - TYPES_SPEC["Float"], - # > Double - TYPES_SPEC["Double"], - # > Decimal - # TYPES_SPEC["Decimal"], - # > String - TYPES_SPEC["String"], - # > Utf8 - TYPES_SPEC["Utf8"], - # > Date - TYPES_SPEC["Date"], - # > Datetime - TYPES_SPEC["Datetime"], - # > Timestamp - TYPES_SPEC["Timestamp"], - # > UUID - # TYPES_SPEC["Uuid"], - ], - DataType.GENERICDATETIME: [ - # > Bool - # TYPES_SPEC["Bool"], - # > INT - TYPES_SPEC["Int8"], - TYPES_SPEC["Int16"], - TYPES_SPEC["Int32"], - TYPES_SPEC["Int64"], - # > UINT - TYPES_SPEC["UInt8"], - TYPES_SPEC["UInt16"], - TYPES_SPEC["UInt32"], - TYPES_SPEC["UInt64"], - # > Float - TYPES_SPEC["Float"], - # > Double - TYPES_SPEC["Double"], - # > Decimal - # TYPES_SPEC["Decimal"], - # > String - TYPES_SPEC["String"], - # > Utf8 - TYPES_SPEC["Utf8"], - # > Date - TYPES_SPEC["Date"], - # > Datetime - TYPES_SPEC["Datetime"], - # > Timestamp - TYPES_SPEC["Timestamp"], - # > UUID - # TYPES_SPEC["Uuid"], - ], - DataType.UUID: [ - # > Bool - # TYPES_SPEC["Bool"], - # > INT - # TYPES_SPEC["Int8"], - # TYPES_SPEC["Int16"], - # TYPES_SPEC["Int32"], - # TYPES_SPEC["Int64"], - # > UINT - # TYPES_SPEC["UInt8"], - # TYPES_SPEC["UInt16"], - # TYPES_SPEC["UInt32"], - # TYPES_SPEC["UInt64"], - # > Float - # TYPES_SPEC["Float"], - # > Double - # TYPES_SPEC["Double"], - # > Decimal - # TYPES_SPEC["Decimal"], - # > String - TYPES_SPEC["String"], - # > Utf8 - TYPES_SPEC["Utf8"], - # > Date - # TYPES_SPEC["Date"], - # > Datetime - # TYPES_SPEC["Datetime"], - # > Timestamp - # TYPES_SPEC["Timestamp"], - # > UUID - # TYPES_SPEC["Uuid"], - ], DataType.ARRAY_STR: [], DataType.ARRAY_INT: [], DataType.ARRAY_FLOAT: [], - # DataType.GEOPOINT: [ - # ], - # DataType.GEOPOLYGON: [ - # ], - # DataType.MARKUP: [ - # ], - # DataType.TREE_STR: [ - # ], } - for yql_dialect in (D.YQL,) + for yql_dialect in (D.YQL, D.YQ, D.YDB) } diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py index 4577ddb8ca..1a25ab8d66 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py @@ -134,7 +134,7 @@ def test_db_cast_ydb(self, dbe: DbEvaluator, data_table: sa.Table) -> None: with pytest.raises(exc.TranslationError): assert dbe.eval('DB_CAST([int_value], "meow")', from_=data_table) == value - def _test_db_cast_ydb_bool( + def _test_db_cast_ydb_func( self, dbe: DbEvaluator, ydb_type_test_data_table: sa.Table, @@ -156,6 +156,147 @@ def _test_db_cast_ydb_bool( with pytest.raises(exc.TranslationError): dbe.eval(query_string, from_=ydb_type_test_data_table) + @pytest.mark.parametrize( + "target,cast_args,ok", + [ + # Bool + ("Bool", None, True), + # Int + ("Int8", None, True), + ("Int16", None, True), + ("Int32", None, True), + ("Int64", None, True), + ("UInt8", None, True), + ("UInt16", None, True), + ("UInt32", None, True), + # Float + ("Float", None, True), + ("Double", None, True), + # String + ("String", None, True), + ("Utf8", None, False), + # Date + ("Date", None, False), + ("Datetime", None, False), + ("Timestamp", None, False), + # Uuid + ("Uuid", None, False), + ], + ) + def test_db_cast_ydb_bool( + self, + dbe: DbEvaluator, + # data_table: sa.Table, + ydb_type_test_data_table: sa.Table, + target: str, + cast_args: tuple[int, int] | None, + ok: bool, + ydb_data_test_table_field_types_patch, + ) -> None: + self._test_db_cast_ydb_func( + dbe=dbe, + ydb_type_test_data_table=ydb_type_test_data_table, + target=target, + cast_args=cast_args, + ok=ok, + ydb_data_test_table_field_types_patch=ydb_data_test_table_field_types_patch, + source_column="bool_value", + ) + + @pytest.mark.parametrize( + "target,cast_args,ok", + [ + # Bool + ("Bool", None, True), + # Int + ("Int8", None, True), + ("Int16", None, True), + ("Int32", None, True), + ("Int64", None, True), + ("UInt8", None, True), + ("UInt16", None, True), + ("UInt32", None, True), + # Float + ("Float", None, True), + ("Double", None, True), + # String + ("String", None, True), + ("Utf8", None, False), + # Date + ("Date", None, True), + ("Datetime", None, True), + ("Timestamp", None, True), + # Uuid + ("Uuid", None, False), + ], + ) + def test_db_cast_ydb_integer( + self, + dbe: DbEvaluator, + # data_table: sa.Table, + ydb_type_test_data_table: sa.Table, + target: str, + cast_args: tuple[int, int] | None, + ok: bool, + ydb_data_test_table_field_types_patch, + ) -> None: + self._test_db_cast_ydb_func( + dbe=dbe, + ydb_type_test_data_table=ydb_type_test_data_table, + target=target, + cast_args=cast_args, + ok=ok, + ydb_data_test_table_field_types_patch=ydb_data_test_table_field_types_patch, + source_column="int64_value", + ) + + @pytest.mark.parametrize( + "target,cast_args,ok", + [ + # Bool + ("Bool", None, True), + # Int + ("Int8", None, True), + ("Int16", None, True), + ("Int32", None, True), + ("Int64", None, True), + ("UInt8", None, True), + ("UInt16", None, True), + ("UInt32", None, True), + # Float + ("Float", None, True), + ("Double", None, True), + # String + ("String", None, True), + ("Utf8", None, False), + # Date + ("Date", None, False), + ("Datetime", None, False), + ("Timestamp", None, False), + # Uuid + ("Uuid", None, False), + ], + ) + def test_db_cast_ydb_float( + self, + dbe: DbEvaluator, + # data_table: sa.Table, + ydb_type_test_data_table: sa.Table, + target: str, + cast_args: tuple[int, int] | None, + ok: bool, + ydb_data_test_table_field_types_patch, + ) -> None: + self._test_db_cast_ydb_func( + dbe=dbe, + ydb_type_test_data_table=ydb_type_test_data_table, + target=target, + cast_args=cast_args, + ok=ok, + ydb_data_test_table_field_types_patch=ydb_data_test_table_field_types_patch, + source_column="float_value", + ) + @pytest.mark.parametrize( "target,cast_args,ok", [ @@ -183,7 +324,7 @@ def _test_db_cast_ydb_bool( ("Uuid", None, True), ], ) - def test_db_cast_ydb_bool( + def test_db_cast_ydb_string( self, dbe: DbEvaluator, # data_table: sa.Table, @@ -193,14 +334,61 @@ def test_db_cast_ydb_bool( ok: bool, ydb_data_test_table_field_types_patch, ) -> None: - self._test_db_cast_ydb_bool( + self._test_db_cast_ydb_func( dbe=dbe, ydb_type_test_data_table=ydb_type_test_data_table, target=target, cast_args=cast_args, ok=ok, ydb_data_test_table_field_types_patch=ydb_data_test_table_field_types_patch, - source_column="bool_value", + source_column="string_value", + ) + + @pytest.mark.parametrize( + "target,cast_args,ok", + [ + # Bool + ("Bool", None, False), + # Int + ("Int8", None, True), + ("Int16", None, True), + ("Int32", None, True), + ("Int64", None, True), + ("UInt8", None, True), + ("UInt16", None, True), + ("UInt32", None, True), + # Float + ("Float", None, True), + ("Double", None, True), + # String + ("String", None, True), + ("Utf8", None, True), + # Date + ("Date", None, True), + ("Datetime", None, True), + ("Timestamp", None, True), + # Uuid + ("Uuid", None, False), + ], + ) + def test_db_cast_ydb_date( + self, + dbe: DbEvaluator, + # data_table: sa.Table, + ydb_type_test_data_table: sa.Table, + target: str, + cast_args: tuple[int, int] | None, + ok: bool, + ydb_data_test_table_field_types_patch, + ) -> None: + self._test_db_cast_ydb_func( + dbe=dbe, + ydb_type_test_data_table=ydb_type_test_data_table, + target=target, + cast_args=cast_args, + ok=ok, + ydb_data_test_table_field_types_patch=ydb_data_test_table_field_types_patch, + source_column="date_value", ) @@ -218,7 +406,6 @@ def make_ydb_type_test_data_table( sa.Column("float_value", sa.Float()), sa.Column("string_value", sa.Text()), sa.Column("date_value", sa.Date()), - sa.Column("datetime_value", sa.DateTime()), ] table = self.lowlevel_make_sa_table( @@ -234,7 +421,6 @@ def make_ydb_type_test_data_table( "float_value": 0.1 + 0.2, "string_value": "lobster", "date_value": datetime.date(2000, 1, 2), - "datetime_value": datetime.datetime(2000, 1, 2, 4, 5, 6, 7), }, ] From 9d5872bc88b8b9eddfc59925bfab4e7212ac8477 Mon Sep 17 00:00:00 2001 From: catsona Date: Mon, 29 Sep 2025 20:22:08 +0300 Subject: [PATCH 06/10] cleanup and comments --- .../formula/test_functions_type_conversion.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py index 1a25ab8d66..61aeea085e 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py @@ -157,6 +157,9 @@ def _test_db_cast_ydb_func( dbe.eval(query_string, from_=ydb_type_test_data_table) @pytest.mark.parametrize( + # target - target type for cast + # cast_args - type arguments (for decimal) + # ok - if no exception should occur "target,cast_args,ok", [ # Bool @@ -186,7 +189,6 @@ def _test_db_cast_ydb_func( def test_db_cast_ydb_bool( self, dbe: DbEvaluator, - # data_table: sa.Table, ydb_type_test_data_table: sa.Table, target: str, cast_args: tuple[int, int] | None, @@ -204,6 +206,9 @@ def test_db_cast_ydb_bool( ) @pytest.mark.parametrize( + # target - target type for cast + # cast_args - type arguments (for decimal) + # ok - if no exception should occur "target,cast_args,ok", [ # Bool @@ -233,7 +238,6 @@ def test_db_cast_ydb_bool( def test_db_cast_ydb_integer( self, dbe: DbEvaluator, - # data_table: sa.Table, ydb_type_test_data_table: sa.Table, target: str, cast_args: tuple[int, int] | None, @@ -251,6 +255,9 @@ def test_db_cast_ydb_integer( ) @pytest.mark.parametrize( + # target - target type for cast + # cast_args - type arguments (for decimal) + # ok - if no exception should occur "target,cast_args,ok", [ # Bool @@ -280,7 +287,6 @@ def test_db_cast_ydb_integer( def test_db_cast_ydb_float( self, dbe: DbEvaluator, - # data_table: sa.Table, ydb_type_test_data_table: sa.Table, target: str, cast_args: tuple[int, int] | None, @@ -298,6 +304,9 @@ def test_db_cast_ydb_float( ) @pytest.mark.parametrize( + # target - target type for cast + # cast_args - type arguments (for decimal) + # ok - if no exception should occur "target,cast_args,ok", [ # Bool @@ -327,7 +336,6 @@ def test_db_cast_ydb_float( def test_db_cast_ydb_string( self, dbe: DbEvaluator, - # data_table: sa.Table, ydb_type_test_data_table: sa.Table, target: str, cast_args: tuple[int, int] | None, @@ -345,6 +353,9 @@ def test_db_cast_ydb_string( ) @pytest.mark.parametrize( + # target - target type for cast + # cast_args - type arguments (for decimal) + # ok - if no exception should occur "target,cast_args,ok", [ # Bool @@ -374,7 +385,6 @@ def test_db_cast_ydb_string( def test_db_cast_ydb_date( self, dbe: DbEvaluator, - # data_table: sa.Table, ydb_type_test_data_table: sa.Table, target: str, cast_args: tuple[int, int] | None, From f7b7ddc20a3a74dc4128104ada38655f3e6a3fce Mon Sep 17 00:00:00 2001 From: catsona Date: Tue, 30 Sep 2025 11:27:19 +0300 Subject: [PATCH 07/10] remove unsupported test --- .../db/formula/test_functions_array.py | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py deleted file mode 100644 index c2056a6ae3..0000000000 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_array.py +++ /dev/null @@ -1,33 +0,0 @@ -import sqlalchemy as sa - -from dl_formula_testing.evaluator import DbEvaluator -from dl_formula_testing.testcases.functions_array import DefaultArrayFunctionFormulaConnectorTestSuite - -from dl_connector_ydb_tests.db.formula.base import YQLTestBase - - -class ArrayFunctionYDBTestSuite(DefaultArrayFunctionFormulaConnectorTestSuite): - make_decimal_cast = None - make_float_cast = "Double?" - make_float_array_cast = "List?" - make_str_array_cast = "List?" - - def test_startswith_string_array(self, dbe: DbEvaluator, data_table: sa.Table) -> None: - assert dbe.eval("STARTSWITH([arr_str_value], [arr_str_value])", from_=data_table) - assert not dbe.eval( - 'STARTSWITH([arr_str_value], DB_CAST(ARRAY("", "cde", NULL), "varchar[]"))', from_=data_table - ) - - def test_array_contains_all_string_array(self, dbe: DbEvaluator, data_table: sa.Table) -> None: - assert dbe.eval('CONTAINS_ALL([arr_str_value], DB_CAST(ARRAY("cde"), "varchar[]"))', from_=data_table) - assert dbe.eval('CONTAINS_ALL([arr_str_value], DB_CAST(ARRAY("cde", NULL), "varchar[]"))', from_=data_table) - assert not dbe.eval('CONTAINS_ALL(DB_CAST(ARRAY("cde"), "varchar[]"), [arr_str_value])', from_=data_table) - - def test_array_contains_any_string_array(self, dbe: DbEvaluator, data_table: sa.Table) -> None: - assert dbe.eval('CONTAINS_ANY([arr_str_value], DB_CAST(ARRAY("cde"), "varchar[]"))', from_=data_table) - assert dbe.eval('CONTAINS_ANY([arr_str_value], DB_CAST(ARRAY("123", NULL), "varchar[]"))', from_=data_table) - assert dbe.eval('CONTAINS_ANY(DB_CAST(ARRAY("cde"), "varchar[]"), [arr_str_value])', from_=data_table) - - -class TestArrayFunctionYDB(YQLTestBase, ArrayFunctionYDBTestSuite): - pass From e1c7d123a5e2da5b982585c88be5c9f4d340167b Mon Sep 17 00:00:00 2001 From: catsona Date: Thu, 27 Nov 2025 15:49:39 +0300 Subject: [PATCH 08/10] lint-fix --- lib/dl_connector_ydb/pyproject.toml | 3 +-- metapkg/poetry.lock | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/dl_connector_ydb/pyproject.toml b/lib/dl_connector_ydb/pyproject.toml index 475d110b32..afea6497e6 100644 --- a/lib/dl_connector_ydb/pyproject.toml +++ b/lib/dl_connector_ydb/pyproject.toml @@ -19,7 +19,7 @@ dl-formula = {path = "../dl_formula"} dl-formula-ref = {path = "../dl_formula_ref"} dl-i18n = {path = "../dl_i18n"} dl-query-processing = {path = "../dl_query_processing"} -dl-sqlalchemy-ydb = {path = "../../lib/dl_sqlalchemy_ydb"} +dl-sqlalchemy-ydb = {path = "../dl_sqlalchemy_ydb"} dl-type-transformer = {path = "../dl_type_transformer"} dl-utils = {path = "../dl_utils"} grpcio = "*" @@ -41,7 +41,6 @@ dl-core-testing = {path = "../dl_core_testing"} dl-formula-testing = {path = "../dl_formula_testing"} dl-testing = {path = "../dl_testing"} frozendict = "*" -dl-sqlalchemy-ydb = {path = "../../lib/dl_sqlalchemy_ydb"} pytest = "*" requests = "*" diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index e88ee0a605..fd1f8cd17b 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -2569,7 +2569,7 @@ dl-formula = {path = "../dl_formula"} dl-formula-ref = {path = "../dl_formula_ref"} dl-i18n = {path = "../dl_i18n"} dl-query-processing = {path = "../dl_query_processing"} -dl-sqlalchemy-ydb = {path = "../../lib/dl_sqlalchemy_ydb"} +dl-sqlalchemy-ydb = {path = "../dl_sqlalchemy_ydb"} dl-type-transformer = {path = "../dl_type_transformer"} dl-utils = {path = "../dl_utils"} grpcio = "*" From a0aed9e54fdaf14328b4d52219087d51196a9383 Mon Sep 17 00:00:00 2001 From: catsona Date: Thu, 27 Nov 2025 19:56:58 +0300 Subject: [PATCH 09/10] change 0 to 1 --- .../db/formula/test_functions_type_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py index 61aeea085e..08eeeceb97 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py @@ -111,7 +111,7 @@ def test_db_cast_ydb(self, dbe: DbEvaluator, data_table: sa.Table) -> None: dbe.eval('DB_CAST(STR([int_value]), "Utf8")', from_=data_table) # Cast to decimal with correct arguments - assert dbe.eval('DB_CAST([int_value], "Decimal", 5, 0)', from_=data_table) == value + assert dbe.eval('DB_CAST([int_value], "Decimal", 5, 1)', from_=data_table) == value # Invalid number of arguments for Decimal with pytest.raises(exc.TranslationError): From d1714c43bc5eb8e88746ee294f88bf452a2434b2 Mon Sep 17 00:00:00 2001 From: catsona Date: Fri, 28 Nov 2025 14:31:03 +0300 Subject: [PATCH 10/10] cleanup --- .../formula/definitions/functions_type.py | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py index 698b816527..f6899627a8 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py @@ -35,35 +35,6 @@ ] } -BOOL_TYPES_SPEC = [ - TYPES_SPEC["Bool"], -] - -INT_TYPES_SPEC = [ - TYPES_SPEC["Int8"], - TYPES_SPEC["Int16"], - TYPES_SPEC["Int32"], - TYPES_SPEC["Int64"], - TYPES_SPEC["UInt8"], - TYPES_SPEC["UInt16"], - TYPES_SPEC["UInt32"], - TYPES_SPEC["UInt64"], -] - -DECIMAL_TYPES_SPEC = [ - TYPES_SPEC["Decimal"], -] - -STRING_TYPES_SPEC = [ - TYPES_SPEC["Utf8"], - TYPES_SPEC["String"], -] - -FLOAT_TYPES_SPEC = [ - TYPES_SPEC["Double"], - TYPES_SPEC["Float"], -] - class YQLDbCastArgTypes(base.DbCastArgTypes): def __init__(self) -> None: