Skip to content

feat(connectors): BI-6611 DB_CAST for YDB without ARRAY support #1261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
catsona wants to merge 10 commits into
base: main
Choose a base branch
from
Open

feat(connectors): BI-6611 DB_CAST for YDB without ARRAY support #1261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
a358692
feat(connectors): BI-5971 Upgrade YDB SDK dependencies
catsona Sep 25, 2025
19bcc48
cleanup
catsona Sep 26, 2025
2426966
feat(connectors): BI-6611 DB_CAST for YDB
catsona Sep 29, 2025
7da76f5
reorder classes
catsona Sep 29, 2025
1bdcc09
more tests
catsona Sep 29, 2025
9d5872b
cleanup and comments
catsona Sep 29, 2025
f7b7ddc
remove unsupported test
catsona Sep 30, 2025
e1c7d12
lint-fix
catsona Nov 27, 2025
a0aed9e
change 0 to 1
catsona Nov 27, 2025
d1714c4
cleanup
catsona Nov 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
317 changes: 317 additions & 0 deletions lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,326 @@
import sqlalchemy as sa
import ydb_sqlalchemy as ydb_sa

from dl_formula.core.datatype import DataType
from dl_formula.definitions.base import TranslationVariant
import dl_formula.definitions.functions_type as base
import dl_sqlalchemy_ydb.dialect as ydb_dialect

from dl_connector_ydb.formula.constants import YqlDialect as D


V = TranslationVariant.make

TYPES_SPEC = {
wlts.name: wlts
for wlts in [
base.WhitelistTypeSpec(name="Bool", sa_type=sa.BOOLEAN),
base.WhitelistTypeSpec(name="Int8", sa_type=ydb_sa.types.Int8),
base.WhitelistTypeSpec(name="Int16", sa_type=ydb_sa.types.Int16),
base.WhitelistTypeSpec(name="Int32", sa_type=ydb_sa.types.Int32),
base.WhitelistTypeSpec(name="Int64", sa_type=ydb_sa.types.Int64),
base.WhitelistTypeSpec(name="UInt8", sa_type=ydb_sa.types.UInt8),
base.WhitelistTypeSpec(name="UInt16", sa_type=ydb_sa.types.UInt16),
base.WhitelistTypeSpec(name="UInt32", sa_type=ydb_sa.types.UInt32),
base.WhitelistTypeSpec(name="UInt64", sa_type=ydb_sa.types.UInt64),
base.WhitelistTypeSpec(name="Double", sa_type=sa.types.FLOAT),
base.WhitelistTypeSpec(name="Float", sa_type=sa.types.FLOAT),
base.WhitelistTypeSpec(name="Decimal", sa_type=sa.DECIMAL, arg_types=base.DECIMAL_CAST_ARG_T),
base.WhitelistTypeSpec(name="Utf8", sa_type=sa.types.TEXT),
base.WhitelistTypeSpec(name="String", sa_type=sa.types.TEXT),
base.WhitelistTypeSpec(name="Date", sa_type=sa.types.DATE),
base.WhitelistTypeSpec(name="Datetime", sa_type=ydb_dialect.YqlDateTime),
base.WhitelistTypeSpec(name="Timestamp", sa_type=ydb_dialect.YqlTimestamp),
base.WhitelistTypeSpec(name="Uuid", sa_type=sa.types.TEXT),
]
}


class YQLDbCastArgTypes(base.DbCastArgTypes):
def __init__(self) -> None:
# See DbCastArgTypes.__init__
super(base.DbCastArgTypes, self).__init__(
arg_types=[
{
DataType.BOOLEAN,
DataType.INTEGER,
DataType.FLOAT,
DataType.STRING,
DataType.DATE,
DataType.ARRAY_INT,
DataType.ARRAY_FLOAT,
DataType.ARRAY_STR,
},
DataType.CONST_STRING,
]
)


class FuncDbCastYQLBase(base.FuncDbCastBase):
# For numeric types see: https://ydb.tech/docs/en/yql/reference/types/primitive#casting-to-numeric-types
# Type cast tables date: 2025-09-29.
#
# Type Bool Int8 Int16 Int32 Int64 Uint8 Uint16 Uint32 Uint64 Float Double Decimal <- (Target Type)
# Bool — Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] Yes[1] No
# Int8 Yes2 — Yes Yes Yes Yes[3] Yes[3] Yes[3] Yes[3] Yes Yes Yes
# Int16 Yes2 Yes[4] — Yes Yes Yes[3,4] Yes[3] Yes[3] Yes[3] Yes Yes Yes
# Int32 Yes2 Yes[4] Yes[4] — Yes Yes[3,4] Yes[3,4] Yes[3] Yes[3] Yes Yes Yes
# Int64 Yes2 Yes[4] Yes[4] Yes[4] — Yes[3,4] Yes[3,4] Yes[3,4] Yes[3] Yes Yes Yes
# Uint8 Yes2 Yes[4] Yes Yes Yes — Yes Yes Yes Yes Yes Yes
# Uint16 Yes2 Yes[4] Yes[4] Yes Yes Yes[4] — Yes Yes Yes Yes Yes
# Uint32 Yes2 Yes[4] Yes[4] Yes[4] Yes Yes[4] Yes[4] — Yes Yes Yes Yes
# Uint64 Yes2 Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] — Yes Yes Yes
# Float Yes2 Yes[4] Yes[4] Yes[4] Yes[4] Yes[3,4] Yes[3,4] Yes[3,4] Yes[3,4] — Yes No
# Double Yes2 Yes[4] Yes[4] Yes[4] Yes[4] Yes[3,4] Yes[3,4] Yes[3,4] Yes[3,4] Yes — No
# Decimal No Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes —
# String Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes
# Utf8 Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes
# Json No No No No No No No No No No No No
# Yson Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] Yes[5] No
# Uuid No No No No No No No No No No No No
# Date No Yes[4] Yes[4] Yes Yes Yes[4] Yes Yes Yes Yes Yes No
# Datetime No Yes[4] Yes[4] Yes[4] Yes Yes[4] Yes[4] Yes Yes Yes Yes No
# Timestamp No Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes[4] Yes Yes Yes No
# Interval No Yes[4] Yes[4] Yes[4] Yes Yes[3,4] Yes[3,4] Yes[3,4] Yes[3] Yes Yes No
# ^
# |
# (Source Type)
#
# Type String Utf8 Json Yson Uuid
# Bool Yes No No No No
# INT Yes No No No No
# Uint Yes No No No No
# Float Yes No No No No
# Double Yes No No No No
# Decimal Yes No No No No
# String — Yes Yes Yes Yes
# Utf8 Yes — No No No
# Json Yes Yes — No No
# Yson Yes[4] No No No No
# Uuid Yes Yes No No —
# Date Yes Yes No No No
# Datetime Yes Yes No No No
# Timestamp Yes Yes No No No
# Interval Yes Yes No No No
#
# Type Date Datetime Timestamp Interval
# Bool No No No No
# INT Yes Yes Yes Yes
# Uint Yes Yes Yes Yes
# Float No No No No
# Double No No No No
# Decimal No No No No
# String Yes Yes Yes Yes
# Utf8 Yes Yes Yes Yes
# Json No No No No
# Yson No No No No
# Uuid No No No No
# Date — Yes Yes No
# Datetime Yes — Yes No
# Timestamp Yes Yes — No
# Interval No No No —
#
# [1] - True is converted to 1 and False to 0.
# [2] - Any value other than 0 is converted to True, 0 is converted to False.
# [3] - Possible only in case of a non-negative value.
# [4] - Possible only within the valid range.
# [5] - Using the built-in function Yson::ConvertTo.

argument_types = [
YQLDbCastArgTypes(),
]

WHITELISTS = {
yql_dialect: {
# TODO: Decimal
# TODO: DyNumber
# TODO: Json
# TODO: JsonDocument
# TODO: Yson
# TODO: Cast Integer to Interval
# Commented - not supported
DataType.BOOLEAN: [
# > Bool
TYPES_SPEC["Bool"],
# > INT
TYPES_SPEC["Int8"],
TYPES_SPEC["Int16"],
TYPES_SPEC["Int32"],
TYPES_SPEC["Int64"],
# > UINT
TYPES_SPEC["UInt8"],
TYPES_SPEC["UInt16"],
TYPES_SPEC["UInt32"],
TYPES_SPEC["UInt64"],
# > Float
TYPES_SPEC["Float"],
# > Double
TYPES_SPEC["Double"],
# > Decimal
# TYPES_SPEC["Decimal"],
# > String
TYPES_SPEC["String"],
# > Utf8
# TYPES_SPEC["Utf8"],
# > Date
# TYPES_SPEC["Date"],
# > Datetime
# TYPES_SPEC["Datetime"],
# > Timestamp
# TYPES_SPEC["Timestamp"],
# > UUID
# TYPES_SPEC["Uuid"],
],
DataType.INTEGER: [
# Interval can not be casted to Bool
# Interval can not be casted to Decimal
# > Bool
TYPES_SPEC["Bool"],
# > INT
TYPES_SPEC["Int8"],
TYPES_SPEC["Int16"],
TYPES_SPEC["Int32"],
TYPES_SPEC["Int64"],
# > UINT
TYPES_SPEC["UInt8"],
TYPES_SPEC["UInt16"],
TYPES_SPEC["UInt32"],
TYPES_SPEC["UInt64"],
# > Float
TYPES_SPEC["Float"],
# > Double
TYPES_SPEC["Double"],
# > Decimal
TYPES_SPEC["Decimal"],
# > String
TYPES_SPEC["String"],
# > Utf8
# TYPES_SPEC["Utf8"],
# > Date
TYPES_SPEC["Date"],
# > Datetime
TYPES_SPEC["Datetime"],
# > Timestamp
TYPES_SPEC["Timestamp"],
# > UUID
# TYPES_SPEC["Uuid"],
],
DataType.FLOAT: [
# > Bool
TYPES_SPEC["Bool"],
# > INT
TYPES_SPEC["Int8"],
TYPES_SPEC["Int16"],
TYPES_SPEC["Int32"],
TYPES_SPEC["Int64"],
# > UINT
TYPES_SPEC["UInt8"],
TYPES_SPEC["UInt16"],
TYPES_SPEC["UInt32"],
TYPES_SPEC["UInt64"],
# > Float
TYPES_SPEC["Float"],
# > Double
TYPES_SPEC["Double"],
# > Decimal
# TYPES_SPEC["Decimal"],
# > String
TYPES_SPEC["String"],
# > Utf8
# TYPES_SPEC["Utf8"],
# > Date
# TYPES_SPEC["Date"],
# > Datetime
# TYPES_SPEC["Datetime"],
# > Timestamp
# TYPES_SPEC["Timestamp"],
# > UUID
# TYPES_SPEC["Uuid"],
],
DataType.STRING: [
# Utf8 can not be casted to Uuid
# > Bool
TYPES_SPEC["Bool"],
# > INT
TYPES_SPEC["Int8"],
TYPES_SPEC["Int16"],
TYPES_SPEC["Int32"],
TYPES_SPEC["Int64"],
# > UINT
TYPES_SPEC["UInt8"],
TYPES_SPEC["UInt16"],
TYPES_SPEC["UInt32"],
TYPES_SPEC["UInt64"],
# > Float
TYPES_SPEC["Float"],
# > Double
TYPES_SPEC["Double"],
# > Decimal
TYPES_SPEC["Decimal"],
# > String
TYPES_SPEC["String"],
# > Utf8
TYPES_SPEC["Utf8"],
# > Date
TYPES_SPEC["Date"],
# > Datetime
TYPES_SPEC["Datetime"],
# > Timestamp
TYPES_SPEC["Timestamp"],
# > UUID
TYPES_SPEC["Uuid"],
],
DataType.DATE: [
# > Bool
# TYPES_SPEC["Bool"],
# > INT
TYPES_SPEC["Int8"],
TYPES_SPEC["Int16"],
TYPES_SPEC["Int32"],
TYPES_SPEC["Int64"],
# > UINT
TYPES_SPEC["UInt8"],
TYPES_SPEC["UInt16"],
TYPES_SPEC["UInt32"],
TYPES_SPEC["UInt64"],
# > Float
TYPES_SPEC["Float"],
# > Double
TYPES_SPEC["Double"],
# > Decimal
# TYPES_SPEC["Decimal"],
# > String
TYPES_SPEC["String"],
# > Utf8
TYPES_SPEC["Utf8"],
# > Date
TYPES_SPEC["Date"],
# > Datetime
TYPES_SPEC["Datetime"],
# > Timestamp
TYPES_SPEC["Timestamp"],
# > UUID
# TYPES_SPEC["Uuid"],
],
DataType.ARRAY_STR: [],
DataType.ARRAY_INT: [],
DataType.ARRAY_FLOAT: [],
}
for yql_dialect in (D.YQL, D.YQ, D.YDB)
}


class FuncDbCastYQL2(FuncDbCastYQLBase, base.FuncDbCast2):
pass


class FuncDbCastYQL3(FuncDbCastYQLBase, base.FuncDbCast3):
pass


class FuncDbCastYQL4(FuncDbCastYQLBase, base.FuncDbCast4):
pass


DEFINITIONS_TYPE = [
# bool
Expand Down Expand Up @@ -44,6 +357,10 @@
),
# datetimetz
base.FuncDatetimeTZConst.for_dialect(D.YQL),
# db_cast
FuncDbCastYQL2(),
FuncDbCastYQL3(),
FuncDbCastYQL4(),
# float
base.FuncFloatNumber(
variants=[
Expand Down
Loading
Loading