From 771069bbdf9555e23c067fedf19b6fde73476f86 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Mon, 9 Feb 2026 14:38:22 +0000 Subject: [PATCH 1/4] Bump to latest SDK version with Python 3.9 support --- poetry.lock | 129 ++++++++++++++++++++++++++++++++++--------------- pyproject.toml | 6 +-- 2 files changed, 93 insertions(+), 42 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2c495e2..e54536f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -144,9 +144,10 @@ frozenlist = ">=1.1.0" name = "appdirs" version = "1.4.4" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false +optional = true python-versions = "*" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"s3\"" files = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, @@ -555,9 +556,10 @@ files = [ name = "fs" version = "2.4.16" description = "Python's filesystem abstraction layer" -optional = false +optional = true python-versions = "*" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"s3\"" files = [ {file = "fs-2.4.16-py2.py3-none-any.whl", hash = "sha256:660064febbccda264ae0b6bace80a8d1be9e089e0a5eb2427b7d517f9a91545c"}, {file = "fs-2.4.16.tar.gz", hash = "sha256:ae97c7d51213f4b70b6a958292530289090de3a7e15841e108fbe144f069d313"}, @@ -1372,6 +1374,18 @@ files = [ {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] +[[package]] +name = "pathlib-abc" +version = "0.5.2" +description = "Backport of pathlib ABCs" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "pathlib_abc-0.5.2-py3-none-any.whl", hash = "sha256:4c9d94cf1b23af417ce7c0417b43333b06a106c01000b286c99de230d95eefbb"}, + {file = "pathlib_abc-0.5.2.tar.gz", hash = "sha256:fcd56f147234645e2c59c7ae22808b34c364bb231f685ddd9f96885aed78a94c"}, +] + [[package]] name = "pluggy" version = "1.5.0" @@ -1466,28 +1480,44 @@ files = [ [package.dependencies] pyasn1 = ">=0.4.6,<0.7.0" +[[package]] +name = "pygments" +version = "2.19.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "pytest" -version = "7.4.4" +version = "8.4.2" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, - {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, + {file = "pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79"}, + {file = "pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01"}, ] [package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1", markers = "python_version < \"3.11\""} +iniconfig = ">=1" +packaging = ">=20" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] [[package]] name = "python-dateutil" @@ -1790,9 +1820,10 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] name = "setuptools" version = "70.3.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = false +optional = true python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main"] +markers = "extra == \"s3\"" files = [ {file = "setuptools-70.3.0-py3-none-any.whl", hash = "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"}, {file = "setuptools-70.3.0.tar.gz", hash = "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5"}, @@ -1936,49 +1967,47 @@ files = [ [[package]] name = "singer-sdk" -version = "0.41.0" +version = "0.48.1" description = "A framework for building Singer taps" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "singer_sdk-0.41.0-py3-none-any.whl", hash = "sha256:9570377d043239c04d38d4193e0c6e164949d07382234c5895e5ea1ba273e260"}, - {file = "singer_sdk-0.41.0.tar.gz", hash = "sha256:be3a4b0ae034eda445e7dd9378999f9d19d8135fd14c9135e3bc5deaf5dbd3ad"}, + {file = "singer_sdk-0.48.1-py3-none-any.whl", hash = "sha256:63d969e06f69a7a94b84917f9a8c22b8e5bd8b87ee7082c77e2436e0c14cfdc0"}, + {file = "singer_sdk-0.48.1.tar.gz", hash = "sha256:f6ac6ba9a184713b328c0f271b811560b51c25f1eb3d8469abdb2f9a66fd6578"}, ] [package.dependencies] -backoff = {version = ">=2.0.0", markers = "python_version < \"4\""} +backoff = ">=2.0.0,<4" backports-datetime-fromisoformat = {version = ">=2.0.1", markers = "python_version < \"3.11\""} -click = ">=8.0,<9.0" -fs = ">=2.4.16" +click = ">=8.0,<9" fsspec = ">=2024.9.0" -importlib-metadata = {version = "<9.0.0", markers = "python_version < \"3.12\""} +importlib-metadata = {version = ">=5.0", markers = "python_version < \"3.12\""} importlib-resources = {version = ">=5.12.0,<6.2.0 || >6.2.0,<6.3.0 || >6.3.0,<6.3.1 || >6.3.1", markers = "python_version < \"3.10\""} inflection = ">=0.5.1" joblib = ">=1.3.0" jsonpath-ng = ">=1.5.3" jsonschema = ">=4.16.0" packaging = ">=23.1" -pytest = {version = ">=7.2.1", optional = true, markers = "extra == \"docs\" or extra == \"testing\""} +pytest = {version = ">=7.5", optional = true, markers = "extra == \"testing\""} python-dotenv = ">=0.20" -PyYAML = ">=6.0" +pyyaml = ">=6.0" referencing = ">=0.30.0" requests = ">=2.25.1" -setuptools = "<=70.3.0" -simpleeval = ">=0.9.13" +simpleeval = ">=0.9.13,<1.0.1 || >1.0.1" simplejson = ">=3.17.6" -sqlalchemy = ">=1.4,<3.0" -typing-extensions = ">=4.5.0" -urllib3 = ">=1.26,<2" +sqlalchemy = ">=2" +typing-extensions = {version = ">=4.5.0", markers = "python_version < \"3.13\""} +universal-pathlib = ">=0.2.6" [package.extras] -docs = ["furo (>=2024.5.6) ; python_version >= \"3.9\"", "myst-parser (>=3) ; python_version >= \"3.9\"", "pytest (>=7.2.1)", "sphinx (>=7) ; python_version >= \"3.9\"", "sphinx-copybutton (>=0.5.2) ; python_version >= \"3.9\"", "sphinx-inline-tabs (>=2023.4.21) ; python_version >= \"3.9\"", "sphinx-notfound-page (>=1.0.0) ; python_version >= \"3.9\"", "sphinx-reredirects (>=0.1.5) ; python_version >= \"3.9\""] faker = ["faker (>=22.5)"] -jwt = ["PyJWT (>=2.4,<3.0)", "cryptography (>=3.4.6)"] -parquet = ["numpy (>=1.22) ; python_version >= \"3.10\"", "numpy (>=1.22,<1.25) ; python_version == \"3.8\"", "numpy (>=1.22,<2.1) ; python_version == \"3.9\"", "pyarrow (>=13)"] -s3 = ["fs-s3fs (>=1.1.1)", "s3fs (>=2024.9.0)"] +jwt = ["cryptography (>=3.4.6)", "pyjwt (>=2.4.0)"] +msgspec = ["msgspec (>=0.19.0)"] +parquet = ["pyarrow (>=15)"] +s3 = ["s3fs (>=2024.9.0)"] ssh = ["paramiko (>=3.3.0)"] -testing = ["pytest (>=7.2.1)"] +testing = ["pytest (>=7.5)"] [[package]] name = "six" @@ -1986,7 +2015,7 @@ version = "1.16.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -2131,6 +2160,28 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "universal-pathlib" +version = "0.3.9" +description = "pathlib api extended to use fsspec backends" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "universal_pathlib-0.3.9-py3-none-any.whl", hash = "sha256:e4c3bdfd7650f55fb47da0a8a48eae9da6ae7216ea628fe5e1cadcc82d591a0e"}, + {file = "universal_pathlib-0.3.9.tar.gz", hash = "sha256:3a2225bc8cb0ca685a032ff885b945fdb0613b46e8c97d32df6afb6327c6c817"}, +] + +[package.dependencies] +fsspec = ">=2024.5.0" +pathlib-abc = ">=0.5.1,<0.6.0" + +[package.extras] +dev = ["adlfs (>=2024)", "cheroot", "fsspec[adl,gcs,github,http,s3,smb,ssh] (>=2024.5.0)", "gcsfs (>=2024.5.0)", "huggingface_hub", "moto[s3,server]", "pyftpdlib", "s3fs (>=2024.5.0)", "typing_extensions ; python_version < \"3.11\"", "webdav4[fsspec]", "wsgidav"] +dev-third-party = ["pydantic", "pydantic-settings"] +tests = ["mypy (>=1.10.0)", "packaging", "pydantic (>=2)", "pylint (>=2.17.4)", "pytest (>=8)", "pytest-cov (>=4.1.0)", "pytest-mock (>=3.12.0)", "pytest-mypy-plugins (>=3.1.2)", "pytest-sugar (>=0.9.7)"] +typechecking = ["mypy (>=1.10.0)", "pytest-mypy-plugins (>=3.1.2)"] + [[package]] name = "urllib3" version = "1.26.20" @@ -2281,4 +2332,4 @@ s3 = ["fs-s3fs"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.13" -content-hash = "d6236050f1caadde2c9278e4b26aa99c9579229e655c03b9ff1626974b33f873" +content-hash = "de91a187a0707f360545066e33ae94c535a8b790e0596925e340d25d56a52836" diff --git a/pyproject.toml b/pyproject.toml index b3ec00d..d40a31d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,15 +15,15 @@ packages = [ [tool.poetry.dependencies] python = ">=3.9,<3.13" importlib-resources = { version = "==6.4.*", python = "<3.9" } -singer-sdk = "^0.41.0" +singer-sdk = "^0.48.1" fs-s3fs = { version = "^1.1.1", optional = true } sqlalchemy-bigquery = "^1.8.0" google-cloud-bigquery = "^3.25.0" gcsfs = "^2024.9.0.post1" [tool.poetry.group.dev.dependencies] -pytest = "^7.4.4" -singer-sdk = { version="^0.41.0", extras = ["testing"] } +pytest = "^8" +singer-sdk = { version="^0.48.1", extras = ["testing"] } [tool.poetry.extras] s3 = ["fs-s3fs"] From 794d89fe5db6c2b3d9f27f7afb76ca8b5ba5bc37 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Mon, 9 Feb 2026 16:05:34 +0000 Subject: [PATCH 2/4] `singerlib` is now a public module --- tap_bigquery/connector.py | 2 +- tests/test_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_bigquery/connector.py b/tap_bigquery/connector.py index 5452681..68aa3af 100644 --- a/tap_bigquery/connector.py +++ b/tap_bigquery/connector.py @@ -9,7 +9,7 @@ import sqlalchemy from singer_sdk import SQLConnector from singer_sdk import typing as th # JSON schema typing helpers -from singer_sdk._singerlib import CatalogEntry, MetadataMapping, Schema +from singer_sdk.singerlib import CatalogEntry, MetadataMapping, Schema from sqlalchemy_bigquery import ( ARRAY, FLOAT, diff --git a/tests/test_client.py b/tests/test_client.py index 1d01147..927894e 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -8,7 +8,7 @@ from sqlalchemy.types import String, Double, Float from sqlalchemy_bigquery import ARRAY -from singer_sdk._singerlib import Catalog, CatalogEntry +from singer_sdk.singerlib import Catalog, CatalogEntry from tap_bigquery.tap import TapBigQuery from tap_bigquery.client import BigQueryConnector From 8e0de9e9790032a83d7196dc5b9d1ab6fc08b722 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Mon, 9 Feb 2026 22:00:56 +0000 Subject: [PATCH 3/4] Fix failing tests --- tap_bigquery/connector.py | 1 + tests/test_client.py | 6 +++--- tests/utils/mockinspector.py | 17 ++++++++++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tap_bigquery/connector.py b/tap_bigquery/connector.py index 68aa3af..87d22b2 100644 --- a/tap_bigquery/connector.py +++ b/tap_bigquery/connector.py @@ -167,6 +167,7 @@ def discover_catalog_entry( schema_name: str, table_name: str, is_view: bool, # noqa: FBT001 + **kwargs: dict[str, t.Any], # noqa: ARG002 ) -> CatalogEntry: """Create `CatalogEntry` object for the given table or a view. diff --git a/tests/test_client.py b/tests/test_client.py index 927894e..1368b2b 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -34,7 +34,7 @@ def setUp(self): ['mock-schema'], ['mock_table'], { - 'mock-schema.mock_table': [ + ('mock-schema', 'mock_table'): [ { 'name': 'double_field', 'type': Double }, { 'name': 'double_infinity', 'type': Double }, { 'name': 'float_field', 'type': Float }, @@ -79,7 +79,7 @@ def test_record_serialisable_post_processing(self, mock_engine, mock_inspector): ['mock-schema'], ['mock_table'], { - 'mock-schema.mock_table': [ + ('mock-schema', 'mock_table'): [ { 'name': 'string_field', 'type': String(50) }, { 'name': 'float_field', 'type': Float }, { 'name': 'float_none', 'type': Float }, @@ -171,7 +171,7 @@ def test_catalog_supplied(self, mock_engine, mock_inspector, mock_sql_tap): ['mock-schema'], ['mock_table'], { - 'mock-schema.mock_table': [ + ('mock-schema', 'mock_table'): [ { 'name': 'string_field', 'type': String(50) }, ], }, diff --git a/tests/utils/mockinspector.py b/tests/utils/mockinspector.py index efa3ef4..41ec9d7 100644 --- a/tests/utils/mockinspector.py +++ b/tests/utils/mockinspector.py @@ -1,7 +1,9 @@ from __future__ import annotations from typing import List -from sqlalchemy import Inspector, Column + +from sqlalchemy import Column, Inspector + class MockInspector(Inspector): def __init__( @@ -27,7 +29,16 @@ def get_indexes(self, table_name: str, schema: str) -> List[str]: return [] def get_columns(self, table_name: str, schema: str) -> dict[str, Column]: - return self.table_columns[schema + '.' + table_name] + return self.table_columns[(schema, table_name)] def get_pk_constraint(self, table_name: str, schema: str) -> dict: - return {} \ No newline at end of file + return {} + + def get_multi_pk_constraint(self, *args, **kwargs): + return {} + + def get_multi_indexes(self, *args, **kwargs): + return {} + + def get_multi_columns(self, *args, **kwargs): + return self.table_columns From a3810d19f40f8204055c96876a1f058b6b85edc5 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Tue, 10 Feb 2026 15:35:51 +0000 Subject: [PATCH 4/4] Restore previous `discover_catalog_entries` implementation --- tap_bigquery/connector.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tap_bigquery/connector.py b/tap_bigquery/connector.py index 87d22b2..a54f84a 100644 --- a/tap_bigquery/connector.py +++ b/tap_bigquery/connector.py @@ -157,6 +157,33 @@ def to_jsonschema_type( return jsonschema.type_dict return super().to_jsonschema_type(sql_type) + def discover_catalog_entries(self, **kwargs: dict[str, t.Any]) -> list[dict]: # noqa: ARG002 + """Return a list of catalog entries from discovery. + + Returns: + The discovered catalog entries as a list. + """ + result: list[dict] = [] + engine = self._engine + inspected = sqlalchemy.inspect(engine) + for schema_name in self.get_schema_names(engine, inspected): + # Iterate through each table and view + for table_name, is_view in self.get_object_names( + engine, + inspected, + schema_name, + ): + catalog_entry = self.discover_catalog_entry( + engine, + inspected, + schema_name, + table_name, + is_view, + ) + result.append(catalog_entry.to_dict()) + + return result + # TODO this only needs a column filtering capability in the singer-sdk # as sqlalchemy returns additional columns on bigquery for all the json # it has natively understood. @@ -296,4 +323,4 @@ def get_schema_names(self, engine: Engine, inspected: Inspector) -> list[str]: return self.config["filter_schemas"] return super().get_schema_names(engine, inspected) -__all__ = ["TapBigQuery", "BigQueryConnector", "BigQueryStream"] +__all__ = ["BigQueryConnector", "BigQueryStream", "TapBigQuery"]