diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c0d0e84..be4ad8ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ Changelog](http://keepachangelog.com/). ### Fixes +* **Schema inspection** - Ignore temporary schemas ending with _next or _NEXT to prevent intermittent errors when schemas disappear during execution. ([GEM-5](https://linear.app/gemma-analytics/issue/GEM-5), [#11](https://github.com/Gemma-Analytics/permifrost/pull/11)) + ### Changes ## 0.15.4 - (2023-12-04) diff --git a/src/permifrost/snowflake_connector.py b/src/permifrost/snowflake_connector.py index 6ee84765..77d446b9 100644 --- a/src/permifrost/snowflake_connector.py +++ b/src/permifrost/snowflake_connector.py @@ -148,7 +148,11 @@ def show_schemas(self, database: Optional[str] = None) -> List[str]: results = self.run_query(query).fetchall() for result in results: - schema_identifier = f"{result['database_name']}.{result['name']}" + schema_name = result["name"] + # Ignore temporary Snowflake schemas that end with _next or _NEXT + if schema_name.upper().endswith("_NEXT"): + continue + schema_identifier = f"{result['database_name']}.{schema_name}" names.append(SnowflakeConnector.snowflaky(schema_identifier)) return names diff --git a/tests/permifrost/test_snowflake_connector.py b/tests/permifrost/test_snowflake_connector.py index e4fd1de3..3c5a95af 100644 --- a/tests/permifrost/test_snowflake_connector.py +++ b/tests/permifrost/test_snowflake_connector.py @@ -208,7 +208,7 @@ def test_get_current_role(self, mocker): ) assert role == "test_role" - def test_show_schemas(self, mocker): + def test_show_schemas(self, mocker, snowflake_connector_env): mocker.patch("sqlalchemy.create_engine") conn = SnowflakeConnector() conn.run_query = mocker.MagicMock() @@ -233,6 +233,35 @@ def test_show_schemas(self, mocker): 'database_1."CaseSensitiveSchema"', ] + def test_show_schemas_filters_next_suffix( + self, mocker, snowflake_connector_env + ): + mocker.patch("sqlalchemy.create_engine") + conn = SnowflakeConnector() + conn.run_query = mocker.MagicMock() + mocker.patch.object( + conn.run_query(), + "fetchall", + return_value=[ + {"database_name": "DATABASE_1", "name": "SCHEMA_1"}, + {"database_name": "DATABASE_1", "name": "SCHEMA_1_next"}, + {"database_name": "DATABASE_1", "name": "SCHEMA_2_NEXT"}, + {"database_name": "DATABASE_1", "name": "SCHEMA_3_Next"}, + {"database_name": "DATABASE_1", "name": "SCHEMA_4"}, + ], + ) + + schemas = conn.show_schemas("database_1") + + conn.run_query.assert_has_calls( + [mocker.call("SHOW TERSE SCHEMAS IN DATABASE database_1")] + ) + # Should filter out schemas ending with _next, _NEXT, or _Next (case-insensitive) + assert schemas == [ + "database_1.schema_1", + "database_1.schema_4", + ] + def test_show_tables(self, mocker): mocker.patch("sqlalchemy.create_engine") conn = SnowflakeConnector()