feat: Ignore temporary schemas ending with _next or _NEXT

CHANGELOG.md

@@ -12,6 +12,8 @@ Changelog](http://keepachangelog.com/).
 
 ### Fixes
 
+* **Schema inspection** - Ignore temporary schemas ending with _next or _NEXT to prevent intermittent errors when schemas disappear during execution. ([GEM-5](https://linear.app/gemma-analytics/issue/GEM-5), [#11](https://github.com/Gemma-Analytics/permifrost/pull/11))
+
 ### Changes
 
 ## 0.15.4 - (2023-12-04)

src/permifrost/snowflake_connector.py

@@ -148,7 +148,11 @@ def show_schemas(self, database: Optional[str] = None) -> List[str]:
         results = self.run_query(query).fetchall()
 
         for result in results:
-            schema_identifier = f"{result['database_name']}.{result['name']}"
+            schema_name = result["name"]
+            # Ignore temporary Snowflake schemas that end with _next or _NEXT
+            if schema_name.upper().endswith("_NEXT"):
+                continue
+            schema_identifier = f"{result['database_name']}.{schema_name}"
             names.append(SnowflakeConnector.snowflaky(schema_identifier))
 
         return names

tests/permifrost/test_snowflake_connector.py

@@ -208,7 +208,7 @@ def test_get_current_role(self, mocker):
         )
         assert role == "test_role"
 
-    def test_show_schemas(self, mocker):
+    def test_show_schemas(self, mocker, snowflake_connector_env):
         mocker.patch("sqlalchemy.create_engine")
         conn = SnowflakeConnector()
         conn.run_query = mocker.MagicMock()
@@ -233,6 +233,35 @@ def test_show_schemas(self, mocker):
             'database_1."CaseSensitiveSchema"',
         ]
 
+    def test_show_schemas_filters_next_suffix(
+        self, mocker, snowflake_connector_env
+    ):
+        mocker.patch("sqlalchemy.create_engine")
+        conn = SnowflakeConnector()
+        conn.run_query = mocker.MagicMock()
+        mocker.patch.object(
+            conn.run_query(),
+            "fetchall",
+            return_value=[
+                {"database_name": "DATABASE_1", "name": "SCHEMA_1"},
+                {"database_name": "DATABASE_1", "name": "SCHEMA_1_next"},
+                {"database_name": "DATABASE_1", "name": "SCHEMA_2_NEXT"},
+                {"database_name": "DATABASE_1", "name": "SCHEMA_3_Next"},
+                {"database_name": "DATABASE_1", "name": "SCHEMA_4"},
+            ],
+        )
+
+        schemas = conn.show_schemas("database_1")
+
+        conn.run_query.assert_has_calls(
+            [mocker.call("SHOW TERSE SCHEMAS IN DATABASE database_1")]
+        )
+        # Should filter out schemas ending with _next, _NEXT, or _Next (case-insensitive)
+        assert schemas == [
+            "database_1.schema_1",
+            "database_1.schema_4",
+        ]
+
     def test_show_tables(self, mocker):
         mocker.patch("sqlalchemy.create_engine")
         conn = SnowflakeConnector()