Fire off DELETE request to purge any stale introspection records after successful introspection pass. (#73)

Author: James Robinson

noteable_magics/sql/meta_commands.py

@@ -2,9 +2,10 @@
 
 import pathlib
 import re
+import sys
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-import sys
+from datetime import datetime
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 from uuid import UUID
 
@@ -541,6 +542,12 @@ def run(self, invoked_as: str, args: List[str]) -> None:
 
         inspector = self.get_inspector()
 
+        # On successful completion, will tell Gate to delete any now-orphaned
+        # relations from prior introspections older than this timestamp.
+        introspection_started_at = datetime.utcnow()
+
+        # This and delta() just for development timing figures. Could become yet another
+        # timer context manager implementation.
         start = time.monotonic()
 
         def delta() -> float:
@@ -600,12 +607,15 @@ def delta() -> float:
         session.headers.update(auth_header)
 
         if message_queue:
-            # Clear out any prior known relations for this datasource.
-            self.inform_gate_start(session, ds_id)
-
             for message in message_queue:
                 self.inform_gate_relation(session, ds_id, message)
 
+            # Clear out any prior known relations which may not exist anymore in this datasource.
+            #
+            # We do this at the tail end of things, and not the beginning, so as to not eagerly delete
+            # prior known data if we happen to croak due to some unforseen exception while introspecting.
+            self.inform_gate_completed(session, ds_id, introspection_started_at)
+
             print(f'Done storing discovered table and view structures in {delta()}')
 
         # run() contract: return what to bind to the SQL cell variable name, and if display() needs
@@ -821,14 +831,6 @@ def introspect_columns(
 
         return retlist
 
-    def inform_gate_start(self, session: requests.Session, datasource_id: UUID):
-        """Tell gate to forget about any prior structures known for this datasource"""
-
-        # No route implemented for this yet, but need one, otherwise Gate-side will
-        # never forget about dropped tables/views.
-
-        pass
-
     def inform_gate_relation(
         self,
         session: requests.Session,
@@ -851,6 +853,16 @@ def inform_gate_relation(
                 f'Failed storing structure of {relation_description.schema_name}.{relation_description.relation_name}: {resp.status_code}, {resp.text}'
             )
 
+    def inform_gate_completed(
+        self, session: requests.Session, datasource_id: UUID, started_at: datetime
+    ):
+        """Tell gate to forget about any structures known for this datasource older than when we
+        started this introspection run."""
+
+        session.delete(
+            f"http://gate.default/api/v1/datasources/{datasource_id}/schema/relations?older_than={started_at.isoformat()}"
+        )
+
     def get_datasource_id(self) -> UUID:
         """Convert a noteable_magics.sql.connection.Connection's name to the original
         UUID Gate knew it as.

tests/test_sql_magic_meta_commands.py

@@ -1,4 +1,6 @@
 import re
+import urllib.parse
+from datetime import datetime
 from typing import List, Optional, Tuple
 from uuid import uuid4
 
@@ -605,10 +607,20 @@ def patched_requests_mock(self, mocker, patched_jwt, requests_mock):
             status_code=204,
         )
 
+        # And deletes to this one.
+        requests_mock.delete(
+            f"http://gate.default/api/v1/datasources/{COCKROACH_UUID}/schema/relations",
+            status_code=204,
+        )
+
         yield requests_mock
 
     def test_full_introspection(self, sql_magic, capsys, patched_requests_mock):
+
+        introspection_start = datetime.utcnow()
         sql_magic.execute(rf'{COCKROACH_HANDLE} \introspect')
+        introspection_end = datetime.utcnow()
+
         out, err = capsys.readouterr()
 
         assert 'Exception' not in out
@@ -620,9 +632,19 @@ def test_full_introspection(self, sql_magic, capsys, patched_requests_mock):
 
         assert 'Done storing discovered table and view structures' in out
 
-        # One POST per relation discovered (as currently implemented)
-        assert len(patched_requests_mock.request_history) == len(KNOWN_TABLES_AND_KINDS)
-        assert all(req.method == 'POST' for req in patched_requests_mock.request_history)
+        # One POST per relation discovered, and then a single DELETE (as currently implemented)
+        assert len(patched_requests_mock.request_history) == len(KNOWN_TABLES_AND_KINDS) + 1
+        # Remember, booleans are also integers because filthy historical C.
+        assert sum(req.method == 'POST' for req in patched_requests_mock.request_history) == len(
+            KNOWN_TABLES_AND_KINDS
+        )
+
+        # The delete should be the last request, and should specify a since_when timestamp param
+        # which should be between when introspection started and ended (in UTC).
+        delete_request = patched_requests_mock.request_history[-1]
+        assert delete_request.method == 'DELETE'
+        provided_since_when: str = urllib.parse.parse_qs(delete_request.query)['older_than'][0]
+        assert introspection_start < datetime.fromisoformat(provided_since_when) < introspection_end
 
         # Each POST body should be a RelationStructureDescription, whose
         # relation_names should add up to be all the expected ones.