From 39f17304cbc80d7bd60a8a4900f9f9a3d2e5d77b Mon Sep 17 00:00:00 2001 From: Hanqing Date: Wed, 4 Feb 2026 21:41:29 +0800 Subject: [PATCH] fix sticky timestamp issue --- update_utils/update_goldsky.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/update_utils/update_goldsky.py b/update_utils/update_goldsky.py index 5fdb197..fb6769d 100644 --- a/update_utils/update_goldsky.py +++ b/update_utils/update_goldsky.py @@ -113,6 +113,9 @@ def scrape(at_once=1000): print(f"Saving columns: {COLUMNS_TO_SAVE}") while True: + # Capture sticky state at start of loop to determine exit condition later + is_sticky_query = sticky_timestamp is not None + # Build the where clause based on cursor state if sticky_timestamp is not None: # We're in sticky mode: stay at this timestamp and paginate by id @@ -218,7 +221,7 @@ def scrape(at_once=1000): # Save cursor state for efficient resume (no duplicates on restart) save_cursor(last_timestamp, last_id, sticky_timestamp) - if len(df) < at_once and sticky_timestamp is None: + if len(df) < at_once and not is_sticky_query: break # Clear cursor file on successful completion