diff --git a/src/lib/twitter-client-base.ts b/src/lib/twitter-client-base.ts index d74542e..d2101c5 100644 --- a/src/lib/twitter-client-base.ts +++ b/src/lib/twitter-client-base.ts @@ -155,4 +155,78 @@ export abstract class TwitterClientBase { this.clientUserId = result.user.id; } } + + protected async paginateItems(options: { + limit: number; + pageSize: number; + maxPages?: number; + startCursor?: string; + fetchPage: ( + count: number, + cursor?: string, + ) => Promise< + | { success: true; items: T[]; cursor?: string; had404?: boolean } + | { success: false; error: string; had404?: boolean } + >; + }): Promise<{ success: true; items: T[]; nextCursor?: string } | { success: false; error: string }> { + const { limit, pageSize, maxPages, startCursor, fetchPage } = options; + const unlimited = limit === Number.POSITIVE_INFINITY; + const seen = new Set(); + const items: T[] = []; + let cursor = startCursor; + let nextCursor: string | undefined; + let pagesFetched = 0; + + const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { + const firstAttempt = await fetchPage(pageCount, pageCursor); + if (firstAttempt.success) { + return firstAttempt; + } + if (firstAttempt.had404) { + await this.refreshQueryIds(); + const secondAttempt = await fetchPage(pageCount, pageCursor); + if (secondAttempt.success) { + return secondAttempt; + } + return { success: false as const, error: secondAttempt.error }; + } + return { success: false as const, error: firstAttempt.error }; + }; + + while (unlimited || items.length < limit) { + const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - items.length); + const page = await fetchWithRefresh(pageCount, cursor); + if (!page.success) { + return { success: false, error: page.error }; + } + pagesFetched += 1; + + let added = 0; + for (const item of page.items) { + if (seen.has(item.id)) { + continue; + } + seen.add(item.id); + items.push(item); + added += 1; + if (!unlimited && items.length >= limit) { + break; + } + } + + const pageCursor = page.cursor; + if (!pageCursor || pageCursor === cursor || page.items.length === 0 || added === 0) { + nextCursor = undefined; + break; + } + if (maxPages && pagesFetched >= maxPages) { + nextCursor = pageCursor; + break; + } + cursor = pageCursor; + nextCursor = pageCursor; + } + + return { success: true, items, nextCursor }; + } } diff --git a/src/lib/twitter-client-lists.ts b/src/lib/twitter-client-lists.ts index 877d5a3..96e30fa 100644 --- a/src/lib/twitter-client-lists.ts +++ b/src/lib/twitter-client-lists.ts @@ -349,142 +349,94 @@ export function withLists>( options: TimelinePaginationOptions = {}, ): Promise { const features = buildListsFeatures(); - const pageSize = 20; - const seen = new Set(); - const tweets: TweetData[] = []; - let cursor: string | undefined = options.cursor; - let nextCursor: string | undefined; - let pagesFetched = 0; - const { includeRaw = false, maxPages } = options; - - const fetchPage = async (pageCount: number, pageCursor?: string) => { - let lastError: string | undefined; - let had404 = false; - const queryIds = await this.getListTimelineQueryIds(); - - const variables = { - listId, - count: pageCount, - ...(pageCursor ? { cursor: pageCursor } : {}), - }; - - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - features: JSON.stringify(features), - }); - - for (const queryId of queryIds) { - const url = `${TWITTER_API_BASE}/${queryId}/ListLatestTweetsTimeline?${params.toString()}`; - - try { - const response = await this.fetchWithTimeout(url, { - method: 'GET', - headers: this.getHeaders(), - }); - - if (response.status === 404) { - had404 = true; - lastError = `HTTP ${response.status}`; - continue; - } - - if (!response.ok) { - const text = await response.text(); - return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 }; - } + const { includeRaw = false, maxPages, cursor } = options; + + const result = await this.paginateItems({ + limit, + pageSize: 20, + maxPages, + startCursor: cursor, + fetchPage: async (pageCount, pageCursor) => { + let lastError: string | undefined; + let had404 = false; + const queryIds = await this.getListTimelineQueryIds(); + + const variables = { + listId, + count: pageCount, + ...(pageCursor ? { cursor: pageCursor } : {}), + }; - const data = (await response.json()) as { - data?: { - list?: { - tweets_timeline?: { - timeline?: { - instructions?: Array<{ - entries?: Array<{ - content?: { - itemContent?: { - tweet_results?: { - result?: GraphqlTweetResult; + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(features), + }); + + for (const queryId of queryIds) { + const url = `${TWITTER_API_BASE}/${queryId}/ListLatestTweetsTimeline?${params.toString()}`; + + try { + const response = await this.fetchWithTimeout(url, { + method: 'GET', + headers: this.getHeaders(), + }); + + if (response.status === 404) { + had404 = true; + lastError = `HTTP ${response.status}`; + continue; + } + + if (!response.ok) { + const text = await response.text(); + return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 }; + } + + const data = (await response.json()) as { + data?: { + list?: { + tweets_timeline?: { + timeline?: { + instructions?: Array<{ + entries?: Array<{ + content?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; }; }; - }; + }>; }>; - }>; + }; }; }; }; + errors?: Array<{ message: string }>; }; - errors?: Array<{ message: string }>; - }; - if (data.errors && data.errors.length > 0) { - return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 }; - } + if (data.errors && data.errors.length > 0) { + return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 }; + } - const instructions = data.data?.list?.tweets_timeline?.timeline?.instructions; - const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); - const nextCursor = extractCursorFromInstructions(instructions); + const instructions = data.data?.list?.tweets_timeline?.timeline?.instructions; + const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const extractedCursor = extractCursorFromInstructions(instructions); - return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; - } catch (error) { - lastError = error instanceof Error ? error.message : String(error); - } - } - - return { success: false as const, error: lastError ?? 'Unknown error fetching list timeline', had404 }; - }; - - const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { - const firstAttempt = await fetchPage(pageCount, pageCursor); - if (firstAttempt.success) { - return firstAttempt; - } - if (firstAttempt.had404) { - await this.refreshQueryIds(); - const secondAttempt = await fetchPage(pageCount, pageCursor); - if (secondAttempt.success) { - return secondAttempt; + return { success: true as const, items: pageTweets, cursor: extractedCursor, had404 }; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } } - return { success: false as const, error: secondAttempt.error }; - } - return { success: false as const, error: firstAttempt.error }; - }; - - const unlimited = limit === Number.POSITIVE_INFINITY; - while (unlimited || tweets.length < limit) { - const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); - const page = await fetchWithRefresh(pageCount, cursor); - if (!page.success) { - return { success: false, error: page.error }; - } - pagesFetched += 1; - let added = 0; - for (const tweet of page.tweets) { - if (seen.has(tweet.id)) { - continue; - } - seen.add(tweet.id); - tweets.push(tweet); - added += 1; - if (!unlimited && tweets.length >= limit) { - break; - } - } + return { success: false as const, error: lastError ?? 'Unknown error fetching list timeline', had404 }; + }, + }); - const pageCursor = page.cursor; - if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { - nextCursor = undefined; - break; - } - if (maxPages && pagesFetched >= maxPages) { - nextCursor = pageCursor; - break; - } - cursor = pageCursor; - nextCursor = pageCursor; + if (!result.success) { + return result; } - - return { success: true, tweets, nextCursor }; + return { success: true, tweets: result.items, nextCursor: result.nextCursor }; } } diff --git a/src/lib/twitter-client-search.ts b/src/lib/twitter-client-search.ts index 7848d69..8eb845f 100644 --- a/src/lib/twitter-client-search.ts +++ b/src/lib/twitter-client-search.ts @@ -74,84 +74,84 @@ export function withSearch> options: SearchPaginationOptions = {}, ): Promise { const features = buildSearchFeatures(); - const pageSize = 20; - const seen = new Set(); - const tweets: TweetData[] = []; - let cursor: string | undefined = options.cursor; - let nextCursor: string | undefined; - let pagesFetched = 0; - const { includeRaw = false, maxPages } = options; - - const fetchPage = async (pageCount: number, pageCursor?: string) => { - let lastError: string | undefined; - let had404 = false; - const queryIds = await this.getSearchTimelineQueryIds(); - - for (const queryId of queryIds) { - const variables = { - rawQuery: query, - count: pageCount, - querySource: 'typed_query', - product: 'Latest', - ...(pageCursor ? { cursor: pageCursor } : {}), - }; - - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - }); - - const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params.toString()}`; - - try { - const response = await this.fetchWithTimeout(url, { - method: 'POST', - headers: this.getHeaders(), - body: JSON.stringify({ features, queryId }), - }); - - if (response.status === 404) { - had404 = true; - lastError = `HTTP ${response.status}`; - continue; - } + const { includeRaw = false, maxPages, cursor } = options; + + const result = await this.paginateItems({ + limit, + pageSize: 20, + maxPages, + startCursor: cursor, + fetchPage: async (pageCount, pageCursor) => { + let lastError: string | undefined; + let had404 = false; + const queryIds = await this.getSearchTimelineQueryIds(); + + for (const queryId of queryIds) { + const variables = { + rawQuery: query, + count: pageCount, + querySource: 'typed_query', + product: 'Latest', + ...(pageCursor ? { cursor: pageCursor } : {}), + }; - if (!response.ok) { - const text = await response.text(); - const shouldRefreshQueryIds = - (response.status === 400 || response.status === 422) && isQueryIdMismatch(text); - return { - success: false as const, - error: `HTTP ${response.status}: ${text.slice(0, 200)}`, - had404: had404 || shouldRefreshQueryIds, - }; - } + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + }); - const data = (await response.json()) as { - data?: { - search_by_raw_query?: { - search_timeline?: { - timeline?: { - instructions?: Array<{ - entries?: Array<{ - content?: { - itemContent?: { - tweet_results?: { - result?: { - rest_id?: string; - legacy?: { - full_text?: string; - created_at?: string; - reply_count?: number; - retweet_count?: number; - favorite_count?: number; - in_reply_to_status_id_str?: string; - }; - core?: { - user_results?: { - result?: { - legacy?: { - screen_name?: string; - name?: string; + const url = `${TWITTER_API_BASE}/${queryId}/SearchTimeline?${params.toString()}`; + + try { + const response = await this.fetchWithTimeout(url, { + method: 'POST', + headers: this.getHeaders(), + body: JSON.stringify({ features, queryId }), + }); + + if (response.status === 404) { + had404 = true; + lastError = `HTTP ${response.status}`; + continue; + } + + if (!response.ok) { + const text = await response.text(); + const shouldRefreshQueryIds = + (response.status === 400 || response.status === 422) && isQueryIdMismatch(text); + return { + success: false as const, + error: `HTTP ${response.status}: ${text.slice(0, 200)}`, + had404: had404 || shouldRefreshQueryIds, + }; + } + + const data = (await response.json()) as { + data?: { + search_by_raw_query?: { + search_timeline?: { + timeline?: { + instructions?: Array<{ + entries?: Array<{ + content?: { + itemContent?: { + tweet_results?: { + result?: { + rest_id?: string; + legacy?: { + full_text?: string; + created_at?: string; + reply_count?: number; + retweet_count?: number; + favorite_count?: number; + in_reply_to_status_id_str?: string; + }; + core?: { + user_results?: { + result?: { + legacy?: { + screen_name?: string; + name?: string; + }; }; }; }; @@ -159,92 +159,44 @@ export function withSearch> }; }; }; - }; + }>; }>; - }>; + }; }; }; }; + errors?: Array<{ message: string; extensions?: { code?: string } }>; }; - errors?: Array<{ message: string; extensions?: { code?: string } }>; - }; - - if (data.errors && data.errors.length > 0) { - const shouldRefreshQueryIds = data.errors.some( - (error) => error?.extensions?.code === 'GRAPHQL_VALIDATION_FAILED', - ); - return { - success: false as const, - error: data.errors.map((e) => e.message).join(', '), - had404: had404 || shouldRefreshQueryIds, - }; - } - const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions; - const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); - const nextCursor = extractCursorFromInstructions(instructions); - - return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; - } catch (error) { - lastError = error instanceof Error ? error.message : String(error); - } - } + if (data.errors && data.errors.length > 0) { + const shouldRefreshQueryIds = data.errors.some( + (error) => error?.extensions?.code === 'GRAPHQL_VALIDATION_FAILED', + ); + return { + success: false as const, + error: data.errors.map((e) => e.message).join(', '), + had404: had404 || shouldRefreshQueryIds, + }; + } - return { success: false as const, error: lastError ?? 'Unknown error fetching search results', had404 }; - }; + const instructions = data.data?.search_by_raw_query?.search_timeline?.timeline?.instructions; + const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const extractedCursor = extractCursorFromInstructions(instructions); - const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { - const firstAttempt = await fetchPage(pageCount, pageCursor); - if (firstAttempt.success) { - return firstAttempt; - } - if (firstAttempt.had404) { - await this.refreshQueryIds(); - const secondAttempt = await fetchPage(pageCount, pageCursor); - if (secondAttempt.success) { - return secondAttempt; + return { success: true as const, items: pageTweets, cursor: extractedCursor, had404 }; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } } - return { success: false as const, error: secondAttempt.error }; - } - return { success: false as const, error: firstAttempt.error }; - }; - - const unlimited = limit === Number.POSITIVE_INFINITY; - while (unlimited || tweets.length < limit) { - const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); - const page = await fetchWithRefresh(pageCount, cursor); - if (!page.success) { - return { success: false, error: page.error }; - } - pagesFetched += 1; - let added = 0; - for (const tweet of page.tweets) { - if (seen.has(tweet.id)) { - continue; - } - seen.add(tweet.id); - tweets.push(tweet); - added += 1; - if (!unlimited && tweets.length >= limit) { - break; - } - } + return { success: false as const, error: lastError ?? 'Unknown error fetching search results', had404 }; + }, + }); - const pageCursor = page.cursor; - if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { - nextCursor = undefined; - break; - } - if (maxPages && pagesFetched >= maxPages) { - nextCursor = pageCursor; - break; - } - cursor = pageCursor; - nextCursor = pageCursor; + if (!result.success) { + return result; } - - return { success: true, tweets, nextCursor }; + return { success: true, tweets: result.items, nextCursor: result.nextCursor }; } } diff --git a/src/lib/twitter-client-timelines.ts b/src/lib/twitter-client-timelines.ts index 9494312..e4927c4 100644 --- a/src/lib/twitter-client-timelines.ts +++ b/src/lib/twitter-client-timelines.ts @@ -90,156 +90,106 @@ export function withTimelines(); - const tweets: TweetData[] = []; - let cursor: string | undefined = options.cursor; - let nextCursor: string | undefined; - let pagesFetched = 0; - const { includeRaw = false, maxPages } = options; - - const fetchPage = async (pageCount: number, pageCursor?: string) => { - let lastError: string | undefined; - let had404 = false; - const queryIds = await this.getLikesQueryIds(); - - for (const queryId of queryIds) { - const variables = { - userId, - count: pageCount, - includePromotedContent: false, - withClientEventToken: false, - withBirdwatchNotes: false, - withVoice: true, - ...(pageCursor ? { cursor: pageCursor } : {}), - }; + const { includeRaw = false, maxPages, cursor } = options; + + const result = await this.paginateItems({ + limit, + pageSize: 20, + maxPages, + startCursor: cursor, + fetchPage: async (pageCount, pageCursor) => { + let lastError: string | undefined; + let had404 = false; + const queryIds = await this.getLikesQueryIds(); - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - features: JSON.stringify(features), - }); - const url = `${TWITTER_API_BASE}/${queryId}/Likes?${params.toString()}`; + for (const queryId of queryIds) { + const variables = { + userId, + count: pageCount, + includePromotedContent: false, + withClientEventToken: false, + withBirdwatchNotes: false, + withVoice: true, + ...(pageCursor ? { cursor: pageCursor } : {}), + }; - try { - const response = await this.fetchWithTimeout(url, { - method: 'GET', - headers: this.getHeaders(), + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(features), }); + const url = `${TWITTER_API_BASE}/${queryId}/Likes?${params.toString()}`; - if (response.status === 404) { - had404 = true; - lastError = `HTTP ${response.status}`; - continue; - } + try { + const response = await this.fetchWithTimeout(url, { + method: 'GET', + headers: this.getHeaders(), + }); - if (!response.ok) { - const text = await response.text(); - return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 }; - } + if (response.status === 404) { + had404 = true; + lastError = `HTTP ${response.status}`; + continue; + } - const data = (await response.json()) as { - data?: { - user?: { - result?: { - timeline?: { + if (!response.ok) { + const text = await response.text(); + return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 }; + } + + const data = (await response.json()) as { + data?: { + user?: { + result?: { timeline?: { - instructions?: Array<{ - entries?: Array<{ - content?: { - itemContent?: { - tweet_results?: { - result?: GraphqlTweetResult; + timeline?: { + instructions?: Array<{ + entries?: Array<{ + content?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; }; }; - }; + }>; }>; - }>; + }; }; }; }; }; + errors?: Array<{ message: string }>; }; - errors?: Array<{ message: string }>; - }; - const instructions = data.data?.user?.result?.timeline?.timeline?.instructions; - if (data.errors && data.errors.length > 0) { - const message = data.errors.map((e) => e.message).join(', '); - if (!instructions) { - if (message.includes('Query: Unspecified')) { - lastError = message; - continue; + const instructions = data.data?.user?.result?.timeline?.timeline?.instructions; + if (data.errors && data.errors.length > 0) { + const message = data.errors.map((e) => e.message).join(', '); + if (!instructions) { + if (message.includes('Query: Unspecified')) { + had404 = true; // Trigger query ID refresh + lastError = message; + continue; + } + return { success: false as const, error: message, had404 }; } - return { success: false as const, error: message, had404 }; } - } - const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); - const extractedCursor = extractCursorFromInstructions(instructions); - - return { success: true as const, tweets: pageTweets, cursor: extractedCursor, had404 }; - } catch (error) { - lastError = error instanceof Error ? error.message : String(error); - } - } - - return { success: false as const, error: lastError ?? 'Unknown error fetching likes', had404 }; - }; + const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); + const extractedCursor = extractCursorFromInstructions(instructions); - const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { - const firstAttempt = await fetchPage(pageCount, pageCursor); - if (firstAttempt.success) { - return firstAttempt; - } - const shouldRefresh = - firstAttempt.had404 || - (typeof firstAttempt.error === 'string' && firstAttempt.error.includes('Query: Unspecified')); - if (shouldRefresh) { - await this.refreshQueryIds(); - const secondAttempt = await fetchPage(pageCount, pageCursor); - if (secondAttempt.success) { - return secondAttempt; + return { success: true as const, items: pageTweets, cursor: extractedCursor, had404 }; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } } - return { success: false as const, error: secondAttempt.error }; - } - return { success: false as const, error: firstAttempt.error }; - }; - - const unlimited = limit === Number.POSITIVE_INFINITY; - while (unlimited || tweets.length < limit) { - const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); - const page = await fetchWithRefresh(pageCount, cursor); - if (!page.success) { - return { success: false, error: page.error }; - } - pagesFetched += 1; - let added = 0; - for (const tweet of page.tweets) { - if (seen.has(tweet.id)) { - continue; - } - seen.add(tweet.id); - tweets.push(tweet); - added += 1; - if (!unlimited && tweets.length >= limit) { - break; - } - } + return { success: false as const, error: lastError ?? 'Unknown error fetching likes', had404 }; + }, + }); - const pageCursor = page.cursor; - if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { - nextCursor = undefined; - break; - } - if (maxPages && pagesFetched >= maxPages) { - nextCursor = pageCursor; - break; - } - cursor = pageCursor; - nextCursor = pageCursor; + if (!result.success) { + return result; } - - return { success: true, tweets, nextCursor }; + return { success: true, tweets: result.items, nextCursor: result.nextCursor }; } /** @@ -259,208 +209,40 @@ export function withTimelines { const features = buildBookmarksFeatures(); - const pageSize = 20; - const seen = new Set(); - const tweets: TweetData[] = []; - let cursor: string | undefined = options.cursor; - let nextCursor: string | undefined; - let pagesFetched = 0; - const { includeRaw = false, maxPages } = options; - - const fetchPage = async (pageCount: number, pageCursor?: string) => { - let lastError: string | undefined; - let had404 = false; - const queryIds = await this.getBookmarksQueryIds(); - - const variables = { - count: pageCount, - includePromotedContent: false, - withDownvotePerspective: false, - withReactionsMetadata: false, - withReactionsPerspective: false, - ...(pageCursor ? { cursor: pageCursor } : {}), - }; - - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - features: JSON.stringify(features), - }); - - for (const queryId of queryIds) { - const url = `${TWITTER_API_BASE}/${queryId}/Bookmarks?${params.toString()}`; - - try { - this.logBookmarksDebug('request bookmarks page', { - queryId, - pageCount, - hasCursor: Boolean(pageCursor), - }); - const response = await this.fetchWithRetry(url, { - method: 'GET', - headers: this.getHeaders(), - }); - - if (response.status === 404) { - had404 = true; - lastError = `HTTP ${response.status}`; - this.logBookmarksDebug('bookmarks 404', { queryId }); - continue; - } - - if (!response.ok) { - const text = await response.text(); - this.logBookmarksDebug('bookmarks non-200', { - queryId, - status: response.status, - body: text.slice(0, 200), - }); - return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 }; - } - - const data = (await response.json()) as { - data?: { - bookmark_timeline_v2?: { - timeline?: { - instructions?: Array<{ - entries?: Array<{ - content?: { - itemContent?: { - tweet_results?: { - result?: GraphqlTweetResult; - }; - }; - }; - }>; - }>; - }; - }; - }; - errors?: Array<{ message: string }>; - }; - - const instructions = data.data?.bookmark_timeline_v2?.timeline?.instructions; - const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); - const nextCursor = extractCursorFromInstructions(instructions); - - if (data.errors && data.errors.length > 0) { - this.logBookmarksDebug('bookmarks graphql errors (non-fatal)', { queryId, errors: data.errors }); - if (!instructions) { - lastError = data.errors.map((e) => e.message).join(', '); - continue; - } - } - - this.logBookmarksDebug('bookmarks page parsed', { - queryId, - tweets: pageTweets.length, - hasNextCursor: Boolean(nextCursor), - }); - - return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; - } catch (error) { - lastError = error instanceof Error ? error.message : String(error); - this.logBookmarksDebug('bookmarks request error', { queryId, error: lastError }); - } - } - - return { success: false as const, error: lastError ?? 'Unknown error fetching bookmarks', had404 }; - }; - - const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { - const firstAttempt = await fetchPage(pageCount, pageCursor); - if (firstAttempt.success) { - return firstAttempt; - } - if (firstAttempt.had404) { - await this.refreshQueryIds(); - const secondAttempt = await fetchPage(pageCount, pageCursor); - if (secondAttempt.success) { - return secondAttempt; - } - return { success: false as const, error: secondAttempt.error }; - } - return { success: false as const, error: firstAttempt.error }; - }; - - const unlimited = limit === Number.POSITIVE_INFINITY; - while (unlimited || tweets.length < limit) { - const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); - const page = await fetchWithRefresh(pageCount, cursor); - if (!page.success) { - return { success: false, error: page.error }; - } - pagesFetched += 1; - - let added = 0; - for (const tweet of page.tweets) { - if (seen.has(tweet.id)) { - continue; - } - seen.add(tweet.id); - tweets.push(tweet); - added += 1; - if (!unlimited && tweets.length >= limit) { - break; - } - } - - const pageCursor = page.cursor; - if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { - nextCursor = undefined; - break; - } - if (maxPages && pagesFetched >= maxPages) { - nextCursor = pageCursor; - break; - } - cursor = pageCursor; - nextCursor = pageCursor; - } - - return { success: true, tweets, nextCursor }; - } - - private async getBookmarkFolderTimelinePaged( - folderId: string, - limit: number, - options: TimelinePaginationOptions = {}, - ): Promise { - const features = buildBookmarksFeatures(); - const pageSize = 20; - const seen = new Set(); - const tweets: TweetData[] = []; - let cursor: string | undefined = options.cursor; - let nextCursor: string | undefined; - let pagesFetched = 0; - const { includeRaw = false, maxPages } = options; - - const buildVariables = (pageCount: number, pageCursor: string | undefined, includeCount: boolean) => ({ - bookmark_collection_id: folderId, - includePromotedContent: true, - ...(includeCount ? { count: pageCount } : {}), - ...(pageCursor ? { cursor: pageCursor } : {}), - }); + const { includeRaw = false, maxPages, cursor } = options; + + const result = await this.paginateItems({ + limit, + pageSize: 20, + maxPages, + startCursor: cursor, + fetchPage: async (pageCount, pageCursor) => { + let lastError: string | undefined; + let had404 = false; + const queryIds = await this.getBookmarksQueryIds(); - const fetchPage = async (pageCount: number, pageCursor?: string) => { - let lastError: string | undefined; - let had404 = false; - const queryIds = await this.getBookmarkFolderQueryIds(); + const variables = { + count: pageCount, + includePromotedContent: false, + withDownvotePerspective: false, + withReactionsMetadata: false, + withReactionsPerspective: false, + ...(pageCursor ? { cursor: pageCursor } : {}), + }; - const tryOnce = async (variables: Record) => { const params = new URLSearchParams({ variables: JSON.stringify(variables), features: JSON.stringify(features), }); for (const queryId of queryIds) { - const url = `${TWITTER_API_BASE}/${queryId}/BookmarkFolderTimeline?${params.toString()}`; + const url = `${TWITTER_API_BASE}/${queryId}/Bookmarks?${params.toString()}`; try { - this.logBookmarksDebug('request bookmark folder page', { + this.logBookmarksDebug('request bookmarks page', { queryId, pageCount, hasCursor: Boolean(pageCursor), - includeCount: Object.hasOwn(variables, 'count'), }); const response = await this.fetchWithRetry(url, { method: 'GET', @@ -470,13 +252,13 @@ export function withTimelines; }; - const instructions = data.data?.bookmark_collection_timeline?.timeline?.instructions; + const instructions = data.data?.bookmark_timeline_v2?.timeline?.instructions; const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw }); - const nextCursor = extractCursorFromInstructions(instructions); + const extractedCursor = extractCursorFromInstructions(instructions); if (data.errors && data.errors.length > 0) { - this.logBookmarksDebug('bookmark folder graphql errors (non-fatal)', { queryId, errors: data.errors }); + this.logBookmarksDebug('bookmarks graphql errors (non-fatal)', { queryId, errors: data.errors }); if (!instructions) { lastError = data.errors.map((e) => e.message).join(', '); continue; } } - this.logBookmarksDebug('bookmark folder page parsed', { + this.logBookmarksDebug('bookmarks page parsed', { queryId, tweets: pageTweets.length, - hasNextCursor: Boolean(nextCursor), + hasNextCursor: Boolean(extractedCursor), }); - return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 }; + return { success: true as const, items: pageTweets, cursor: extractedCursor, had404 }; } catch (error) { lastError = error instanceof Error ? error.message : String(error); - this.logBookmarksDebug('bookmark folder request error', { queryId, error: lastError }); + this.logBookmarksDebug('bookmarks request error', { queryId, error: lastError }); } } - return { success: false as const, error: lastError ?? 'Unknown error fetching bookmark folder', had404 }; - }; + return { success: false as const, error: lastError ?? 'Unknown error fetching bookmarks', had404 }; + }, + }); - let attempt = await tryOnce(buildVariables(pageCount, pageCursor, true)); - if (!attempt.success && attempt.error?.includes('Variable "$count"')) { - attempt = await tryOnce(buildVariables(pageCount, pageCursor, false)); - } + if (!result.success) { + return result; + } + return { success: true, tweets: result.items, nextCursor: result.nextCursor }; + } - if (!attempt.success && attempt.error?.includes('Variable "$cursor"') && pageCursor) { - return { - success: false as const, - error: 'Bookmark folder pagination rejected the cursor parameter', - had404: attempt.had404, - }; - } + private async getBookmarkFolderTimelinePaged( + folderId: string, + limit: number, + options: TimelinePaginationOptions = {}, + ): Promise { + const features = buildBookmarksFeatures(); + const { includeRaw = false, maxPages, cursor } = options; - return attempt; - }; + const buildVariables = (pageCount: number, pageCursor: string | undefined, includeCount: boolean) => ({ + bookmark_collection_id: folderId, + includePromotedContent: true, + ...(includeCount ? { count: pageCount } : {}), + ...(pageCursor ? { cursor: pageCursor } : {}), + }); - const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => { - const firstAttempt = await fetchPage(pageCount, pageCursor); - if (firstAttempt.success) { - return firstAttempt; - } - if (firstAttempt.had404) { - await this.refreshQueryIds(); - const secondAttempt = await fetchPage(pageCount, pageCursor); - if (secondAttempt.success) { - return secondAttempt; + const result = await this.paginateItems({ + limit, + pageSize: 20, + maxPages, + startCursor: cursor, + fetchPage: async (pageCount, pageCursor) => { + let lastError: string | undefined; + let had404 = false; + const queryIds = await this.getBookmarkFolderQueryIds(); + + const tryOnce = async (variables: Record) => { + const params = new URLSearchParams({ + variables: JSON.stringify(variables), + features: JSON.stringify(features), + }); + + for (const queryId of queryIds) { + const url = `${TWITTER_API_BASE}/${queryId}/BookmarkFolderTimeline?${params.toString()}`; + + try { + this.logBookmarksDebug('request bookmark folder page', { + queryId, + pageCount, + hasCursor: Boolean(pageCursor), + includeCount: Object.hasOwn(variables, 'count'), + }); + const response = await this.fetchWithRetry(url, { + method: 'GET', + headers: this.getHeaders(), + }); + + if (response.status === 404) { + had404 = true; + lastError = `HTTP ${response.status}`; + this.logBookmarksDebug('bookmark folder 404', { queryId }); + continue; + } + + if (!response.ok) { + const text = await response.text(); + this.logBookmarksDebug('bookmark folder non-200', { + queryId, + status: response.status, + body: text.slice(0, 200), + }); + return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 }; + } + + const data = (await response.json()) as { + data?: { + bookmark_collection_timeline?: { + timeline?: { + instructions?: Array<{ + entries?: Array<{ + content?: { + itemContent?: { + tweet_results?: { + result?: GraphqlTweetResult; + }; + }; + }; + }>; + }>; + }; + }; + }; + errors?: Array<{ message: string }>; + }; + + const instructions = data.data?.bookmark_collection_timeline?.timeline?.instructions; + const pageTweets = parseTweetsFromInstructions(instructions, { + quoteDepth: this.quoteDepth, + includeRaw, + }); + const extractedCursor = extractCursorFromInstructions(instructions); + + if (data.errors && data.errors.length > 0) { + this.logBookmarksDebug('bookmark folder graphql errors (non-fatal)', { queryId, errors: data.errors }); + if (!instructions) { + lastError = data.errors.map((e) => e.message).join(', '); + continue; + } + } + + this.logBookmarksDebug('bookmark folder page parsed', { + queryId, + tweets: pageTweets.length, + hasNextCursor: Boolean(extractedCursor), + }); + + return { success: true as const, items: pageTweets, cursor: extractedCursor, had404 }; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + this.logBookmarksDebug('bookmark folder request error', { queryId, error: lastError }); + } + } + + return { success: false as const, error: lastError ?? 'Unknown error fetching bookmark folder', had404 }; + }; + + let attempt = await tryOnce(buildVariables(pageCount, pageCursor, true)); + if (!attempt.success && attempt.error?.includes('Variable "$count"')) { + attempt = await tryOnce(buildVariables(pageCount, pageCursor, false)); } - return { success: false as const, error: secondAttempt.error }; - } - return { success: false as const, error: firstAttempt.error }; - }; - - const unlimited = limit === Number.POSITIVE_INFINITY; - while (unlimited || tweets.length < limit) { - const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length); - const page = await fetchWithRefresh(pageCount, cursor); - if (!page.success) { - return { success: false, error: page.error }; - } - pagesFetched += 1; - let added = 0; - for (const tweet of page.tweets) { - if (seen.has(tweet.id)) { - continue; + if (!attempt.success && attempt.error?.includes('Variable "$cursor"') && pageCursor) { + return { + success: false as const, + error: 'Bookmark folder pagination rejected the cursor parameter', + had404: attempt.had404, + }; } - seen.add(tweet.id); - tweets.push(tweet); - added += 1; - if (!unlimited && tweets.length >= limit) { - break; + + if (attempt.success) { + return { success: true as const, items: attempt.items, cursor: attempt.cursor, had404: attempt.had404 }; } - } + return { success: false as const, error: attempt.error, had404: attempt.had404 }; + }, + }); - const pageCursor = page.cursor; - if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) { - nextCursor = undefined; - break; - } - if (maxPages && pagesFetched >= maxPages) { - nextCursor = pageCursor; - break; - } - cursor = pageCursor; - nextCursor = pageCursor; + if (!result.success) { + return result; } - - return { success: true, tweets, nextCursor }; + return { success: true, tweets: result.items, nextCursor: result.nextCursor }; } private async fetchWithRetry(url: string, init: RequestInit): Promise {