Skip to content

refactor: extract pagination helper to reduce code duplication #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
odysseus0 wants to merge 1 commit into
base: main
Choose a base branch
from
+496 −658
Open

refactor: extract pagination helper to reduce code duplication #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions src/lib/twitter-client-base.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,4 +155,78 @@ export abstract class TwitterClientBase {
this.clientUserId = result.user.id;
}
}

protected async paginateItems<T extends { id: string }>(options: {
limit: number;
pageSize: number;
maxPages?: number;
startCursor?: string;
fetchPage: (
count: number,
cursor?: string,
) => Promise<
| { success: true; items: T[]; cursor?: string; had404?: boolean }
| { success: false; error: string; had404?: boolean }
>;
}): Promise<{ success: true; items: T[]; nextCursor?: string } | { success: false; error: string }> {
const { limit, pageSize, maxPages, startCursor, fetchPage } = options;
const unlimited = limit === Number.POSITIVE_INFINITY;
const seen = new Set<string>();
const items: T[] = [];
let cursor = startCursor;
let nextCursor: string | undefined;
let pagesFetched = 0;

const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
const firstAttempt = await fetchPage(pageCount, pageCursor);
if (firstAttempt.success) {
return firstAttempt;
}
if (firstAttempt.had404) {
await this.refreshQueryIds();
const secondAttempt = await fetchPage(pageCount, pageCursor);
if (secondAttempt.success) {
return secondAttempt;
}
return { success: false as const, error: secondAttempt.error };
}
return { success: false as const, error: firstAttempt.error };
};

while (unlimited || items.length < limit) {
const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - items.length);
const page = await fetchWithRefresh(pageCount, cursor);
if (!page.success) {
return { success: false, error: page.error };
}
pagesFetched += 1;

let added = 0;
for (const item of page.items) {
if (seen.has(item.id)) {
continue;
}
seen.add(item.id);
items.push(item);
added += 1;
if (!unlimited && items.length >= limit) {
break;
}
}

const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor || page.items.length === 0 || added === 0) {
nextCursor = undefined;
break;
}
if (maxPages && pagesFetched >= maxPages) {
nextCursor = pageCursor;
break;
}
cursor = pageCursor;
nextCursor = pageCursor;
}

return { success: true, items, nextCursor };
}
}
194 changes: 73 additions & 121 deletions src/lib/twitter-client-lists.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,142 +349,94 @@ export function withLists<TBase extends AbstractConstructor<TwitterClientBase>>(
options: TimelinePaginationOptions = {},
): Promise<SearchResult> {
const features = buildListsFeatures();
const pageSize = 20;
const seen = new Set<string>();
const tweets: TweetData[] = [];
let cursor: string | undefined = options.cursor;
let nextCursor: string | undefined;
let pagesFetched = 0;
const { includeRaw = false, maxPages } = options;

const fetchPage = async (pageCount: number, pageCursor?: string) => {
let lastError: string | undefined;
let had404 = false;
const queryIds = await this.getListTimelineQueryIds();

const variables = {
listId,
count: pageCount,
...(pageCursor ? { cursor: pageCursor } : {}),
};

const params = new URLSearchParams({
variables: JSON.stringify(variables),
features: JSON.stringify(features),
});

for (const queryId of queryIds) {
const url = `${TWITTER_API_BASE}/${queryId}/ListLatestTweetsTimeline?${params.toString()}`;

try {
const response = await this.fetchWithTimeout(url, {
method: 'GET',
headers: this.getHeaders(),
});

if (response.status === 404) {
had404 = true;
lastError = `HTTP ${response.status}`;
continue;
}

if (!response.ok) {
const text = await response.text();
return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
}
const { includeRaw = false, maxPages, cursor } = options;

const result = await this.paginateItems<TweetData>({
limit,
pageSize: 20,
maxPages,
startCursor: cursor,
fetchPage: async (pageCount, pageCursor) => {
let lastError: string | undefined;
let had404 = false;
const queryIds = await this.getListTimelineQueryIds();

const variables = {
listId,
count: pageCount,
...(pageCursor ? { cursor: pageCursor } : {}),
};

const data = (await response.json()) as {
data?: {
list?: {
tweets_timeline?: {
timeline?: {
instructions?: Array<{
entries?: Array<{
content?: {
itemContent?: {
tweet_results?: {
result?: GraphqlTweetResult;
const params = new URLSearchParams({
variables: JSON.stringify(variables),
features: JSON.stringify(features),
});

for (const queryId of queryIds) {
const url = `${TWITTER_API_BASE}/${queryId}/ListLatestTweetsTimeline?${params.toString()}`;

try {
const response = await this.fetchWithTimeout(url, {
method: 'GET',
headers: this.getHeaders(),
});

if (response.status === 404) {
had404 = true;
lastError = `HTTP ${response.status}`;
continue;
}

if (!response.ok) {
const text = await response.text();
return { success: false as const, error: `HTTP ${response.status}: ${text.slice(0, 200)}`, had404 };
}

const data = (await response.json()) as {
data?: {
list?: {
tweets_timeline?: {
timeline?: {
instructions?: Array<{
entries?: Array<{
content?: {
itemContent?: {
tweet_results?: {
result?: GraphqlTweetResult;
};
};
};
};
}>;
}>;
}>;
};
};
};
};
errors?: Array<{ message: string }>;
};
errors?: Array<{ message: string }>;
};

if (data.errors && data.errors.length > 0) {
return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 };
}
if (data.errors && data.errors.length > 0) {
return { success: false as const, error: data.errors.map((e) => e.message).join(', '), had404 };
}

const instructions = data.data?.list?.tweets_timeline?.timeline?.instructions;
const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
const nextCursor = extractCursorFromInstructions(instructions);
const instructions = data.data?.list?.tweets_timeline?.timeline?.instructions;
const pageTweets = parseTweetsFromInstructions(instructions, { quoteDepth: this.quoteDepth, includeRaw });
const extractedCursor = extractCursorFromInstructions(instructions);

return { success: true as const, tweets: pageTweets, cursor: nextCursor, had404 };
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
}
}

return { success: false as const, error: lastError ?? 'Unknown error fetching list timeline', had404 };
};

const fetchWithRefresh = async (pageCount: number, pageCursor?: string) => {
const firstAttempt = await fetchPage(pageCount, pageCursor);
if (firstAttempt.success) {
return firstAttempt;
}
if (firstAttempt.had404) {
await this.refreshQueryIds();
const secondAttempt = await fetchPage(pageCount, pageCursor);
if (secondAttempt.success) {
return secondAttempt;
return { success: true as const, items: pageTweets, cursor: extractedCursor, had404 };
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
}
}
return { success: false as const, error: secondAttempt.error };
}
return { success: false as const, error: firstAttempt.error };
};

const unlimited = limit === Number.POSITIVE_INFINITY;
while (unlimited || tweets.length < limit) {
const pageCount = unlimited ? pageSize : Math.min(pageSize, limit - tweets.length);
const page = await fetchWithRefresh(pageCount, cursor);
if (!page.success) {
return { success: false, error: page.error };
}
pagesFetched += 1;

let added = 0;
for (const tweet of page.tweets) {
if (seen.has(tweet.id)) {
continue;
}
seen.add(tweet.id);
tweets.push(tweet);
added += 1;
if (!unlimited && tweets.length >= limit) {
break;
}
}
return { success: false as const, error: lastError ?? 'Unknown error fetching list timeline', had404 };
},
});

const pageCursor = page.cursor;
if (!pageCursor || pageCursor === cursor || page.tweets.length === 0 || added === 0) {
nextCursor = undefined;
break;
}
if (maxPages && pagesFetched >= maxPages) {
nextCursor = pageCursor;
break;
}
cursor = pageCursor;
nextCursor = pageCursor;
if (!result.success) {
return result;
}

return { success: true, tweets, nextCursor };
return { success: true, tweets: result.items, nextCursor: result.nextCursor };
}
}

Expand Down
Loading