Skip to content

feat: add pagination control to home timeline methods #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
the-vampiire wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add pagination control to home timeline methods #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions src/lib/twitter-client-home.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ function isQueryIdMismatch(errors: Array<{ message?: string }>): boolean {
export interface HomeTimelineFetchOptions {
/** Include raw GraphQL response in `_raw` field */
includeRaw?: boolean;
/** Cursor for pagination - continue from this position */
cursor?: string;
/** Maximum number of pages to fetch (returns nextCursor if more available) */
maxPages?: number;
}

export interface TwitterClientHomeMethods {
Expand Down Expand Up @@ -59,12 +63,14 @@ export function withHome<TBase extends AbstractConstructor<TwitterClientBase>>(
count: number,
options: HomeTimelineFetchOptions,
): Promise<SearchResult> {
const { includeRaw = false } = options;
const { includeRaw = false, cursor: externalCursor, maxPages } = options;
const features = buildHomeTimelineFeatures();
const pageSize = 20;
const seen = new Set<string>();
const tweets: TweetData[] = [];
let cursor: string | undefined;
let cursor: string | undefined = externalCursor;
let nextCursor: string | undefined;
let pagesFetched = 0;

const fetchPage = async (pageCount: number, pageCursor?: string) => {
let lastError: string | undefined;
Expand Down Expand Up @@ -170,6 +176,7 @@ export function withHome<TBase extends AbstractConstructor<TwitterClientBase>>(
while (tweets.length < count) {
const pageCount = Math.min(pageSize, count - tweets.length);
const page = await fetchWithRefresh(pageCount, cursor);
pagesFetched += 1;
if (!page.success) {
return { success: false, error: page.error };
}
Expand All @@ -188,12 +195,18 @@ export function withHome<TBase extends AbstractConstructor<TwitterClientBase>>(
}

if (!page.cursor || page.cursor === cursor || page.tweets.length === 0 || added === 0) {
nextCursor = undefined;
break;
}
if (maxPages && pagesFetched >= maxPages) {
nextCursor = page.cursor;
break;
}
cursor = page.cursor;
nextCursor = page.cursor;
}

return { success: true, tweets };
return { success: true, tweets, nextCursor };
}
}

Expand Down
8 changes: 2 additions & 6 deletions tests/cli-shared.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it, vi } from 'vitest';
Expand Down Expand Up @@ -40,11 +40,7 @@ describe('cli shared', () => {
const tempHome = mkdtempSync(join(tmpdir(), 'bird-home-'));
const configDir = join(tempHome, '.config', 'bird');
mkdirSync(configDir, { recursive: true });
writeFileSync(
join(configDir, 'config.json5'),
'{ chromeProfileDir: "/tmp/Brave/Profile 1" }',
'utf8',
);
writeFileSync(join(configDir, 'config.json5'), '{ chromeProfileDir: "/tmp/Brave/Profile 1" }', 'utf8');
process.env.HOME = tempHome;

try {
Expand Down
198 changes: 198 additions & 0 deletions tests/twitter-client.home-timeline.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,5 +340,203 @@ describe('TwitterClient home timeline', () => {
expect(result.tweets).toHaveLength(1);
expect(mockFetch).toHaveBeenCalledTimes(2);
});

it('respects maxPages option', async () => {
// First page with cursor
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: {
home: {
home_timeline_urt: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: 'tweet1',
legacy: { full_text: 'First tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' },
core: {
user_results: {
result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } },
},
},
},
},
},
},
},
{
content: {
cursorType: 'Bottom',
value: 'cursor-page-2',
},
},
],
},
],
},
},
},
}),
});

const client = new TwitterClient({ cookies: validCookies });
const result = await client.getHomeTimeline(10, { maxPages: 1 });

expect(result.success).toBe(true);
expect(result.tweets).toHaveLength(1);
expect(result.nextCursor).toBe('cursor-page-2');
expect(mockFetch).toHaveBeenCalledTimes(1);
});

it('resumes from provided cursor', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: {
home: {
home_timeline_urt: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: 'tweet2',
legacy: {
full_text: 'Second page tweet',
created_at: 'Mon Jan 06 00:00:00 +0000 2025',
},
core: {
user_results: {
result: { rest_id: 'u2', legacy: { screen_name: 'user2', name: 'User 2' } },
},
},
},
},
},
},
},
],
},
],
},
},
},
}),
});

const client = new TwitterClient({ cookies: validCookies });
const result = await client.getHomeTimeline(1, { cursor: 'resume-cursor' });

expect(result.success).toBe(true);
expect(result.tweets).toHaveLength(1);
expect(result.tweets?.[0].id).toBe('tweet2');

// Verify cursor was passed in the request
const url = mockFetch.mock.calls[0][0] as string;
expect(url).toContain('resume-cursor');
});

it('returns undefined nextCursor when no more pages', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: {
home: {
home_timeline_urt: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: 'tweet1',
legacy: { full_text: 'Only tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' },
core: {
user_results: {
result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } },
},
},
},
},
},
},
},
// No cursor entry - end of timeline
],
},
],
},
},
},
}),
});

const client = new TwitterClient({ cookies: validCookies });
const result = await client.getHomeTimeline(1);

expect(result.success).toBe(true);
expect(result.tweets).toHaveLength(1);
expect(result.nextCursor).toBeUndefined();
});

it('works with getHomeLatestTimeline too', async () => {
mockFetch.mockResolvedValueOnce({
ok: true,
json: async () => ({
data: {
home: {
home_timeline_urt: {
instructions: [
{
entries: [
{
content: {
itemContent: {
tweet_results: {
result: {
rest_id: 'latest1',
legacy: { full_text: 'Latest tweet', created_at: 'Mon Jan 06 00:00:00 +0000 2025' },
core: {
user_results: {
result: { rest_id: 'u1', legacy: { screen_name: 'user1', name: 'User 1' } },
},
},
},
},
},
},
},
{
content: {
cursorType: 'Bottom',
value: 'latest-cursor',
},
},
],
},
],
},
},
},
}),
});

const client = new TwitterClient({ cookies: validCookies });
const result = await client.getHomeLatestTimeline(10, { maxPages: 1 });

expect(result.success).toBe(true);
expect(result.tweets).toHaveLength(1);
expect(result.nextCursor).toBe('latest-cursor');
});
});
});