Skip to content

fix(intercept): IIFE wrapping for installInterceptor/getInterceptedRequests #100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jackwener merged 2 commits into from
Mar 19, 2026
+106 −88
Merged

fix(intercept): IIFE wrapping for installInterceptor/getInterceptedRequests #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7faa23f
fix(intercept): use evaluate() for IIFE wrapping in installIntercepto…
Astro-Han Mar 19, 2026
f9d304d
fix(twitter): SPA navigation, data path, and author resolution for IN…
Astro-Han Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 8 additions & 11 deletions src/browser/page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,21 +260,18 @@ export class Page implements IPage {

async installInterceptor(pattern: string): Promise<void> {
const { generateInterceptorJs } = await import('../interceptor.js');
await sendCommand('exec', {
code: generateInterceptorJs(JSON.stringify(pattern), {
arrayName: '__opencli_xhr',
patchGuard: '__opencli_interceptor_patched',
}),
...this._tabOpt(),
});
// Must use evaluate() so wrapForEval() converts the arrow function into an IIFE;
// sendCommand('exec') sends the code as-is, and CDP never executes a bare arrow.
await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), {
arrayName: '__opencli_xhr',
patchGuard: '__opencli_interceptor_patched',
}));
}

async getInterceptedRequests(): Promise<any[]> {
const { generateReadInterceptedJs } = await import('../interceptor.js');
const result = await sendCommand('exec', {
code: generateReadInterceptedJs('__opencli_xhr'),
...this._tabOpt(),
});
// Same as installInterceptor: must go through evaluate() for IIFE wrapping
const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
return (result as any[]) || [];
}
}
Expand Down
65 changes: 36 additions & 29 deletions src/clis/twitter/followers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,65 +15,73 @@ cli({
func: async (page, kwargs) => {
let targetUser = kwargs.user;

// If no user is specified, we must figure out the logged-in user's handle
// If no user is specified, figure out the logged-in user's handle
if (!targetUser) {
await page.goto('https://x.com/home');
// wait for home page navigation
await page.wait(5);

const href = await page.evaluate(`() => {
const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
return link ? link.getAttribute('href') : null;
}`);

if (!href) {
throw new Error('Could not find logged-in user profile link. Are you logged in?');
}
targetUser = href.replace('/', '');
}

// 1. Navigate to user profile page
// 1. Navigate to profile page
await page.goto(`https://x.com/${targetUser}`);
await page.wait(3);

// 2. Inject interceptor for the followers GraphQL API
// 2. Install interceptor BEFORE SPA navigation.
// goto() resets JS context, but SPA click preserves it.
await page.installInterceptor('Followers');

// 3. Click the followers link inside the profile page
await page.evaluate(`() => {
const target = '${targetUser}';
const link = document.querySelector('a[href="/' + target + '/followers"]');
if (link) link.click();

// 3. Click the followers link via SPA navigation (preserves interceptor).
// Twitter uses /verified_followers instead of /followers now.
const safeUser = JSON.stringify(targetUser);
const clicked = await page.evaluate(`() => {
const target = ${safeUser};
const selectors = [
'a[href="/' + target + '/verified_followers"]',
'a[href="/' + target + '/followers"]',
];
for (const sel of selectors) {
const link = document.querySelector(sel);
if (link) { link.click(); return true; }
}
return false;
}`);
await page.wait(3);
if (!clicked) {
throw new Error('Could not find followers link on profile page. Twitter may have changed the layout.');
}
await page.wait(5);

// 4. Trigger API by scrolling
// 4. Scroll to trigger pagination API calls
await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });

// 4. Retrieve data from opencli's registered interceptors
const allRequests = await page.getInterceptedRequests();
const requestList = Array.isArray(allRequests) ? allRequests : [];
// 5. Retrieve intercepted data
const requests = await page.getInterceptedRequests();
const requestList = Array.isArray(requests) ? requests : [];

if (requestList.length === 0) {
return [];
}

const requests = requestList.filter((r: any) => r?.url?.includes('Followers'));
if (!requests || requests.length === 0) {
return [];
}

let results: any[] = [];
for (const req of requests) {
for (const req of requestList) {
try {
let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
// GraphQL response: { data: { user: { result: { timeline: ... } } } }
let instructions = req.data?.user?.result?.timeline?.timeline?.instructions;
if (!instructions) continue;

let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
if (!addEntries) {
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
}

if (!addEntries) continue;

for (const entry of addEntries.entries) {
Expand All @@ -82,10 +90,9 @@ cli({
const item = entry.content?.itemContent?.user_results?.result;
if (!item || item.__typename !== 'User') continue;

// Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state
const core = item.core || {};
const legacy = item.legacy || {};

results.push({
screen_name: core.screen_name || legacy.screen_name || 'unknown',
name: core.name || legacy.name || 'unknown',
Expand All @@ -98,7 +105,7 @@ cli({
}
}

// Deduplicate by screen_name in case multiple scrolls caught the same
// Deduplicate by screen_name
const unique = new Map();
results.forEach(r => unique.set(r.screen_name, r));
const deduplicatedResults = Array.from(unique.values());
Expand Down
45 changes: 25 additions & 20 deletions src/clis/twitter/following.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,60 +15,66 @@ cli({
func: async (page, kwargs) => {
let targetUser = kwargs.user;

// If no user is specified, we must figure out the logged-in user's handle
// If no user is specified, figure out the logged-in user's handle
if (!targetUser) {
await page.goto('https://x.com/home');
// wait for home page navigation
await page.wait(5);

const href = await page.evaluate(`() => {
const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
return link ? link.getAttribute('href') : null;
}`);

if (!href) {
throw new Error('Could not find logged-in user profile link. Are you logged in?');
}
targetUser = href.replace('/', '');
}

// 1. Navigate to user profile page
// 1. Navigate to profile page
await page.goto(`https://x.com/${targetUser}`);
await page.wait(3);

// 2. Inject interceptor for Following GraphQL API
// 2. Install interceptor BEFORE SPA navigation.
// goto() resets JS context, but SPA click preserves it.
await page.installInterceptor('Following');

// 3. Click the following link inside the profile page
await page.evaluate(`() => {
const target = '${targetUser}';

// 3. Click the following link via SPA navigation (preserves interceptor)
const safeUser = JSON.stringify(targetUser);
const clicked = await page.evaluate(`() => {
const target = ${safeUser};
const link = document.querySelector('a[href="/' + target + '/following"]');
if (link) link.click();
if (link) { link.click(); return true; }
return false;
}`);
await page.wait(3);
if (!clicked) {
throw new Error('Could not find following link on profile page. Twitter may have changed the layout.');
}
await page.wait(5);

// 4. Trigger API by scrolling
// 4. Scroll to trigger pagination API calls
await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });

// 4. Retrieve data from opencli's registered interceptors
// 5. Retrieve intercepted data
const requests = await page.getInterceptedRequests();
const requestList = Array.isArray(requests) ? requests : [];

if (requestList.length === 0) {
return [];
}

let results: any[] = [];
for (const req of requestList) {
try {
let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
// GraphQL response: { data: { user: { result: { timeline: ... } } } }
let instructions = req.data?.user?.result?.timeline?.timeline?.instructions;
if (!instructions) continue;

let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
if (!addEntries) {
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
}

if (!addEntries) continue;

for (const entry of addEntries.entries) {
Expand All @@ -77,10 +83,9 @@ cli({
const item = entry.content?.itemContent?.user_results?.result;
if (!item || item.__typename !== 'User') continue;

// Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state
const core = item.core || {};
const legacy = item.legacy || {};

results.push({
screen_name: core.screen_name || legacy.screen_name || 'unknown',
name: core.name || legacy.name || 'unknown',
Expand All @@ -93,7 +98,7 @@ cli({
}
}

// Deduplicate by screen_name in case multiple scrolls caught the same
// Deduplicate by screen_name
const unique = new Map();
results.forEach(r => unique.set(r.screen_name, r));
const deduplicatedResults = Array.from(unique.values());
Expand Down
61 changes: 34 additions & 27 deletions src/clis/twitter/notifications.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,43 +12,51 @@ cli({
],
columns: ['id', 'action', 'author', 'text', 'url'],
func: async (page, kwargs) => {
// 1. Navigate directly to notifications
await page.goto('https://x.com/notifications');
// 1. Navigate to home first (we need a loaded Twitter page for SPA navigation)
await page.goto('https://x.com/home');
await page.wait(3);

// 2. Install interceptor after page load (must be after goto, not before,
// because goto triggers a full navigation that resets the JS context).
// Note: this misses the initial request fired during hydration;
// we rely on scroll-triggered pagination to capture data.
// 2. Install interceptor BEFORE SPA navigation
await page.installInterceptor('NotificationsTimeline');

// 3. Scroll to trigger API calls (load more notifications)
// 3. SPA navigate to notifications via history API
await page.evaluate(`() => {
window.history.pushState({}, '', '/notifications');
window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
}`);
await page.wait(5);

// Verify SPA navigation succeeded
const currentUrl = await page.evaluate('() => window.location.pathname');
if (currentUrl !== '/notifications') {
throw new Error('SPA navigation to notifications failed. Twitter may have changed its routing.');
}

// 4. Scroll to trigger pagination
await page.autoScroll({ times: 2, delayMs: 2000 });

// 4. Retrieve data
// 5. Retrieve data
const requests = await page.getInterceptedRequests();
if (!requests || requests.length === 0) return [];

let results: any[] = [];
const seen = new Set<string>();
for (const req of requests) {
try {
// GraphQL response: { data: { viewer: ... } } (one level of .data)
let instructions: any[] = [];
if (req.data?.data?.viewer?.timeline_response?.timeline?.instructions) {
instructions = req.data.data.viewer.timeline_response.timeline.instructions;
} else if (req.data?.data?.viewer_v2?.user_results?.result?.notification_timeline?.timeline?.instructions) {
instructions = req.data.data.viewer_v2.user_results.result.notification_timeline.timeline.instructions;
} else if (req.data?.data?.timeline?.instructions) {
instructions = req.data.data.timeline.instructions;
if (req.data?.viewer?.timeline_response?.timeline?.instructions) {
instructions = req.data.viewer.timeline_response.timeline.instructions;
} else if (req.data?.viewer_v2?.user_results?.result?.notification_timeline?.timeline?.instructions) {
instructions = req.data.viewer_v2.user_results.result.notification_timeline.timeline.instructions;
} else if (req.data?.timeline?.instructions) {
instructions = req.data.timeline.instructions;
}

let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');

// Sometimes it's the first object without a 'type' field but has 'entries'
if (!addEntries) {
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
}

if (!addEntries) continue;

for (const entry of addEntries.entries) {
Expand All @@ -66,24 +74,22 @@ cli({

function processNotificationItem(itemContent: any, entryId: string) {
if (!itemContent) return;

// Twitter wraps standard notifications

let item = itemContent?.notification_results?.result || itemContent?.tweet_results?.result || itemContent;

let actionText = 'Notification';
let author = 'unknown';
let text = '';
let urlStr = '';

if (item.__typename === 'TimelineNotification') {
// Greet likes, retweet, mentions
text = item.rich_message?.text || item.message?.text || '';
const fromUser = item.template?.from_users?.[0]?.user_results?.result;
author = fromUser?.legacy?.screen_name || fromUser?.core?.screen_name || 'unknown';
// Twitter moved screen_name from legacy to core
author = fromUser?.core?.screen_name || fromUser?.legacy?.screen_name || 'unknown';
urlStr = item.notification_url?.url || '';
actionText = item.notification_icon || 'Activity';

// If there's an attached tweet

const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result;
if (targetTweet) {
const targetText = targetTweet.note_tweet?.note_tweet_results?.result?.text || targetTweet.legacy?.full_text || '';
Expand All @@ -93,14 +99,15 @@ cli({
}
}
} else if (item.__typename === 'TweetNotification') {
// Direct mention/reply
const tweet = item.tweet_result?.result;
author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown';
const tweetUser = tweet?.core?.user_results?.result;
author = tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown';
text = tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text || item.message?.text || '';
actionText = 'Mention/Reply';
urlStr = `https://x.com/i/status/${tweet?.rest_id}`;
} else if (item.__typename === 'Tweet') {
author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown';
const tweetUser = item.core?.user_results?.result;
author = tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown';
text = item.note_tweet?.note_tweet_results?.result?.text || item.legacy?.full_text || '';
actionText = 'Mention';
urlStr = `https://x.com/i/status/${item.rest_id}`;
Expand Down
4 changes: 3 additions & 1 deletion src/clis/twitter/search.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,11 @@ cli({
if (!tweet.rest_id || seen.has(tweet.rest_id)) continue;
seen.add(tweet.rest_id);

// Twitter moved screen_name from legacy to core
const tweetUser = tweet.core?.user_results?.result;
results.push({
id: tweet.rest_id,
author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown',
text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
likes: tweet.legacy?.favorite_count || 0,
views: tweet.views?.count || '0',
Expand Down
Loading