From 7faa23ff245a2ca37d1c7a6897c2ffa86198877e Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 19 Mar 2026 23:38:03 +0800 Subject: [PATCH 1/2] fix(intercept): use evaluate() for IIFE wrapping in installInterceptor/getInterceptedRequests Root cause: daemon migration changed these methods from this.evaluate() to direct sendCommand('exec'), losing the wrapForEval() IIFE wrapping. CDP received bare arrow functions that were never invoked. Fixes #98 --- src/browser/page.ts | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/browser/page.ts b/src/browser/page.ts index fafc01d..754b999 100644 --- a/src/browser/page.ts +++ b/src/browser/page.ts @@ -260,21 +260,18 @@ export class Page implements IPage { async installInterceptor(pattern: string): Promise { const { generateInterceptorJs } = await import('../interceptor.js'); - await sendCommand('exec', { - code: generateInterceptorJs(JSON.stringify(pattern), { - arrayName: '__opencli_xhr', - patchGuard: '__opencli_interceptor_patched', - }), - ...this._tabOpt(), - }); + // Must use evaluate() so wrapForEval() converts the arrow function into an IIFE; + // sendCommand('exec') sends the code as-is, and CDP never executes a bare arrow. + await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), { + arrayName: '__opencli_xhr', + patchGuard: '__opencli_interceptor_patched', + })); } async getInterceptedRequests(): Promise { const { generateReadInterceptedJs } = await import('../interceptor.js'); - const result = await sendCommand('exec', { - code: generateReadInterceptedJs('__opencli_xhr'), - ...this._tabOpt(), - }); + // Same as installInterceptor: must go through evaluate() for IIFE wrapping + const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr')); return (result as any[]) || []; } } From f9d304df55ec8c223933292ca0d240105917325c Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 19 Mar 2026 23:38:53 +0800 Subject: [PATCH 2/2] fix(twitter): SPA navigation, data path, and author resolution for INTERCEPT commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - followers/following: install interceptor on profile page, then click followers/following link (SPA navigation preserves JS context). Use JSON.stringify for targetUser to prevent injection. Throw on navigation failure. Update selector: /verified_followers. - notifications: install interceptor on home, then pushState+popstate to /notifications. Validate navigation URL. - search: fix author resolution (core.screen_name, not legacy). - All: fix GraphQL data path (remove extra .data level), update author resolution to try core.screen_name before legacy.screen_name. - followers: remove erroneous .filter(r => r?.url) — interceptor stores response body JSON, URL filtering happens at capture time. --- src/clis/twitter/followers.ts | 65 +++++++++++++++++-------------- src/clis/twitter/following.ts | 45 +++++++++++---------- src/clis/twitter/notifications.ts | 61 ++++++++++++++++------------- src/clis/twitter/search.ts | 4 +- 4 files changed, 98 insertions(+), 77 deletions(-) diff --git a/src/clis/twitter/followers.ts b/src/clis/twitter/followers.ts index ac83ce3..d379b34 100644 --- a/src/clis/twitter/followers.ts +++ b/src/clis/twitter/followers.ts @@ -15,65 +15,73 @@ cli({ func: async (page, kwargs) => { let targetUser = kwargs.user; - // If no user is specified, we must figure out the logged-in user's handle + // If no user is specified, figure out the logged-in user's handle if (!targetUser) { await page.goto('https://x.com/home'); - // wait for home page navigation await page.wait(5); - + const href = await page.evaluate(`() => { const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]'); return link ? link.getAttribute('href') : null; }`); - + if (!href) { throw new Error('Could not find logged-in user profile link. Are you logged in?'); } targetUser = href.replace('/', ''); } - // 1. Navigate to user profile page + // 1. Navigate to profile page await page.goto(`https://x.com/${targetUser}`); await page.wait(3); - // 2. Inject interceptor for the followers GraphQL API + // 2. Install interceptor BEFORE SPA navigation. + // goto() resets JS context, but SPA click preserves it. await page.installInterceptor('Followers'); - - // 3. Click the followers link inside the profile page - await page.evaluate(`() => { - const target = '${targetUser}'; - const link = document.querySelector('a[href="/' + target + '/followers"]'); - if (link) link.click(); + + // 3. Click the followers link via SPA navigation (preserves interceptor). + // Twitter uses /verified_followers instead of /followers now. + const safeUser = JSON.stringify(targetUser); + const clicked = await page.evaluate(`() => { + const target = ${safeUser}; + const selectors = [ + 'a[href="/' + target + '/verified_followers"]', + 'a[href="/' + target + '/followers"]', + ]; + for (const sel of selectors) { + const link = document.querySelector(sel); + if (link) { link.click(); return true; } + } + return false; }`); - await page.wait(3); + if (!clicked) { + throw new Error('Could not find followers link on profile page. Twitter may have changed the layout.'); + } + await page.wait(5); - // 4. Trigger API by scrolling + // 4. Scroll to trigger pagination API calls await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 }); - // 4. Retrieve data from opencli's registered interceptors - const allRequests = await page.getInterceptedRequests(); - const requestList = Array.isArray(allRequests) ? allRequests : []; - + // 5. Retrieve intercepted data + const requests = await page.getInterceptedRequests(); + const requestList = Array.isArray(requests) ? requests : []; + if (requestList.length === 0) { return []; } - - const requests = requestList.filter((r: any) => r?.url?.includes('Followers')); - if (!requests || requests.length === 0) { - return []; - } let results: any[] = []; - for (const req of requests) { + for (const req of requestList) { try { - let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions; + // GraphQL response: { data: { user: { result: { timeline: ... } } } } + let instructions = req.data?.user?.result?.timeline?.timeline?.instructions; if (!instructions) continue; let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries'); if (!addEntries) { addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries)); } - + if (!addEntries) continue; for (const entry of addEntries.entries) { @@ -82,10 +90,9 @@ cli({ const item = entry.content?.itemContent?.user_results?.result; if (!item || item.__typename !== 'User') continue; - // Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state const core = item.core || {}; const legacy = item.legacy || {}; - + results.push({ screen_name: core.screen_name || legacy.screen_name || 'unknown', name: core.name || legacy.name || 'unknown', @@ -98,7 +105,7 @@ cli({ } } - // Deduplicate by screen_name in case multiple scrolls caught the same + // Deduplicate by screen_name const unique = new Map(); results.forEach(r => unique.set(r.screen_name, r)); const deduplicatedResults = Array.from(unique.values()); diff --git a/src/clis/twitter/following.ts b/src/clis/twitter/following.ts index 1781c44..69b5322 100644 --- a/src/clis/twitter/following.ts +++ b/src/clis/twitter/following.ts @@ -15,45 +15,50 @@ cli({ func: async (page, kwargs) => { let targetUser = kwargs.user; - // If no user is specified, we must figure out the logged-in user's handle + // If no user is specified, figure out the logged-in user's handle if (!targetUser) { await page.goto('https://x.com/home'); - // wait for home page navigation await page.wait(5); - + const href = await page.evaluate(`() => { const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]'); return link ? link.getAttribute('href') : null; }`); - + if (!href) { throw new Error('Could not find logged-in user profile link. Are you logged in?'); } targetUser = href.replace('/', ''); } - // 1. Navigate to user profile page + // 1. Navigate to profile page await page.goto(`https://x.com/${targetUser}`); await page.wait(3); - // 2. Inject interceptor for Following GraphQL API + // 2. Install interceptor BEFORE SPA navigation. + // goto() resets JS context, but SPA click preserves it. await page.installInterceptor('Following'); - - // 3. Click the following link inside the profile page - await page.evaluate(`() => { - const target = '${targetUser}'; + + // 3. Click the following link via SPA navigation (preserves interceptor) + const safeUser = JSON.stringify(targetUser); + const clicked = await page.evaluate(`() => { + const target = ${safeUser}; const link = document.querySelector('a[href="/' + target + '/following"]'); - if (link) link.click(); + if (link) { link.click(); return true; } + return false; }`); - await page.wait(3); + if (!clicked) { + throw new Error('Could not find following link on profile page. Twitter may have changed the layout.'); + } + await page.wait(5); - // 4. Trigger API by scrolling + // 4. Scroll to trigger pagination API calls await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 }); - // 4. Retrieve data from opencli's registered interceptors + // 5. Retrieve intercepted data const requests = await page.getInterceptedRequests(); const requestList = Array.isArray(requests) ? requests : []; - + if (requestList.length === 0) { return []; } @@ -61,14 +66,15 @@ cli({ let results: any[] = []; for (const req of requestList) { try { - let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions; + // GraphQL response: { data: { user: { result: { timeline: ... } } } } + let instructions = req.data?.user?.result?.timeline?.timeline?.instructions; if (!instructions) continue; let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries'); if (!addEntries) { addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries)); } - + if (!addEntries) continue; for (const entry of addEntries.entries) { @@ -77,10 +83,9 @@ cli({ const item = entry.content?.itemContent?.user_results?.result; if (!item || item.__typename !== 'User') continue; - // Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state const core = item.core || {}; const legacy = item.legacy || {}; - + results.push({ screen_name: core.screen_name || legacy.screen_name || 'unknown', name: core.name || legacy.name || 'unknown', @@ -93,7 +98,7 @@ cli({ } } - // Deduplicate by screen_name in case multiple scrolls caught the same + // Deduplicate by screen_name const unique = new Map(); results.forEach(r => unique.set(r.screen_name, r)); const deduplicatedResults = Array.from(unique.values()); diff --git a/src/clis/twitter/notifications.ts b/src/clis/twitter/notifications.ts index 64bf2af..001a2ec 100644 --- a/src/clis/twitter/notifications.ts +++ b/src/clis/twitter/notifications.ts @@ -12,20 +12,30 @@ cli({ ], columns: ['id', 'action', 'author', 'text', 'url'], func: async (page, kwargs) => { - // 1. Navigate directly to notifications - await page.goto('https://x.com/notifications'); + // 1. Navigate to home first (we need a loaded Twitter page for SPA navigation) + await page.goto('https://x.com/home'); await page.wait(3); - // 2. Install interceptor after page load (must be after goto, not before, - // because goto triggers a full navigation that resets the JS context). - // Note: this misses the initial request fired during hydration; - // we rely on scroll-triggered pagination to capture data. + // 2. Install interceptor BEFORE SPA navigation await page.installInterceptor('NotificationsTimeline'); - // 3. Scroll to trigger API calls (load more notifications) + // 3. SPA navigate to notifications via history API + await page.evaluate(`() => { + window.history.pushState({}, '', '/notifications'); + window.dispatchEvent(new PopStateEvent('popstate', { state: {} })); + }`); + await page.wait(5); + + // Verify SPA navigation succeeded + const currentUrl = await page.evaluate('() => window.location.pathname'); + if (currentUrl !== '/notifications') { + throw new Error('SPA navigation to notifications failed. Twitter may have changed its routing.'); + } + + // 4. Scroll to trigger pagination await page.autoScroll({ times: 2, delayMs: 2000 }); - // 4. Retrieve data + // 5. Retrieve data const requests = await page.getInterceptedRequests(); if (!requests || requests.length === 0) return []; @@ -33,22 +43,20 @@ cli({ const seen = new Set(); for (const req of requests) { try { + // GraphQL response: { data: { viewer: ... } } (one level of .data) let instructions: any[] = []; - if (req.data?.data?.viewer?.timeline_response?.timeline?.instructions) { - instructions = req.data.data.viewer.timeline_response.timeline.instructions; - } else if (req.data?.data?.viewer_v2?.user_results?.result?.notification_timeline?.timeline?.instructions) { - instructions = req.data.data.viewer_v2.user_results.result.notification_timeline.timeline.instructions; - } else if (req.data?.data?.timeline?.instructions) { - instructions = req.data.data.timeline.instructions; + if (req.data?.viewer?.timeline_response?.timeline?.instructions) { + instructions = req.data.viewer.timeline_response.timeline.instructions; + } else if (req.data?.viewer_v2?.user_results?.result?.notification_timeline?.timeline?.instructions) { + instructions = req.data.viewer_v2.user_results.result.notification_timeline.timeline.instructions; + } else if (req.data?.timeline?.instructions) { + instructions = req.data.timeline.instructions; } let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries'); - - // Sometimes it's the first object without a 'type' field but has 'entries' if (!addEntries) { addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries)); } - if (!addEntries) continue; for (const entry of addEntries.entries) { @@ -66,24 +74,22 @@ cli({ function processNotificationItem(itemContent: any, entryId: string) { if (!itemContent) return; - - // Twitter wraps standard notifications + let item = itemContent?.notification_results?.result || itemContent?.tweet_results?.result || itemContent; let actionText = 'Notification'; let author = 'unknown'; let text = ''; let urlStr = ''; - + if (item.__typename === 'TimelineNotification') { - // Greet likes, retweet, mentions text = item.rich_message?.text || item.message?.text || ''; const fromUser = item.template?.from_users?.[0]?.user_results?.result; - author = fromUser?.legacy?.screen_name || fromUser?.core?.screen_name || 'unknown'; + // Twitter moved screen_name from legacy to core + author = fromUser?.core?.screen_name || fromUser?.legacy?.screen_name || 'unknown'; urlStr = item.notification_url?.url || ''; actionText = item.notification_icon || 'Activity'; - - // If there's an attached tweet + const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result; if (targetTweet) { const targetText = targetTweet.note_tweet?.note_tweet_results?.result?.text || targetTweet.legacy?.full_text || ''; @@ -93,14 +99,15 @@ cli({ } } } else if (item.__typename === 'TweetNotification') { - // Direct mention/reply const tweet = item.tweet_result?.result; - author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown'; + const tweetUser = tweet?.core?.user_results?.result; + author = tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown'; text = tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text || item.message?.text || ''; actionText = 'Mention/Reply'; urlStr = `https://x.com/i/status/${tweet?.rest_id}`; } else if (item.__typename === 'Tweet') { - author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown'; + const tweetUser = item.core?.user_results?.result; + author = tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown'; text = item.note_tweet?.note_tweet_results?.result?.text || item.legacy?.full_text || ''; actionText = 'Mention'; urlStr = `https://x.com/i/status/${item.rest_id}`; diff --git a/src/clis/twitter/search.ts b/src/clis/twitter/search.ts index 5dc7e4a..b5f9697 100644 --- a/src/clis/twitter/search.ts +++ b/src/clis/twitter/search.ts @@ -91,9 +91,11 @@ cli({ if (!tweet.rest_id || seen.has(tweet.rest_id)) continue; seen.add(tweet.rest_id); + // Twitter moved screen_name from legacy to core + const tweetUser = tweet.core?.user_results?.result; results.push({ id: tweet.rest_id, - author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown', + author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown', text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '', likes: tweet.legacy?.favorite_count || 0, views: tweet.views?.count || '0',