From 4db0ba008fa88895e0d106ff186af354d53c05c2 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 19:59:52 +0000 Subject: [PATCH 01/17] refactor: remove deprecated auth types and deprecation tests Co-Authored-By: Claude Opus 4.6 --- src/types.ts | 8 +- test/unit/deprecation.test.ts | 143 ---------------------------------- 2 files changed, 1 insertion(+), 150 deletions(-) delete mode 100644 test/unit/deprecation.test.ts diff --git a/src/types.ts b/src/types.ts index 12509ec..4adca54 100644 --- a/src/types.ts +++ b/src/types.ts @@ -8,12 +8,7 @@ export interface VideoUrl { export enum ErrorClassification { LOGIN_WALL = 'login_wall', - /** - * @deprecated Private tweet extraction is experimental (ALPHA). - * May not work reliably. Use at your own risk. - */ - PROTECTED_ACCOUNT = 'protected_account', - NO_VIDEO_FOUND = 'no_video_found', +NO_VIDEO_FOUND = 'no_video_found', INVALID_URL = 'invalid_url', PARSE_ERROR = 'parse_error', EXTRACTION_ERROR = 'extraction_error', @@ -40,7 +35,6 @@ export interface ExtractOptions { url?: string; timeout?: number; headed?: boolean; - profileDir?: string; browserChannel?: 'chrome' | 'chromium' | 'msedge'; browserExecutablePath?: string; debugArtifactsDir?: string; diff --git a/test/unit/deprecation.test.ts b/test/unit/deprecation.test.ts deleted file mode 100644 index 34879f5..0000000 --- a/test/unit/deprecation.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; - -describe('Deprecation Notice Tests', () => { - - describe('Static Analysis - JSDoc Comments', () => { - it('should have @deprecated comment on isPrivateTweet function', async () => { - const utilsFile = await Bun.file('./src/utils.ts').text(); - expect(utilsFile).toContain('@deprecated'); - expect(utilsFile).toContain('isPrivateTweet'); - expect(utilsFile).toContain('ALPHA'); - }); - - it('should have @deprecated comment on PROTECTED_ACCOUNT enum', async () => { - const typesFile = await Bun.file('./src/types.ts').text(); - expect(typesFile).toContain('@deprecated'); - expect(typesFile).toContain('PROTECTED_ACCOUNT'); - expect(typesFile).toContain('ALPHA'); - }); - - it('should have @deprecated comment on verifyAuth method', async () => { - const extractorFile = await Bun.file('./src/extractor.ts').text(); - expect(extractorFile).toContain('@deprecated'); - expect(extractorFile).toContain('verifyAuth'); - expect(extractorFile).toContain('ALPHA'); - }); - }); - - describe('Runtime - Console Warnings', () => { - let originalWarn: typeof console.warn; - const warnings: string[] = []; - - beforeEach(() => { - originalWarn = console.warn; - warnings.length = 0; - console.warn = (...args: any[]) => { - warnings.push(args.join(' ')); - }; - }); - - afterEach(() => { - console.warn = originalWarn; - }); - - it('should log warning when verifyAuth is called', async () => { - const { VideoExtractor } = await import('../../src/extractor.ts'); - const extractor = new VideoExtractor({ profileDir: '/tmp/test-profile' }); - // Call verifyAuth which should log warning - await extractor.verifyAuth(); - - const hasWarning = warnings.some(w => - w.includes('DEPRECATED') && w.includes('verifyAuth') - ); - expect(hasWarning).toBe(true); - }); - - it('should log warning mentioning experimental/alpha status for verifyAuth', async () => { - const { VideoExtractor } = await import('../../src/extractor.ts'); - const extractor = new VideoExtractor({ profileDir: '/tmp/test-profile' }); - await extractor.verifyAuth(); - - const hasAlphaWarning = warnings.some(w => - w.includes('experimental') || w.includes('alpha') - ); - expect(hasAlphaWarning).toBe(true); - }); - - it('should NOT log warning when isPrivateTweet is called (called during normal operation)', async () => { - const { isPrivateTweet } = await import('../../src/utils.ts'); - isPrivateTweet(''); - - const hasWarning = warnings.some(w => - w.includes('DEPRECATED') && w.includes('isPrivateTweet') - ); - expect(hasWarning).toBe(false); - }); - }); - - describe('CLI - Flag Deprecation Warnings', () => { - it('should show deprecation warning when using --login flag', async () => { - const process = Bun.spawn( - ['bun', 'run', './src/index.ts', '--login', '--help'], - { - stdout: 'pipe', - stderr: 'pipe', - } - ); - - await process.exited; - const stdout = await new Response(process.stdout).text(); - - expect(stdout).toMatch(/EXPERIMENTAL ALPHA/i); - }); - - it('should show deprecation warning when using --verify-auth flag', async () => { - const process = Bun.spawn( - ['bun', 'run', './src/index.ts', '--help'], - { - stdout: 'pipe', - stderr: 'pipe', - } - ); - - await process.exited; - const stdout = await new Response(process.stdout).text(); - - expect(stdout).toMatch(/EXPERIMENTAL ALPHA/i); - expect(stdout).toMatch(/verify-auth/i); - }); - }); - - describe('Documentation - README Section', () => { - it('should have experimental alpha section in README', async () => { - const readme = await Bun.file('./README.md').text(); - expect(readme).toMatch(/experimental\s+alpha/i); - expect(readme).toMatch(/alpha\s+features/i); - }); - - it('should mention private tweet features as alpha in README', async () => { - const readme = await Bun.file('./README.md').text(); - expect(readme).toMatch(/private.*tweet/i); - expect(readme).toMatch(/authentication/i); - }); - }); - - describe('Integration - Combined Deprecation Check', () => { - it('should have consistent deprecation messaging across all files', async () => { - const [utils, types, extractor, readme] = await Promise.all([ - Bun.file('./src/utils.ts').text(), - Bun.file('./src/types.ts').text(), - Bun.file('./src/extractor.ts').text(), - Bun.file('./README.md').text(), - ]); - - const allFiles = [utils, types, extractor, readme]; - - allFiles.forEach(file => { - if (file.includes('@deprecated') || file.includes('deprecated')) { - expect(file).toMatch(/alpha|experimental/i); - } - }); - }); - }); -}); From 456749b43af2e269c20296e88d6356c7f7dde7ce Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:00:38 +0000 Subject: [PATCH 02/17] refactor: remove isPrivateTweet and auth cookie utils Co-Authored-By: Claude Opus 4.6 --- src/utils.ts | 36 ----- test/unit/auth.test.ts | 292 ----------------------------------------- 2 files changed, 328 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 12129cc..37f2388 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -78,26 +78,6 @@ export async function commandExists(command: string): Promise { } } -/** - * @deprecated Private tweet detection is experimental (ALPHA). - * May produce false positives/negatives. Use at your own risk. - */ -export function isPrivateTweet(html: string): boolean { - const privateIndicators = [ - 'this tweet is from an account that is', - 'protected tweets', - 'you are not authorized to view', - 'these tweets are protected', - 'only followers can see', - 'this tweet is protected', - ]; - - const lowerHtml = html.toLowerCase(); - return privateIndicators.some(indicator => - lowerHtml.includes(indicator.toLowerCase()) - ); -} - export function hasLoginWall(html: string): boolean { const loginIndicators = [ 'log in', @@ -166,19 +146,3 @@ export function formatTime(seconds: number): string { return `${mins}:${secs.toString().padStart(2, '0')}`; } -export function hasCookie(cookies: any[], name: string): boolean { - return cookies.some(cookie => cookie.name === name); -} - -export function findAuthCookies(cookies: any[]): string[] { - const authCookieNames = [ - 'auth_token', - 'auth_multi_select', - 'personalization_id', - 'ct0', - ]; - - return cookies - .filter(cookie => authCookieNames.includes(cookie.name)) - .map(cookie => cookie.name); -} diff --git a/test/unit/auth.test.ts b/test/unit/auth.test.ts index 39a4ccc..6d58941 100644 --- a/test/unit/auth.test.ts +++ b/test/unit/auth.test.ts @@ -1,9 +1,6 @@ import { describe, it, expect } from 'bun:test'; import { hasLoginWall, - isPrivateTweet, - hasCookie, - findAuthCookies, } from '../../src/utils.ts'; /** @@ -114,71 +111,6 @@ const loginWallFixtures = { `, }; -// Protected/Private tweet scenarios -const protectedTweetFixtures = { - protectedAccount: ` - - -
-

This tweet is from an account that is protected

-
- - - `, - - protectedTweets: ` - - -
-

These tweets are protected

-

Only followers can see this content

-
- - - `, - - notAuthorized: ` - - -
-

You are not authorized to view this tweet

-
- - - `, - - tweetProtected: ` - - -
-

This tweet is protected

-
- - - `, - - onlyFollowersCanSee: ` - - -
-

Only followers can see posts from this account

-
- - - `, - - caseInsensitiveProtected: ` - - -
-

PROTECTED TWEETS

-

YOU ARE NOT AUTHORIZED TO VIEW THIS CONTENT

-
- - - `, -}; - // Edge cases and false positives to avoid const nonAuthFixtures = { publicTweet: ` @@ -192,18 +124,6 @@ const nonAuthFixtures = { `, - publicTweetWithComments: ` - - -
-

Check out this video! To sign up for alerts, click here.

- -

Log in to comment on this tweet

-
- - - `, - articleWithLoginText: ` @@ -234,14 +154,6 @@ const nonAuthFixtures = { `, - - whitespaceSensitive: ` - - -
Content here with no auth keywords
- - - `, }; // Tests for hasLoginWall() @@ -300,207 +212,3 @@ describe('hasLoginWall Detection', () => { }); }); }); - -// Tests for isPrivateTweet() -describe('Private Tweet Detection', () => { - describe('Basic Protected Tweet Detection', () => { - it('should detect protected accounts', () => { - expect(isPrivateTweet(protectedTweetFixtures.protectedAccount)).toBe(true); - }); - - it('should detect "these tweets are protected" messages', () => { - expect(isPrivateTweet(protectedTweetFixtures.protectedTweets)).toBe(true); - }); - - it('should detect "not authorized" messages', () => { - expect(isPrivateTweet(protectedTweetFixtures.notAuthorized)).toBe(true); - }); - - it('should detect "this tweet is protected" messages', () => { - expect(isPrivateTweet(protectedTweetFixtures.tweetProtected)).toBe(true); - }); - - it('should detect "only followers can see" messages', () => { - expect(isPrivateTweet(protectedTweetFixtures.onlyFollowersCanSee)).toBe(true); - }); - }); - - describe('Case Insensitivity', () => { - it('should detect protected tweets regardless of case', () => { - expect(isPrivateTweet(protectedTweetFixtures.caseInsensitiveProtected)).toBe(true); - }); - }); - - describe('Avoiding False Positives', () => { - it('should not flag public tweets as private', () => { - expect(isPrivateTweet(nonAuthFixtures.publicTweet)).toBe(false); - }); - - it('should not flag public tweets even with "sign up" text', () => { - expect(isPrivateTweet(nonAuthFixtures.publicTweetWithComments)).toBe(false); - }); - - it('should not flag help articles as private tweets', () => { - expect(isPrivateTweet(nonAuthFixtures.helpDocumentation)).toBe(false); - }); - - it('should not flag empty content as private', () => { - expect(isPrivateTweet(nonAuthFixtures.emptyContent)).toBe(false); - }); - }); - - describe('Interaction with Login Walls', () => { - it('should distinguish protected tweets from login walls', () => { - // Public tweet with login prompt should be detected as login wall, not private tweet - expect(isPrivateTweet(nonAuthFixtures.publicTweetWithComments)).toBe(false); - }); - }); -}); - -// Tests for auth cookie logic -describe('Auth Cookie Functions', () => { - describe('hasCookie', () => { - it('should find a cookie by name', () => { - const cookies = [ - { name: 'auth_token', value: 'token123' }, - { name: 'other_cookie', value: 'value456' }, - ]; - expect(hasCookie(cookies, 'auth_token')).toBe(true); - }); - - it('should return false when cookie is not present', () => { - const cookies = [ - { name: 'session_id', value: 'sess123' }, - { name: 'other_cookie', value: 'value456' }, - ]; - expect(hasCookie(cookies, 'auth_token')).toBe(false); - }); - - it('should handle empty cookie array', () => { - expect(hasCookie([], 'auth_token')).toBe(false); - }); - - it('should be case sensitive for cookie names', () => { - const cookies = [ - { name: 'Auth_Token', value: 'token123' }, - ]; - expect(hasCookie(cookies, 'auth_token')).toBe(false); - expect(hasCookie(cookies, 'Auth_Token')).toBe(true); - }); - }); - - describe('findAuthCookies', () => { - it('should find all auth-related cookies', () => { - const cookies = [ - { name: 'auth_token', value: 'token123' }, - { name: 'auth_multi_select', value: 'select456' }, - { name: 'personalization_id', value: 'pers789' }, - { name: 'ct0', value: 'csrf_token' }, - { name: 'other_cookie', value: 'other' }, - ]; - const result = findAuthCookies(cookies); - expect(result.sort()).toEqual(['auth_multi_select', 'auth_token', 'ct0', 'personalization_id'].sort()); - }); - - it('should return empty array when no auth cookies present', () => { - const cookies = [ - { name: 'session_id', value: 'sess123' }, - { name: 'other_cookie', value: 'value456' }, - ]; - expect(findAuthCookies(cookies)).toEqual([]); - }); - - it('should handle empty cookie array', () => { - expect(findAuthCookies([])).toEqual([]); - }); - - it('should find auth_token specifically', () => { - const cookies = [ - { name: 'auth_token', value: 'token123' }, - ]; - expect(findAuthCookies(cookies)).toContain('auth_token'); - }); - - it('should find auth_multi_select cookie', () => { - const cookies = [ - { name: 'auth_multi_select', value: 'select456' }, - ]; - expect(findAuthCookies(cookies)).toContain('auth_multi_select'); - }); - - it('should find personalization_id cookie', () => { - const cookies = [ - { name: 'personalization_id', value: 'pers789' }, - ]; - expect(findAuthCookies(cookies)).toContain('personalization_id'); - }); - - it('should find ct0 (CSRF) cookie', () => { - const cookies = [ - { name: 'ct0', value: 'csrf_token' }, - ]; - expect(findAuthCookies(cookies)).toContain('ct0'); - }); - - it('should return only the cookie names, not values', () => { - const cookies = [ - { name: 'auth_token', value: 'token123' }, - { name: 'auth_multi_select', value: 'select456' }, - ]; - const result = findAuthCookies(cookies); - result.forEach(name => { - expect(typeof name).toBe('string'); - expect(name).not.toContain('token123'); - expect(name).not.toContain('select456'); - }); - }); - - it('should handle cookies with extra properties', () => { - const cookies = [ - { name: 'auth_token', value: 'token123', domain: '.x.com', path: '/' }, - { name: 'other', value: 'other_value', domain: '.x.com' }, - ]; - const result = findAuthCookies(cookies); - expect(result).toContain('auth_token'); - expect(result).not.toContain('other'); - }); - }); -}); - -// Integration-style tests combining auth detection functions -describe('Auth Check Integration', () => { - it('should correctly classify a login wall scenario', () => { - const loginWallHtml = loginWallFixtures.loginToFollowAccount; - expect(hasLoginWall(loginWallHtml)).toBe(true); - expect(isPrivateTweet(loginWallHtml)).toBe(false); - }); - - it('should correctly classify a protected account scenario', () => { - const protectedHtml = protectedTweetFixtures.protectedAccount; - expect(isPrivateTweet(protectedHtml)).toBe(true); - expect(hasLoginWall(protectedHtml)).toBe(false); - }); - - it('should correctly classify public content', () => { - const publicHtml = nonAuthFixtures.publicTweet; - expect(hasLoginWall(publicHtml)).toBe(false); - expect(isPrivateTweet(publicHtml)).toBe(false); - }); - - it('should correctly identify when auth cookies are present', () => { - const cookies = [ - { name: 'auth_token', value: 'valid_token' }, - { name: 'ct0', value: 'csrf_token' }, - ]; - expect(hasCookie(cookies, 'auth_token')).toBe(true); - expect(findAuthCookies(cookies).length).toBeGreaterThan(0); - }); - - it('should correctly identify when auth cookies are absent', () => { - const cookies = [ - { name: 'session_id', value: 'session123' }, - ]; - expect(hasCookie(cookies, 'auth_token')).toBe(false); - expect(findAuthCookies(cookies).length).toBe(0); - }); -}); From 7561dfdb1694a8e01ae4026e514c1476a1e7ee88 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:02:29 +0000 Subject: [PATCH 03/17] refactor: remove auth methods from VideoExtractor, accept external page Co-Authored-By: Claude Opus 4.6 --- src/extractor.ts | 182 +++--------------------------------------- test/unit/url.test.ts | 32 -------- 2 files changed, 11 insertions(+), 203 deletions(-) diff --git a/src/extractor.ts b/src/extractor.ts index d76cefa..b28fe0f 100644 --- a/src/extractor.ts +++ b/src/extractor.ts @@ -8,7 +8,6 @@ import { generateFilename, getVideoFormat, hasLoginWall, - isPrivateTweet, isValidTwitterUrl, parseTweetUrl, } from './utils.ts'; @@ -25,7 +24,6 @@ type ExtractCandidate = { export class VideoExtractor { private timeout: number; private headed: boolean; - private profileDir?: string; private debugArtifactsDir?: string; private browserChannel?: 'chrome' | 'chromium' | 'msedge'; private browserExecutablePath?: string; @@ -33,13 +31,12 @@ export class VideoExtractor { constructor(options: ExtractOptions) { this.timeout = options.timeout || 30000; this.headed = options.headed || false; - this.profileDir = options.profileDir; this.debugArtifactsDir = options.debugArtifactsDir; this.browserChannel = options.browserChannel; this.browserExecutablePath = options.browserExecutablePath; } - async extract(url: string): Promise { + async extract(url: string, externalPage?: Page): Promise { console.log(`\ud83c\udfac Extracting video from: ${url}`); if (!isValidTwitterUrl(url)) { @@ -66,9 +63,14 @@ export class VideoExtractor { let browser: Browser | null = null; let context: BrowserContext | null = null; let page: Page | null = null; + const usingExternalPage = !!externalPage; try { - ({ browser, context, page } = await this.createContextAndPage(chromium)); + if (usingExternalPage) { + page = externalPage; + } else { + ({ browser, context, page } = await this.createContextAndPage(chromium)); + } const candidates = new Set(); page.on('response', async (resp) => { @@ -85,20 +87,9 @@ export class VideoExtractor { const pageHtml = await page.content(); - // WARNING: Private tweet detection is experimental (ALPHA) - if (isPrivateTweet(pageHtml)) { - const debugInfo = await this.saveDebugArtifacts(page, pageHtml, 'protected-account'); - return { - videoUrl: null, - error: 'This tweet is private or protected. Only public tweets can be extracted.', - errorClassification: ErrorClassification.PROTECTED_ACCOUNT, - debugInfo, - }; - } - const loginWall = hasLoginWall(pageHtml); if (loginWall) { - console.log('\u26a0\ufe0f Login wall detected; trying to extract anyway (use --login/--profile for best results)...'); + console.log('\u26a0\ufe0f Login wall detected; trying to extract anyway...'); } // Try to trigger media loading. @@ -117,7 +108,7 @@ export class VideoExtractor { return { videoUrl: null, error: loginWall - ? 'No video URL found. This tweet likely requires authentication. Run: x-dl --login --profile ~/.x-dl-profile' + ? 'No video URL found. This tweet likely requires authentication. Try: x-dl cdp ' : 'Failed to extract video URL.', errorClassification: loginWall ? ErrorClassification.LOGIN_WALL : ErrorClassification.NO_VIDEO_FOUND, debugInfo, @@ -139,40 +130,8 @@ export class VideoExtractor { debugInfo, }; } finally { - await this.safeClose({ browser, context }); - } - } - - async downloadAuthenticated(videoUrl: string, outputPath: string): Promise { - if (!this.profileDir) { - throw new Error('Authenticated download requested but no profileDir provided'); - } - - const { chromium } = await import('playwright'); - - console.log(`\ud83d\udd10 Authenticated download via Playwright: ${videoUrl}`); - const startTime = Date.now(); - - let context: BrowserContext | null = null; - - try { - context = await this.createPersistentContext(chromium, true); - - const resp = await context.request.get(videoUrl); - if (!resp.ok()) { - throw new Error(`HTTP error! status: ${resp.status()}`); - } - - const bytes = await resp.body(); - await Bun.write(outputPath, bytes); - - const elapsedSec = (Date.now() - startTime) / 1000; - console.log(`\u2705 Download completed in ${elapsedSec.toFixed(1)}s`); - - return outputPath; - } finally { - if (context) { - await context.close().catch(() => undefined); + if (!usingExternalPage) { + await this.safeClose({ browser, context }); } } } @@ -180,22 +139,6 @@ export class VideoExtractor { private async createContextAndPage( chromium: typeof import('playwright').chromium ): Promise<{ browser: Browser | null; context: BrowserContext; page: Page }> { - if (this.profileDir) { - const launchOptions: any = { - headless: !this.headed, - }; - - if (this.browserExecutablePath) { - launchOptions.executablePath = this.browserExecutablePath; - } else if (this.browserChannel) { - launchOptions.channel = this.browserChannel; - } - - const context = await chromium.launchPersistentContext(this.profileDir, launchOptions); - const page = await context.newPage(); - return { browser: null, context, page }; - } - const launchOptions: any = { headless: !this.headed }; if (this.browserExecutablePath) { @@ -210,21 +153,6 @@ export class VideoExtractor { return { browser, context, page }; } - private async createPersistentContext( - chromium: typeof import('playwright').chromium, - headless: boolean = true - ): Promise { - const launchOptions: any = { headless }; - - if (this.browserExecutablePath) { - launchOptions.executablePath = this.browserExecutablePath; - } else if (this.browserChannel) { - launchOptions.channel = this.browserChannel; - } - - return await chromium.launchPersistentContext(this.profileDir!, launchOptions); - } - private async safeClose({ browser, context, @@ -532,92 +460,4 @@ export class VideoExtractor { } } - /** - * @deprecated Authentication for private tweets is experimental (ALPHA). - * May not bypass login walls reliably. Use at your own risk. - */ - async verifyAuth(): Promise<{ - hasAuthToken: boolean; - canAccessHome: boolean; - authCookies: string[]; - message: string; - }> { - console.warn('[DEPRECATED] verifyAuth is experimental and may not work reliably.'); - if (!this.profileDir) { - return { - hasAuthToken: false, - canAccessHome: false, - authCookies: [], - message: 'No profile directory specified', - }; - } - - const { chromium } = await import('playwright'); - - let context: BrowserContext | null = null; - let page: Page | null = null; - - try { - context = await this.createPersistentContext(chromium, true); - - // Check for auth cookies - const cookies = await context.cookies(); - const authTokenCookie = cookies.find(c => c.name === 'auth_token'); - const authCookieNames = cookies - .filter(c => ['auth_token', 'auth_multi_select', 'personalization_id', 'ct0'].includes(c.name)) - .map(c => c.name); - - const hasAuthToken = !!authTokenCookie; - - // Try to load X.com/home - page = await context.newPage(); - let canAccessHome = false; - let message = ''; - - try { - await page.goto('https://x.com/home', { - waitUntil: 'domcontentloaded', - timeout: this.timeout, - }); - - const pageHtml = await page.content(); - const loginWallDetected = hasLoginWall(pageHtml); - - if (loginWallDetected) { - canAccessHome = false; - message = 'Login wall detected at X.com/home - authentication may be invalid or expired'; - } else if (pageHtml.includes('Home') && hasAuthToken) { - canAccessHome = true; - message = 'Authentication is valid and X.com/home is accessible'; - } else if (!loginWallDetected && pageHtml.includes('Home')) { - canAccessHome = true; - message = 'X.com/home loaded successfully (page loaded but no auth token present)'; - } else if (!loginWallDetected) { - canAccessHome = true; - message = 'X.com/home loaded (no login wall detected, but auth token not present)'; - } else { - canAccessHome = false; - message = 'Unable to verify access status'; - } - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - canAccessHome = false; - message = `Failed to access X.com/home: ${errorMsg}`; - } - - return { - hasAuthToken, - canAccessHome, - authCookies: authCookieNames, - message, - }; - } finally { - if (page) { - await page.close().catch(() => undefined); - } - if (context) { - await context.close().catch(() => undefined); - } - } - } } diff --git a/test/unit/url.test.ts b/test/unit/url.test.ts index 11270cd..b19483e 100644 --- a/test/unit/url.test.ts +++ b/test/unit/url.test.ts @@ -4,7 +4,6 @@ import { parseTweetUrl, generateFilename, sanitizeFilename, - isPrivateTweet, hasVideo, getVideoFormat, selectBestMp4, @@ -128,37 +127,6 @@ describe('Filename Sanitization', () => { }); }); -describe('Private Tweet Detection', () => { - it('should detect protected tweet indicators', () => { - const protectedHtml = 'This tweet is from an account that is protected'; - expect(isPrivateTweet(protectedHtml)).toBe(true); - - const privateHtml = 'These tweets are protected'; - expect(isPrivateTweet(privateHtml)).toBe(true); - - const notAuthorizedHtml = 'You are not authorized to view this tweet'; - expect(isPrivateTweet(notAuthorizedHtml)).toBe(true); - }); - - it('should not flag public tweets as private', () => { - const publicHtml = 'This is a great tweet with a video'; - expect(isPrivateTweet(publicHtml)).toBe(false); - }); - - it('should be case insensitive', () => { - expect(isPrivateTweet('PROTECTED TWEETS')).toBe(true); - expect(isPrivateTweet('YOU ARE NOT AUTHORIZED TO VIEW')).toBe(true); - }); - - it('should not flag login walls as private tweets', () => { - const loginHtml = 'Log in to follow this account'; - expect(isPrivateTweet(loginHtml)).toBe(false); - - const signUpHtml = 'Sign up to follow'; - expect(isPrivateTweet(signUpHtml)).toBe(false); - }); -}); - describe('Video Detection', () => { it('should detect video elements', () => { const videoHtml = ''; From f59d9b32c15964446998328535ca8fd45e02119f Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:04:41 +0000 Subject: [PATCH 04/17] refactor: remove --profile, --login, --verify-auth CLI flags Co-Authored-By: Claude Opus 4.6 --- src/index.ts | 133 +++++---------------------------------------------- 1 file changed, 12 insertions(+), 121 deletions(-) diff --git a/src/index.ts b/src/index.ts index 0ca7c86..f4bbc19 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,9 +16,6 @@ interface CliOptions { quality?: 'best' | 'worst'; timeout?: number; headed?: boolean; - profile?: string; - login?: boolean; - verifyAuth?: boolean; browserChannel?: 'chrome' | 'chromium' | 'msedge'; browserExecutablePath?: string; clipFrom?: string; @@ -30,18 +27,6 @@ interface InstallCliOptions { help?: boolean; } -const DEFAULT_PROFILE_DIR = path.join(os.homedir(), '.x-dl-profile'); - -function expandHomeDir(p: string): string { - if (p.startsWith('~/')) { - return path.join(os.homedir(), p.slice(2)); - } - if (p === '~') { - return os.homedir(); - } - return p; -} - function getDefaultDownloadsDir(): string { const platform = os.platform(); @@ -97,21 +82,6 @@ function parseArgs(args: string[]): CliOptions { case '--headed': options.headed = true; break; - case '--profile': { - if (!nextArg || nextArg.startsWith('-')) { - options.profile = DEFAULT_PROFILE_DIR; - } else { - options.profile = nextArg; - i++; - } - break; - } - case '--login': - options.login = true; - break; - case '--verify-auth': - options.verifyAuth = true; - break; case '--browser-channel': if (nextArg === 'chrome' || nextArg === 'chromium' || nextArg === 'msedge') { options.browserChannel = nextArg; @@ -189,11 +159,8 @@ OPTIONS: --quality Video quality preference (default: best) --timeout Page load timeout in seconds (default: 30) --headed Show browser window for debugging - --profile [dir] Persistent profile dir for authenticated extraction (default: ~/.x-dl-profile) - --login Open X in a persistent profile and wait for you to log in (EXPERIMENTAL ALPHA) --browser-channel Browser channel: chrome, chromium, or msedge (default: chromium) --browser-executable-path Path to browser executable (optional, overrides channel) - --verify-auth Check authentication status (EXPERIMENTAL ALPHA) --from Clip start time (e.g., 00:30) --to Clip end time (e.g., 01:30) --version, -v Show version information @@ -203,12 +170,18 @@ INSTALL: x-dl install Install Playwright Chromium only x-dl install --with-deps Install Chromium + ffmpeg + Linux deps (may require sudo on Linux) -AUTH EXAMPLES: - # Create/reuse a persistent login session - ${commandName} --login --profile ~/.x-dl-profile +CDP MODE (Private Tweets): + ${commandName} cdp Connect to Chrome and download (port 9222) + ${commandName} cdp --port 9333 Use custom debugging port + + Requires Chrome v144+ with remote debugging enabled. + See: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session + + 1. Open Chrome -> chrome://inspect/#remote-debugging -> Enable + 2. Run: ${commandName} cdp - # Extract using the authenticated profile - ${commandName} --profile ~/.x-dl-profile https://x.com/user/status/123 + If Chrome is not running, x-dl will launch it automatically. + If not logged into X/Twitter, x-dl will open a login page automatically. BROWSER EXAMPLES: # Use Chrome instead of Chromium @@ -290,45 +263,6 @@ function getOutputPath(tweetUrl: string, options: CliOptions, preferredExtension return `${options.output}/${filename}`; } -async function waitForEnter(): Promise { - process.stdin.resume(); - return new Promise((resolve) => { - process.stdin.once('data', () => resolve()); - }); -} - -async function runLoginFlow( - profileDir: string, - browserOptions?: { browserChannel?: string; browserExecutablePath?: string } -): Promise { - const { chromium } = await import('playwright'); - - console.log(`\n🔐 Login mode`); - console.log(`📁 Profile: ${profileDir}`); - console.log('🌐 Opening https://x.com/home ...'); - console.log('\nLog in to X in the opened browser, then press Enter here to close.\n'); - - const launchOptions: any = { - headless: false, - }; - - if (browserOptions?.browserExecutablePath) { - launchOptions.executablePath = browserOptions.browserExecutablePath; - } else if (browserOptions?.browserChannel) { - launchOptions.channel = browserOptions.browserChannel; - } - - const context = await chromium.launchPersistentContext(profileDir, launchOptions); - - try { - const page = await context.newPage(); - await page.goto('https://x.com/home', { waitUntil: 'domcontentloaded', timeout: 60000 }); - await waitForEnter(); - } finally { - await context.close(); - } -} - async function handleInstallMode(args: string[]): Promise { const options: InstallCliOptions = {}; @@ -373,9 +307,7 @@ async function main(): Promise { const args = parseArgs(argv); - const needsDependencies = args.login || args.verifyAuth || args.url; - - if (!needsDependencies) { + if (!args.url) { const commandName = getCommandName(); console.error('❌ Error: No URL provided'); console.error(`\nUsage: ${commandName} [options]`); @@ -397,47 +329,15 @@ async function main(): Promise { console.warn('⚠️ ffmpeg is not available. HLS (m3u8) downloads will not work.'); } - if (args.login) { - console.warn('[DEPRECATED] --login is an experimental alpha feature.'); - const profileDir = expandHomeDir(args.profile || DEFAULT_PROFILE_DIR); - await runLoginFlow(profileDir, { - browserChannel: args.browserChannel, - browserExecutablePath: args.browserExecutablePath, - }); - process.exit(0); - } - - if (args.verifyAuth) { - console.warn('[DEPRECATED] verify-auth is an experimental alpha feature.'); - const profileDir = expandHomeDir(args.profile || DEFAULT_PROFILE_DIR); - const extractor = new VideoExtractor({ - profileDir, - browserChannel: args.browserChannel, - browserExecutablePath: args.browserExecutablePath, - }); - const result = await extractor.verifyAuth(); - - console.log('\nAuth Status:'); - console.log(`- Auth token present: ${result.hasAuthToken ? 'Yes' : 'No'}`); - console.log(`- Can access X.com/home: ${result.canAccessHome ? 'Yes' : 'No'}`); - console.log(`- Auth cookies found: ${result.authCookies.join(', ') || 'None'}`); - console.log(`\n${result.message}\n`); - - process.exit(result.canAccessHome && result.hasAuthToken ? 0 : 1); - } - if (!isValidTwitterUrl(args.url)) { console.error('❌ Error: Invalid X/Twitter URL'); console.error('Please provide a valid tweet URL like: https://x.com/user/status/123456\n'); process.exit(1); } - const profileDir = args.profile ? expandHomeDir(args.profile) : undefined; - const extractor = new VideoExtractor({ timeout: args.timeout, headed: args.headed, - profileDir, browserChannel: args.browserChannel, browserExecutablePath: args.browserExecutablePath, }); @@ -575,15 +475,6 @@ async function main(): Promise { console.log(`\n\n✅ Video saved to: ${outputPath}\n`); } catch (error) { const message = error instanceof Error ? error.message : String(error); - const isAuthFailure = message.includes('status: 401') || message.includes('status: 403'); - - if (isAuthFailure && profileDir) { - console.log('\n\n🔐 Direct download was blocked; retrying with authenticated Playwright request...'); - await extractor.downloadAuthenticated(result.videoUrl.url, outputPath); - console.log(`\n\n✅ Video saved to: ${outputPath}\n`); - process.exit(0); - } - process.stdout.write('\r\x1b[K'); console.error(`❌ Download failed: ${message}\n`); process.exit(1); From 61e2ac912c542a9068126c904fa42764081efd31 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:05:35 +0000 Subject: [PATCH 05/17] feat: add CDP module with Chrome detection, connection, and login flow Co-Authored-By: Claude Opus 4.6 --- src/cdp.ts | 219 ++++++++++++++++++++++++++++++++++++++++++ test/unit/cdp.test.ts | 28 ++++++ 2 files changed, 247 insertions(+) create mode 100644 src/cdp.ts create mode 100644 test/unit/cdp.test.ts diff --git a/src/cdp.ts b/src/cdp.ts new file mode 100644 index 0000000..b4a9ada --- /dev/null +++ b/src/cdp.ts @@ -0,0 +1,219 @@ +import os from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs'; +import type { Browser, BrowserContext, Page } from 'playwright'; + +const CHROME_PATHS_MACOS = [ + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', +]; + +const CHROME_PATHS_LINUX = [ + '/usr/bin/google-chrome', + '/usr/bin/google-chrome-stable', + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + '/snap/bin/chromium', +]; + +export function findChromePath(): string | null { + const platform = os.platform(); + const candidates = platform === 'darwin' ? CHROME_PATHS_MACOS : CHROME_PATHS_LINUX; + + for (const p of candidates) { + if (fs.existsSync(p)) return p; + } + return null; +} + +export function findChromeProfileDir(): string { + const platform = os.platform(); + if (platform === 'darwin') { + return path.join(os.homedir(), 'Library', 'Application Support', 'Google', 'Chrome'); + } + return path.join(os.homedir(), '.config', 'google-chrome'); +} + +export interface CdpConnection { + browser: Browser; + context: BrowserContext; + page: Page; + launchedByUs: boolean; + cleanup: () => Promise; +} + +export async function connectOverCdp(port: number = 9222): Promise { + const { chromium } = await import('playwright'); + + // Try connecting to an already-running Chrome + try { + const browser = await chromium.connectOverCDP(`http://localhost:${port}`); + const context = browser.contexts()[0] || await browser.newContext(); + const page = await context.newPage(); + + return { + browser, + context, + page, + launchedByUs: false, + cleanup: async () => { + await page.close().catch(() => {}); + // Don't close browser — user's Chrome stays running + }, + }; + } catch { + // Connection failed — try launching Chrome ourselves + } + + const chromePath = findChromePath(); + if (!chromePath) { + throw new Error( + 'Google Chrome not found.\n' + + 'CDP mode requires Google Chrome installed on your system.' + ); + } + + const profileDir = findChromeProfileDir(); + const debuggingPort = port; + + // Launch Chrome headlessly with the user's profile + const chromeProcess = Bun.spawn([ + chromePath, + `--remote-debugging-port=${debuggingPort}`, + `--user-data-dir=${profileDir}`, + '--headless=new', + '--no-first-run', + '--no-default-browser-check', + ], { + stdout: 'ignore', + stderr: 'ignore', + }); + + // Wait for Chrome to be ready + let browser: Browser | null = null; + for (let i = 0; i < 20; i++) { + try { + browser = await chromium.connectOverCDP(`http://localhost:${debuggingPort}`); + break; + } catch { + await new Promise(r => setTimeout(r, 500)); + } + } + + if (!browser) { + chromeProcess.kill(); + throw new Error( + `Could not connect to Chrome on port ${port}.\n\n` + + 'To use CDP mode, enable remote debugging in Chrome (v144+):\n' + + ' 1. Open Chrome and go to chrome://inspect/#remote-debugging\n' + + ' 2. Enable incoming debugging connections\n' + + ' 3. Then run: x-dl cdp \n\n' + + 'Learn more: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session' + ); + } + + const context = browser.contexts()[0] || await browser.newContext(); + const page = await context.newPage(); + + return { + browser, + context, + page, + launchedByUs: true, + cleanup: async () => { + await page.close().catch(() => {}); + await browser!.close().catch(() => {}); + chromeProcess.kill(); + }, + }; +} + +export async function handleCdpLogin( + connection: CdpConnection, + port: number +): Promise { + const { chromium } = await import('playwright'); + + if (connection.launchedByUs) { + // Kill headless instance, relaunch headed + await connection.cleanup(); + + const chromePath = findChromePath()!; + const profileDir = findChromeProfileDir(); + + console.log('🔐 Not logged into X/Twitter. Opening Chrome for login...'); + console.log('⚠️ Don\'t close the Chrome window — log in, and x-dl will continue automatically.'); + + const chromeProcess = Bun.spawn([ + chromePath, + `--remote-debugging-port=${port}`, + `--user-data-dir=${profileDir}`, + '--no-first-run', + '--no-default-browser-check', + ], { + stdout: 'ignore', + stderr: 'ignore', + }); + + // Wait for Chrome to be ready + let browser: Browser | null = null; + for (let i = 0; i < 20; i++) { + try { + browser = await chromium.connectOverCDP(`http://localhost:${port}`); + break; + } catch { + await new Promise(r => setTimeout(r, 500)); + } + } + + if (!browser) { + chromeProcess.kill(); + throw new Error('Failed to relaunch Chrome for login.'); + } + + const context = browser.contexts()[0] || await browser.newContext(); + const page = await context.newPage(); + await page.goto('https://x.com/i/flow/login', { waitUntil: 'domcontentloaded', timeout: 30000 }); + + // Poll for auth_token cookie + await waitForAuthCookie(context); + + console.log('✅ Login detected! Continuing download...'); + + // Close headed, relaunch headless, return new connection + await page.close().catch(() => {}); + await browser.close().catch(() => {}); + chromeProcess.kill(); + + return connectOverCdp(port); + } else { + // Chrome was already running — open a login tab + console.log('🔐 Not logged into X/Twitter. A login tab has been opened in Chrome.'); + + const loginPage = await connection.context.newPage(); + await loginPage.goto('https://x.com/i/flow/login', { waitUntil: 'domcontentloaded', timeout: 30000 }); + + // Poll for auth_token cookie + await waitForAuthCookie(connection.context); + + console.log('✅ Login detected! Continuing download...'); + + await loginPage.close().catch(() => {}); + + // Return a fresh page in the same connection + const page = await connection.context.newPage(); + return { + ...connection, + page, + }; + } +} + +async function waitForAuthCookie(context: BrowserContext, timeoutMs: number = 300000): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const cookies = await context.cookies('https://x.com'); + if (cookies.some(c => c.name === 'auth_token')) return; + await new Promise(r => setTimeout(r, 2000)); + } + throw new Error('Login timed out (5 minutes). Please try again.'); +} diff --git a/test/unit/cdp.test.ts b/test/unit/cdp.test.ts new file mode 100644 index 0000000..e57a634 --- /dev/null +++ b/test/unit/cdp.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect } from 'bun:test'; +import { findChromePath, findChromeProfileDir, CdpConnection } from '../../src/cdp.ts'; + +describe('Chrome Detection', () => { + it('should return a path string or null from findChromePath', () => { + const result = findChromePath(); + expect(result === null || typeof result === 'string').toBe(true); + }); + + it('should return a profile dir string from findChromeProfileDir', () => { + const result = findChromeProfileDir(); + expect(typeof result).toBe('string'); + expect(result.length).toBeGreaterThan(0); + }); +}); + +describe('CdpConnection type', () => { + it('should export CdpConnection interface fields', () => { + const dummy: CdpConnection = { + browser: null as any, + context: null as any, + page: null as any, + launchedByUs: false, + cleanup: async () => {}, + }; + expect(dummy.launchedByUs).toBe(false); + }); +}); From 3b69a08e8af8f72846c8eb6355aa218b60a4d599 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:16:45 +0000 Subject: [PATCH 06/17] feat: add x-dl cdp subcommand for private tweet downloads Co-Authored-By: Claude Opus 4.6 --- src/index.ts | 281 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 280 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index f4bbc19..7248f81 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,8 +6,9 @@ import path from 'node:path'; import { VideoExtractor } from './extractor.ts'; import { downloadVideo } from './downloader.ts'; import { ensurePlaywrightReady, runInstall } from './installer.ts'; -import { generateFilename, isValidTwitterUrl, parseTweetUrl, formatBytes } from './utils.ts'; +import { generateFilename, isValidTwitterUrl, parseTweetUrl, formatBytes, hasLoginWall } from './utils.ts'; import { downloadHlsWithFfmpeg, clipLocalFile, mmssToSeconds } from './ffmpeg.ts'; +import { connectOverCdp, handleCdpLogin } from './cdp.ts'; interface CliOptions { url?: string; @@ -263,6 +264,279 @@ function getOutputPath(tweetUrl: string, options: CliOptions, preferredExtension return `${options.output}/${filename}`; } +interface CdpCliOptions { + url?: string; + output?: string; + urlOnly?: boolean; + quality?: 'best' | 'worst'; + timeout?: number; + port?: number; + clipFrom?: string; + clipTo?: string; +} + +function parseCdpArgs(args: string[]): CdpCliOptions { + const options: CdpCliOptions = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + const nextArg = args[i + 1]; + + switch (arg) { + case '--output': + case '-o': + options.output = nextArg; + i++; + break; + case '--url-only': + options.urlOnly = true; + break; + case '--quality': + if (nextArg === 'best' || nextArg === 'worst') { + options.quality = nextArg; + i++; + } + break; + case '--timeout': + if (!nextArg || nextArg.startsWith('-')) { + console.error('❌ Error: --timeout requires a numeric value'); + process.exit(1); + } + const timeoutSeconds = parseInt(nextArg, 10); + if (isNaN(timeoutSeconds) || timeoutSeconds <= 0) { + console.error('❌ Error: --timeout must be a positive number'); + process.exit(1); + } + options.timeout = timeoutSeconds * 1000; + i++; + break; + case '--port': + if (!nextArg || nextArg.startsWith('-')) { + console.error('❌ Error: --port requires a numeric value'); + process.exit(1); + } + const port = parseInt(nextArg, 10); + if (isNaN(port) || port <= 0) { + console.error('❌ Error: --port must be a positive number'); + process.exit(1); + } + options.port = port; + i++; + break; + case '--from': + if (!nextArg || nextArg.startsWith('-')) { + console.error('❌ Error: --from requires a time value (e.g., --from 00:30)'); + process.exit(1); + } + if (!/^\d{2}:\d{2}$/.test(nextArg)) { + console.error(`❌ Error: --from must be in MM:SS format (got: ${nextArg})`); + process.exit(1); + } + options.clipFrom = nextArg; + i++; + break; + case '--to': + if (!nextArg || nextArg.startsWith('-')) { + console.error('❌ Error: --to requires a time value (e.g., --to 01:30)'); + process.exit(1); + } + if (!/^\d{2}:\d{2}$/.test(nextArg)) { + console.error(`❌ Error: --to must be in MM:SS format (got: ${nextArg})`); + process.exit(1); + } + options.clipTo = nextArg; + i++; + break; + default: + if (!arg.startsWith('-') && !options.url) { + options.url = arg; + } + break; + } + } + + return options; +} + +async function handleCdpMode(argv: string[]): Promise { + const args = parseCdpArgs(argv); + const commandName = getCommandName(); + const port = args.port || 9222; + + if (!args.url) { + console.error('❌ Error: No URL provided'); + console.error(`\nUsage: ${commandName} cdp [options]`); + console.error(`Run: ${commandName} --help for more information\n`); + process.exit(1); + } + + if (!isValidTwitterUrl(args.url)) { + console.error('❌ Error: Invalid X/Twitter URL'); + console.error('Please provide a valid tweet URL like: https://x.com/user/status/123456\n'); + process.exit(1); + } + + console.log('🎬 x-dl - X/Twitter Video Extractor (CDP mode)\n'); + + const installed = await ensurePlaywrightReady(); + if (!installed) { + console.error('\n❌ Playwright is required. Try: bunx playwright install chromium\n'); + process.exit(1); + } + + let connection = await connectOverCdp(port); + + try { + const extractor = new VideoExtractor({ + timeout: args.timeout, + }); + + let result = await extractor.extract(args.url, connection.page); + + // If login wall detected, trigger login flow and retry + if (!result.videoUrl && result.errorClassification === 'login_wall') { + connection = await handleCdpLogin(connection, port); + result = await extractor.extract(args.url, connection.page); + } + + if (result.error || !result.videoUrl) { + console.error(`\n❌ ${result.error || 'Failed to extract video'}\n`); + process.exit(1); + } + + if (args.urlOnly) { + console.log(`\n${result.videoUrl.url}\n`); + process.exit(0); + } + + let defaultExtension = 'mp4'; + if (result.videoUrl.format === 'm3u8') { + defaultExtension = 'mp4'; + } else if (result.videoUrl.format !== 'unknown') { + defaultExtension = result.videoUrl.format; + } + + const cliOpts: CliOptions = { output: args.output }; + const basePath = getOutputPath(args.url, cliOpts, defaultExtension); + const isClipping = args.clipFrom || args.clipTo; + + if (args.clipFrom && args.clipTo) { + const fromSecs = mmssToSeconds(args.clipFrom); + const toSecs = mmssToSeconds(args.clipTo); + if (toSecs <= fromSecs) { + console.error('❌ Error: --to must be after --from'); + process.exit(1); + } + } + + const outputPath = isClipping + ? path.join(path.dirname(basePath), `${path.basename(basePath, path.extname(basePath))}_clip${path.extname(basePath)}`) + : basePath; + + if (result.videoUrl.format === 'm3u8') { + const { ensureFfmpegReady } = await import('./installer.ts'); + const ffmpegReady = await ensureFfmpegReady(); + + if (!ffmpegReady) { + console.error('\n❌ ffmpeg is required to download HLS (m3u8) videos.'); + console.error('Please install ffmpeg:'); + console.error(' macOS: brew install ffmpeg'); + console.error(' Linux: sudo apt-get install ffmpeg'); + console.error(`\nPlaylist URL:\n${result.videoUrl.url}\n`); + process.exit(1); + } + + try { + const fromSecs = args.clipFrom ? mmssToSeconds(args.clipFrom) : undefined; + const toSecs = args.clipTo ? mmssToSeconds(args.clipTo) : undefined; + const durationSecs = toSecs !== undefined ? toSecs - (fromSecs ?? 0) : undefined; + + await downloadHlsWithFfmpeg({ + playlistUrl: result.videoUrl.url, + outputPath, + clipFromSecs: fromSecs, + clipDurationSecs: durationSecs, + }); + console.log(`\n✅ Video saved to: ${outputPath}\n`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stdout.write('\r\x1b[K'); + console.error(`❌ HLS download failed: ${message}\n`); + process.exit(1); + } + return; + } + + if (isClipping) { + const { ensureFfmpegReady: ensureFfmpegReadyForClip } = await import('./installer.ts'); + const ffmpegReady = await ensureFfmpegReadyForClip(); + if (!ffmpegReady) { + console.error('\n❌ ffmpeg is required to clip videos.'); + console.error('Please install ffmpeg:'); + console.error(' macOS: brew install ffmpeg'); + console.error(' Linux: sudo apt-get install ffmpeg'); + process.exit(1); + } + + const osModule = await import('node:os'); + const fsModule = await import('node:fs'); + const tmpPath = path.join(osModule.tmpdir(), `x-dl-tmp-${Date.now()}.mp4`); + + try { + await downloadVideo({ + url: result.videoUrl.url, + outputPath: tmpPath, + onProgress: (progress, downloaded, total) => { + process.stdout.write( + `\r⏳ Downloading: ${progress.toFixed(1)}% (${formatBytes(downloaded)}/${formatBytes(total)})` + ); + }, + }); + process.stdout.write('\n'); + + await clipLocalFile({ + inputPath: tmpPath, + outputPath, + clipFrom: args.clipFrom, + clipTo: args.clipTo, + }); + + console.log(`\n✅ Video saved to: ${outputPath}\n`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stdout.write('\r\x1b[K'); + console.error(`❌ Failed: ${message}\n`); + if (fsModule.existsSync(tmpPath)) fsModule.unlinkSync(tmpPath); + process.exit(1); + } finally { + if (fsModule.existsSync(tmpPath)) fsModule.unlinkSync(tmpPath); + } + return; + } + + try { + await downloadVideo({ + url: result.videoUrl.url, + outputPath, + onProgress: (progress, downloaded, total) => { + process.stdout.write( + `\r⏳ Progress: ${progress.toFixed(1)}% (${formatBytes(downloaded)}/${formatBytes(total)})` + ); + }, + }); + + console.log(`\n\n✅ Video saved to: ${outputPath}\n`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + process.stdout.write('\r\x1b[K'); + console.error(`❌ Download failed: ${message}\n`); + process.exit(1); + } + } finally { + await connection.cleanup(); + } +} + async function handleInstallMode(args: string[]): Promise { const options: InstallCliOptions = {}; @@ -305,6 +579,11 @@ async function main(): Promise { return; } + if (argv[0] === 'cdp') { + await handleCdpMode(argv.slice(1)); + return; + } + const args = parseArgs(argv); if (!args.url) { From a9684f1392913ad0b6a6a75f6297e09c6d82e0f5 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:17:15 +0000 Subject: [PATCH 07/17] test: add CDP subcommand CLI tests, remove old auth flag tests Co-Authored-By: Claude Opus 4.6 --- test/unit/cli.test.ts | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/unit/cli.test.ts b/test/unit/cli.test.ts index 3bca389..2d7de36 100644 --- a/test/unit/cli.test.ts +++ b/test/unit/cli.test.ts @@ -80,6 +80,67 @@ describe('CLI Commands', () => { }); }); + describe('CDP subcommand', () => { + it('should show CDP info in help output', async () => { + const process = Bun.spawn(['bun', './bin/xld', '--help'], { + cwd: import.meta.dir + '/../../', + stdout: 'pipe', + stderr: 'pipe', + }); + + const output = await new Response(process.stdout).text(); + await process.exited; + + expect(output).toContain('cdp'); + expect(output).toContain('CDP MODE'); + expect(output).toContain('chrome://inspect'); + }); + + it('should error with no URL for cdp subcommand', async () => { + const process = Bun.spawn(['bun', './bin/xld', 'cdp'], { + cwd: import.meta.dir + '/../../', + stdout: 'pipe', + stderr: 'pipe', + }); + + const stderr = await new Response(process.stderr).text(); + const exitCode = await process.exited; + + expect(exitCode).not.toBe(0); + expect(stderr).toContain('No URL provided'); + }); + + it('should error with invalid URL for cdp subcommand', async () => { + const process = Bun.spawn(['bun', './bin/xld', 'cdp', 'https://example.com'], { + cwd: import.meta.dir + '/../../', + stdout: 'pipe', + stderr: 'pipe', + }); + + const stderr = await new Response(process.stderr).text(); + const exitCode = await process.exited; + + expect(exitCode).not.toBe(0); + expect(stderr).toContain('Invalid'); + }); + + it('should not show deprecated auth flags in help', async () => { + const process = Bun.spawn(['bun', './bin/xld', '--help'], { + cwd: import.meta.dir + '/../../', + stdout: 'pipe', + stderr: 'pipe', + }); + + const output = await new Response(process.stdout).text(); + await process.exited; + + expect(output).not.toContain('--profile'); + expect(output).not.toContain('--login'); + expect(output).not.toContain('--verify-auth'); + expect(output).not.toContain('EXPERIMENTAL ALPHA'); + }); + }); + describe('x-dl alias (original name)', () => { it('should spawn x-dl with --help and display usage information', async () => { const process = Bun.spawn(['bun', './bin/x-dl', '--help'], { From 02b419cc7ccee93eb66a959ab16c1828ff7fb1ff Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:18:53 +0000 Subject: [PATCH 08/17] docs: update README with CDP mode, remove deprecated auth docs Co-Authored-By: Claude Opus 4.6 --- README.md | 89 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 5ea01ad..320e434 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Extract videos from X (formerly Twitter) tweets. - ✅ Automatic format selection (highest quality) - ✅ Download videos directly or just get the URL - ✅ Clip videos to a specific time range (`--from` and `--to`) -- ⚠️ Downloading videos from private tweets (experimental alpha features) +- ✅ Download videos from private tweets via CDP mode (connects to your Chrome) - ❌ Windows support ## Quick Install @@ -59,13 +59,13 @@ x-dl https://x.com/user/status/123456 - **Download:** - mp4/webm/gif files: direct download - HLS (m3u8) playlists: downloads via ffmpeg to produce mp4 - - If direct download fails with 401/403 auth errors and `--profile` is used, automatically retries using authenticated Playwright requests + - In CDP mode, uses your Chrome's authenticated session for private tweets - **Clipping:** - `--from` and `--to` (MM:SS format) trim videos to a specific time range - HLS streams are clipped during download with ffmpeg re-encoding - MP4 streams download full video, then clip locally - Clipped files get a `_clip` suffix in the filename -- **Auth:** with `--profile`, Playwright reuses cookies/session from a persistent profile directory +- **Auth:** CDP mode connects to your real Chrome browser, reusing your logged-in session - **ffmpeg:** checked at runtime and auto-installed when possible Examples: @@ -76,11 +76,8 @@ x-dl --url-only https://x.com/WesRoth/status/2013693268190437410 ``` ```bash -# Log in once (interactive browser), saving cookies to a profile dir (alpha) -x-dl --login --profile ~/.x-dl-profile - -# Then extract using the logged-in session -x-dl --profile ~/.x-dl-profile --url-only https://x.com/WesRoth/status/2013693268190437410 +# Download a private tweet using CDP mode (connects to your Chrome) +x-dl cdp https://x.com/user/status/123456 ``` ## Installation @@ -162,8 +159,6 @@ x-dl install --with-deps | `--quality ` | Video quality preference (default: best) | | `--timeout ` | Page load timeout in seconds (default: 30) | | `--headed` | Show browser window for debugging | -| `--profile [dir]` | Use a persistent browser profile for authenticated extraction (default: `~/.x-dl-profile`) | -| `--login` | Open X in a persistent profile and wait for you to log in | | `--from ` | Clip start time in minutes and seconds (e.g. `00:30`) | | `--to ` | Clip end time in minutes and seconds (e.g. `01:30`) | | `--help, -h` | Show help message | @@ -192,15 +187,17 @@ x-dl --url-only https://x.com/user/status/123456 x-dl --headed https://x.com/user/status/123456 ``` -**Login once, then reuse the session (alpha):** +**Download a private tweet via CDP mode:** ```bash -# Log in interactively (creates/uses the profile dir) -x-dl --login --profile ~/.x-dl-profile +# Connects to your Chrome browser (must have remote debugging enabled) +x-dl cdp https://x.com/user/status/123456 -# Extract using the logged-in session -x-dl --profile ~/.x-dl-profile https://x.com/user/status/123456 +# Use a custom debugging port +x-dl cdp --port 9333 https://x.com/user/status/123456 ``` +See [CDP Mode](#cdp-mode-private-tweets) below for setup instructions. + **Custom timeout:** ```bash x-dl --timeout 60 https://x.com/user/status/123456 @@ -282,21 +279,49 @@ When extracting a video, the tool will: ✅ Video saved to: ~/Downloads/Remotion_2013626968386765291_clip.mp4 ``` +## CDP Mode (Private Tweets) + +CDP mode connects to your real Chrome browser via the Chrome DevTools Protocol, using your logged-in session to download private or login-walled tweets. + +### Setup + +1. **Chrome v144+** is required +2. Enable remote debugging in Chrome: + - Open Chrome and go to `chrome://inspect/#remote-debugging` + - Enable incoming debugging connections +3. Run: `x-dl cdp ` + +### How It Works + +- If Chrome is already running with remote debugging, x-dl connects to it +- If Chrome is not running, x-dl launches it headlessly with your profile +- If you're not logged into X/Twitter, x-dl opens Chrome with a login page and waits for you to log in + +### Examples + +```bash +# Download a private tweet +x-dl cdp https://x.com/user/status/123456 + +# Use a custom debugging port +x-dl cdp --port 9333 https://x.com/user/status/123456 + +# Just get the URL +x-dl cdp --url-only https://x.com/user/status/123456 + +# Clip a private tweet +x-dl cdp --from 00:30 --to 01:30 https://x.com/user/status/123456 +``` + +Learn more: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session + ## Limitations -- **Public tweets only**: Private or protected tweets cannot be extracted - **Clipping requires ffmpeg**: `--from` and `--to` require ffmpeg for processing - **Clipping time format**: Times must be in MM:SS format (e.g., `00:30`, not `0:30` or `30`) - -- **Public tweets only**: Private or protected tweets cannot be extracted - **Time-limited URLs**: Video URLs may expire after some time - **Rate limiting**: X may rate-limit excessive requests -- **Login walls**: Use `--login` and `--profile` to extract login-walled tweets (alpha) - -**How to tell if a tweet can be extracted:** -1. Try opening the tweet in an incognito/private browser window -2. If you see a "Sign up" or "Log in" prompt, this tool cannot extract it -3. If the content loads without login, extraction should work +- **CDP mode requires Chrome**: CDP mode needs Google Chrome installed (not Chromium) ## Testing @@ -341,6 +366,7 @@ Use `--headed` mode to see the browser for debugging. │ ├── index.ts # CLI entry point │ ├── extractor.ts # Video extraction logic │ ├── downloader.ts # Download logic (Bun fetch) + │ ├── cdp.ts # Chrome DevTools Protocol connection │ ├── ffmpeg.ts # HLS download via ffmpeg │ ├── installer.ts # Dependency management (Playwright + ffmpeg) │ ├── types.ts # TypeScript interfaces @@ -390,19 +416,16 @@ bun run src/index.ts The tool will verify ffmpeg capabilities automatically. -### Authenticated extraction doesn't work - -- Run `x-dl --login --profile ~/.x-dl-profile` and make sure you can view the tweet in that browser -- Then rerun extraction with `--profile ~/.x-dl-profile` +### CDP mode can't connect to Chrome -Security note: your profile directory contains authentication cookies. +- Make sure Chrome v144+ is installed +- Enable remote debugging: `chrome://inspect/#remote-debugging` +- If Chrome is already running without remote debugging, restart it or use `--port` with a different port +- Check that port 9222 (default) is not blocked by a firewall ### "This tweet is private or protected" -Only public tweets can be extracted. Verify that: -- The account is not private/protected -- You're not trying to access sensitive content -- The tweet is publicly accessible +Use CDP mode to download private tweets: `x-dl cdp `. This uses your Chrome's logged-in session. ### "No video found in this tweet" From a1c30fcc85dec4bdd4a2bc140afee63b66ae14ac Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:32:31 +0000 Subject: [PATCH 09/17] fix: handle Chrome running without remote debugging, clean error messages Co-Authored-By: Claude Opus 4.6 --- src/cdp.ts | 84 ++++++++++++++++++++++++++++++++++------------------ src/index.ts | 9 +++++- 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/src/cdp.ts b/src/cdp.ts index b4a9ada..b485994 100644 --- a/src/cdp.ts +++ b/src/cdp.ts @@ -41,29 +41,53 @@ export interface CdpConnection { cleanup: () => Promise; } +function isChromeRunning(): boolean { + try { + const result = Bun.spawnSync(['pgrep', '-f', 'Google Chrome'], { + stdout: 'pipe', + stderr: 'ignore', + }); + return result.exitCode === 0; + } catch { + return false; + } +} + +async function tryConnectPlaywright( + chromium: typeof import('playwright').chromium, + port: number, + timeoutMs: number = 5000, +): Promise { + try { + return await chromium.connectOverCDP(`http://localhost:${port}`, { timeout: timeoutMs }); + } catch { + return null; + } +} + export async function connectOverCdp(port: number = 9222): Promise { const { chromium } = await import('playwright'); - // Try connecting to an already-running Chrome - try { - const browser = await chromium.connectOverCDP(`http://localhost:${port}`); - const context = browser.contexts()[0] || await browser.newContext(); + // Try connecting to Chrome launched with --remote-debugging-port + const existing = await tryConnectPlaywright(chromium, port); + if (existing) { + const context = existing.contexts()[0] || await existing.newContext(); const page = await context.newPage(); return { - browser, + browser: existing, context, page, launchedByUs: false, cleanup: async () => { await page.close().catch(() => {}); - // Don't close browser — user's Chrome stays running }, }; - } catch { - // Connection failed — try launching Chrome ourselves } + // Check if Chrome is running without --remote-debugging-port + const chromeRunning = isChromeRunning(); + const chromePath = findChromePath(); if (!chromePath) { throw new Error( @@ -72,13 +96,23 @@ export async function connectOverCdp(port: number = 9222): Promise' + ); + } + + // Chrome is not running — launch it headlessly with the user's profile const profileDir = findChromeProfileDir(); - const debuggingPort = port; - // Launch Chrome headlessly with the user's profile const chromeProcess = Bun.spawn([ chromePath, - `--remote-debugging-port=${debuggingPort}`, + `--remote-debugging-port=${port}`, `--user-data-dir=${profileDir}`, '--headless=new', '--no-first-run', @@ -91,23 +125,18 @@ export async function connectOverCdp(port: number = 9222): Promise setTimeout(r, 500)); - } + browser = await tryConnectPlaywright(chromium, port); + if (browser) break; + await new Promise(r => setTimeout(r, 500)); } if (!browser) { chromeProcess.kill(); throw new Error( - `Could not connect to Chrome on port ${port}.\n\n` + - 'To use CDP mode, enable remote debugging in Chrome (v144+):\n' + - ' 1. Open Chrome and go to chrome://inspect/#remote-debugging\n' + - ' 2. Enable incoming debugging connections\n' + - ' 3. Then run: x-dl cdp \n\n' + - 'Learn more: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session' + `Could not launch Chrome with remote debugging on port ${port}.\n\n` + + 'Try launching Chrome manually:\n' + + ` /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=${port}\n\n` + + 'Then run: x-dl cdp ' ); } @@ -157,12 +186,9 @@ export async function handleCdpLogin( // Wait for Chrome to be ready let browser: Browser | null = null; for (let i = 0; i < 20; i++) { - try { - browser = await chromium.connectOverCDP(`http://localhost:${port}`); - break; - } catch { - await new Promise(r => setTimeout(r, 500)); - } + browser = await tryConnectPlaywright(chromium, port); + if (browser) break; + await new Promise(r => setTimeout(r, 500)); } if (!browser) { diff --git a/src/index.ts b/src/index.ts index 7248f81..9a66a89 100644 --- a/src/index.ts +++ b/src/index.ts @@ -384,7 +384,14 @@ async function handleCdpMode(argv: string[]): Promise { process.exit(1); } - let connection = await connectOverCdp(port); + let connection; + try { + connection = await connectOverCdp(port); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`❌ ${message}\n`); + process.exit(1); + } try { const extractor = new VideoExtractor({ From 684b50021b46b8636d6493dd550b56fe7f0d9da8 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:52:46 +0000 Subject: [PATCH 10/17] feat: replace CDP module with launchPersistentContext approach Replace connectOverCDP (which hangs on Chrome 146) with Playwright's launchPersistentContext using channel: 'chrome' and a dedicated profile at ~/.x-dl-chrome-profile. Remove --port flag, update help text and tests. Co-Authored-By: Claude Opus 4.6 --- src/cdp.ts | 245 -------------------------------------- src/index.ts | 36 ++---- src/private.ts | 75 ++++++++++++ test/unit/cdp.test.ts | 28 ----- test/unit/cli.test.ts | 2 +- test/unit/private.test.ts | 10 ++ 6 files changed, 94 insertions(+), 302 deletions(-) delete mode 100644 src/cdp.ts create mode 100644 src/private.ts delete mode 100644 test/unit/cdp.test.ts create mode 100644 test/unit/private.test.ts diff --git a/src/cdp.ts b/src/cdp.ts deleted file mode 100644 index b485994..0000000 --- a/src/cdp.ts +++ /dev/null @@ -1,245 +0,0 @@ -import os from 'node:os'; -import path from 'node:path'; -import fs from 'node:fs'; -import type { Browser, BrowserContext, Page } from 'playwright'; - -const CHROME_PATHS_MACOS = [ - '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', -]; - -const CHROME_PATHS_LINUX = [ - '/usr/bin/google-chrome', - '/usr/bin/google-chrome-stable', - '/usr/bin/chromium-browser', - '/usr/bin/chromium', - '/snap/bin/chromium', -]; - -export function findChromePath(): string | null { - const platform = os.platform(); - const candidates = platform === 'darwin' ? CHROME_PATHS_MACOS : CHROME_PATHS_LINUX; - - for (const p of candidates) { - if (fs.existsSync(p)) return p; - } - return null; -} - -export function findChromeProfileDir(): string { - const platform = os.platform(); - if (platform === 'darwin') { - return path.join(os.homedir(), 'Library', 'Application Support', 'Google', 'Chrome'); - } - return path.join(os.homedir(), '.config', 'google-chrome'); -} - -export interface CdpConnection { - browser: Browser; - context: BrowserContext; - page: Page; - launchedByUs: boolean; - cleanup: () => Promise; -} - -function isChromeRunning(): boolean { - try { - const result = Bun.spawnSync(['pgrep', '-f', 'Google Chrome'], { - stdout: 'pipe', - stderr: 'ignore', - }); - return result.exitCode === 0; - } catch { - return false; - } -} - -async function tryConnectPlaywright( - chromium: typeof import('playwright').chromium, - port: number, - timeoutMs: number = 5000, -): Promise { - try { - return await chromium.connectOverCDP(`http://localhost:${port}`, { timeout: timeoutMs }); - } catch { - return null; - } -} - -export async function connectOverCdp(port: number = 9222): Promise { - const { chromium } = await import('playwright'); - - // Try connecting to Chrome launched with --remote-debugging-port - const existing = await tryConnectPlaywright(chromium, port); - if (existing) { - const context = existing.contexts()[0] || await existing.newContext(); - const page = await context.newPage(); - - return { - browser: existing, - context, - page, - launchedByUs: false, - cleanup: async () => { - await page.close().catch(() => {}); - }, - }; - } - - // Check if Chrome is running without --remote-debugging-port - const chromeRunning = isChromeRunning(); - - const chromePath = findChromePath(); - if (!chromePath) { - throw new Error( - 'Google Chrome not found.\n' + - 'CDP mode requires Google Chrome installed on your system.' - ); - } - - if (chromeRunning) { - // Chrome is running but not with remote debugging on this port. - // We can't reuse the profile (it's locked), so ask the user to restart Chrome. - throw new Error( - 'Chrome is running but not with remote debugging enabled.\n\n' + - 'Please close Chrome and let x-dl launch it, or restart Chrome with:\n' + - ` /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=${port}\n\n` + - 'Then run: x-dl cdp ' - ); - } - - // Chrome is not running — launch it headlessly with the user's profile - const profileDir = findChromeProfileDir(); - - const chromeProcess = Bun.spawn([ - chromePath, - `--remote-debugging-port=${port}`, - `--user-data-dir=${profileDir}`, - '--headless=new', - '--no-first-run', - '--no-default-browser-check', - ], { - stdout: 'ignore', - stderr: 'ignore', - }); - - // Wait for Chrome to be ready - let browser: Browser | null = null; - for (let i = 0; i < 20; i++) { - browser = await tryConnectPlaywright(chromium, port); - if (browser) break; - await new Promise(r => setTimeout(r, 500)); - } - - if (!browser) { - chromeProcess.kill(); - throw new Error( - `Could not launch Chrome with remote debugging on port ${port}.\n\n` + - 'Try launching Chrome manually:\n' + - ` /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=${port}\n\n` + - 'Then run: x-dl cdp ' - ); - } - - const context = browser.contexts()[0] || await browser.newContext(); - const page = await context.newPage(); - - return { - browser, - context, - page, - launchedByUs: true, - cleanup: async () => { - await page.close().catch(() => {}); - await browser!.close().catch(() => {}); - chromeProcess.kill(); - }, - }; -} - -export async function handleCdpLogin( - connection: CdpConnection, - port: number -): Promise { - const { chromium } = await import('playwright'); - - if (connection.launchedByUs) { - // Kill headless instance, relaunch headed - await connection.cleanup(); - - const chromePath = findChromePath()!; - const profileDir = findChromeProfileDir(); - - console.log('🔐 Not logged into X/Twitter. Opening Chrome for login...'); - console.log('⚠️ Don\'t close the Chrome window — log in, and x-dl will continue automatically.'); - - const chromeProcess = Bun.spawn([ - chromePath, - `--remote-debugging-port=${port}`, - `--user-data-dir=${profileDir}`, - '--no-first-run', - '--no-default-browser-check', - ], { - stdout: 'ignore', - stderr: 'ignore', - }); - - // Wait for Chrome to be ready - let browser: Browser | null = null; - for (let i = 0; i < 20; i++) { - browser = await tryConnectPlaywright(chromium, port); - if (browser) break; - await new Promise(r => setTimeout(r, 500)); - } - - if (!browser) { - chromeProcess.kill(); - throw new Error('Failed to relaunch Chrome for login.'); - } - - const context = browser.contexts()[0] || await browser.newContext(); - const page = await context.newPage(); - await page.goto('https://x.com/i/flow/login', { waitUntil: 'domcontentloaded', timeout: 30000 }); - - // Poll for auth_token cookie - await waitForAuthCookie(context); - - console.log('✅ Login detected! Continuing download...'); - - // Close headed, relaunch headless, return new connection - await page.close().catch(() => {}); - await browser.close().catch(() => {}); - chromeProcess.kill(); - - return connectOverCdp(port); - } else { - // Chrome was already running — open a login tab - console.log('🔐 Not logged into X/Twitter. A login tab has been opened in Chrome.'); - - const loginPage = await connection.context.newPage(); - await loginPage.goto('https://x.com/i/flow/login', { waitUntil: 'domcontentloaded', timeout: 30000 }); - - // Poll for auth_token cookie - await waitForAuthCookie(connection.context); - - console.log('✅ Login detected! Continuing download...'); - - await loginPage.close().catch(() => {}); - - // Return a fresh page in the same connection - const page = await connection.context.newPage(); - return { - ...connection, - page, - }; - } -} - -async function waitForAuthCookie(context: BrowserContext, timeoutMs: number = 300000): Promise { - const start = Date.now(); - while (Date.now() - start < timeoutMs) { - const cookies = await context.cookies('https://x.com'); - if (cookies.some(c => c.name === 'auth_token')) return; - await new Promise(r => setTimeout(r, 2000)); - } - throw new Error('Login timed out (5 minutes). Please try again.'); -} diff --git a/src/index.ts b/src/index.ts index 9a66a89..147e685 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import { downloadVideo } from './downloader.ts'; import { ensurePlaywrightReady, runInstall } from './installer.ts'; import { generateFilename, isValidTwitterUrl, parseTweetUrl, formatBytes, hasLoginWall } from './utils.ts'; import { downloadHlsWithFfmpeg, clipLocalFile, mmssToSeconds } from './ffmpeg.ts'; -import { connectOverCdp, handleCdpLogin } from './cdp.ts'; +import { launchPrivateBrowser, handlePrivateLogin } from './private.ts'; interface CliOptions { url?: string; @@ -172,17 +172,12 @@ INSTALL: x-dl install --with-deps Install Chromium + ffmpeg + Linux deps (may require sudo on Linux) CDP MODE (Private Tweets): - ${commandName} cdp Connect to Chrome and download (port 9222) - ${commandName} cdp --port 9333 Use custom debugging port + ${commandName} cdp Use Chrome to download private tweets - Requires Chrome v144+ with remote debugging enabled. - See: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session + First run will open Chrome for you to log in to X/Twitter. + Subsequent runs reuse the saved session (~/.x-dl-chrome-profile). - 1. Open Chrome -> chrome://inspect/#remote-debugging -> Enable - 2. Run: ${commandName} cdp - - If Chrome is not running, x-dl will launch it automatically. - If not logged into X/Twitter, x-dl will open a login page automatically. + Requires Google Chrome installed on your system. BROWSER EXAMPLES: # Use Chrome instead of Chromium @@ -270,7 +265,6 @@ interface CdpCliOptions { urlOnly?: boolean; quality?: 'best' | 'worst'; timeout?: number; - port?: number; clipFrom?: string; clipTo?: string; } @@ -310,19 +304,6 @@ function parseCdpArgs(args: string[]): CdpCliOptions { options.timeout = timeoutSeconds * 1000; i++; break; - case '--port': - if (!nextArg || nextArg.startsWith('-')) { - console.error('❌ Error: --port requires a numeric value'); - process.exit(1); - } - const port = parseInt(nextArg, 10); - if (isNaN(port) || port <= 0) { - console.error('❌ Error: --port must be a positive number'); - process.exit(1); - } - options.port = port; - i++; - break; case '--from': if (!nextArg || nextArg.startsWith('-')) { console.error('❌ Error: --from requires a time value (e.g., --from 00:30)'); @@ -361,7 +342,6 @@ function parseCdpArgs(args: string[]): CdpCliOptions { async function handleCdpMode(argv: string[]): Promise { const args = parseCdpArgs(argv); const commandName = getCommandName(); - const port = args.port || 9222; if (!args.url) { console.error('❌ Error: No URL provided'); @@ -376,7 +356,7 @@ async function handleCdpMode(argv: string[]): Promise { process.exit(1); } - console.log('🎬 x-dl - X/Twitter Video Extractor (CDP mode)\n'); + console.log('🎬 x-dl - X/Twitter Video Extractor (private mode)\n'); const installed = await ensurePlaywrightReady(); if (!installed) { @@ -386,7 +366,7 @@ async function handleCdpMode(argv: string[]): Promise { let connection; try { - connection = await connectOverCdp(port); + connection = await launchPrivateBrowser(); } catch (error) { const message = error instanceof Error ? error.message : String(error); console.error(`❌ ${message}\n`); @@ -402,7 +382,7 @@ async function handleCdpMode(argv: string[]): Promise { // If login wall detected, trigger login flow and retry if (!result.videoUrl && result.errorClassification === 'login_wall') { - connection = await handleCdpLogin(connection, port); + connection = await handlePrivateLogin(connection); result = await extractor.extract(args.url, connection.page); } diff --git a/src/private.ts b/src/private.ts new file mode 100644 index 0000000..eb30b62 --- /dev/null +++ b/src/private.ts @@ -0,0 +1,75 @@ +import path from 'node:path'; +import os from 'node:os'; +import type { BrowserContext, Page } from 'playwright'; + +const DEFAULT_PROFILE_DIR = path.join(os.homedir(), '.x-dl-chrome-profile'); + +export interface PrivateConnection { + context: BrowserContext; + page: Page; + cleanup: () => Promise; +} + +export function getProfileDir(): string { + return DEFAULT_PROFILE_DIR; +} + +export async function launchPrivateBrowser(options?: { + headed?: boolean; +}): Promise { + const { chromium } = await import('playwright'); + + const context = await chromium.launchPersistentContext(DEFAULT_PROFILE_DIR, { + channel: 'chrome', + headless: !(options?.headed), + }); + + const page = await context.newPage(); + + return { + context, + page, + cleanup: async () => { + await page.close().catch(() => {}); + await context.close().catch(() => {}); + }, + }; +} + +export async function handlePrivateLogin( + connection: PrivateConnection +): Promise { + // Close headless, relaunch headed for login + await connection.cleanup(); + + console.log('🔐 Not logged into X/Twitter. Opening Chrome for login...'); + console.log('⚠️ Log in, and x-dl will continue automatically.'); + + const headed = await launchPrivateBrowser({ headed: true }); + + await headed.page.goto('https://x.com/i/flow/login', { + waitUntil: 'domcontentloaded', + timeout: 30000, + }); + + // Poll for auth_token cookie + const start = Date.now(); + const timeoutMs = 300000; // 5 minutes + while (Date.now() - start < timeoutMs) { + const cookies = await headed.context.cookies('https://x.com'); + if (cookies.some(c => c.name === 'auth_token')) break; + await new Promise(r => setTimeout(r, 2000)); + } + + const cookies = await headed.context.cookies('https://x.com'); + if (!cookies.some(c => c.name === 'auth_token')) { + await headed.cleanup(); + throw new Error('Login timed out (5 minutes). Please try again.'); + } + + console.log('✅ Login detected! Continuing download...'); + + // Close headed, relaunch headless — cookies are persisted in the profile dir + await headed.cleanup(); + return launchPrivateBrowser(); +} diff --git a/test/unit/cdp.test.ts b/test/unit/cdp.test.ts deleted file mode 100644 index e57a634..0000000 --- a/test/unit/cdp.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, it, expect } from 'bun:test'; -import { findChromePath, findChromeProfileDir, CdpConnection } from '../../src/cdp.ts'; - -describe('Chrome Detection', () => { - it('should return a path string or null from findChromePath', () => { - const result = findChromePath(); - expect(result === null || typeof result === 'string').toBe(true); - }); - - it('should return a profile dir string from findChromeProfileDir', () => { - const result = findChromeProfileDir(); - expect(typeof result).toBe('string'); - expect(result.length).toBeGreaterThan(0); - }); -}); - -describe('CdpConnection type', () => { - it('should export CdpConnection interface fields', () => { - const dummy: CdpConnection = { - browser: null as any, - context: null as any, - page: null as any, - launchedByUs: false, - cleanup: async () => {}, - }; - expect(dummy.launchedByUs).toBe(false); - }); -}); diff --git a/test/unit/cli.test.ts b/test/unit/cli.test.ts index 2d7de36..f9237b6 100644 --- a/test/unit/cli.test.ts +++ b/test/unit/cli.test.ts @@ -93,7 +93,7 @@ describe('CLI Commands', () => { expect(output).toContain('cdp'); expect(output).toContain('CDP MODE'); - expect(output).toContain('chrome://inspect'); + expect(output).toContain('.x-dl-chrome-profile'); }); it('should error with no URL for cdp subcommand', async () => { diff --git a/test/unit/private.test.ts b/test/unit/private.test.ts new file mode 100644 index 0000000..593c40b --- /dev/null +++ b/test/unit/private.test.ts @@ -0,0 +1,10 @@ +import { describe, it, expect } from 'bun:test'; +import { getProfileDir } from '../../src/private.ts'; + +describe('Private Browser Profile', () => { + it('should return a profile dir string from getProfileDir', () => { + const result = getProfileDir(); + expect(typeof result).toBe('string'); + expect(result).toContain('.x-dl-chrome-profile'); + }); +}); From 086b26e99a8a39c15de4f273b5c3cfee992c5f25 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:53:27 +0000 Subject: [PATCH 11/17] docs: update README for persistent context approach Remove references to --port, chrome://inspect, and remote debugging. Document the new persistent profile flow at ~/.x-dl-chrome-profile. Co-Authored-By: Claude Opus 4.6 --- README.md | 42 ++++++++++++------------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 320e434..df05ad8 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ x-dl https://x.com/user/status/123456 - HLS streams are clipped during download with ffmpeg re-encoding - MP4 streams download full video, then clip locally - Clipped files get a `_clip` suffix in the filename -- **Auth:** CDP mode connects to your real Chrome browser, reusing your logged-in session +- **Auth:** CDP mode uses a persistent Chrome profile, reusing your logged-in session - **ffmpeg:** checked at runtime and auto-installed when possible Examples: @@ -189,11 +189,7 @@ x-dl --headed https://x.com/user/status/123456 **Download a private tweet via CDP mode:** ```bash -# Connects to your Chrome browser (must have remote debugging enabled) x-dl cdp https://x.com/user/status/123456 - -# Use a custom debugging port -x-dl cdp --port 9333 https://x.com/user/status/123456 ``` See [CDP Mode](#cdp-mode-private-tweets) below for setup instructions. @@ -281,21 +277,14 @@ When extracting a video, the tool will: ## CDP Mode (Private Tweets) -CDP mode connects to your real Chrome browser via the Chrome DevTools Protocol, using your logged-in session to download private or login-walled tweets. +CDP mode uses Google Chrome with a dedicated profile to download private or login-walled tweets. ### Setup -1. **Chrome v144+** is required -2. Enable remote debugging in Chrome: - - Open Chrome and go to `chrome://inspect/#remote-debugging` - - Enable incoming debugging connections -3. Run: `x-dl cdp ` - -### How It Works - -- If Chrome is already running with remote debugging, x-dl connects to it -- If Chrome is not running, x-dl launches it headlessly with your profile -- If you're not logged into X/Twitter, x-dl opens Chrome with a login page and waits for you to log in +1. **Google Chrome** must be installed +2. Run: `x-dl cdp ` +3. First time: Chrome opens for you to log in to X/Twitter +4. After login: x-dl downloads the video and saves your session for next time ### Examples @@ -303,17 +292,11 @@ CDP mode connects to your real Chrome browser via the Chrome DevTools Protocol, # Download a private tweet x-dl cdp https://x.com/user/status/123456 -# Use a custom debugging port -x-dl cdp --port 9333 https://x.com/user/status/123456 - # Just get the URL x-dl cdp --url-only https://x.com/user/status/123456 - -# Clip a private tweet -x-dl cdp --from 00:30 --to 01:30 https://x.com/user/status/123456 ``` -Learn more: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session +Session data is stored in `~/.x-dl-chrome-profile`. Delete this directory to log out. ## Limitations @@ -366,7 +349,7 @@ Use `--headed` mode to see the browser for debugging. │ ├── index.ts # CLI entry point │ ├── extractor.ts # Video extraction logic │ ├── downloader.ts # Download logic (Bun fetch) - │ ├── cdp.ts # Chrome DevTools Protocol connection + │ ├── private.ts # Private tweet browser session (persistent Chrome profile) │ ├── ffmpeg.ts # HLS download via ffmpeg │ ├── installer.ts # Dependency management (Playwright + ffmpeg) │ ├── types.ts # TypeScript interfaces @@ -416,12 +399,11 @@ bun run src/index.ts The tool will verify ffmpeg capabilities automatically. -### CDP mode can't connect to Chrome +### CDP mode doesn't work -- Make sure Chrome v144+ is installed -- Enable remote debugging: `chrome://inspect/#remote-debugging` -- If Chrome is already running without remote debugging, restart it or use `--port` with a different port -- Check that port 9222 (default) is not blocked by a firewall +- Make sure Google Chrome is installed (not just Chromium) +- Try deleting `~/.x-dl-chrome-profile` and logging in again +- Use `--headed` if you need to debug: the browser window will stay visible ### "This tweet is private or protected" From 6802570010e3c03105eb1a1685d5e60c29124cbe Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 20:57:57 +0000 Subject: [PATCH 12/17] fix: use real Chrome for login to avoid Twitter automation detection Twitter blocks login in Playwright-controlled browsers. Instead, launch real Chrome (not Playwright) for the login flow using the same profile dir. User logs in normally, closes Chrome, then Playwright picks up the persisted cookies headlessly for extraction. Co-Authored-By: Claude Opus 4.6 --- src/private.ts | 78 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/src/private.ts b/src/private.ts index eb30b62..bccf87a 100644 --- a/src/private.ts +++ b/src/private.ts @@ -1,9 +1,31 @@ import path from 'node:path'; import os from 'node:os'; +import fs from 'node:fs'; import type { BrowserContext, Page } from 'playwright'; const DEFAULT_PROFILE_DIR = path.join(os.homedir(), '.x-dl-chrome-profile'); +const CHROME_PATHS_MACOS = [ + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', +]; + +const CHROME_PATHS_LINUX = [ + '/usr/bin/google-chrome', + '/usr/bin/google-chrome-stable', + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + '/snap/bin/chromium', +]; + +function findChromePath(): string | null { + const platform = os.platform(); + const candidates = platform === 'darwin' ? CHROME_PATHS_MACOS : CHROME_PATHS_LINUX; + for (const p of candidates) { + if (fs.existsSync(p)) return p; + } + return null; +} + export interface PrivateConnection { context: BrowserContext; page: Page; @@ -39,37 +61,47 @@ export async function launchPrivateBrowser(options?: { export async function handlePrivateLogin( connection: PrivateConnection ): Promise { - // Close headless, relaunch headed for login + // Close Playwright so it releases the profile directory lock await connection.cleanup(); - console.log('🔐 Not logged into X/Twitter. Opening Chrome for login...'); - console.log('⚠️ Log in, and x-dl will continue automatically.'); + const chromePath = findChromePath(); + if (!chromePath) { + throw new Error( + 'Google Chrome not found.\n' + + 'CDP mode requires Google Chrome installed on your system.' + ); + } - const headed = await launchPrivateBrowser({ headed: true }); + console.log('🔐 Not logged into X/Twitter. Opening Chrome for login...'); + console.log('⚠️ Log in to X/Twitter, then close Chrome to continue.\n'); - await headed.page.goto('https://x.com/i/flow/login', { - waitUntil: 'domcontentloaded', - timeout: 30000, + // Launch real Chrome (not Playwright-controlled) so Twitter doesn't detect automation + const chromeProcess = Bun.spawn([ + chromePath, + `--user-data-dir=${DEFAULT_PROFILE_DIR}`, + '--no-first-run', + '--no-default-browser-check', + 'https://x.com/i/flow/login', + ], { + stdout: 'ignore', + stderr: 'ignore', }); - // Poll for auth_token cookie - const start = Date.now(); - const timeoutMs = 300000; // 5 minutes - while (Date.now() - start < timeoutMs) { - const cookies = await headed.context.cookies('https://x.com'); - if (cookies.some(c => c.name === 'auth_token')) break; - await new Promise(r => setTimeout(r, 2000)); - } + // Wait for the user to close Chrome + await chromeProcess.exited; - const cookies = await headed.context.cookies('https://x.com'); + console.log('✅ Chrome closed. Checking login status...'); + + // Relaunch via Playwright headless — cookies are persisted in the profile dir + const newConnection = await launchPrivateBrowser(); + + // Verify login succeeded by checking cookies + const cookies = await newConnection.context.cookies('https://x.com'); if (!cookies.some(c => c.name === 'auth_token')) { - await headed.cleanup(); - throw new Error('Login timed out (5 minutes). Please try again.'); + await newConnection.cleanup(); + throw new Error('Login not detected. Please try again.'); } - console.log('✅ Login detected! Continuing download...'); - - // Close headed, relaunch headless — cookies are persisted in the profile dir - await headed.cleanup(); - return launchPrivateBrowser(); + console.log('✅ Login detected! Continuing download...\n'); + return newConnection; } From e149e2487d6100ddd85f7bf235c04ef11bf01f03 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 21:05:40 +0000 Subject: [PATCH 13/17] feat: add x-dl login subcommand to open Chrome for X/Twitter login Opens real Chrome with the dedicated profile dir so the user can log in without Playwright automation detection. Session persists for cdp mode. Co-Authored-By: Claude Opus 4.6 --- src/index.ts | 39 ++++++++++++++++++++++++++++++++++++++- src/private.ts | 2 +- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 147e685..f8b8577 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import { downloadVideo } from './downloader.ts'; import { ensurePlaywrightReady, runInstall } from './installer.ts'; import { generateFilename, isValidTwitterUrl, parseTweetUrl, formatBytes, hasLoginWall } from './utils.ts'; import { downloadHlsWithFfmpeg, clipLocalFile, mmssToSeconds } from './ffmpeg.ts'; -import { launchPrivateBrowser, handlePrivateLogin } from './private.ts'; +import { launchPrivateBrowser, handlePrivateLogin, findChromePath, getProfileDir } from './private.ts'; interface CliOptions { url?: string; @@ -171,6 +171,10 @@ INSTALL: x-dl install Install Playwright Chromium only x-dl install --with-deps Install Chromium + ffmpeg + Linux deps (may require sudo on Linux) +LOGIN: + ${commandName} login Open Chrome to log in to X/Twitter + Session is saved to ~/.x-dl-chrome-profile + CDP MODE (Private Tweets): ${commandName} cdp Use Chrome to download private tweets @@ -558,6 +562,34 @@ async function handleInstallMode(args: string[]): Promise { } } +async function handleLoginMode(): Promise { + const chromePath = findChromePath(); + if (!chromePath) { + console.error('❌ Google Chrome not found.\n'); + console.error('CDP mode requires Google Chrome installed on your system.'); + process.exit(1); + } + + const profileDir = getProfileDir(); + console.log('🔐 Opening Chrome for X/Twitter login...'); + console.log(`📁 Session will be saved to: ${profileDir}`); + console.log('⚠️ Log in, then close Chrome to finish.\n'); + + const chromeProcess = Bun.spawn([ + chromePath, + `--user-data-dir=${profileDir}`, + '--no-first-run', + '--no-default-browser-check', + 'https://x.com/i/flow/login', + ], { + stdout: 'ignore', + stderr: 'ignore', + }); + + await chromeProcess.exited; + console.log('✅ Chrome closed. You can now use: x-dl cdp \n'); +} + async function main(): Promise { const argv = process.argv.slice(2); @@ -566,6 +598,11 @@ async function main(): Promise { return; } + if (argv[0] === 'login') { + await handleLoginMode(); + return; + } + if (argv[0] === 'cdp') { await handleCdpMode(argv.slice(1)); return; diff --git a/src/private.ts b/src/private.ts index bccf87a..a055c65 100644 --- a/src/private.ts +++ b/src/private.ts @@ -17,7 +17,7 @@ const CHROME_PATHS_LINUX = [ '/snap/bin/chromium', ]; -function findChromePath(): string | null { +export function findChromePath(): string | null { const platform = os.platform(); const candidates = platform === 'darwin' ? CHROME_PATHS_MACOS : CHROME_PATHS_LINUX; for (const p of candidates) { From 9a3c5eaa719720bd2c4f461b82af427238c55a26 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 21:14:44 +0000 Subject: [PATCH 14/17] fix: use Playwright with stealth flags for login and extraction Real Chrome and Playwright don't share cookie storage, so using real Chrome for login and Playwright for extraction doesn't work. Instead, use Playwright for both but with --disable-blink-features=AutomationControlled and navigator.webdriver override to avoid Twitter's automation detection. Login subcommand now uses Playwright headed mode. Co-Authored-By: Claude Opus 4.6 --- src/index.ts | 47 ++++++++++++++++++++++--------------- src/private.ts | 63 ++++++++++++++++++++++++-------------------------- 2 files changed, 59 insertions(+), 51 deletions(-) diff --git a/src/index.ts b/src/index.ts index f8b8577..8cb9436 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ import { downloadVideo } from './downloader.ts'; import { ensurePlaywrightReady, runInstall } from './installer.ts'; import { generateFilename, isValidTwitterUrl, parseTweetUrl, formatBytes, hasLoginWall } from './utils.ts'; import { downloadHlsWithFfmpeg, clipLocalFile, mmssToSeconds } from './ffmpeg.ts'; -import { launchPrivateBrowser, handlePrivateLogin, findChromePath, getProfileDir } from './private.ts'; +import { launchPrivateBrowser, handlePrivateLogin, getProfileDir } from './private.ts'; interface CliOptions { url?: string; @@ -563,31 +563,42 @@ async function handleInstallMode(args: string[]): Promise { } async function handleLoginMode(): Promise { - const chromePath = findChromePath(); - if (!chromePath) { - console.error('❌ Google Chrome not found.\n'); - console.error('CDP mode requires Google Chrome installed on your system.'); + const installed = await ensurePlaywrightReady(); + if (!installed) { + console.error('\n❌ Playwright is required. Try: bunx playwright install chromium\n'); process.exit(1); } const profileDir = getProfileDir(); console.log('🔐 Opening Chrome for X/Twitter login...'); console.log(`📁 Session will be saved to: ${profileDir}`); - console.log('⚠️ Log in, then close Chrome to finish.\n'); - - const chromeProcess = Bun.spawn([ - chromePath, - `--user-data-dir=${profileDir}`, - '--no-first-run', - '--no-default-browser-check', - 'https://x.com/i/flow/login', - ], { - stdout: 'ignore', - stderr: 'ignore', + console.log('⚠️ Log in, and x-dl will detect it automatically.\n'); + + const connection = await launchPrivateBrowser({ headed: true }); + + await connection.page.goto('https://x.com/i/flow/login', { + waitUntil: 'domcontentloaded', + timeout: 30000, }); - await chromeProcess.exited; - console.log('✅ Chrome closed. You can now use: x-dl cdp \n'); + // Poll for auth_token cookie (5 min timeout) + const start = Date.now(); + const timeoutMs = 300000; + while (Date.now() - start < timeoutMs) { + const cookies = await connection.context.cookies('https://x.com'); + if (cookies.some(c => c.name === 'auth_token')) break; + await new Promise(r => setTimeout(r, 2000)); + } + + const cookies = await connection.context.cookies('https://x.com'); + await connection.cleanup(); + + if (!cookies.some(c => c.name === 'auth_token')) { + console.error('❌ Login timed out (5 minutes). Please try again.\n'); + process.exit(1); + } + + console.log('✅ Login successful! You can now use: x-dl cdp \n'); } async function main(): Promise { diff --git a/src/private.ts b/src/private.ts index a055c65..3c2105d 100644 --- a/src/private.ts +++ b/src/private.ts @@ -44,9 +44,16 @@ export async function launchPrivateBrowser(options?: { const context = await chromium.launchPersistentContext(DEFAULT_PROFILE_DIR, { channel: 'chrome', headless: !(options?.headed), + args: [ + '--disable-blink-features=AutomationControlled', + ], }); + // Remove navigator.webdriver flag to avoid detection const page = await context.newPage(); + await page.addInitScript(() => { + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + }); return { context, @@ -61,47 +68,37 @@ export async function launchPrivateBrowser(options?: { export async function handlePrivateLogin( connection: PrivateConnection ): Promise { - // Close Playwright so it releases the profile directory lock + // Close headless, relaunch headed for login await connection.cleanup(); - const chromePath = findChromePath(); - if (!chromePath) { - throw new Error( - 'Google Chrome not found.\n' + - 'CDP mode requires Google Chrome installed on your system.' - ); - } - console.log('🔐 Not logged into X/Twitter. Opening Chrome for login...'); - console.log('⚠️ Log in to X/Twitter, then close Chrome to continue.\n'); - - // Launch real Chrome (not Playwright-controlled) so Twitter doesn't detect automation - const chromeProcess = Bun.spawn([ - chromePath, - `--user-data-dir=${DEFAULT_PROFILE_DIR}`, - '--no-first-run', - '--no-default-browser-check', - 'https://x.com/i/flow/login', - ], { - stdout: 'ignore', - stderr: 'ignore', - }); + console.log('⚠️ Log in, and x-dl will continue automatically.'); - // Wait for the user to close Chrome - await chromeProcess.exited; + const headed = await launchPrivateBrowser({ headed: true }); - console.log('✅ Chrome closed. Checking login status...'); + await headed.page.goto('https://x.com/i/flow/login', { + waitUntil: 'domcontentloaded', + timeout: 30000, + }); - // Relaunch via Playwright headless — cookies are persisted in the profile dir - const newConnection = await launchPrivateBrowser(); + // Poll for auth_token cookie + const start = Date.now(); + const timeoutMs = 300000; // 5 minutes + while (Date.now() - start < timeoutMs) { + const cookies = await headed.context.cookies('https://x.com'); + if (cookies.some(c => c.name === 'auth_token')) break; + await new Promise(r => setTimeout(r, 2000)); + } - // Verify login succeeded by checking cookies - const cookies = await newConnection.context.cookies('https://x.com'); + const cookies = await headed.context.cookies('https://x.com'); if (!cookies.some(c => c.name === 'auth_token')) { - await newConnection.cleanup(); - throw new Error('Login not detected. Please try again.'); + await headed.cleanup(); + throw new Error('Login timed out (5 minutes). Please try again.'); } - console.log('✅ Login detected! Continuing download...\n'); - return newConnection; + console.log('✅ Login detected! Continuing download...'); + + // Close headed, relaunch headless — cookies are persisted in the profile dir + await headed.cleanup(); + return launchPrivateBrowser(); } From b57eb1502c8aa4818b8120222f1ba4da5c530f1e Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 21:20:27 +0000 Subject: [PATCH 15/17] docs: add login subcommand to README Co-Authored-By: Claude Opus 4.6 --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index df05ad8..b75449b 100644 --- a/README.md +++ b/README.md @@ -282,9 +282,17 @@ CDP mode uses Google Chrome with a dedicated profile to download private or logi ### Setup 1. **Google Chrome** must be installed -2. Run: `x-dl cdp ` -3. First time: Chrome opens for you to log in to X/Twitter -4. After login: x-dl downloads the video and saves your session for next time +2. Log in first: `x-dl login` +3. Download private tweets: `x-dl cdp ` + +### Login + +```bash +# Open Chrome to log in to X/Twitter (session is saved for future use) +x-dl login +``` + +Chrome opens with the X/Twitter login page. Log in normally — x-dl detects the login automatically and closes the browser. ### Examples From 02b98fdba43d6b866f73b067af6322161ed6b0c2 Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Fri, 27 Mar 2026 21:34:46 +0000 Subject: [PATCH 16/17] fix: ensure browser cleanup on all exit paths in CDP and login modes Replace process.exit() calls inside try blocks with return + exitCode pattern so the finally block always runs connection.cleanup(). Also wrap handleLoginMode in try/finally. Co-Authored-By: Claude Opus 4.6 --- src/index.ts | 61 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/src/index.ts b/src/index.ts index 8cb9436..0913f51 100644 --- a/src/index.ts +++ b/src/index.ts @@ -377,6 +377,7 @@ async function handleCdpMode(argv: string[]): Promise { process.exit(1); } + let exitCode = 0; try { const extractor = new VideoExtractor({ timeout: args.timeout, @@ -392,12 +393,13 @@ async function handleCdpMode(argv: string[]): Promise { if (result.error || !result.videoUrl) { console.error(`\n❌ ${result.error || 'Failed to extract video'}\n`); - process.exit(1); + exitCode = 1; + return; } if (args.urlOnly) { console.log(`\n${result.videoUrl.url}\n`); - process.exit(0); + return; } let defaultExtension = 'mp4'; @@ -416,7 +418,8 @@ async function handleCdpMode(argv: string[]): Promise { const toSecs = mmssToSeconds(args.clipTo); if (toSecs <= fromSecs) { console.error('❌ Error: --to must be after --from'); - process.exit(1); + exitCode = 1; + return; } } @@ -434,7 +437,8 @@ async function handleCdpMode(argv: string[]): Promise { console.error(' macOS: brew install ffmpeg'); console.error(' Linux: sudo apt-get install ffmpeg'); console.error(`\nPlaylist URL:\n${result.videoUrl.url}\n`); - process.exit(1); + exitCode = 1; + return; } try { @@ -453,7 +457,7 @@ async function handleCdpMode(argv: string[]): Promise { const message = error instanceof Error ? error.message : String(error); process.stdout.write('\r\x1b[K'); console.error(`❌ HLS download failed: ${message}\n`); - process.exit(1); + exitCode = 1; } return; } @@ -466,7 +470,8 @@ async function handleCdpMode(argv: string[]): Promise { console.error('Please install ffmpeg:'); console.error(' macOS: brew install ffmpeg'); console.error(' Linux: sudo apt-get install ffmpeg'); - process.exit(1); + exitCode = 1; + return; } const osModule = await import('node:os'); @@ -498,7 +503,7 @@ async function handleCdpMode(argv: string[]): Promise { process.stdout.write('\r\x1b[K'); console.error(`❌ Failed: ${message}\n`); if (fsModule.existsSync(tmpPath)) fsModule.unlinkSync(tmpPath); - process.exit(1); + exitCode = 1; } finally { if (fsModule.existsSync(tmpPath)) fsModule.unlinkSync(tmpPath); } @@ -521,10 +526,11 @@ async function handleCdpMode(argv: string[]): Promise { const message = error instanceof Error ? error.message : String(error); process.stdout.write('\r\x1b[K'); console.error(`❌ Download failed: ${message}\n`); - process.exit(1); + exitCode = 1; } } finally { await connection.cleanup(); + if (exitCode !== 0) process.exit(exitCode); } } @@ -576,29 +582,32 @@ async function handleLoginMode(): Promise { const connection = await launchPrivateBrowser({ headed: true }); - await connection.page.goto('https://x.com/i/flow/login', { - waitUntil: 'domcontentloaded', - timeout: 30000, - }); + try { + await connection.page.goto('https://x.com/i/flow/login', { + waitUntil: 'domcontentloaded', + timeout: 30000, + }); + + // Poll for auth_token cookie (5 min timeout) + const start = Date.now(); + const timeoutMs = 300000; + while (Date.now() - start < timeoutMs) { + const cookies = await connection.context.cookies('https://x.com'); + if (cookies.some(c => c.name === 'auth_token')) break; + await new Promise(r => setTimeout(r, 2000)); + } - // Poll for auth_token cookie (5 min timeout) - const start = Date.now(); - const timeoutMs = 300000; - while (Date.now() - start < timeoutMs) { const cookies = await connection.context.cookies('https://x.com'); - if (cookies.some(c => c.name === 'auth_token')) break; - await new Promise(r => setTimeout(r, 2000)); - } - const cookies = await connection.context.cookies('https://x.com'); - await connection.cleanup(); + if (!cookies.some(c => c.name === 'auth_token')) { + console.error('❌ Login timed out (5 minutes). Please try again.\n'); + process.exit(1); + } - if (!cookies.some(c => c.name === 'auth_token')) { - console.error('❌ Login timed out (5 minutes). Please try again.\n'); - process.exit(1); + console.log('✅ Login successful! You can now use: x-dl cdp \n'); + } finally { + await connection.cleanup(); } - - console.log('✅ Login successful! You can now use: x-dl cdp \n'); } async function main(): Promise { From 56a3a978bd983e0398a9ed8250e074eec1e4445e Mon Sep 17 00:00:00 2001 From: Richard Oliver Bray Date: Sat, 28 Mar 2026 16:04:49 +0000 Subject: [PATCH 17/17] refactor: rename externalPage to authenticatedPage for clarity Co-Authored-By: Claude Opus 4.6 --- src/extractor.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/extractor.ts b/src/extractor.ts index b28fe0f..3896dfb 100644 --- a/src/extractor.ts +++ b/src/extractor.ts @@ -36,7 +36,7 @@ export class VideoExtractor { this.browserExecutablePath = options.browserExecutablePath; } - async extract(url: string, externalPage?: Page): Promise { + async extract(url: string, authenticatedPage?: Page): Promise { console.log(`\ud83c\udfac Extracting video from: ${url}`); if (!isValidTwitterUrl(url)) { @@ -63,11 +63,11 @@ export class VideoExtractor { let browser: Browser | null = null; let context: BrowserContext | null = null; let page: Page | null = null; - const usingExternalPage = !!externalPage; + const usingAuthenticatedPage = !!authenticatedPage; try { - if (usingExternalPage) { - page = externalPage; + if (usingAuthenticatedPage) { + page = authenticatedPage; } else { ({ browser, context, page } = await this.createContextAndPage(chromium)); } @@ -130,7 +130,7 @@ export class VideoExtractor { debugInfo, }; } finally { - if (!usingExternalPage) { + if (!usingAuthenticatedPage) { await this.safeClose({ browser, context }); } }