Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
.DS_Store
# Dependencies
node_modules/

Expand All @@ -22,3 +23,6 @@ test-results/

# OpenCode
.opencode/

# Docs
docs/
75 changes: 75 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# CLAUDE.md

## Project overview

x-dl is a CLI tool that extracts and downloads videos from X/Twitter tweets. It uses Playwright (Chromium) to open tweets, captures video URLs from network requests, and downloads via direct fetch or ffmpeg for HLS streams.

## Tech stack

- **Runtime/tooling**: Bun (runtime, package manager, bundler, test runner)
- **Language**: TypeScript
- **Browser automation**: Playwright (Chromium)
- **Video processing**: ffmpeg (for HLS/m3u8 streams)

## Project structure

```
src/
index.ts # CLI entry point and orchestration
extractor.ts # Playwright-based video URL extraction
downloader.ts # Direct HTTP video download with progress
ffmpeg.ts # HLS download via ffmpeg with spinner
installer.ts # Dependency installation (Playwright, ffmpeg)
utils.ts # URL parsing, filename generation, helpers
types.ts # TypeScript type definitions
bin/ # CLI entry scripts (x-dl, xld)
test/
unit/ # Unit tests (bun:test)
integration/ # Integration tests with mock server
e2e/ # End-to-end download tests (shell script)
```

## Common commands

```sh
bun install # Install dependencies
bun test # Run all tests
bun test test/unit/ # Unit tests only
bun run dev -- <url> # Run from source
./test/e2e/download.sh # Run e2e download tests (requires network)
```

## Building

```sh
bun run build # Bundle for Bun runtime → dist/
bun run build:macos-arm64 # Compile standalone binary (Apple Silicon)
bun run build:macos-intel # Compile standalone binary (Intel Mac)
bun run build:linux-x64 # Compile standalone binary (Linux)
```

## Local testing with standalone binary

The compiled binary installs to `~/.local/bin/x-dl`. To test a local build:

```sh
# Remove the existing binary
rm ~/.local/bin/x-dl

# Build for Apple Silicon
bun run build:macos-arm64

# Install the new binary
cp dist/xld-macos-apple-silicon ~/.local/bin/x-dl

# Test it
x-dl <tweet-url>
```

## Code conventions

- No external logging library — uses `process.stdout.write` for inline progress/spinners and `console.log`/`console.error` for line output
- Spinner animation runs on a fast interval (80ms); file-size polling runs on a slower interval (2s)
- All spinner/progress lines must be cleared (`\r\x1b[K`) before printing errors on every exit path
- Stream readers must be released in `finally` blocks
- Tests use `bun:test` for unit/integration; e2e tests are a standalone shell script
34 changes: 19 additions & 15 deletions src/downloader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,23 +31,27 @@ export async function downloadVideo(options: DownloadOptions): Promise<string> {
const chunks: Uint8Array[] = [];
let downloaded = 0;
let lastProgressUpdate = 0;

while (true) {
const { done, value } = await reader.read();

if (done) break;

chunks.push(value);
downloaded += value.length;

if (onProgress && total > 0) {
const now = Date.now();
if (now - lastProgressUpdate > 500 || downloaded === total) {
const progress = (downloaded / total) * 100;
onProgress(progress, downloaded, total);
lastProgressUpdate = now;

try {
while (true) {
const { done, value } = await reader.read();

if (done) break;

chunks.push(value);
downloaded += value.length;

if (onProgress && total > 0) {
const now = Date.now();
if (now - lastProgressUpdate > 500 || downloaded === total) {
const progress = (downloaded / total) * 100;
onProgress(progress, downloaded, total);
lastProgressUpdate = now;
}
}
}
} finally {
reader.cancel().catch(() => {});
}

const blob = new Blob(chunks);
Expand Down
14 changes: 13 additions & 1 deletion src/ffmpeg.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,12 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis
const spinnerChars = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
let spinnerIndex = 0;

const pollInterval = setInterval(() => {
const spinnerInterval = setInterval(() => {
process.stdout.write(`\r${spinnerChars[spinnerIndex]} Downloading HLS...`);
spinnerIndex = (spinnerIndex + 1) % spinnerChars.length;
}, 80);

const pollInterval = setInterval(() => {
try {
const now = Date.now();

Expand All @@ -75,49 +77,59 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis
lastProgressTime = now;
}
else if (now - lastProgressTime > noProgressTimeoutMs) {
clearInterval(spinnerInterval);
clearInterval(pollInterval);
clearTimeout(timeoutHandle);
rejected = true;
ffmpeg.kill();
process.stdout.write('\r\x1b[K');
reject(new Error(`FFMPEG stuck: no progress for ${noProgressTimeoutMs / 1000} seconds`));
}
}
else if (now - lastProgressTime > noProgressTimeoutMs) {
clearInterval(spinnerInterval);
clearInterval(pollInterval);
clearTimeout(timeoutHandle);
rejected = true;
ffmpeg.kill();
process.stdout.write('\r\x1b[K');
reject(new Error(`FFMPEG stuck: no progress for ${noProgressTimeoutMs / 1000} seconds`));
}
} catch (_err) {
}
}, 2000);

const timeoutHandle = setTimeout(() => {
clearInterval(spinnerInterval);
clearInterval(pollInterval);
rejected = true;
ffmpeg.kill();
process.stdout.write('\r\x1b[K');
reject(new Error(`FFMPEG download timed out after ${timeoutMs / 1000} seconds`));
}, timeoutMs);

ffmpeg.on('close', (code) => {
clearInterval(spinnerInterval);
clearInterval(pollInterval);
clearTimeout(timeoutHandle);

if (code === 0 && !rejected) {
process.stdout.write('\r✅ HLS download completed\n');
resolve(outputPath);
} else if (!rejected) {
process.stdout.write('\r\x1b[K');
const error = stderr.trim() || `ffmpeg exited with code ${code ?? 'null (signal)'}`;
reject(new Error(`Failed to download HLS: ${error}`));
}
});

ffmpeg.on('error', (err) => {
clearInterval(spinnerInterval);
clearInterval(pollInterval);
clearTimeout(timeoutHandle);
if (!rejected) {
rejected = true;
process.stdout.write('\r\x1b[K');
reject(new Error(`Failed to start ffmpeg: ${err.message}`));
}
});
Expand Down
6 changes: 4 additions & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,8 @@ async function main(): Promise<void> {
console.log(`\n✅ Video saved to: ${outputPath}\n`);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
console.error(`\n\n❌ HLS download failed: ${message}\n`);
process.stdout.write('\r\x1b[K');
console.error(`❌ HLS download failed: ${message}\n`);
process.exit(1);
}
return;
Expand Down Expand Up @@ -478,7 +479,8 @@ async function main(): Promise<void> {
process.exit(0);
}

console.error(`\n\n❌ Download failed: ${message}\n`);
process.stdout.write('\r\x1b[K');
console.error(`❌ Download failed: ${message}\n`);
process.exit(1);
}
}
Expand Down
Loading