diff --git a/.gitignore b/.gitignore index 7ed451f..1b739a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store # Dependencies node_modules/ @@ -22,3 +23,6 @@ test-results/ # OpenCode .opencode/ + +# Docs +docs/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a63dac4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,75 @@ +# CLAUDE.md + +## Project overview + +x-dl is a CLI tool that extracts and downloads videos from X/Twitter tweets. It uses Playwright (Chromium) to open tweets, captures video URLs from network requests, and downloads via direct fetch or ffmpeg for HLS streams. + +## Tech stack + +- **Runtime/tooling**: Bun (runtime, package manager, bundler, test runner) +- **Language**: TypeScript +- **Browser automation**: Playwright (Chromium) +- **Video processing**: ffmpeg (for HLS/m3u8 streams) + +## Project structure + +``` +src/ + index.ts # CLI entry point and orchestration + extractor.ts # Playwright-based video URL extraction + downloader.ts # Direct HTTP video download with progress + ffmpeg.ts # HLS download via ffmpeg with spinner + installer.ts # Dependency installation (Playwright, ffmpeg) + utils.ts # URL parsing, filename generation, helpers + types.ts # TypeScript type definitions +bin/ # CLI entry scripts (x-dl, xld) +test/ + unit/ # Unit tests (bun:test) + integration/ # Integration tests with mock server + e2e/ # End-to-end download tests (shell script) +``` + +## Common commands + +```sh +bun install # Install dependencies +bun test # Run all tests +bun test test/unit/ # Unit tests only +bun run dev -- # Run from source +./test/e2e/download.sh # Run e2e download tests (requires network) +``` + +## Building + +```sh +bun run build # Bundle for Bun runtime → dist/ +bun run build:macos-arm64 # Compile standalone binary (Apple Silicon) +bun run build:macos-intel # Compile standalone binary (Intel Mac) +bun run build:linux-x64 # Compile standalone binary (Linux) +``` + +## Local testing with standalone binary + +The compiled binary installs to `~/.local/bin/x-dl`. To test a local build: + +```sh +# Remove the existing binary +rm ~/.local/bin/x-dl + +# Build for Apple Silicon +bun run build:macos-arm64 + +# Install the new binary +cp dist/xld-macos-apple-silicon ~/.local/bin/x-dl + +# Test it +x-dl +``` + +## Code conventions + +- No external logging library — uses `process.stdout.write` for inline progress/spinners and `console.log`/`console.error` for line output +- Spinner animation runs on a fast interval (80ms); file-size polling runs on a slower interval (2s) +- All spinner/progress lines must be cleared (`\r\x1b[K`) before printing errors on every exit path +- Stream readers must be released in `finally` blocks +- Tests use `bun:test` for unit/integration; e2e tests are a standalone shell script diff --git a/src/downloader.ts b/src/downloader.ts index 84893a4..8f23e4f 100644 --- a/src/downloader.ts +++ b/src/downloader.ts @@ -31,23 +31,27 @@ export async function downloadVideo(options: DownloadOptions): Promise { const chunks: Uint8Array[] = []; let downloaded = 0; let lastProgressUpdate = 0; - - while (true) { - const { done, value } = await reader.read(); - - if (done) break; - - chunks.push(value); - downloaded += value.length; - - if (onProgress && total > 0) { - const now = Date.now(); - if (now - lastProgressUpdate > 500 || downloaded === total) { - const progress = (downloaded / total) * 100; - onProgress(progress, downloaded, total); - lastProgressUpdate = now; + + try { + while (true) { + const { done, value } = await reader.read(); + + if (done) break; + + chunks.push(value); + downloaded += value.length; + + if (onProgress && total > 0) { + const now = Date.now(); + if (now - lastProgressUpdate > 500 || downloaded === total) { + const progress = (downloaded / total) * 100; + onProgress(progress, downloaded, total); + lastProgressUpdate = now; + } } } + } finally { + reader.cancel().catch(() => {}); } const blob = new Blob(chunks); diff --git a/src/ffmpeg.ts b/src/ffmpeg.ts index 62e7c0e..bdd332d 100644 --- a/src/ffmpeg.ts +++ b/src/ffmpeg.ts @@ -60,10 +60,12 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis const spinnerChars = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; let spinnerIndex = 0; - const pollInterval = setInterval(() => { + const spinnerInterval = setInterval(() => { process.stdout.write(`\r${spinnerChars[spinnerIndex]} Downloading HLS...`); spinnerIndex = (spinnerIndex + 1) % spinnerChars.length; + }, 80); + const pollInterval = setInterval(() => { try { const now = Date.now(); @@ -75,18 +77,22 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis lastProgressTime = now; } else if (now - lastProgressTime > noProgressTimeoutMs) { + clearInterval(spinnerInterval); clearInterval(pollInterval); clearTimeout(timeoutHandle); rejected = true; ffmpeg.kill(); + process.stdout.write('\r\x1b[K'); reject(new Error(`FFMPEG stuck: no progress for ${noProgressTimeoutMs / 1000} seconds`)); } } else if (now - lastProgressTime > noProgressTimeoutMs) { + clearInterval(spinnerInterval); clearInterval(pollInterval); clearTimeout(timeoutHandle); rejected = true; ffmpeg.kill(); + process.stdout.write('\r\x1b[K'); reject(new Error(`FFMPEG stuck: no progress for ${noProgressTimeoutMs / 1000} seconds`)); } } catch (_err) { @@ -94,13 +100,16 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis }, 2000); const timeoutHandle = setTimeout(() => { + clearInterval(spinnerInterval); clearInterval(pollInterval); rejected = true; ffmpeg.kill(); + process.stdout.write('\r\x1b[K'); reject(new Error(`FFMPEG download timed out after ${timeoutMs / 1000} seconds`)); }, timeoutMs); ffmpeg.on('close', (code) => { + clearInterval(spinnerInterval); clearInterval(pollInterval); clearTimeout(timeoutHandle); @@ -108,16 +117,19 @@ export async function downloadHlsWithFfmpeg(options: DownloadHlsOptions): Promis process.stdout.write('\r✅ HLS download completed\n'); resolve(outputPath); } else if (!rejected) { + process.stdout.write('\r\x1b[K'); const error = stderr.trim() || `ffmpeg exited with code ${code ?? 'null (signal)'}`; reject(new Error(`Failed to download HLS: ${error}`)); } }); ffmpeg.on('error', (err) => { + clearInterval(spinnerInterval); clearInterval(pollInterval); clearTimeout(timeoutHandle); if (!rejected) { rejected = true; + process.stdout.write('\r\x1b[K'); reject(new Error(`Failed to start ffmpeg: ${err.message}`)); } }); diff --git a/src/index.ts b/src/index.ts index 1f226e3..f79bea4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -449,7 +449,8 @@ async function main(): Promise { console.log(`\n✅ Video saved to: ${outputPath}\n`); } catch (error) { const message = error instanceof Error ? error.message : String(error); - console.error(`\n\n❌ HLS download failed: ${message}\n`); + process.stdout.write('\r\x1b[K'); + console.error(`❌ HLS download failed: ${message}\n`); process.exit(1); } return; @@ -478,7 +479,8 @@ async function main(): Promise { process.exit(0); } - console.error(`\n\n❌ Download failed: ${message}\n`); + process.stdout.write('\r\x1b[K'); + console.error(`❌ Download failed: ${message}\n`); process.exit(1); } } diff --git a/test/e2e/download.sh b/test/e2e/download.sh new file mode 100755 index 0000000..d9a1f96 --- /dev/null +++ b/test/e2e/download.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# End-to-end download tests for x-dl +# Usage: ./test/e2e/download.sh +# +# Requires: bun, ffmpeg, Playwright Chromium installed +# These tests hit real X/Twitter URLs so they need network access. +# Note: Many tweets require authentication; tests use known public HLS tweets. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BIN="bun run $SCRIPT_DIR/../../bin/x-dl" +TMPDIR_BASE=$(mktemp -d) +PASS=0 +FAIL=0 + +# Known working public tweet with HLS video +PUBLIC_HLS_URL="https://x.com/thorstenball/status/2022310010391302259" + +cleanup() { + rm -rf "$TMPDIR_BASE" +} +trap cleanup EXIT + +pass() { + echo " ✅ PASS: $1" + PASS=$((PASS + 1)) +} + +fail() { + echo " ❌ FAIL: $1 — $2" + FAIL=$((FAIL + 1)) +} + +run_test() { + local name="$1" + shift + echo "" + echo "━━━ $name ━━━" + "$@" +} + +# ────────────────────────────────────────────── +# Test 1: Download HLS tweet via ffmpeg +# ────────────────────────────────────────────── +test_hls_download() { + local outdir="$TMPDIR_BASE/hls" + mkdir -p "$outdir" + + local output + output=$($BIN "$PUBLIC_HLS_URL" -o "$outdir/hls.mp4" 2>&1) || true + + if [[ -f "$outdir/hls.mp4" ]]; then + local size + size=$(stat -f%z "$outdir/hls.mp4" 2>/dev/null || stat -c%s "$outdir/hls.mp4" 2>/dev/null) + if [[ "$size" -gt 100000 ]]; then + pass "HLS video downloaded (${size} bytes)" + else + fail "HLS video too small" "${size} bytes" + fi + else + fail "HLS file not created" "$(echo "$output" | tail -3)" + fi + + # Verify spinner resolved with completion message + if echo "$output" | grep -q 'HLS download completed'; then + pass "HLS spinner resolved with completion message" + else + fail "HLS spinner did not show completion" "$(echo "$output" | tail -3)" + fi + + # Verify final output contains saved message + if echo "$output" | grep -qE '✅.*saved'; then + pass "Output shows saved confirmation" + else + fail "Missing saved confirmation" "$(echo "$output" | tail -3)" + fi +} + +# ────────────────────────────────────────────── +# Test 2: Custom output path with -o flag +# ────────────────────────────────────────────── +test_custom_output_path() { + local outdir="$TMPDIR_BASE/custom" + mkdir -p "$outdir" + + local output + output=$($BIN "$PUBLIC_HLS_URL" -o "$outdir/my_custom_video.mp4" 2>&1) || true + + if [[ -f "$outdir/my_custom_video.mp4" ]]; then + pass "Custom output path works" + else + fail "Custom output path file not found" "$(echo "$output" | tail -3)" + fi +} + +# ────────────────────────────────────────────── +# Test 3: Invalid URL shows error, no hanging +# ────────────────────────────────────────────── +test_invalid_url() { + local url="https://x.com/nobody/status/99999999999999999" + local outdir="$TMPDIR_BASE/invalid" + mkdir -p "$outdir" + + local output + local exit_code=0 + output=$($BIN "$url" -o "$outdir/bad.mp4" 2>&1) || exit_code=$? + + if [[ "$exit_code" -ne 0 ]]; then + pass "Invalid URL exits with non-zero code ($exit_code)" + else + fail "Invalid URL should fail" "exited 0" + fi + + # Ensure no spinner characters left in final output line + local last_line + last_line=$(echo "$output" | tail -1) + if echo "$last_line" | grep -qE '[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏].*Downloading'; then + fail "Spinner frame leaked into final output" "$last_line" + else + pass "No spinner leak in output" + fi +} + +# ────────────────────────────────────────────── +# Test 4: Not a video tweet shows clear error +# ────────────────────────────────────────────── +test_not_a_video() { + # A text-only tweet (Elon's first tweet) + local url="https://x.com/elonmusk/status/1" + local outdir="$TMPDIR_BASE/novideo" + mkdir -p "$outdir" + + local output + local exit_code=0 + output=$($BIN "$url" -o "$outdir/nope.mp4" 2>&1) || exit_code=$? + + if [[ "$exit_code" -ne 0 ]]; then + pass "Non-video tweet exits with error" + else + fail "Non-video tweet should fail" "exited 0" + fi +} + +# ────────────────────────────────────────────── +# Test 5: No orphan ffmpeg processes after download +# ────────────────────────────────────────────── +test_clean_exit() { + # Check that previous HLS tests didn't leave orphan ffmpeg processes + sleep 2 + local orphans + orphans=$(pgrep -cf 'ffmpeg.*m3u8' 2>/dev/null || echo "0") + + if [[ "$orphans" -eq 0 ]]; then + pass "No orphan ffmpeg processes" + else + fail "Orphan ffmpeg processes detected" "count=$orphans" + fi +} + +# ────────────────────────────────────────────── +# Run all tests +# ────────────────────────────────────────────── +echo "🧪 x-dl end-to-end tests" +echo " tmp dir: $TMPDIR_BASE" + +run_test "HLS (ffmpeg) download" test_hls_download +run_test "Custom output path" test_custom_output_path +run_test "Invalid URL handling" test_invalid_url +run_test "Non-video tweet" test_not_a_video +run_test "Clean exit (no orphans)" test_clean_exit + +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━" +echo "Results: $PASS passed, $FAIL failed" +echo "━━━━━━━━━━━━━━━━━━━━━━━" + +if [[ "$FAIL" -gt 0 ]]; then + exit 1 +fi