diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8786b8c..b17e347 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -23,10 +23,8 @@ jobs: - run: pnpm install --frozen-lockfile env: NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} - - run: pnpm --filter @spool-lab/connector-sdk build - run: pnpm --filter @spool-lab/core build - run: pnpm --filter @spool-lab/core test - - run: pnpm --filter @spool-lab/connector-sdk test - run: pnpm --filter @spool-lab/cli build - run: pnpm --filter @spool-lab/cli test @@ -52,9 +50,6 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} run: pnpm install --frozen-lockfile - - name: Build SDK - run: pnpm --filter @spool-lab/connector-sdk build - - name: Build core run: pnpm --filter @spool-lab/core build diff --git a/packages/connector-sdk/README.md b/packages/connector-sdk/README.md deleted file mode 100644 index 6108a26..0000000 --- a/packages/connector-sdk/README.md +++ /dev/null @@ -1,239 +0,0 @@ -# @spool-lab/connector-sdk - -The plugin contract for [Spool](https://spool.pro) connectors. A Spool connector is a small npm package that knows how to pull items from one source — a remote API, a browser session, a local database, a CLI tool — and hand them to Spool's sync engine as `CapturedItem`s. The host app indexes them, makes them searchable, and feeds them to AI agents. - -This package is zero-dependency types + a handful of helpers. You depend on it to write a connector; the Spool app provides the runtime implementations of every capability. - -## Minimal connector - -Three files and ~40 lines of code. - -**`package.json`** — your connector is identified by `spool.type: "connector"`: - -```json -{ - "name": "@you/connector-example", - "version": "0.1.0", - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": ["dist"], - "peerDependencies": { - "@spool-lab/connector-sdk": "^0.1.0" - }, - "spool": { - "type": "connector", - "connectors": [ - { - "id": "example", - "platform": "example", - "label": "Example", - "description": "One line about what this captures", - "color": "#000000", - "ephemeral": false, - "capabilities": ["fetch", "log"] - } - ] - } -} -``` - -**`src/index.ts`** — implement `Connector`: - -```ts -import type { - Connector, - ConnectorCapabilities, - AuthStatus, - FetchContext, - PageResult, -} from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode } from '@spool-lab/connector-sdk' - -export class ExampleConnector implements Connector { - readonly id = 'example' - readonly platform = 'example' - readonly label = 'Example' - readonly description = 'One line about what this captures' - readonly color = '#000000' - readonly ephemeral = false - - constructor(private readonly caps: ConnectorCapabilities) {} - - async checkAuth(): Promise { - return { ok: true } - } - - async fetchPage(ctx: FetchContext): Promise { - const page = ctx.cursor ? parseInt(ctx.cursor, 10) : 1 - const res = await this.caps.fetch( - `https://example.com/api/items?page=${page}`, - ) - if (!res.ok) { - throw new SyncError(SyncErrorCode.API_UNEXPECTED_STATUS, `status ${res.status}`) - } - const data = await res.json() as Array<{ id: string; title: string; url: string }> - - const items = data.map(d => ({ - url: d.url, - title: d.title, - contentText: d.title, - author: null, - platform: 'example', - platformId: d.id, - contentType: 'post', - thumbnailUrl: null, - metadata: {}, - capturedAt: new Date().toISOString(), - rawJson: JSON.stringify(d), - })) - - // Stop forward sync when we reach a known item - if (ctx.phase === 'forward' && ctx.sinceItemId) { - const idx = items.findIndex(i => i.platformId === ctx.sinceItemId) - if (idx >= 0) return { items: items.slice(0, idx), nextCursor: null } - } - - return { items, nextCursor: items.length === 0 ? null : String(page + 1) } - } -} - -export const connectors = [ExampleConnector] -``` - -**`tsconfig.json`** — emit ESM + d.ts: - -```json -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "outDir": "./dist", - "rootDir": "./src", - "declaration": true, - "strict": true - }, - "include": ["src"] -} -``` - -`pnpm build && pnpm publish --access public`. Users install it with: - -``` -spool://connector/install/@you/connector-example -``` - -The app downloads the tarball, extracts it into `~/.spool/connectors/node_modules/`, and — because you're not `@spool-lab/*` — prompts the user to trust the package first. - -## Core contract - -### `Connector` - -The interface every connector implements. Fields (`id`, `platform`, `label`, `description`, `color`, `ephemeral`) are copied from the manifest and used by the UI. Two methods do real work: - -- **`checkAuth()`** returns `{ ok: true }` when you can reach the source, or `{ ok: false, error, message, hint }` when you can't. Also returns a `setup: SetupStep[]` array if the connector uses the prerequisites system (see below). -- **`fetchPage(ctx)`** returns one page of items and a cursor for the next. The sync engine calls this in two phases: `forward` (pull new items newer than the last head anchor) and `backfill` (walk history). Honor `ctx.sinceItemId` in the forward phase to stop early. - -### `CapturedItem` - -The canonical data unit: - -```ts -interface CapturedItem { - url: string - title: string - contentText: string - author: string | null - platform: string - platformId: string | null // dedup key, stable per-platform - contentType: string // 'post' | 'video' | 'repo' | ... - thumbnailUrl: string | null - metadata: Record - capturedAt: string // ISO 8601 - rawJson: string | null // source response for future re-parsing -} -``` - -### Capabilities - -You don't call `fetch`, read cookies, run subprocesses, or touch the filesystem directly. Instead you declare what you need in the manifest and Spool injects implementations via `ConnectorCapabilities`: - -| Capability | Use for | -|---|---| -| `fetch` | Proxy-aware HTTP. Respects the user's system proxy, Electron's net module. | -| `cookies:chrome` | RFC 6265 cookie lookup from Chrome's profile — enables "use my logged-in session" connectors. | -| `exec` | Run an external CLI (`yt-dlp`, `gh`, `opencli`). Returns `{ exitCode, stdout, stderr }`. | -| `sqlite` | Read-only access to a local SQLite database — for connectors that wrap a native app's store. | -| `log` | Structured logging with per-connector prefix. | -| `prerequisites` | Enable the Setup card (see below). | - -Declaring `capabilities: ["fetch", "log"]` in the manifest gates what's available at runtime; requesting a capability you didn't declare terminates the connector. This is the security boundary. - -### Prerequisites (optional) - -If your connector needs a CLI, a browser extension, or a logged-in session before it can work, declare it in the manifest: - -```json -"prerequisites": [ - { - "id": "yt-dlp", - "name": "yt-dlp", - "kind": "cli", - "detect": { - "type": "exec", - "command": "yt-dlp", - "args": ["--version"], - "versionRegex": "(\\d{4}\\.\\d{2}\\.\\d{2})" - }, - "minVersion": "2024.01.01", - "install": { - "kind": "cli", - "command": { - "darwin": "brew install yt-dlp", - "linux": "pip install -U yt-dlp", - "win32": "pip install -U yt-dlp" - } - }, - "docsUrl": "https://github.com/yt-dlp/yt-dlp" - } -] -``` - -Spool's Setup card renders each step with a status pill + one-click install button. Your `checkAuth()` can delegate: - -```ts -import { checkAuthViaPrerequisites } from '@spool-lab/connector-sdk' - -async checkAuth() { - return checkAuthViaPrerequisites(this.caps) -} -``` - -## Helpers - -- `SyncError(code, message)` — throw from `fetchPage` with one of the `SyncErrorCode` values to get proper retry/backoff behavior. -- `parseCliJsonOutput(stdout, platform, contentType)` — converts `yt-dlp`-style one-JSON-per-line output into `CapturedItem[]`. -- `abortableSleep(ms, signal)` — honor `ctx.signal` in retry/backoff loops so cancellation responds quickly. - -## Multi-connector packages - -One package can ship several connectors that share prerequisites (e.g. GitHub Stars + Notifications share `gh auth`). Declare `spool.connectors` as an array with multiple entries and export `connectors: [A, B]` from your entry. - -## Reference - -- Architecture + authoring guide: [`docs/connector-sync-architecture.md`](https://github.com/spool-lab/spool/blob/main/docs/connector-sync-architecture.md) -- First-party examples: [`packages/connectors/*`](https://github.com/spool-lab/spool/tree/main/packages/connectors) — Reddit, GitHub, Hacker News, Twitter Bookmarks, Typeless, Xiaohongshu -- Community example: [`@graydawnc/connector-youtube`](https://www.npmjs.com/package/@graydawnc/connector-youtube) - -## Versioning - -`0.x` while the contract is stabilizing — minor bumps may include breaking changes, patch bumps are safe. - -## License - -MIT - -## Trademark - -Spool™ is a trademark of TypeSafe Limited. The MIT License covers the source code only and does not grant permission to use the Spool name or logo. diff --git a/packages/connector-sdk/package.json b/packages/connector-sdk/package.json deleted file mode 100644 index b065fa5..0000000 --- a/packages/connector-sdk/package.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "@spool-lab/connector-sdk", - "version": "0.1.2", - "description": "Public plugin contract for Spool connectors.", - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "exports": { - ".": { - "import": "./dist/index.js", - "types": "./dist/index.d.ts" - } - }, - "files": [ - "dist", - "README.md" - ], - "keywords": [ - "spool", - "spool-connector", - "plugin-sdk" - ], - "repository": { - "type": "git", - "url": "https://github.com/spool-lab/spool.git", - "directory": "packages/connector-sdk" - }, - "license": "MIT", - "publishConfig": { - "access": "public" - }, - "scripts": { - "build": "tsc", - "dev": "tsc --watch", - "test": "vitest run", - "clean": "rm -rf dist", - "prepack": "pnpm run build" - }, - "devDependencies": { - "@types/node": "^22.15.3", - "typescript": "^5.7.3", - "vitest": "^3.1.2" - } -} diff --git a/packages/connector-sdk/src/capabilities.ts b/packages/connector-sdk/src/capabilities.ts deleted file mode 100644 index a076887..0000000 --- a/packages/connector-sdk/src/capabilities.ts +++ /dev/null @@ -1,161 +0,0 @@ -// ── Fetch ─────────────────────────────────────────────────────────────────── - -/** - * Proxy-aware HTTP fetch. Shape-compatible with the standard `fetch` global. - * Connector authors can use it exactly like `fetch(url, init)`. - * - * Convention (not type-enforced): use only `status`, `ok`, `headers`, - * `text()`, `json()`, `arrayBuffer()` on the Response. Streaming APIs - * (`body` as ReadableStream, FormData bodies) are not guaranteed to work - * across all injected implementations. - */ -export type FetchCapability = typeof globalThis.fetch - -// ── Cookies ───────────────────────────────────────────────────────────────── - -export interface CookiesCapability { - /** Returns decrypted cookies matching the query. */ - get(query: CookieQuery): Promise -} - -export interface CookieQuery { - /** v1 only supports 'chrome'. Future versions may add 'safari' | 'firefox'. */ - browser: 'chrome' - /** Chrome profile directory name; defaults to 'Default'. */ - profile?: string - /** Filter cookies by URL (host + path matching). */ - url: string -} - -export interface Cookie { - name: string - /** Already-decrypted plaintext value. */ - value: string - domain: string - path: string - /** Unix timestamp (seconds); null = session cookie. */ - expires: number | null - secure: boolean - httpOnly: boolean -} - -// ── Log ───────────────────────────────────────────────────────────────────── - -export interface LogCapability { - debug(msg: string, fields?: LogFields): void - info(msg: string, fields?: LogFields): void - warn(msg: string, fields?: LogFields): void - error(msg: string, fields?: LogFields): void - - /** - * Run an async block inside a tracing span. The span is automatically - * closed when the promise settles (including on exception). Span duration - * and attributes are forwarded to the framework's OpenTelemetry exporter - * when one is configured. - */ - span( - name: string, - fn: () => Promise, - opts?: { attributes?: LogFields } - ): Promise -} - -export type LogFields = Record - -// ── SQLite ────────────────────────────────────────────────────────────────── - -/** Values accepted as bind parameters in SQLite queries. */ -export type SqliteBindValue = string | number | bigint | Buffer | null - -/** - * A prepared statement bound to a specific SQL query. - * Generic parameter `T` is the expected row shape — declared at the - * `prepare()` call site, same pattern as `better-sqlite3`. - */ -export interface SqliteStatement { - /** Execute the query and return all matching rows. */ - all(...params: SqliteBindValue[]): T[] - /** Execute the query and return the first matching row, or undefined. */ - get(...params: SqliteBindValue[]): T | undefined -} - -/** - * A readonly handle to a SQLite database file. - * Connectors receive this from `caps.sqlite.openReadonly()`. - */ -export interface SqliteDatabase { - /** Prepare a SQL statement. `T` is the expected row type. */ - prepare(sql: string): SqliteStatement - /** Close the database connection. Must be called when done. */ - close(): void -} - -/** - * Capability for reading local SQLite database files. - * The app injects a `better-sqlite3`-backed implementation; connectors - * see only these interfaces and carry no native dependency. - */ -export interface SqliteCapability { - /** - * Open a database file in readonly mode. - * Throws if the file does not exist or is not a valid SQLite database. - */ - openReadonly(path: string): SqliteDatabase -} - -// ── Exec ─────────────────────────────────────────────────────────────────── - -export interface ExecResult { - stdout: string - stderr: string - exitCode: number -} - -export interface ExecCapability { - run(bin: string, args: string[], opts?: { timeout?: number }): Promise - // TODO: timeout contract is undefined — callers currently sniff the error - // message string for "timeout". A future version should throw a recognizable - // error: either an AbortError (err.name === 'AbortError') or attach a - // structured flag ({ timedOut: true }) so callers can reliably distinguish - // timeout from ENOENT/EACCES without message parsing. -} - -// ── Prerequisites ───────────────────────────────────────────────────────────── - -export interface PrerequisitesCapability { - check(): Promise -} - -// ── Bundle ────────────────────────────────────────────────────────────────── - -/** - * The full set of capabilities passed to a connector's constructor. - * v1.0: 4 capabilities. Future versions may add more via additive, non-breaking - * extension — connectors only receive what they declared in spool.capabilities. - */ -export interface ConnectorCapabilities { - fetch: FetchCapability - cookies: CookiesCapability - log: LogCapability - sqlite: SqliteCapability - exec: ExecCapability - prerequisites?: PrerequisitesCapability -} - -// ── Manifest allowed values ──────────────────────────────────────────────── - -/** - * The complete set of capability strings allowed in a connector's - * `spool.capabilities` manifest field as of SDK v1. Future versions add to - * this set (additive, non-breaking). - */ -export const KNOWN_CAPABILITIES_V1 = [ - 'fetch', - 'cookies:chrome', - 'log', - 'sqlite', - 'exec', - 'prerequisites', -] as const - -export type KnownCapabilityV1 = typeof KNOWN_CAPABILITIES_V1[number] diff --git a/packages/connector-sdk/src/captured-item.ts b/packages/connector-sdk/src/captured-item.ts deleted file mode 100644 index c45323d..0000000 --- a/packages/connector-sdk/src/captured-item.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Canonical data unit flowing through the connector system. - * Every item a connector produces and every item stored in Spool's DB - * starts as a CapturedItem. - */ -export interface CapturedItem { - /** Original URL on the source platform. */ - url: string - /** Display title (truncated for long content). */ - title: string - /** Full text content of the item. */ - contentText: string - /** Author handle or name. null if unknown. */ - author: string | null - /** Platform identifier: 'twitter', 'github', 'reddit', etc. */ - platform: string - /** Platform-specific unique ID used for dedup. null = no stable ID. */ - platformId: string | null - /** Content type for rendering: 'tweet', 'repo', 'video', 'post', 'page'. */ - contentType: string - /** Preview image URL. null if none. */ - thumbnailUrl: string | null - /** Extensible bag for platform-specific structured data. */ - metadata: Record - /** When the item was created on the platform (ISO 8601). */ - capturedAt: string - /** Raw API response for future re-parsing. null to skip storing raw. */ - rawJson: string | null -} diff --git a/packages/connector-sdk/src/cli-parser.test.ts b/packages/connector-sdk/src/cli-parser.test.ts deleted file mode 100644 index cdec07a..0000000 --- a/packages/connector-sdk/src/cli-parser.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { parseCliJsonOutput } from './cli-parser.js' - -describe('parseCliJsonOutput', () => { - it('parses a JSON array of items', () => { - const input = JSON.stringify([ - { id: '1', title: 'Hello', url: 'https://example.com', created_at: '2026-01-01T00:00:00Z' }, - { id: '2', title: 'World', url: 'https://example.com/2', created_at: '2026-01-02T00:00:00Z' }, - ]) - const items = parseCliJsonOutput(input, 'test') - expect(items.length).toBe(2) - expect(items[0].title).toBe('Hello') - expect(items[0].platformId).toBe('1') - expect(items[0].platform).toBe('test') - }) - - it('parses newline-delimited JSON', () => { - const input = '{"id":"1","title":"A","url":"https://a.com"}\n{"id":"2","title":"B","url":"https://b.com"}' - const items = parseCliJsonOutput(input, 'test') - expect(items.length).toBe(2) - }) - - it('extracts fields from flat JSON objects', () => { - const input = JSON.stringify([{ - id: 12345, - full_name: 'user/repo', - html_url: 'https://github.com/user/repo', - description: 'A cool repo', - owner: { login: 'user' }, - created_at: '2026-03-15T10:00:00Z', - }]) - const items = parseCliJsonOutput(input, 'github', 'repo') - expect(items.length).toBe(1) - expect(items[0].url).toBe('https://github.com/user/repo') - expect(items[0].title).toBe('user/repo') - expect(items[0].capturedAt).toBe('2026-03-15T10:00:00Z') - expect(items[0].contentType).toBe('repo') - expect(items[0].author).toBe('user') - }) - - it('returns empty array for empty input', () => { - expect(parseCliJsonOutput('', 'test')).toEqual([]) - expect(parseCliJsonOutput(' \n ', 'test')).toEqual([]) - }) - - it('skips non-JSON lines in NDJSON', () => { - const input = '{"id":"1","title":"ok","url":"https://ok.com"}\nthis is not json\n{"id":"2","title":"also ok","url":"https://ok.com"}' - const items = parseCliJsonOutput(input, 'test') - expect(items.length).toBe(2) - }) - - it('reads url field with fallbacks', () => { - expect(parseCliJsonOutput(JSON.stringify([{ id: '1', url: 'https://a.com' }]), 'test')[0].url).toBe('https://a.com') - expect(parseCliJsonOutput(JSON.stringify([{ id: '1', link: 'https://b.com' }]), 'test')[0].url).toBe('https://b.com') - expect(parseCliJsonOutput(JSON.stringify([{ id: '1', html_url: 'https://c.com' }]), 'test')[0].url).toBe('https://c.com') - }) - - it('uses contentType from caller, defaults to page', () => { - const input = JSON.stringify([{ id: '1', title: 'Post', url: 'https://x.com' }]) - expect(parseCliJsonOutput(input, 'github', 'repo')[0].contentType).toBe('repo') - expect(parseCliJsonOutput(input, 'twitter', 'tweet')[0].contentType).toBe('tweet') - expect(parseCliJsonOutput(input, 'unknown')[0].contentType).toBe('page') - }) -}) diff --git a/packages/connector-sdk/src/cli-parser.ts b/packages/connector-sdk/src/cli-parser.ts deleted file mode 100644 index 65109ed..0000000 --- a/packages/connector-sdk/src/cli-parser.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { CapturedItem } from './captured-item.js' - -interface ParseOptions { - platform: string - contentType?: string -} - -function parseOneItem(raw: Record, opts: ParseOptions): CapturedItem { - const url = String(raw['url'] ?? raw['link'] ?? raw['html_url'] ?? '') - const title = String(raw['title'] ?? raw['name'] ?? raw['full_name'] ?? '') - const contentText = String( - raw['content'] ?? raw['description'] ?? raw['text'] ?? raw['body'] - ?? raw['summary'] ?? '', - ) - - const authorRaw = raw['author'] ?? raw['user'] ?? raw['owner'] ?? null - let author: string | null = null - if (typeof authorRaw === 'string') { - author = authorRaw - } else if (authorRaw && typeof authorRaw === 'object' && 'login' in (authorRaw as any)) { - author = String((authorRaw as any).login) - } - - const capturedAt = String( - raw['created_at'] ?? raw['date'] ?? raw['timestamp'] ?? new Date().toISOString(), - ) - - const platformId = raw['id'] ?? raw['platform_id'] ?? null - const thumbnailUrl = raw['thumbnail'] ?? raw['thumbnail_url'] ?? null - - return { - url, - title, - contentText: contentText || title, - author, - platform: opts.platform, - platformId: platformId != null ? String(platformId) : null, - contentType: opts.contentType ?? 'page', - thumbnailUrl: typeof thumbnailUrl === 'string' ? thumbnailUrl : null, - metadata: {}, - capturedAt, - rawJson: JSON.stringify(raw), - } -} - -export function parseCliJsonOutput(stdout: string, platform: string, contentType?: string): CapturedItem[] { - const opts: ParseOptions = contentType ? { platform, contentType } : { platform } - const trimmed = stdout.trim() - if (!trimmed) return [] - - try { - const parsed = JSON.parse(trimmed) - if (Array.isArray(parsed)) { - return parsed.map(item => parseOneItem(item, opts)) - } - return [parseOneItem(parsed as Record, opts)] - } catch {} - - // Newline-delimited JSON fallback - const items: CapturedItem[] = [] - for (const line of trimmed.split('\n')) { - const l = line.trim() - if (!l) continue - try { - items.push(parseOneItem(JSON.parse(l) as Record, opts)) - } catch {} - } - return items -} diff --git a/packages/connector-sdk/src/connector.ts b/packages/connector-sdk/src/connector.ts deleted file mode 100644 index 41e5dd1..0000000 --- a/packages/connector-sdk/src/connector.ts +++ /dev/null @@ -1,153 +0,0 @@ -import type { SyncErrorCode } from './errors.js' -import type { CapturedItem } from './captured-item.js' - -// ── Prerequisites ───────────────────────────────────────────────────────────── - -export type PrerequisiteKind = 'cli' | 'browser-extension' | 'site-session' - -export interface Prerequisite { - id: string - name: string - kind: PrerequisiteKind - requires?: string[] - detect: Detect - install: Install - minVersion?: string - docsUrl?: string -} - -export interface Detect { - type: 'exec' - command: string - args: string[] - versionRegex?: string - matchStdout?: string - timeoutMs?: number -} - -export type Install = - | { - kind: 'cli' - command: Partial> - requiresManual?: boolean - } - | { - kind: 'browser-extension' - webstoreUrl?: string - manual?: ManualInstall - } - | { - kind: 'site-session' - openUrl: string - } - -/** - * Steps for manual extension install. By convention: - * - steps[0]: download step (rendered with a "Download" button wired to downloadUrl) - * - steps[1]: unzip step (user action, no button) - * - steps[2]: open chrome://extensions step (rendered with an "Open" button) - * - steps[3+]: any additional user actions - */ -export interface ManualInstall { - downloadUrl: string - steps: string[] -} - -export type SetupStatus = 'ok' | 'missing' | 'outdated' | 'error' | 'pending' - -export interface SetupStep { - id: string - label: string - kind: PrerequisiteKind - status: SetupStatus - hint?: string - detectedVersion?: string - minVersion?: string - install?: Install - docsUrl?: string -} - -// ── Auth ───────────────────────────────────────────────────────────────────── - -export interface AuthStatus { - ok: boolean - error?: SyncErrorCode - message?: string - /** Actionable guidance for the user. */ - hint?: string - setup?: SetupStep[] -} - -// ── Page result ────────────────────────────────────────────────────────────── - -export interface PageResult { - items: CapturedItem[] - /** Cursor for the next page. null = no more data. */ - nextCursor: string | null -} - -// ── Fetch context ──────────────────────────────────────────────────────────── - -export interface FetchContext { - /** Pagination cursor. null = start from the newest page. */ - cursor: string | null - /** Platform ID of the newest known item (head anchor). null = no anchor. */ - sinceItemId: string | null - /** Which sync phase is requesting this page. */ - phase: 'forward' | 'backfill' - /** - * AbortSignal that fires when the sync engine wants to stop this sync. - * Connectors should pass it through to their fetch calls and respect it - * in retry/backoff loops (use `abortableSleep(ms, signal)`). - * - * Ignoring this signal is valid — the engine still interrupts at its own - * layer — but cancellation response time will be slower. - * - * Optional until Task 5 wires the engine to always provide it. - */ - signal?: AbortSignal -} - -// ── checkAuthViaPrerequisites helper ───────────────────────────────────────── - -import type { ConnectorCapabilities } from './capabilities.js' - -export async function checkAuthViaPrerequisites(caps: ConnectorCapabilities): Promise { - if (!caps.prerequisites) { - return { - ok: false, - message: 'Prerequisites capability not wired', - } - } - const setup = await caps.prerequisites.check() - return { ok: setup.every(s => s.status === 'ok'), setup } -} - -// ── Connector interface ────────────────────────────────────────────────────── - -export interface Connector { - /** Unique identifier, e.g. 'twitter-bookmarks'. */ - readonly id: string - /** Platform name for grouping, e.g. 'twitter'. */ - readonly platform: string - /** Human-readable label, e.g. 'X Bookmarks'. */ - readonly label: string - /** Short description for the connector picker. */ - readonly description: string - /** UI color for badges/dots. */ - readonly color: string - /** Ephemeral = cache (full-replace), persistent = user-owned (dual-frontier). */ - readonly ephemeral: boolean - - /** Check if authentication / prerequisites are available. */ - checkAuth(opts?: Record): Promise - - /** - * Fetch one page of data. - * - * The sync engine calls this repeatedly, following nextCursor. - * The connector handles API-level retries (429, 5xx) internally. - * If retries are exhausted, throw a SyncError. - */ - fetchPage(ctx: FetchContext): Promise -} diff --git a/packages/connector-sdk/src/errors.ts b/packages/connector-sdk/src/errors.ts deleted file mode 100644 index 0e6fa36..0000000 --- a/packages/connector-sdk/src/errors.ts +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Enumerated sync error codes. - * - * Every error that can occur during sync maps to one of these codes so the UI - * can display a specific, actionable message and track failure patterns. - */ -export enum SyncErrorCode { - // ── Auth ──────────────────────────────────────────────────────────── - AUTH_CHROME_NOT_FOUND = 'AUTH_CHROME_NOT_FOUND', - AUTH_NOT_LOGGED_IN = 'AUTH_NOT_LOGGED_IN', - AUTH_COOKIE_DECRYPT_FAILED = 'AUTH_COOKIE_DECRYPT_FAILED', - AUTH_KEYCHAIN_DENIED = 'AUTH_KEYCHAIN_DENIED', - AUTH_SESSION_EXPIRED = 'AUTH_SESSION_EXPIRED', - AUTH_UNKNOWN = 'AUTH_UNKNOWN', - - // ── Network / API ────────────────────────────────────────────────── - API_RATE_LIMITED = 'API_RATE_LIMITED', - API_SERVER_ERROR = 'API_SERVER_ERROR', - NETWORK_OFFLINE = 'NETWORK_OFFLINE', - NETWORK_TIMEOUT = 'NETWORK_TIMEOUT', - API_PARSE_ERROR = 'API_PARSE_ERROR', - API_UNEXPECTED_STATUS = 'API_UNEXPECTED_STATUS', - - // ── Sync engine ──────────────────────────────────────────────────── - SYNC_MAX_PAGES = 'SYNC_MAX_PAGES', - SYNC_TIMEOUT = 'SYNC_TIMEOUT', - SYNC_CANCELLED = 'SYNC_CANCELLED', - - // ── Storage ──────────────────────────────────────────────────────── - DB_WRITE_ERROR = 'DB_WRITE_ERROR', - - // ── Connector ────────────────────────────────────────────────────── - CONNECTOR_ERROR = 'CONNECTOR_ERROR', -} - -/** - * Human-readable hints for each error code. - * Shown in the sync UI so the user knows what happened and how to fix it. - */ -export const SYNC_ERROR_HINTS: Record = { - [SyncErrorCode.AUTH_CHROME_NOT_FOUND]: - 'Chrome is not installed, or the cookies database was not found. Install Google Chrome and open it at least once.', - [SyncErrorCode.AUTH_NOT_LOGGED_IN]: - 'You are not logged into this platform in Chrome. Open Chrome, log in, then retry.', - [SyncErrorCode.AUTH_COOKIE_DECRYPT_FAILED]: - 'Could not decrypt Chrome cookies. Try closing Chrome completely and retrying. If using a non-default profile, check your connector settings.', - [SyncErrorCode.AUTH_KEYCHAIN_DENIED]: - 'Could not read the Chrome encryption key from macOS Keychain. You may need to grant access in System Settings > Privacy > Keychain.', - [SyncErrorCode.AUTH_SESSION_EXPIRED]: - 'Your session expired during sync. Open Chrome, visit the platform to refresh your login, then retry.', - [SyncErrorCode.AUTH_UNKNOWN]: - 'Authentication failed for an unknown reason. Check that you are logged in via Chrome and retry.', - [SyncErrorCode.API_RATE_LIMITED]: - 'The platform rate-limited requests. Spool will retry automatically after a cooldown.', - [SyncErrorCode.API_SERVER_ERROR]: - 'The platform returned a server error. This is usually temporary — Spool will retry later.', - [SyncErrorCode.NETWORK_OFFLINE]: - 'No internet connection. Spool will sync when connectivity is restored.', - [SyncErrorCode.NETWORK_TIMEOUT]: - 'The request timed out. Check your internet connection. Spool will retry later.', - [SyncErrorCode.API_PARSE_ERROR]: - 'The platform returned an unexpected response format. This may indicate an API change — check for Spool updates.', - [SyncErrorCode.API_UNEXPECTED_STATUS]: - 'The platform returned an unexpected HTTP status. This may be temporary — Spool will retry later.', - [SyncErrorCode.SYNC_MAX_PAGES]: - 'Sync stopped after reaching the page limit. Remaining data will be fetched in the next cycle.', - [SyncErrorCode.SYNC_TIMEOUT]: - 'Sync stopped after reaching the time limit. Remaining data will be fetched in the next cycle.', - [SyncErrorCode.SYNC_CANCELLED]: - 'Sync was cancelled. It will resume in the next cycle.', - [SyncErrorCode.DB_WRITE_ERROR]: - 'Failed to write to the local database. Check disk space and permissions for ~/.spool/', - [SyncErrorCode.CONNECTOR_ERROR]: - 'The connector encountered an error. Check the error details below.', -} - -const RETRYABLE_CODES = new Set([ - SyncErrorCode.API_RATE_LIMITED, - SyncErrorCode.API_SERVER_ERROR, - SyncErrorCode.NETWORK_OFFLINE, - SyncErrorCode.NETWORK_TIMEOUT, - SyncErrorCode.SYNC_MAX_PAGES, - SyncErrorCode.SYNC_TIMEOUT, - SyncErrorCode.SYNC_CANCELLED, -]) - -/** - * Structural shape of a SyncError, independent of class identity. Used by - * {@link isSyncError} so a SyncError thrown from a differently-loaded copy of - * this SDK (e.g. one bundled inside a connector tarball) still gets recognized. - */ -export interface SyncErrorShape { - readonly _tag: 'SyncError' - readonly code: SyncErrorCode - readonly message: string - readonly cause?: unknown -} - -/** - * Structural check for SyncError that survives module duplication. Matches on - * the `_tag` field rather than class identity, so it works across copies of - * this SDK loaded from different paths (host vs. bundled-in-connector). - */ -export function isSyncError(e: unknown): e is SyncErrorShape { - return typeof e === 'object' && e !== null - && (e as { _tag?: unknown })._tag === 'SyncError' -} - -/** - * Error thrown by connectors and the sync engine. Tagged with a machine-readable - * SyncErrorCode so the framework and UI can classify and respond. - * - * This is a plain class, not `Data.TaggedError` — the SDK cannot depend on - * `effect`. `@spool-lab/core` wraps this in an `Effect.Cause` boundary internally. - */ -export class SyncError extends Error { - readonly _tag = 'SyncError' as const - readonly code: SyncErrorCode - override readonly cause?: unknown - - constructor(code: SyncErrorCode, message?: string, cause?: unknown) { - super(message ?? SYNC_ERROR_HINTS[code]) - this.name = 'SyncError' - this.code = code - this.cause = cause - } - - static from(e: unknown): SyncError { - if (e instanceof SyncError) return e - if (isSyncError(e)) return new SyncError(e.code, e.message, e.cause) - return new SyncError( - SyncErrorCode.CONNECTOR_ERROR, - e instanceof Error ? e.message : String(e), - e, - ) - } - - /** Whether this error indicates the connector needs re-authentication. */ - get needsReauth(): boolean { - return this.code.startsWith('AUTH_') - } - - /** Whether this error is transient and the sync can be retried. */ - get retryable(): boolean { - return RETRYABLE_CODES.has(this.code) - } -} diff --git a/packages/connector-sdk/src/index.ts b/packages/connector-sdk/src/index.ts deleted file mode 100644 index 86373bc..0000000 --- a/packages/connector-sdk/src/index.ts +++ /dev/null @@ -1,46 +0,0 @@ -// Public plugin contract types -export type { - Connector, - AuthStatus, - PageResult, - FetchContext, - Prerequisite, - PrerequisiteKind, - Detect, - Install, - ManualInstall, - SetupStep, - SetupStatus, -} from './connector.js' -export { checkAuthViaPrerequisites } from './connector.js' -export type { CapturedItem } from './captured-item.js' -export type { SyncState } from './sync-state.js' - -// Error types -export { SyncError, SyncErrorCode, SYNC_ERROR_HINTS, isSyncError } from './errors.js' - -// Capabilities -export type { - FetchCapability, - CookiesCapability, - Cookie, - CookieQuery, - LogCapability, - LogFields, - SqliteCapability, - SqliteDatabase, - SqliteStatement, - SqliteBindValue, - ConnectorCapabilities, - KnownCapabilityV1, - ExecCapability, - ExecResult, - PrerequisitesCapability, -} from './capabilities.js' -export { KNOWN_CAPABILITIES_V1 } from './capabilities.js' - -// Utilities -export { abortableSleep } from './utils.js' - -// CLI parsing helper -export { parseCliJsonOutput } from './cli-parser.js' diff --git a/packages/connector-sdk/src/sync-state.ts b/packages/connector-sdk/src/sync-state.ts deleted file mode 100644 index fc78421..0000000 --- a/packages/connector-sdk/src/sync-state.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { SyncErrorCode } from './errors.js' - -/** - * Per-connector sync state. Persisted in SQLite by the framework. - * Plugins do not mutate this directly — the framework manages it. - * - * This type is exported from the SDK for reference (docs, type imports) - * but plugins never construct or modify SyncState instances. - */ -export interface SyncState { - connectorId: string - // Head frontier (newest end) - headCursor: string | null - headItemId: string | null - // Tail frontier (oldest end) - tailCursor: string | null - tailComplete: boolean - // Metadata - lastForwardSyncAt: string | null - lastBackfillSyncAt: string | null - totalSynced: number - consecutiveErrors: number - enabled: boolean - /** Per-connector config overrides (e.g. Chrome profile). */ - configJson: Record - /** When the last error occurred (ISO 8601). Used as the backoff base time. - * Cleared on successful sync. */ - lastErrorAt: string | null - /** Last error code, null if last sync succeeded. */ - lastErrorCode: SyncErrorCode | null - /** Last error message for UI display. */ - lastErrorMessage: string | null -} diff --git a/packages/connector-sdk/src/utils.test.ts b/packages/connector-sdk/src/utils.test.ts deleted file mode 100644 index bf74fad..0000000 --- a/packages/connector-sdk/src/utils.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { abortableSleep } from './utils.js' - -describe('abortableSleep', () => { - it('resolves after the specified duration when signal is not aborted', async () => { - const start = Date.now() - await abortableSleep(50) - const elapsed = Date.now() - start - expect(elapsed).toBeGreaterThanOrEqual(45) - expect(elapsed).toBeLessThan(200) - }) - - it('rejects immediately if signal is already aborted', async () => { - const ac = new AbortController() - ac.abort(new Error('pre-aborted')) - await expect(abortableSleep(5000, ac.signal)).rejects.toThrow('pre-aborted') - }) - - it('rejects when signal fires during sleep', async () => { - const ac = new AbortController() - const sleepPromise = abortableSleep(5000, ac.signal) - setTimeout(() => ac.abort(new Error('cancelled')), 20) - const start = Date.now() - await expect(sleepPromise).rejects.toThrow('cancelled') - const elapsed = Date.now() - start - expect(elapsed).toBeLessThan(200) - }) - - it('does not leak timeout when signal fires', async () => { - for (let i = 0; i < 100; i++) { - const ac = new AbortController() - const p = abortableSleep(10_000, ac.signal).catch(() => {}) - ac.abort() - await p - } - }) - - it('does not leak listener when timeout completes', async () => { - const ac = new AbortController() - await abortableSleep(10, ac.signal) - ac.abort() - }) -}) diff --git a/packages/connector-sdk/src/utils.ts b/packages/connector-sdk/src/utils.ts deleted file mode 100644 index 78acc37..0000000 --- a/packages/connector-sdk/src/utils.ts +++ /dev/null @@ -1,17 +0,0 @@ -export function abortableSleep(ms: number, signal?: AbortSignal): Promise { - return new Promise((resolve, reject) => { - if (signal?.aborted) { - return reject(signal.reason) - } - let timeout: ReturnType | undefined - const onAbort = () => { - if (timeout) clearTimeout(timeout) - reject(signal!.reason) - } - timeout = setTimeout(() => { - signal?.removeEventListener('abort', onAbort) - resolve() - }, ms) - signal?.addEventListener('abort', onAbort, { once: true }) - }) -} diff --git a/packages/connector-sdk/tsconfig.json b/packages/connector-sdk/tsconfig.json deleted file mode 100644 index 3709739..0000000 --- a/packages/connector-sdk/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist", - "rootDir": "./src", - "isolatedModules": true, - "forceConsistentCasingInFileNames": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules", "src/**/*.test.ts"] -} diff --git a/packages/connectors/.gitkeep b/packages/connectors/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/packages/connectors/github/package.json b/packages/connectors/github/package.json deleted file mode 100644 index c158905..0000000 --- a/packages/connectors/github/package.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "name": "@spool-lab/connector-github", - "version": "0.1.3", - "description": "GitHub Stars and Notifications for Spool", - "keywords": [ - "spool-connector", - "github" - ], - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist" - ], - "scripts": { - "build": "tsc", - "clean": "rm -rf dist", - "prepack": "pnpm run build" - }, - "dependencies": { - "@spool-lab/connector-sdk": "workspace:^" - }, - "bundledDependencies": [ - "@spool-lab/connector-sdk" - ], - "devDependencies": { - "@types/node": "^22.15.3", - "typescript": "^5.7.3" - }, - "spool": { - "type": "connector", - "connectors": [ - { - "id": "github-stars", - "platform": "github", - "label": "GitHub Stars", - "description": "Repos you recently starred on GitHub", - "color": "#333333", - "ephemeral": false, - "capabilities": [ - "exec", - "log" - ] - }, - { - "id": "github-notifications", - "platform": "github", - "label": "GitHub Notifications", - "description": "Your GitHub notifications", - "color": "#333333", - "ephemeral": true, - "capabilities": [ - "exec", - "log" - ] - } - ] - } -} diff --git a/packages/connectors/github/src/index.ts b/packages/connectors/github/src/index.ts deleted file mode 100644 index 6cef20f..0000000 --- a/packages/connectors/github/src/index.ts +++ /dev/null @@ -1,144 +0,0 @@ -import type { - Connector, - ConnectorCapabilities, - AuthStatus, - PageResult, - FetchContext, - CapturedItem, -} from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode, parseCliJsonOutput } from '@spool-lab/connector-sdk' - -async function checkGhAuth(caps: ConnectorCapabilities): Promise { - try { - const result = await caps.exec.run('gh', ['auth', 'status']) - if (result.exitCode === 0) return { ok: true } - return { - ok: false, - error: SyncErrorCode.AUTH_NOT_LOGGED_IN, - message: 'gh CLI not authenticated', - hint: 'Install GitHub CLI and run: gh auth login', - } - } catch { - return { - ok: false, - error: SyncErrorCode.AUTH_NOT_LOGGED_IN, - message: 'gh CLI not found', - hint: 'Install GitHub CLI (https://cli.github.com) and run: gh auth login', - } - } -} - -export class GitHubStarsConnector implements Connector { - readonly id = 'github-stars' - readonly platform = 'github' - readonly label = 'GitHub Stars' - readonly description = 'Repos you recently starred on GitHub' - readonly color = '#333333' - readonly ephemeral = false - - constructor(private readonly caps: ConnectorCapabilities) {} - - async checkAuth(): Promise { - return checkGhAuth(this.caps) - } - - async fetchPage(ctx: FetchContext): Promise { - const page = ctx.cursor ? parseInt(ctx.cursor, 10) : 1 - - const result = await this.caps.exec.run('gh', [ - 'api', `/user/starred?per_page=100&page=${page}`, - '-H', 'Accept: application/vnd.github.v3.star+json', - ]) - - if (result.exitCode !== 0) { - throw new SyncError(SyncErrorCode.API_UNEXPECTED_STATUS, `gh api failed: ${result.stderr.slice(0, 300)}`) - } - - // Starred repos API returns { starred_at, repo: {...} } — flatten before parsing - let stdout = result.stdout - try { - const parsed = JSON.parse(stdout) - if (Array.isArray(parsed) && parsed[0]?.repo) { - stdout = JSON.stringify(parsed.map((s: any) => ({ - ...s.repo, - url: s.repo.html_url ?? s.repo.url, - created_at: s.starred_at ?? s.repo.created_at, - }))) - } - } catch {} - const items = parseCliJsonOutput(stdout, 'github', 'repo') - - if (items.length === 0) { - return { items: [], nextCursor: null } - } - - // Stop forward sync when we reach a known item - if (ctx.phase === 'forward' && ctx.sinceItemId) { - const anchorIdx = items.findIndex(i => i.platformId === ctx.sinceItemId) - if (anchorIdx >= 0) { - return { items: items.slice(0, anchorIdx), nextCursor: null } - } - } - - return { - items, - nextCursor: items.length >= 100 ? String(page + 1) : null, - } - } -} - -export class GitHubNotificationsConnector implements Connector { - readonly id = 'github-notifications' - readonly platform = 'github' - readonly label = 'GitHub Notifications' - readonly description = 'Your GitHub notifications' - readonly color = '#333333' - readonly ephemeral = true - - constructor(private readonly caps: ConnectorCapabilities) {} - - async checkAuth(): Promise { - return checkGhAuth(this.caps) - } - - async fetchPage(ctx: FetchContext): Promise { - const result = await this.caps.exec.run('gh', ['api', '/notifications']) - - if (result.exitCode !== 0) { - throw new SyncError(SyncErrorCode.API_UNEXPECTED_STATUS, `gh api failed: ${result.stderr.slice(0, 300)}`) - } - - let parsed: any[] - try { - parsed = JSON.parse(result.stdout.trim()) - if (!Array.isArray(parsed)) parsed = [] - } catch { - return { items: [], nextCursor: null } - } - - const items: CapturedItem[] = parsed.map((n: any) => ({ - url: n.subject?.url - ? `https://github.com/${n.repository?.full_name ?? ''}` - : `https://github.com/notifications`, - title: n.subject?.title ?? 'Notification', - contentText: `${n.reason}: ${n.subject?.title ?? ''} (${n.repository?.full_name ?? ''})`, - author: null, - platform: 'github', - platformId: n.id ? String(n.id) : null, - contentType: 'notification', - thumbnailUrl: n.repository?.owner?.avatar_url ?? null, - metadata: { - reason: n.reason, - repository: n.repository?.full_name, - type: n.subject?.type, - unread: n.unread, - }, - capturedAt: n.updated_at ?? new Date().toISOString(), - rawJson: JSON.stringify(n), - })) - - return { items, nextCursor: null } - } -} - -export const connectors = [GitHubStarsConnector, GitHubNotificationsConnector] diff --git a/packages/connectors/github/tsconfig.json b/packages/connectors/github/tsconfig.json deleted file mode 100644 index a84e1f5..0000000 --- a/packages/connectors/github/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "lib": ["es2022"], - "outDir": "./dist", - "rootDir": "./src", - "strict": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connectors/hackernews-hot/package.json b/packages/connectors/hackernews-hot/package.json deleted file mode 100644 index fe14ec0..0000000 --- a/packages/connectors/hackernews-hot/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "@spool-lab/connector-hackernews-hot", - "version": "0.1.2", - "description": "Top stories on Hacker News right now for Spool", - "keywords": [ - "spool-connector", - "hackernews" - ], - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist" - ], - "scripts": { - "prepack": "pnpm run build", - "build": "tsc", - "clean": "rm -rf dist" - }, - "dependencies": { - "@spool-lab/connector-sdk": "workspace:^" - }, - "bundledDependencies": [ - "@spool-lab/connector-sdk" - ], - "devDependencies": { - "@types/node": "^22.15.3", - "typescript": "^5.7.3" - }, - "spool": { - "type": "connector", - "id": "hackernews-hot", - "platform": "hackernews", - "label": "Hacker News Hot", - "description": "Top stories on Hacker News right now", - "color": "#FF6600", - "ephemeral": true, - "capabilities": [ - "fetch", - "log" - ] - } -} diff --git a/packages/connectors/hackernews-hot/src/index.ts b/packages/connectors/hackernews-hot/src/index.ts deleted file mode 100644 index 9dd66a3..0000000 --- a/packages/connectors/hackernews-hot/src/index.ts +++ /dev/null @@ -1,99 +0,0 @@ -import type { - Connector, - ConnectorCapabilities, - AuthStatus, - PageResult, - FetchContext, - CapturedItem, -} from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode } from '@spool-lab/connector-sdk' - -const HN_API = 'https://hacker-news.firebaseio.com/v0' -const TOP_N = 30 - -interface HNStory { - id: number - type: string - by?: string - time: number - title: string - url?: string - text?: string - score: number - descendants?: number -} - -export default class HackerNewsHotConnector implements Connector { - readonly id = 'hackernews-hot' - readonly platform = 'hackernews' - readonly label = 'Hacker News Hot' - readonly description = 'Top stories on Hacker News right now' - readonly color = '#FF6600' - readonly ephemeral = true - - constructor(private readonly caps: ConnectorCapabilities) {} - - async checkAuth(): Promise { - return { ok: true } - } - - async fetchPage(ctx: FetchContext): Promise { - const ids = await this.fetchTopStoryIds(ctx.signal) - const stories = await this.fetchStories(ids.slice(0, TOP_N), ctx.signal) - const items: CapturedItem[] = stories.map(story => ({ - url: story.url ?? `https://news.ycombinator.com/item?id=${story.id}`, - title: story.title, - contentText: story.text ?? story.title, - author: story.by ?? null, - platform: 'hackernews', - platformId: String(story.id), - contentType: 'story', - thumbnailUrl: null, - metadata: { - score: story.score, - descendants: story.descendants ?? 0, - type: story.type, - }, - capturedAt: new Date(story.time * 1000).toISOString(), - rawJson: JSON.stringify(story), - })) - return { items, nextCursor: null } - } - - private async fetchTopStoryIds(signal?: AbortSignal): Promise { - const res = await this.caps.fetch(`${HN_API}/topstories.json`, { signal }) - if (!res.ok) { - throw new SyncError( - res.status >= 500 ? SyncErrorCode.API_SERVER_ERROR : SyncErrorCode.API_UNEXPECTED_STATUS, - `HN API returned ${res.status}`, - ) - } - return await res.json() as number[] - } - - private async fetchStories(ids: number[], signal?: AbortSignal): Promise { - const results = await Promise.allSettled( - ids.map(async (id) => { - const res = await this.caps.fetch(`${HN_API}/item/${id}.json`, { signal }) - if (!res.ok) { - this.caps.log.warn('failed to fetch HN item', { - id, - status: res.status, - error: res.status === 429 ? 'rate limited' : `HTTP ${res.status}`, - }) - return null - } - return await res.json() as HNStory - }), - ) - const stories: HNStory[] = [] - for (const r of results) { - if (r.status === 'fulfilled' && r.value) { - stories.push(r.value) - } else if (r.status === 'rejected') { - this.caps.log.warn('failed to fetch HN item', { error: String(r.reason) }) - } - } - return stories - } -} diff --git a/packages/connectors/hackernews-hot/tsconfig.json b/packages/connectors/hackernews-hot/tsconfig.json deleted file mode 100644 index a84e1f5..0000000 --- a/packages/connectors/hackernews-hot/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "lib": ["es2022"], - "outDir": "./dist", - "rootDir": "./src", - "strict": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connectors/reddit/package.json b/packages/connectors/reddit/package.json deleted file mode 100644 index 5a6856a..0000000 --- a/packages/connectors/reddit/package.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "name": "@spool-lab/connector-reddit", - "version": "0.1.2", - "description": "Reddit Saved and Upvoted posts for Spool", - "keywords": [ - "spool-connector", - "reddit" - ], - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist" - ], - "scripts": { - "build": "tsc", - "clean": "rm -rf dist", - "prepack": "pnpm run build" - }, - "dependencies": { - "@spool-lab/connector-sdk": "workspace:^" - }, - "bundledDependencies": [ - "@spool-lab/connector-sdk" - ], - "devDependencies": { - "@types/node": "^22.15.3", - "typescript": "^5.7.3" - }, - "spool": { - "type": "connector", - "connectors": [ - { - "id": "reddit-saved", - "platform": "reddit", - "label": "Reddit Saved", - "description": "Posts and comments you saved on Reddit", - "color": "#FF4500", - "ephemeral": false, - "capabilities": [ - "fetch", - "cookies:chrome", - "log" - ] - }, - { - "id": "reddit-upvoted", - "platform": "reddit", - "label": "Reddit Upvoted", - "description": "Posts you upvoted on Reddit", - "color": "#FF4500", - "ephemeral": false, - "capabilities": [ - "fetch", - "cookies:chrome", - "log" - ] - } - ] - } -} diff --git a/packages/connectors/reddit/src/fetch.ts b/packages/connectors/reddit/src/fetch.ts deleted file mode 100644 index 61506cf..0000000 --- a/packages/connectors/reddit/src/fetch.ts +++ /dev/null @@ -1,227 +0,0 @@ -import type { FetchCapability, Cookie, CapturedItem } from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode, abortableSleep } from '@spool-lab/connector-sdk' - -const USER_AGENT = - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36' - -const PAGE_SIZE = 100 -const RELEVANT_COOKIE_NAMES = new Set(['reddit_session', 'loid', 'token_v2', 'edgebucket']) - -export interface RedditAuth { - cookieHeader: string -} - -export function buildAuth(cookies: Cookie[]): RedditAuth | null { - const parts: string[] = [] - let hasSession = false - for (const c of cookies) { - if (!RELEVANT_COOKIE_NAMES.has(c.name)) continue - if (c.name === 'reddit_session') hasSession = true - parts.push(`${c.name}=${c.value}`) - } - return hasSession ? { cookieHeader: parts.join('; ') } : null -} - -export interface RedditClient { - cookieHeader: string - fetch: FetchCapability - signal: AbortSignal -} - -interface RedditThing { - kind: string - data: Record -} - -interface RedditListing { - data: { - after: string | null - children: RedditThing[] - } -} - -export interface RedditPage { - items: CapturedItem[] - nextCursor: string | null -} - -function headers(cookieHeader: string): Record { - return { - cookie: cookieHeader, - 'user-agent': USER_AGENT, - accept: 'application/json', - } -} - -async function fetchJson(url: string, client: RedditClient): Promise { - const { cookieHeader, fetch: fetchFn, signal } = client - let lastCause: 'rate-limit' | 'server-error' | null = null - - for (let attempt = 0; attempt < 4; attempt++) { - if (signal.aborted) throw signal.reason - - let response: Response - try { - response = await fetchFn(url, { headers: headers(cookieHeader), signal }) - } catch (err) { - if (signal.aborted) throw signal.reason - const message = err instanceof Error ? err.message : String(err) - if (message.includes('ENOTFOUND') || message.includes('ENETUNREACH')) { - throw new SyncError(SyncErrorCode.NETWORK_OFFLINE, message, err) - } - if (message.includes('ETIMEDOUT') || message.includes('timeout')) { - throw new SyncError(SyncErrorCode.NETWORK_TIMEOUT, message, err) - } - throw new SyncError(SyncErrorCode.CONNECTOR_ERROR, message, err) - } - - if (response.status === 429) { - lastCause = 'rate-limit' - await abortableSleep(Math.min(15 * Math.pow(2, attempt), 120) * 1000, signal) - continue - } - if (response.status >= 500) { - lastCause = 'server-error' - await abortableSleep(5000 * (attempt + 1), signal) - continue - } - if (response.status === 401 || response.status === 403) { - throw new SyncError( - SyncErrorCode.AUTH_SESSION_EXPIRED, - `Reddit returned ${response.status}. Your session may have expired — open reddit.com in Chrome and log in again.`, - ) - } - if (!response.ok) { - const text = await response.text().catch(() => '') - throw new SyncError( - SyncErrorCode.API_UNEXPECTED_STATUS, - `Reddit returned ${response.status}: ${text.slice(0, 300)}`, - ) - } - - try { - return await response.json() - } catch (err) { - throw new SyncError(SyncErrorCode.API_PARSE_ERROR, 'Failed to parse Reddit response as JSON', err) - } - } - - throw new SyncError( - lastCause === 'rate-limit' ? SyncErrorCode.API_RATE_LIMITED : SyncErrorCode.API_SERVER_ERROR, - `${lastCause === 'rate-limit' ? 'Rate limited' : 'Server errors'} after 4 retry attempts.`, - ) -} - -export async function fetchUsername(client: RedditClient): Promise { - const json = (await fetchJson('https://old.reddit.com/api/me.json', client)) as any - const name = json?.data?.name - if (typeof name !== 'string' || !name) { - throw new SyncError( - SyncErrorCode.AUTH_NOT_LOGGED_IN, - 'Reddit did not return a username — you may not be logged in. Open reddit.com in Chrome, log in, then retry.', - ) - } - return name -} - -// Reddit uses sentinel strings like 'self', 'default', 'nsfw', 'spoiler', 'image' -// in the thumbnail field when there is no preview. Filter those out. -function validThumbnail(url: unknown): string | null { - if (typeof url !== 'string') return null - if (!url.startsWith('http')) return null - return url -} - -function thingToItem(thing: RedditThing): CapturedItem | null { - const d = thing.data - const platformId = typeof d.name === 'string' ? d.name : null - if (!platformId) return null - - const permalink = typeof d.permalink === 'string' ? `https://www.reddit.com${d.permalink}` : null - const capturedAt = typeof d.created_utc === 'number' - ? new Date(d.created_utc * 1000).toISOString() - : new Date().toISOString() - const author = typeof d.author === 'string' ? d.author : null - - const baseMetadata = { - subreddit: d.subreddit, - subredditPrefixed: d.subreddit_name_prefixed, - score: d.score, - permalink, - } - - if (thing.kind === 't3') { - const title = typeof d.title === 'string' ? d.title : '(untitled)' - const selftext = typeof d.selftext === 'string' ? d.selftext : '' - const externalUrl = typeof d.url === 'string' ? d.url : null - return { - url: externalUrl ?? permalink ?? `https://www.reddit.com/${platformId}`, - title, - contentText: selftext || title, - author, - platform: 'reddit', - platformId, - contentType: 'post', - thumbnailUrl: validThumbnail(d.thumbnail), - metadata: { - ...baseMetadata, - numComments: d.num_comments, - externalUrl, - isSelf: d.is_self, - over18: d.over_18, - domain: d.domain, - }, - capturedAt, - rawJson: JSON.stringify(thing), - } - } - - if (thing.kind === 't1') { - const body = typeof d.body === 'string' ? d.body : '' - const linkTitle = typeof d.link_title === 'string' ? d.link_title : '' - const title = body.length > 120 ? body.slice(0, 117) + '...' : body || linkTitle || '(comment)' - return { - url: permalink ?? `https://www.reddit.com/${platformId}`, - title, - contentText: body, - author, - platform: 'reddit', - platformId, - contentType: 'comment', - thumbnailUrl: null, - metadata: { - ...baseMetadata, - linkTitle, - linkId: d.link_id, - linkPermalink: d.link_permalink, - }, - capturedAt, - rawJson: JSON.stringify(thing), - } - } - - return null -} - -function parseListing(json: unknown): RedditPage { - const listing = json as RedditListing | undefined - const children = listing?.data?.children ?? [] - const items: CapturedItem[] = [] - for (const thing of children) { - const item = thingToItem(thing) - if (item) items.push(item) - } - return { items, nextCursor: listing?.data?.after ?? null } -} - -export async function fetchListingPage( - listing: 'saved' | 'upvoted', - username: string, - cursor: string | null, - client: RedditClient, -): Promise { - const params = new URLSearchParams({ limit: String(PAGE_SIZE), raw_json: '1' }) - if (cursor) params.set('after', cursor) - const url = `https://old.reddit.com/user/${encodeURIComponent(username)}/${listing}.json?${params}` - return parseListing(await fetchJson(url, client)) -} diff --git a/packages/connectors/reddit/src/index.ts b/packages/connectors/reddit/src/index.ts deleted file mode 100644 index 50ad5ff..0000000 --- a/packages/connectors/reddit/src/index.ts +++ /dev/null @@ -1,104 +0,0 @@ -import type { - Connector, - ConnectorCapabilities, - AuthStatus, - PageResult, - FetchContext, -} from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode } from '@spool-lab/connector-sdk' -import { buildAuth, fetchUsername, fetchListingPage } from './fetch.js' - -interface RedditSession { - cookieHeader: string - username: string -} - -async function readCookieHeader(caps: ConnectorCapabilities): Promise { - const cookies = await caps.cookies.get({ browser: 'chrome', url: 'https://reddit.com' }) - const auth = buildAuth(cookies) - if (!auth) { - throw new SyncError( - SyncErrorCode.AUTH_NOT_LOGGED_IN, - 'No reddit_session cookie found in Chrome. Log into reddit.com in Chrome and retry.', - ) - } - return auth.cookieHeader -} - -abstract class RedditListingConnector implements Connector { - abstract readonly id: string - abstract readonly label: string - abstract readonly description: string - abstract readonly listing: 'saved' | 'upvoted' - - readonly platform = 'reddit' - readonly color = '#FF4500' - readonly ephemeral = false - - private cached: RedditSession | null = null - - constructor(protected readonly caps: ConnectorCapabilities) {} - - async checkAuth(): Promise { - try { - await readCookieHeader(this.caps) - return { ok: true } - } catch (err) { - if (err instanceof SyncError) { - return { ok: false, error: err.code, message: err.message, hint: err.message } - } - return { - ok: false, - error: SyncErrorCode.AUTH_UNKNOWN, - message: err instanceof Error ? err.message : String(err), - hint: 'Check that Chrome is installed and you are logged into reddit.com.', - } - } - } - - async fetchPage(ctx: FetchContext): Promise { - const signal = ctx.signal ?? new AbortController().signal - try { - if (!this.cached) { - const cookieHeader = await readCookieHeader(this.caps) - const client = { cookieHeader, fetch: this.caps.fetch, signal } - this.cached = { cookieHeader, username: await fetchUsername(client) } - } - const client = { cookieHeader: this.cached.cookieHeader, fetch: this.caps.fetch, signal } - - const page = await this.caps.log.span( - 'fetchPage', - () => fetchListingPage(this.listing, this.cached!.username, ctx.cursor, client), - { attributes: { 'reddit.listing': this.listing, 'reddit.phase': ctx.phase, 'reddit.cursor': ctx.cursor ?? 'initial' } }, - ) - - if (ctx.phase === 'forward' && ctx.sinceItemId) { - const anchorIdx = page.items.findIndex(i => i.platformId === ctx.sinceItemId) - if (anchorIdx >= 0) { - return { items: page.items.slice(0, anchorIdx), nextCursor: null } - } - } - - return page - } catch (err) { - if (err instanceof SyncError && err.needsReauth) this.cached = null - throw err - } - } -} - -export class RedditSavedConnector extends RedditListingConnector { - readonly id = 'reddit-saved' - readonly label = 'Reddit Saved' - readonly description = 'Posts and comments you saved on Reddit' - readonly listing = 'saved' -} - -export class RedditUpvotedConnector extends RedditListingConnector { - readonly id = 'reddit-upvoted' - readonly label = 'Reddit Upvoted' - readonly description = 'Posts you upvoted on Reddit' - readonly listing = 'upvoted' -} - -export const connectors = [RedditSavedConnector, RedditUpvotedConnector] diff --git a/packages/connectors/reddit/tsconfig.json b/packages/connectors/reddit/tsconfig.json deleted file mode 100644 index a84e1f5..0000000 --- a/packages/connectors/reddit/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "lib": ["es2022"], - "outDir": "./dist", - "rootDir": "./src", - "strict": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connectors/twitter-bookmarks/.npmignore b/packages/connectors/twitter-bookmarks/.npmignore deleted file mode 100644 index 1c69a1d..0000000 --- a/packages/connectors/twitter-bookmarks/.npmignore +++ /dev/null @@ -1,4 +0,0 @@ -src/ -tsconfig.json -*.log -node_modules/ diff --git a/packages/connectors/twitter-bookmarks/package.json b/packages/connectors/twitter-bookmarks/package.json deleted file mode 100644 index d2cce15..0000000 --- a/packages/connectors/twitter-bookmarks/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "@spool-lab/connector-twitter-bookmarks", - "version": "0.1.1", - "description": "Your saved tweets on X (Twitter Bookmarks) for Spool", - "keywords": [ - "spool-connector" - ], - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist" - ], - "scripts": { - "build": "tsc", - "clean": "rm -rf dist", - "prepack": "pnpm run build" - }, - "dependencies": { - "@spool-lab/connector-sdk": "workspace:^" - }, - "bundledDependencies": [ - "@spool-lab/connector-sdk" - ], - "devDependencies": { - "@types/node": "^22.15.3", - "typescript": "^5.7.3" - }, - "spool": { - "type": "connector", - "id": "twitter-bookmarks", - "platform": "twitter", - "label": "X Bookmarks", - "description": "Your saved tweets on X", - "color": "#1DA1F2", - "ephemeral": false, - "capabilities": [ - "fetch", - "cookies:chrome", - "log" - ] - } -} diff --git a/packages/connectors/twitter-bookmarks/src/graphql-fetch.ts b/packages/connectors/twitter-bookmarks/src/graphql-fetch.ts deleted file mode 100644 index d9e8528..0000000 --- a/packages/connectors/twitter-bookmarks/src/graphql-fetch.ts +++ /dev/null @@ -1,288 +0,0 @@ -import type { FetchCapability, CapturedItem } from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode, abortableSleep } from '@spool-lab/connector-sdk' - -// ── Constants ─────────────────────────────────────────────────────────────── - -const X_PUBLIC_BEARER = - 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' - -const BOOKMARKS_QUERY_ID = 'Z9GWmP0kP2dajyckAaDUBw' -const BOOKMARKS_OPERATION = 'Bookmarks' - -const GRAPHQL_FEATURES = { - graphql_timeline_v2_bookmark_timeline: true, - rweb_tipjar_consumption_enabled: true, - responsive_web_graphql_exclude_directive_enabled: true, - verified_phone_label_enabled: false, - creator_subscriptions_tweet_preview_api_enabled: true, - responsive_web_graphql_timeline_navigation_enabled: true, - responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, - communities_web_enable_tweet_community_results_fetch: true, - c9s_tweet_anatomy_moderator_badge_enabled: true, - articles_preview_enabled: true, - responsive_web_edit_tweet_api_enabled: true, - tweetypie_unmention_optimization_enabled: true, - responsive_web_uc_gql_enabled: true, - vibe_api_enabled: true, - responsive_web_text_conversations_enabled: false, - freedom_of_speech_not_reach_fetch_enabled: true, - longform_notetweets_rich_text_read_enabled: true, - longform_notetweets_inline_media_enabled: true, - responsive_web_enhance_cards_enabled: false, - tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, - responsive_web_media_download_video_enabled: false, -} - -// ── URL & Headers ─────────────────────────────────────────────────────────── - -function buildUrl(cursor?: string): string { - const variables: Record = { count: 20 } - if (cursor) variables.cursor = cursor - const params = new URLSearchParams({ - variables: JSON.stringify(variables), - features: JSON.stringify(GRAPHQL_FEATURES), - }) - return `https://x.com/i/api/graphql/${BOOKMARKS_QUERY_ID}/${BOOKMARKS_OPERATION}?${params}` -} - -function buildHeaders(csrfToken: string, cookieHeader?: string): Record { - return { - authorization: `Bearer ${X_PUBLIC_BEARER}`, - 'x-csrf-token': csrfToken, - 'x-twitter-auth-type': 'OAuth2Session', - 'x-twitter-active-user': 'yes', - 'content-type': 'application/json', - 'user-agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36', - cookie: cookieHeader ?? `ct0=${csrfToken}`, - } -} - -// ── Response Parsing ──────────────────────────────────────────────────────── - -export interface BookmarkPageResult { - items: CapturedItem[] - nextCursor: string | null -} - -function convertTweetToItem(tweetResult: any, now: string): CapturedItem | null { - const tweet = tweetResult.tweet ?? tweetResult - const legacy = tweet?.legacy - if (!legacy) return null - - const tweetId = legacy.id_str ?? tweet?.rest_id - if (!tweetId) return null - - const userResult = tweet?.core?.user_results?.result - const authorHandle = - userResult?.core?.screen_name ?? userResult?.legacy?.screen_name - const authorName = - userResult?.core?.name ?? userResult?.legacy?.name - const authorProfileImageUrl = - userResult?.avatar?.image_url ?? - userResult?.legacy?.profile_image_url_https ?? - userResult?.legacy?.profile_image_url - - const mediaEntities = - legacy?.extended_entities?.media ?? legacy?.entities?.media ?? [] - const media: string[] = mediaEntities - .map((m: any) => m.media_url_https ?? m.media_url) - .filter(Boolean) - const mediaObjects = mediaEntities.map((m: any) => ({ - type: m.type, - url: m.media_url_https ?? m.media_url, - expandedUrl: m.expanded_url, - width: m.original_info?.width, - height: m.original_info?.height, - altText: m.ext_alt_text, - })) - - const urlEntities = legacy?.entities?.urls ?? [] - const links: string[] = urlEntities - .map((u: any) => u.expanded_url) - .filter((u: string | undefined) => u && !u.includes('t.co')) - - const authorSnapshot = userResult - ? { - id: userResult.rest_id, - handle: authorHandle, - name: authorName, - profileImageUrl: authorProfileImageUrl, - bio: userResult?.legacy?.description, - followerCount: userResult?.legacy?.followers_count, - followingCount: userResult?.legacy?.friends_count, - isVerified: Boolean( - userResult?.is_blue_verified ?? userResult?.legacy?.verified, - ), - location: - typeof userResult?.location === 'object' - ? userResult.location.location - : userResult?.legacy?.location, - } - : undefined - - const engagement = { - likeCount: legacy.favorite_count, - repostCount: legacy.retweet_count, - replyCount: legacy.reply_count, - quoteCount: legacy.quote_count, - bookmarkCount: legacy.bookmark_count, - viewCount: tweet?.views?.count ? Number(tweet.views.count) : undefined, - } - - const text = legacy.full_text ?? legacy.text ?? '' - const url = `https://x.com/${authorHandle ?? '_'}/status/${tweetId}` - - return { - url, - title: text.length > 120 ? text.slice(0, 117) + '...' : text, - contentText: text, - author: authorHandle ?? null, - platform: 'twitter', - platformId: tweetId, - contentType: 'tweet', - thumbnailUrl: authorProfileImageUrl ?? null, - metadata: { - authorSnapshot, - engagement, - media, - mediaObjects, - links, - language: legacy.lang, - conversationId: legacy.conversation_id_str, - inReplyToStatusId: legacy.in_reply_to_status_id_str, - quotedStatusId: legacy.quoted_status_id_str, - postedAt: legacy.created_at, - sourceApp: legacy.source, - }, - capturedAt: legacy.created_at - ? new Date(legacy.created_at).toISOString() - : now, - rawJson: JSON.stringify(tweetResult), - } -} - -export function parseBookmarksResponse(json: any, now?: string): BookmarkPageResult { - const ts = now ?? new Date().toISOString() - const instructions = - json?.data?.bookmark_timeline_v2?.timeline?.instructions ?? [] - const entries: any[] = [] - for (const inst of instructions) { - if (inst.type === 'TimelineAddEntries' && Array.isArray(inst.entries)) { - entries.push(...inst.entries) - } - } - - const items: CapturedItem[] = [] - let nextCursor: string | null = null - - for (const entry of entries) { - if (entry.entryId?.startsWith('cursor-bottom')) { - nextCursor = entry.content?.value ?? null - continue - } - - const tweetResult = entry?.content?.itemContent?.tweet_results?.result - if (!tweetResult) continue - - const item = convertTweetToItem(tweetResult, ts) - if (item) items.push(item) - } - - return { items, nextCursor } -} - -// ── Fetch with Retry ──────────────────────────────────────────────────────── - -export async function fetchBookmarkPage( - csrfToken: string, - cursor: string | null, - opts: { - cookieHeader: string - fetch: FetchCapability - signal: AbortSignal - }, -): Promise { - const { cookieHeader, fetch: fetchFn, signal } = opts - let lastError: Error | undefined - - for (let attempt = 0; attempt < 4; attempt++) { - if (signal.aborted) { - throw signal.reason - } - - let response: Response - try { - response = await fetchFn( - buildUrl(cursor ?? undefined), - { headers: buildHeaders(csrfToken, cookieHeader), signal }, - ) - } catch (err) { - if (signal.aborted) { - throw signal.reason - } - const message = err instanceof Error ? err.message : String(err) - if (message.includes('ENOTFOUND') || message.includes('ENETUNREACH')) { - throw new SyncError(SyncErrorCode.NETWORK_OFFLINE, message, err) - } - if (message.includes('ETIMEDOUT') || message.includes('timeout')) { - throw new SyncError(SyncErrorCode.NETWORK_TIMEOUT, message, err) - } - throw new SyncError(SyncErrorCode.CONNECTOR_ERROR, message, err) - } - - if (response.status === 429) { - const waitSec = Math.min(15 * Math.pow(2, attempt), 120) - lastError = new Error(`Rate limited (429) on attempt ${attempt + 1}`) - await abortableSleep(waitSec * 1000, signal) - continue - } - - if (response.status >= 500) { - lastError = new Error(`Server error (${response.status}) on attempt ${attempt + 1}`) - await abortableSleep(5000 * (attempt + 1), signal) - continue - } - - if (response.status === 401 || response.status === 403) { - throw new SyncError( - SyncErrorCode.AUTH_SESSION_EXPIRED, - `X API returned ${response.status}. Your session may have expired.`, - ) - } - - if (!response.ok) { - const text = await response.text().catch(() => '') - throw new SyncError( - SyncErrorCode.API_UNEXPECTED_STATUS, - `X GraphQL API returned ${response.status}: ${text.slice(0, 300)}`, - ) - } - - let json: unknown - try { - json = await response.json() - } catch (err) { - throw new SyncError( - SyncErrorCode.API_PARSE_ERROR, - 'Failed to parse X GraphQL response as JSON', - err, - ) - } - - try { - return parseBookmarksResponse(json) - } catch (err) { - throw new SyncError( - SyncErrorCode.API_PARSE_ERROR, - `Failed to parse bookmarks from GraphQL response: ${err instanceof Error ? err.message : String(err)}`, - err, - ) - } - } - - if (lastError?.message.includes('429')) { - throw new SyncError(SyncErrorCode.API_RATE_LIMITED, 'Rate limited after 4 retry attempts.') - } - throw new SyncError(SyncErrorCode.API_SERVER_ERROR, 'Server errors after 4 retry attempts.') -} diff --git a/packages/connectors/twitter-bookmarks/src/index.ts b/packages/connectors/twitter-bookmarks/src/index.ts deleted file mode 100644 index ab8bac3..0000000 --- a/packages/connectors/twitter-bookmarks/src/index.ts +++ /dev/null @@ -1,101 +0,0 @@ -import type { - Connector, - ConnectorCapabilities, - AuthStatus, - PageResult, - FetchContext, - Cookie, -} from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode } from '@spool-lab/connector-sdk' -import { fetchBookmarkPage } from './graphql-fetch.js' - -interface TwitterAuth { - csrfToken: string - cookieHeader: string -} - -function buildAuth(cookies: Cookie[]): TwitterAuth | null { - const ct0 = cookies.find(c => c.name === 'ct0') - if (!ct0) return null - const authToken = cookies.find(c => c.name === 'auth_token') - const parts = [`ct0=${ct0.value}`] - if (authToken) parts.push(`auth_token=${authToken.value}`) - return { csrfToken: ct0.value, cookieHeader: parts.join('; ') } -} - -export default class TwitterBookmarksConnector implements Connector { - readonly id = 'twitter-bookmarks' - readonly platform = 'twitter' - readonly label = 'X Bookmarks' - readonly description = 'Your saved tweets on X' - readonly color = '#1DA1F2' - readonly ephemeral = false - - private cachedAuth: TwitterAuth | null = null - - constructor(private readonly caps: ConnectorCapabilities) {} - - async checkAuth(): Promise { - try { - await this.readAuth() - return { ok: true } - } catch (err) { - if (err instanceof SyncError) { - return { - ok: false, - error: err.code, - message: err.message, - hint: err.message, - } - } - return { - ok: false, - error: SyncErrorCode.AUTH_UNKNOWN, - message: err instanceof Error ? err.message : String(err), - hint: 'Check that Chrome is installed and you are logged into X.', - } - } - } - - async fetchPage(ctx: FetchContext): Promise { - if (!this.cachedAuth) { - this.cachedAuth = await this.readAuth() - } - - try { - const result = await this.caps.log.span( - 'fetchPage', - () => fetchBookmarkPage(this.cachedAuth!.csrfToken, ctx.cursor, { - cookieHeader: this.cachedAuth!.cookieHeader, - fetch: this.caps.fetch, - signal: ctx.signal!, - }), - { attributes: { 'twitter.phase': ctx.phase, 'twitter.cursor': ctx.cursor ?? 'initial' } }, - ) - - return { items: result.items, nextCursor: result.nextCursor } - } catch (err) { - // Invalidate cached auth on session expiry so next cycle re-reads cookies - if (err instanceof SyncError && err.needsReauth) { - this.cachedAuth = null - } - throw err - } - } - - private async readAuth(): Promise { - const xCookies = await this.caps.cookies.get({ browser: 'chrome', url: 'https://x.com' }) - const auth = buildAuth(xCookies) - if (auth) return auth - - // Fallback: twitter.com domain (some accounts still use this) - const twCookies = await this.caps.cookies.get({ browser: 'chrome', url: 'https://twitter.com' }) - const fallback = buildAuth(twCookies) - if (fallback) return fallback - - throw new SyncError( - SyncErrorCode.AUTH_NOT_LOGGED_IN, - 'No ct0 CSRF cookie found for x.com or twitter.com in Chrome. Log into X in Chrome and retry.', - ) - } -} diff --git a/packages/connectors/twitter-bookmarks/tsconfig.json b/packages/connectors/twitter-bookmarks/tsconfig.json deleted file mode 100644 index a84e1f5..0000000 --- a/packages/connectors/twitter-bookmarks/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "lib": ["es2022"], - "outDir": "./dist", - "rootDir": "./src", - "strict": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules"] -} diff --git a/packages/connectors/typeless/package.json b/packages/connectors/typeless/package.json deleted file mode 100644 index fbf257e..0000000 --- a/packages/connectors/typeless/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@spool-lab/connector-typeless", - "version": "0.1.2", - "description": "Your voice transcripts from Typeless for Spool", - "keywords": [ - "spool-connector", - "typeless", - "voice" - ], - "type": "module", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "files": [ - "dist" - ], - "scripts": { - "prepack": "pnpm run build", - "build": "tsc", - "clean": "rm -rf dist", - "test": "vitest run" - }, - "dependencies": { - "@spool-lab/connector-sdk": "workspace:^" - }, - "bundledDependencies": [ - "@spool-lab/connector-sdk" - ], - "devDependencies": { - "@types/better-sqlite3": "^7.6.13", - "@types/node": "^22.15.3", - "better-sqlite3": "^11.9.1", - "typescript": "^5.7.3", - "vitest": "^3.1.1" - }, - "spool": { - "type": "connector", - "id": "typeless", - "platform": "typeless", - "label": "Typeless Voice", - "description": "Your voice transcripts from Typeless", - "color": "#1D1A1A", - "ephemeral": false, - "capabilities": [ - "sqlite", - "log" - ] - } -} diff --git a/packages/connectors/typeless/src/db-reader.ts b/packages/connectors/typeless/src/db-reader.ts deleted file mode 100644 index d80c4fe..0000000 --- a/packages/connectors/typeless/src/db-reader.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { SqliteDatabase } from '@spool-lab/connector-sdk' -import { homedir } from 'node:os' -import { join } from 'node:path' - -export const DEFAULT_DB_PATH = join( - homedir(), - 'Library', - 'Application Support', - 'Typeless', - 'typeless.db', -) - -export const PAGE_SIZE = 25 - -export interface TypelessRow { - id: string - refined_text: string | null - edited_text: string | null - status: string - mode: string | null - duration: number | null - detected_language: string | null - audio_local_path: string | null - focused_app_name: string | null - focused_app_bundle_id: string | null - focused_app_window_title: string | null - focused_app_window_web_url: string | null - focused_app_window_web_domain: string | null - focused_app_window_web_title: string | null - created_at: string -} - -const SELECT_COLS = ` - id, refined_text, edited_text, status, mode, duration, detected_language, - audio_local_path, focused_app_name, focused_app_bundle_id, - focused_app_window_title, focused_app_window_web_url, - focused_app_window_web_domain, focused_app_window_web_title, created_at -` - -const WHERE_TRANSCRIBED = ` - status = 'transcript' - AND refined_text IS NOT NULL - AND refined_text != '' -` - -export function fetchTranscriptPage( - db: SqliteDatabase, - cursor: string | null, -): TypelessRow[] { - if (cursor === null) { - return db - .prepare( - `SELECT ${SELECT_COLS} FROM history - WHERE ${WHERE_TRANSCRIBED} - ORDER BY created_at DESC - LIMIT ?`, - ) - .all(PAGE_SIZE) - } - - return db - .prepare( - `SELECT ${SELECT_COLS} FROM history - WHERE ${WHERE_TRANSCRIBED} - AND created_at < ? - ORDER BY created_at DESC - LIMIT ?`, - ) - .all(cursor, PAGE_SIZE) -} diff --git a/packages/connectors/typeless/src/index.test.ts b/packages/connectors/typeless/src/index.test.ts deleted file mode 100644 index 8dc68b5..0000000 --- a/packages/connectors/typeless/src/index.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import Database from 'better-sqlite3' -import { mkdtempSync, rmSync } from 'node:fs' -import { join } from 'node:path' -import { tmpdir } from 'node:os' -import type { ConnectorCapabilities, FetchContext, SqliteCapability, SqliteDatabase, SqliteStatement } from '@spool-lab/connector-sdk' -import TypelessConnector from './index.js' - -// ── Mock capabilities ───────────────────────────────────────────────────────── - -function makeMockSqlite(): SqliteCapability { - return { - openReadonly(path: string): SqliteDatabase { - const db = new Database(path, { readonly: true, fileMustExist: true }) - return { - prepare(sql: string): SqliteStatement { - const stmt = db.prepare(sql) - return { - all: (...params) => stmt.all(...params) as T[], - get: (...params) => stmt.get(...params) as T | undefined, - } - }, - close: () => { db.close() }, - } - }, - } -} - -const noop = () => {} -const mockCaps: ConnectorCapabilities = { - fetch: (() => { throw new Error('fetch not available') }) as any, - cookies: { get: async () => [] }, - sqlite: makeMockSqlite(), - log: { - debug: noop, - info: noop, - warn: noop, - error: noop, - span: async (_name: string, fn: () => Promise) => fn(), - }, -} - -function makeCtx(cursor: string | null = null): FetchContext { - return { cursor, sinceItemId: null, phase: 'backfill' } -} - -// ── Helpers ─────────────────────────────────────────────────────────────────── - -function makeTmpDb(): { dbPath: string; db: Database.Database; cleanup: () => void } { - const dir = mkdtempSync(join(tmpdir(), 'spool-typeless-test-')) - const dbPath = join(dir, 'typeless.db') - const db = new Database(dbPath) - - db.exec(` - CREATE TABLE history ( - id TEXT PRIMARY KEY NOT NULL, - refined_text TEXT, - edited_text TEXT, - status TEXT NOT NULL DEFAULT 'transcript', - mode TEXT DEFAULT 'voice_transcript', - duration REAL, - detected_language TEXT, - audio_local_path TEXT, - focused_app_name TEXT, - focused_app_bundle_id TEXT, - focused_app_window_title TEXT, - focused_app_window_web_url TEXT, - focused_app_window_web_domain TEXT, - focused_app_window_web_title TEXT, - created_at TEXT NOT NULL - ) - `) - - return { - dbPath, - db, - cleanup: () => { - db.close() - rmSync(dir, { recursive: true, force: true }) - }, - } -} - -const insertRow = ( - db: Database.Database, - overrides: Partial<{ - id: string - refined_text: string | null - edited_text: string | null - status: string - mode: string - duration: number - detected_language: string - audio_local_path: string | null - focused_app_name: string - focused_app_bundle_id: string - focused_app_window_title: string - focused_app_window_web_url: string | null - focused_app_window_web_domain: string | null - focused_app_window_web_title: string | null - created_at: string - }> = {}, -) => { - const row = { - id: 'test-id-1', - refined_text: 'Hello world', - edited_text: null, - status: 'transcript', - mode: 'voice_transcript', - duration: 2.5, - detected_language: 'en', - audio_local_path: '/tmp/test.ogg', - focused_app_name: 'iTerm2', - focused_app_bundle_id: 'com.googlecode.iterm2', - focused_app_window_title: 'spool dev', - focused_app_window_web_url: null, - focused_app_window_web_domain: null, - focused_app_window_web_title: null, - created_at: '2026-01-01T10:00:00.000Z', - ...overrides, - } - db.prepare(` - INSERT INTO history ( - id, refined_text, edited_text, status, mode, duration, detected_language, - audio_local_path, focused_app_name, focused_app_bundle_id, focused_app_window_title, - focused_app_window_web_url, focused_app_window_web_domain, focused_app_window_web_title, - created_at - ) VALUES ( - @id, @refined_text, @edited_text, @status, @mode, @duration, @detected_language, - @audio_local_path, @focused_app_name, @focused_app_bundle_id, @focused_app_window_title, - @focused_app_window_web_url, @focused_app_window_web_domain, @focused_app_window_web_title, - @created_at - ) - `).run(row) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -describe('TypelessConnector.checkAuth', () => { - it('returns ok:true when db exists and is readable', async () => { - const { dbPath, cleanup } = makeTmpDb() - try { - const connector = new TypelessConnector(mockCaps, { dbPath }) - const result = await connector.checkAuth() - expect(result.ok).toBe(true) - } finally { - cleanup() - } - }) - - it('returns ok:false with a hint when db is missing', async () => { - const connector = new TypelessConnector(mockCaps, { dbPath: '/nonexistent/path/typeless.db' }) - const result = await connector.checkAuth() - expect(result.ok).toBe(false) - expect(result.hint).toContain('typeless.com') - }) -}) - -describe('TypelessConnector.fetchPage', () => { - let dbPath: string - let db: Database.Database - let cleanup: () => void - - beforeEach(() => { - const tmp = makeTmpDb() - dbPath = tmp.dbPath - db = tmp.db - cleanup = tmp.cleanup - }) - - afterEach(() => { - cleanup() - }) - - it('returns empty page when no transcripts exist', async () => { - const connector = new TypelessConnector(mockCaps, { dbPath }) - const result = await connector.fetchPage(makeCtx()) - expect(result.items).toHaveLength(0) - expect(result.nextCursor).toBeNull() - }) - - it('returns a CapturedItem with correct shape', async () => { - insertRow(db, { - id: 'abc-123', - refined_text: 'Let me check the build status', - focused_app_name: 'iTerm2', - focused_app_window_title: 'spool dev', - }) - - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - - expect(items).toHaveLength(1) - const item = items[0]! - - expect(item.platformId).toBe('abc-123') - expect(item.platform).toBe('typeless') - expect(item.contentType).toBe('voice_transcript') - expect(item.author).toBeNull() - expect(item.url).toBe('file:///tmp/test.ogg') - expect(item.capturedAt).toBe('2026-01-01T10:00:00.000Z') - }) - - it('uses edited_text over refined_text when both are present', async () => { - insertRow(db, { - refined_text: 'AI polished version', - edited_text: 'User corrected version', - }) - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - expect(items[0]!.contentText).toContain('User corrected version') - expect(items[0]!.contentText).not.toContain('AI polished version') - }) - - it('includes context in contentText', async () => { - insertRow(db, { - refined_text: 'Ship it', - focused_app_name: 'Chrome', - focused_app_window_web_domain: 'remotion.dev', - focused_app_window_title: 'Remotion docs', - }) - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - const text = items[0]!.contentText - expect(text).toContain('Ship it') - expect(text).toContain('Chrome') - expect(text).toContain('remotion.dev') - }) - - it('truncates title at 80 characters', async () => { - const long = 'a'.repeat(100) - insertRow(db, { refined_text: long }) - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - expect(items[0]!.title.length).toBeLessThanOrEqual(82) - expect(items[0]!.title).toContain('…') - }) - - it('falls back to typeless:// URL when audio_local_path is null', async () => { - insertRow(db, { id: 'no-audio', audio_local_path: null }) - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - expect(items[0]!.url).toBe('typeless://transcript/no-audio') - }) - - it('filters out dismissed and error rows', async () => { - insertRow(db, { id: 'dismissed-1', status: 'dismissed', refined_text: 'bye' }) - insertRow(db, { id: 'error-1', status: 'error', refined_text: 'oops' }) - insertRow(db, { id: 'good-1', status: 'transcript', refined_text: 'hello' }) - - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - expect(items).toHaveLength(1) - expect(items[0]!.platformId).toBe('good-1') - }) - - it('filters out rows with empty refined_text', async () => { - insertRow(db, { id: 'empty-1', refined_text: '' }) - insertRow(db, { id: 'null-1', refined_text: null }) - insertRow(db, { id: 'real-1', refined_text: 'real content' }) - - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - expect(items).toHaveLength(1) - expect(items[0]!.platformId).toBe('real-1') - }) - - it('paginates using cursor (created_at)', async () => { - insertRow(db, { id: 'row-c', created_at: '2026-01-03T00:00:00.000Z', refined_text: 'third' }) - insertRow(db, { id: 'row-b', created_at: '2026-01-02T00:00:00.000Z', refined_text: 'second' }) - insertRow(db, { id: 'row-a', created_at: '2026-01-01T00:00:00.000Z', refined_text: 'first' }) - - const connector = new TypelessConnector(mockCaps, { dbPath }) - - const page2 = await connector.fetchPage(makeCtx('2026-01-02T00:00:00.000Z')) - expect(page2.items).toHaveLength(1) - expect(page2.items[0]!.platformId).toBe('row-a') - expect(page2.nextCursor).toBeNull() - }) - - it('returns nextCursor when a full page is returned', async () => { - for (let i = 0; i < 26; i++) { - const ts = new Date(2026, 0, i + 1).toISOString() - insertRow(db, { - id: `row-${i}`, - refined_text: `transcript ${i}`, - created_at: ts, - }) - } - - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items, nextCursor } = await connector.fetchPage(makeCtx()) - expect(items).toHaveLength(25) - expect(nextCursor).not.toBeNull() - expect(nextCursor).toBe(items[24]!.capturedAt) - }) - - it('stores context fields in metadata', async () => { - insertRow(db, { - focused_app_name: 'Notion', - focused_app_window_web_url: 'https://notion.so/my-page', - focused_app_window_web_domain: 'notion.so', - duration: 5.2, - detected_language: 'zh', - }) - const connector = new TypelessConnector(mockCaps, { dbPath }) - const { items } = await connector.fetchPage(makeCtx()) - const meta = items[0]!.metadata - expect(meta['focused_app']).toBe('Notion') - expect(meta['focused_app_window_web_url']).toBe('https://notion.so/my-page') - expect(meta['duration']).toBe(5.2) - expect(meta['detected_language']).toBe('zh') - }) -}) diff --git a/packages/connectors/typeless/src/index.ts b/packages/connectors/typeless/src/index.ts deleted file mode 100644 index e0ed6a9..0000000 --- a/packages/connectors/typeless/src/index.ts +++ /dev/null @@ -1,112 +0,0 @@ -import type { - Connector, - ConnectorCapabilities, - AuthStatus, - PageResult, - FetchContext, - CapturedItem, -} from '@spool-lab/connector-sdk' -import { SyncError, SyncErrorCode } from '@spool-lab/connector-sdk' -import { - fetchTranscriptPage, - DEFAULT_DB_PATH, - PAGE_SIZE, - type TypelessRow, -} from './db-reader.js' - -export default class TypelessConnector implements Connector { - readonly id = 'typeless' - readonly platform = 'typeless' - readonly label = 'Typeless Voice' - readonly description = 'Your voice transcripts from Typeless' - readonly color = '#1D1A1A' - readonly ephemeral = false - - private readonly dbPath: string - - constructor( - private readonly caps: ConnectorCapabilities, - opts?: { dbPath?: string }, - ) { - this.dbPath = opts?.dbPath ?? DEFAULT_DB_PATH - } - - async checkAuth(): Promise { - try { - const db = this.caps.sqlite.openReadonly(this.dbPath) - db.close() - return { ok: true } - } catch (err) { - return { - ok: false, - error: SyncErrorCode.CONNECTOR_ERROR, - message: err instanceof Error ? err.message : String(err), - hint: 'Typeless not found. Install Typeless (typeless.com), record at least once, then retry.', - } - } - } - - async fetchPage(ctx: FetchContext): Promise { - let db: ReturnType | null = null - try { - db = this.caps.sqlite.openReadonly(this.dbPath) - const rows = fetchTranscriptPage(db, ctx.cursor) - const items = rows.map(rowToCapturedItem) - const nextCursor = - rows.length === PAGE_SIZE ? (rows[rows.length - 1]?.created_at ?? null) : null - return { items, nextCursor } - } catch (err) { - if (err instanceof SyncError) throw err - throw new SyncError( - SyncErrorCode.CONNECTOR_ERROR, - err instanceof Error ? err.message : String(err), - ) - } finally { - db?.close() - } - } -} - -function rowToCapturedItem(row: TypelessRow): CapturedItem { - const transcript = (row.edited_text?.trim() || row.refined_text?.trim()) ?? '' - - const contextParts: string[] = [] - if (row.focused_app_name) contextParts.push(row.focused_app_name) - if (row.focused_app_window_title) contextParts.push(row.focused_app_window_title) - if (row.focused_app_window_web_domain) contextParts.push(row.focused_app_window_web_domain) - if (row.focused_app_window_web_title) contextParts.push(row.focused_app_window_web_title) - - const contentText = - contextParts.length > 0 - ? `${transcript}\n${contextParts.join(' · ')}` - : transcript - - const title = - transcript.length > 80 ? `${transcript.slice(0, 80)}…` : transcript - - const url = row.audio_local_path - ? `file://${row.audio_local_path}` - : `typeless://transcript/${row.id}` - - return { - url, - title, - contentText, - author: null, - platform: 'typeless', - platformId: row.id, - contentType: row.mode ?? 'voice_transcript', - thumbnailUrl: null, - metadata: { - duration: row.duration, - detected_language: row.detected_language, - focused_app: row.focused_app_name, - focused_app_bundle_id: row.focused_app_bundle_id, - focused_app_window_title: row.focused_app_window_title, - focused_app_window_web_url: row.focused_app_window_web_url, - focused_app_window_web_domain: row.focused_app_window_web_domain, - }, - capturedAt: row.created_at, - rawJson: JSON.stringify(row), - } -} diff --git a/packages/connectors/typeless/tsconfig.json b/packages/connectors/typeless/tsconfig.json deleted file mode 100644 index d68bda3..0000000 --- a/packages/connectors/typeless/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "lib": ["es2022"], - "outDir": "./dist", - "rootDir": "./src", - "strict": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules", "src/**/*.test.ts"] -} diff --git a/packages/connectors/xiaohongshu/README.md b/packages/connectors/xiaohongshu/README.md deleted file mode 100644 index 6007008..0000000 --- a/packages/connectors/xiaohongshu/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# @spool-lab/connector-xiaohongshu - -Xiaohongshu (小红书) connector for Spool. - -## Prerequisites - -- [OpenCLI](https://github.com/jackwener/opencli) >= 0.3.0 -- OpenCLI Browser Bridge (Chrome extension) -- Logged into xiaohongshu.com in Chrome - -Spool will guide you through installing each when you enable the connector. - -## Sub-connectors - -- `xiaohongshu-notes` — your published notes (persistent) diff --git a/packages/connectors/xiaohongshu/package.json b/packages/connectors/xiaohongshu/package.json deleted file mode 100644 index 2fff226..0000000 --- a/packages/connectors/xiaohongshu/package.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "name": "@spool-lab/connector-xiaohongshu", - "version": "0.1.2", - "description": "Xiaohongshu (小红书) creator notes via opencli", - "type": "module", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "files": [ - "dist", - "README.md" - ], - "scripts": { - "build": "tsc -p tsconfig.json", - "test": "vitest run" - }, - "dependencies": { - "@spool-lab/connector-sdk": "workspace:*" - }, - "bundledDependencies": [ - "@spool-lab/connector-sdk" - ], - "devDependencies": { - "@types/node": "^22.15.3", - "typescript": "^5.4.0", - "vitest": "^3.2.4" - }, - "spool": { - "type": "connector", - "prerequisites": [ - { - "id": "opencli", - "name": "OpenCLI", - "kind": "cli", - "detect": { - "type": "exec", - "command": "opencli", - "args": [ - "--version" - ], - "versionRegex": "v?(\\d+\\.\\d+\\.\\d+)", - "timeoutMs": 5000 - }, - "minVersion": "0.3.0", - "install": { - "kind": "cli", - "command": { - "darwin": "npm i -g @jackwener/opencli", - "linux": "npm i -g @jackwener/opencli", - "win32": "npm i -g @jackwener/opencli" - } - }, - "docsUrl": "https://github.com/jackwener/opencli" - }, - { - "id": "opencli-extension", - "name": "Browser Bridge", - "kind": "browser-extension", - "requires": [ - "opencli" - ], - "detect": { - "type": "exec", - "command": "opencli", - "args": [ - "doctor" - ], - "matchStdout": "\\[OK\\].*Extension", - "timeoutMs": 10000 - }, - "install": { - "kind": "browser-extension", - "manual": { - "downloadUrl": "https://github.com/jackwener/opencli/releases/latest", - "steps": [ - "Download and unzip the extension", - "Paste chrome://extensions into Chrome's address bar", - "Enable Developer mode (top-right toggle)", - "Click 'Load unpacked' and choose the unzipped folder", - "Keep the folder in place — moving or deleting it will break the extension" - ] - } - } - }, - { - "id": "xhs-login", - "name": "Logged into Xiaohongshu", - "kind": "site-session", - "requires": [ - "opencli-extension" - ], - "detect": { - "type": "exec", - "command": "opencli", - "args": [ - "doctor" - ], - "matchStdout": "\\[OK\\].*Connectivity", - "timeoutMs": 10000 - }, - "install": { - "kind": "site-session", - "openUrl": "https://www.xiaohongshu.com" - } - } - ], - "connectors": [ - { - "id": "xiaohongshu-notes", - "platform": "xiaohongshu", - "label": "Xiaohongshu Notes", - "description": "Notes you have published", - "color": "#FF2442", - "ephemeral": false, - "capabilities": [ - "exec", - "log", - "prerequisites" - ] - } - ] - } -} diff --git a/packages/connectors/xiaohongshu/src/index.test.ts b/packages/connectors/xiaohongshu/src/index.test.ts deleted file mode 100644 index f5ddd9d..0000000 --- a/packages/connectors/xiaohongshu/src/index.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { describe, it, expect, vi } from 'vitest' -import { XhsNotesConnector } from './index.js' -import type { ConnectorCapabilities } from '@spool-lab/connector-sdk' - -function mockCaps(runImpl: (cmd: string, args: string[]) => Promise<{ exitCode: number; stdout: string; stderr: string }>): ConnectorCapabilities { - return { - exec: { run: vi.fn().mockImplementation(runImpl) }, - } as unknown as ConnectorCapabilities -} - -function jsonLines(items: Array>): string { - return items.map(i => JSON.stringify(i)).join('\n') -} - -function mkItem(id: string) { - return { id, title: `note ${id}`, url: `https://xhs.test/${id}`, ts: Date.now() } -} - -describe('XhsNotesConnector.fetchPage', () => { - it('single-shot fetch returns items with no nextCursor', async () => { - const caps = mockCaps(async () => ({ exitCode: 0, stdout: jsonLines(Array.from({ length: 5 }, (_, i) => mkItem(`a${i}`))), stderr: '' })) - const c = new XhsNotesConnector(caps) - const r = await c.fetchPage({ cursor: null, sinceItemId: null, phase: 'forward' } as any) - expect(r.items).toHaveLength(5) - expect(r.nextCursor).toBeNull() - }) - - it('never passes --cursor / --page / --offset to opencli (unsupported flags)', async () => { - const caps = mockCaps(async () => ({ exitCode: 0, stdout: jsonLines([mkItem('x')]), stderr: '' })) - const c = new XhsNotesConnector(caps) - await c.fetchPage({ cursor: '2', sinceItemId: null, phase: 'forward' } as any) - const callArgs = (caps.exec!.run as any).mock.calls[0]![1] as string[] - expect(callArgs).not.toContain('--cursor') - expect(callArgs).not.toContain('--page') - expect(callArgs).not.toContain('--offset') - }) - - it('returns nextCursor null even when items reach limit (opencli has no pagination)', async () => { - const items = Array.from({ length: 20 }, (_, i) => mkItem(`b${i}`)) - const caps = mockCaps(async () => ({ exitCode: 0, stdout: jsonLines(items), stderr: '' })) - const c = new XhsNotesConnector(caps) - const r = await c.fetchPage({ cursor: null, sinceItemId: null, phase: 'forward' } as any) - expect(r.nextCursor).toBeNull() - }) - - it('throws SyncError when opencli exits non-zero', async () => { - const caps = mockCaps(async () => ({ exitCode: 1, stdout: '', stderr: 'connection failed' })) - const c = new XhsNotesConnector(caps) - await expect(c.fetchPage({ cursor: null, sinceItemId: null, phase: 'forward' } as any)).rejects.toThrow(/connection failed/) - }) - - it('treats opencli "no X found" exit as empty result, not error', async () => { - const caps = mockCaps(async () => ({ - exitCode: 1, - stdout: '', - stderr: 'No notes found. Are you logged into creator.xiaohongshu.com?', - })) - const c = new XhsNotesConnector(caps) - const r = await c.fetchPage({ cursor: null, sinceItemId: null, phase: 'forward' } as any) - expect(r.items).toEqual([]) - expect(r.nextCursor).toBeNull() - }) -}) diff --git a/packages/connectors/xiaohongshu/src/index.ts b/packages/connectors/xiaohongshu/src/index.ts deleted file mode 100644 index 42f6e3c..0000000 --- a/packages/connectors/xiaohongshu/src/index.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { - Connector, - ConnectorCapabilities, - AuthStatus, - PageResult, - FetchContext, -} from '@spool-lab/connector-sdk' -import { checkAuthViaPrerequisites, SyncError, SyncErrorCode, parseCliJsonOutput } from '@spool-lab/connector-sdk' - -// opencli xiaohongshu subcommands return a single snapshot of the current -// top-N items and don't accept any cursor/page/offset flag. We always do a -// single-shot fetch with --limit. -const PAGE_LIMIT = 100 - -abstract class XhsBaseConnector implements Connector { - abstract readonly id: string - abstract readonly label: string - abstract readonly description: string - abstract readonly ephemeral: boolean - abstract readonly subcommand: string - - readonly platform = 'xiaohongshu' - readonly color = '#FF2442' - - constructor(protected readonly caps: ConnectorCapabilities) {} - - async checkAuth(): Promise { - return checkAuthViaPrerequisites(this.caps) - } - - async fetchPage(_ctx: FetchContext): Promise { - const args = ['xiaohongshu', this.subcommand, '-f', 'json', '--limit', String(PAGE_LIMIT)] - const result = await this.caps.exec.run('opencli', args, { timeout: 30_000 }) - if (result.exitCode !== 0) { - // opencli treats "no rows" as an error; recognize that pattern as an - // empty result so an account with zero items shows "0 items" instead - // of red error state. - if (/no\s+\w+\s+found/i.test(result.stderr)) { - return { items: [], nextCursor: null } - } - throw new SyncError( - SyncErrorCode.API_UNEXPECTED_STATUS, - `opencli ${this.subcommand} failed (exit ${result.exitCode}): ${result.stderr.slice(0, 200)}`, - ) - } - const items = parseCliJsonOutput(result.stdout, 'xiaohongshu', 'post') - return { items, nextCursor: null } - } -} - -export class XhsNotesConnector extends XhsBaseConnector { - readonly id = 'xiaohongshu-notes' - readonly label = 'Xiaohongshu Notes' - readonly description = 'Notes you have published' - readonly ephemeral = false - readonly subcommand = 'creator-notes' -} - -// Feed and Notifications sub-connectors intentionally omitted for now: -// - feed: opencli reads from page Pinia store without scrolling, so item -// count fluctuates with whatever the store happens to hold (~20 items). -// - notifications: opencli currently emits only `{rank: N}` placeholders -// without stable IDs/content, plus session detach issues. -// Both will return when upstream behavior stabilizes. - -export const connectors = [XhsNotesConnector] diff --git a/packages/connectors/xiaohongshu/tsconfig.json b/packages/connectors/xiaohongshu/tsconfig.json deleted file mode 100644 index d68bda3..0000000 --- a/packages/connectors/xiaohongshu/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "nodenext", - "moduleResolution": "nodenext", - "lib": ["es2022"], - "outDir": "./dist", - "rootDir": "./src", - "strict": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "isolatedModules": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["dist", "node_modules", "src/**/*.test.ts"] -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a3ab55..acca143 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -147,108 +147,6 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/debug@4.1.13)(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0) - packages/connector-sdk: - devDependencies: - '@types/node': - specifier: ^22.15.3 - version: 22.19.17 - typescript: - specifier: ^5.7.3 - version: 5.9.3 - vitest: - specifier: ^3.1.2 - version: 3.2.4(@types/debug@4.1.13)(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0) - - packages/connectors/github: - dependencies: - '@spool-lab/connector-sdk': - specifier: workspace:^ - version: link:../../connector-sdk - devDependencies: - '@types/node': - specifier: ^22.15.3 - version: 22.19.17 - typescript: - specifier: ^5.7.3 - version: 5.9.3 - - packages/connectors/hackernews-hot: - dependencies: - '@spool-lab/connector-sdk': - specifier: workspace:^ - version: link:../../connector-sdk - devDependencies: - '@types/node': - specifier: ^22.15.3 - version: 22.19.17 - typescript: - specifier: ^5.7.3 - version: 5.9.3 - - packages/connectors/reddit: - dependencies: - '@spool-lab/connector-sdk': - specifier: workspace:^ - version: link:../../connector-sdk - devDependencies: - '@types/node': - specifier: ^22.15.3 - version: 22.19.17 - typescript: - specifier: ^5.7.3 - version: 5.9.3 - - packages/connectors/twitter-bookmarks: - dependencies: - '@spool-lab/connector-sdk': - specifier: workspace:^ - version: link:../../connector-sdk - devDependencies: - '@types/node': - specifier: ^22.15.3 - version: 22.19.17 - typescript: - specifier: ^5.7.3 - version: 5.9.3 - - packages/connectors/typeless: - dependencies: - '@spool-lab/connector-sdk': - specifier: workspace:^ - version: link:../../connector-sdk - devDependencies: - '@types/better-sqlite3': - specifier: ^7.6.13 - version: 7.6.13 - '@types/node': - specifier: ^22.15.3 - version: 22.19.17 - better-sqlite3: - specifier: ^11.9.1 - version: 11.10.0 - typescript: - specifier: ^5.7.3 - version: 5.9.3 - vitest: - specifier: ^3.1.1 - version: 3.2.4(@types/debug@4.1.13)(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0) - - packages/connectors/xiaohongshu: - dependencies: - '@spool-lab/connector-sdk': - specifier: workspace:* - version: link:../../connector-sdk - devDependencies: - '@types/node': - specifier: ^22.15.3 - version: 22.19.17 - typescript: - specifier: ^5.4.0 - version: 5.9.3 - vitest: - specifier: ^3.2.4 - version: 3.2.4(@types/debug@4.1.13)(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0) - packages/core: dependencies: better-sqlite3: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a54142c..f3ad0e5 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,5 @@ packages: - "packages/*" - - "packages/connectors/*" # Temporarily excluded: depends on private @void-sdk/* packages in GitHub Packages, # which 403 for contributors without NODE_AUTH_TOKEN. Re-include once Void SDK is public. - "!packages/landing" diff --git a/scripts/pack-connector.sh b/scripts/pack-connector.sh deleted file mode 100755 index 8f29442..0000000 --- a/scripts/pack-connector.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Pack a connector into a publish-ready tarball with @spool-lab/connector-sdk -# bundled inside it (bundledDependencies). Uses `pnpm deploy` to materialize -# deps into a hoisted node_modules, then `npm pack` to produce the tarball. -# -# `pnpm pack` alone errors with ERR_PNPM_BUNDLED_DEPENDENCIES_WITHOUT_HOISTED -# because the workspace uses isolated node-linker. -# -# Usage: -# scripts/pack-connector.sh [out-dir] -# e.g. -# scripts/pack-connector.sh twitter-bookmarks /tmp/out -# -set -euo pipefail - -PLUGIN="${1:-}" -OUT_DIR="${2:-$(mktemp -d -t spool-pack-XXXXXX)}" -if [[ -z "$PLUGIN" ]]; then - echo "usage: $0 [out-dir]" >&2 - exit 2 -fi - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -FULL_NAME="@spool-lab/connector-$PLUGIN" - -if [[ ! -d "$REPO_ROOT/packages/connectors/$PLUGIN" ]]; then - echo "plugin dir not found: packages/connectors/$PLUGIN" >&2 - exit 2 -fi - -mkdir -p "$OUT_DIR" -STAGE="$(mktemp -d -t spool-pack-stage-XXXXXX)" -trap 'rm -rf "$STAGE"' EXIT - -echo "==> Building $FULL_NAME and SDK" -pnpm --filter "$FULL_NAME" --filter "@spool-lab/connector-sdk" build - -echo "==> Deploying $FULL_NAME to $STAGE (hoisted, prod-only)" -pnpm --filter "$FULL_NAME" deploy --prod --config.node-linker=hoisted "$STAGE" - -echo "==> Packing tarball into $OUT_DIR" -# --ignore-scripts: the staged dir doesn't have the workspace scaffolding that -# prepack's `pnpm run build` expects. Build already ran above. -(cd "$STAGE" && npm pack --ignore-scripts --pack-destination "$OUT_DIR" >/dev/null) - -TARBALL="$(ls "$OUT_DIR"/spool-lab-connector-"$PLUGIN"-*.tgz 2>/dev/null | head -1)" -if [[ -z "$TARBALL" ]]; then - echo "tarball not found in $OUT_DIR:" >&2 - ls "$OUT_DIR" >&2 - exit 1 -fi - -# Sanity: confirm SDK was bundled -if ! tar -tzf "$TARBALL" | grep -q "package/node_modules/@spool-lab/connector-sdk/package.json"; then - echo "SDK was not bundled into $TARBALL — bundledDependencies not honored?" >&2 - exit 1 -fi - -echo "==> $TARBALL"