diff --git a/README.md b/README.md index ea49d30..d4b4817 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Node.js](https://img.shields.io/badge/node-%3E%3D22-brightgreen?logo=node.js)](https://nodejs.org) [![TypeScript](https://img.shields.io/badge/TypeScript-5.x-3178C6?logo=typescript&logoColor=white)](https://www.typescriptlang.org) [![PostgreSQL](https://img.shields.io/badge/PostgreSQL-pgvector-4169E1?logo=postgresql&logoColor=white)](https://www.postgresql.org) -[![Supabase](https://img.shields.io/badge/Supabase-3FCF8E?logo=supabase&logoColor=white)](https://supabase.com) +[![Neon](https://img.shields.io/badge/Neon-000000?logo=neon&logoColor=white)](https://neon.tech) [![OpenAI](https://img.shields.io/badge/OpenAI-412991?logo=openai&logoColor=white)](https://openai.com) [![Ollama](https://img.shields.io/badge/Ollama-000000?logo=ollama&logoColor=white)](https://ollama.com) [![MCP](https://img.shields.io/badge/MCP-Compatible-8A2BE2)](https://modelcontextprotocol.io) @@ -83,14 +83,20 @@ pnpm setup # Interactive setup for credentials pnpm dev # Start the server ``` -### 2. Set Up Supabase +### 2. Set Up Your Database -1. Create a free project at [supabase.com](https://supabase.com) -2. Run `scripts/setup-db.sql` in the SQL Editor (or `setup-db-ollama.sql` for Ollama with `nomic-embed-text`, or `setup-db-ollama-v2.sql` for `nomic-embed-text-v2-moe`) -3. (Optional) For memory tools, also run `scripts/setup-db-memory.sql` (or `setup-db-memory-ollama.sql`) -4. (Optional) For conversation tools, also run `scripts/setup-db-conversation.sql` (or `setup-db-conversation-ollama.sql` / `setup-db-conversation-ollama-v2.sql`) -5. Run `scripts/security-rls.sql` for security hardening -6. Copy your project URL and service role key to `.env` +1. Create a free project at [neon.tech](https://neon.tech) +2. Copy the **pooled connection string** from the Neon dashboard into `DATABASE_URL` in `.env` +3. Run the base schema against your database: + + ```bash + psql $DATABASE_URL -f scripts/setup-db.sql + ``` + + (Use `setup-db-ollama.sql` for Ollama / `setup-db-ollama-v2.sql` for `nomic-embed-text-v2-moe` / `setup-db-google.sql` for Google AI) +4. (Optional) For memory tools: `psql $DATABASE_URL -f scripts/setup-db-memory.sql` +5. (Optional) For conversation tools: `psql $DATABASE_URL -f scripts/setup-db-conversation.sql` +6. (Optional) For Row Level Security hardening: `psql $DATABASE_URL -f scripts/security-rls.sql` ### 3. Connect Claude Desktop @@ -154,7 +160,7 @@ pnpm upload -- ./converted/ | Guide | Description | |-------|-------------| -| [Supabase Requirements](docs/guides/supabase-requirements.mdx) | Compute tiers, storage estimates, and database sizing | +| [Database Sizing](docs/guides/supabase-requirements.mdx) | Vector dimensions, index counts, and storage estimates by embedding provider | | [CLI Tools](docs/cli/) | Batch conversion and upload from command line | | [Security](docs/guides/security-hardening.mdx) | Row Level Security and access controls | @@ -164,8 +170,8 @@ pnpm upload -- ./converted/ | Variable | Required | Description | |----------|----------|-------------| -| `SUPABASE_URL` | Yes | `https://your-project.supabase.co` | -| `SUPABASE_SERVICE_KEY` | Yes | Service role key | +| `DATABASE_URL` | Yes | Neon (or any PostgreSQL) pooled connection string | +| `DATABASE_URL_UNPOOLED` | No | Direct connection for schema migrations (optional) | | `EMBEDDING_PROVIDER` | No | `openai` (default), `ollama`, or `google` | | `OPENAI_API_KEY` | If OpenAI | For text-embedding-3-small (1536d) | | `OLLAMA_BASE_URL` | If Ollama | Default: `http://localhost:11434` | @@ -176,18 +182,25 @@ pnpm upload -- ./converted/ | `PORT` | No | Default: 3000 | | `LOG_LEVEL` | No | debug, info, warn, error | | `ALLOWED_ORIGINS` | No | Comma-separated CORS origins | -| `ENABLE_MEMORY` | No | Enable memory tools (default: true); requires `setup-db-memory.sql` or `setup-db-memory-ollama.sql` | -| `ENABLE_CONVERSATIONS` | No | Enable conversation memory tools (default: true); requires `setup-db-conversation.sql` or `setup-db-conversation-ollama.sql` or `setup-db-conversation-ollama-v2.sql` | +| `ENABLE_MEMORY` | No | Enable memory tools (default: true); requires `setup-db-memory.sql` | +| `ENABLE_CONVERSATIONS` | No | Enable conversation memory tools (default: true); requires `setup-db-conversation.sql` | | `ENABLE_INSIGHTS` | No | Enable proactive insight tools (default: true) | | `ENABLE_MEMORY_EXTRACTION` | No | Enable LLM-based memory extraction (default: false) | | `ANTHROPIC_API_KEY` | If extraction | Required for `extract_memories` tool | | `EXTRACTION_MODEL` | No | Model for extraction (default: claude-haiku-4-5-20250501) | | `INSIGHT_MODEL` | No | Model for insight synthesis (default: claude-sonnet-4-6-20250514) | | `COMPACT_RESPONSES` | No | Token-efficient responses (default: true) | -| `DATABASE_URL` | No | Direct Postgres connection for pg_analyze tools | -| `PG_REPORT_DIR` | No | Analysis report directory (default: ./reports/pg-analysis) | - -## MCP Tools (25 tools) +| `CHUNKING_MODE` | No | `fixed` (default) or `semantic` (embedding-based splits) | +| `SEMANTIC_SIMILARITY_THRESHOLD` | No | Semantic split sensitivity 0–1 (default: 0.5) | +| `REDIS_URL` | No | Redis URL for shared rate limiting across instances | +| `GOOGLE_CLIENT_ID` | OAuth | OAuth 2.0 client ID (all four OAuth vars required together) | +| `GOOGLE_CLIENT_SECRET` | OAuth | OAuth 2.0 client secret | +| `OAUTH_JWT_SECRET` | OAuth | Min 32-char secret for JWT signing | +| `OAUTH_ALLOWED_EMAILS` | OAuth | Comma-separated email allowlist (optional) | +| `OAUTH_SERVER_URL` | OAuth | Public server URL for OAuth redirect | +| `PG_REPORT_DIR` | No | pg_analyze report directory (default: ./reports/pg-analysis) | + +## MCP Tools (26 tools) Read-only tools (`search`, `get_document`, `list_documents`, `query_memory`, `query_conversations`, `get_stats`, `health_check`) include `outputSchema` and return `structuredContent` for programmatic consumption alongside the text `content` response. @@ -218,9 +231,10 @@ Enable with `ENABLE_MEMORY=true` (default). Requires `scripts/setup-db-memory.sq Enable with `ENABLE_CONVERSATIONS=true` (default). Requires running one of the conversation schema scripts: -- `scripts/setup-db-conversation.sql` (OpenAI embeddings) +- `scripts/setup-db-conversation.sql` (OpenAI embeddings, 1536d) - `scripts/setup-db-conversation-ollama.sql` (Ollama v1 - nomic-embed-text, 1024d) - `scripts/setup-db-conversation-ollama-v2.sql` (Ollama v2 - nomic-embed-text-v2-moe, 768d) +- `scripts/setup-db-conversation-google.sql` (Google AI - text-embedding-004, 768d) | Tool | Description | |------|-------------| @@ -256,7 +270,7 @@ Enable with `ENABLE_INSIGHTS=true` (default). ### Postgres Analysis Tools -Enabled when `DATABASE_URL` is configured. Connects directly to Postgres (independent of Supabase client). +Enabled when `DATABASE_URL` is configured. Connects directly to Postgres. | Tool | Description | |------|-------------| @@ -343,7 +357,7 @@ pnpm docs:dev # Run docs site ### Local Database (Optional) -Run PostgreSQL + pgvector locally instead of using Supabase: +Run PostgreSQL + pgvector locally: ```bash # Start local Postgres with pgvector @@ -387,8 +401,7 @@ OLLAMA_MODEL=nomic-embed-text | Issue | Solution | |-------|----------| -| Invalid Supabase URL | Format: `https://your-project.supabase.co` (no trailing slash) | -| Missing service role key | Use service role key from Settings > API, not anon key | +| Can't connect to database | Check `DATABASE_URL` is set to your Neon pooled connection string | | No search results | Check `chunks` table has embeddings; lower `minScore` | | MCP tools not in Claude | Restart Claude Desktop; check `curl http://localhost:3000/health` | | Rate limit exceeded | API: 100/min, Upload: 10/min | diff --git a/docs/ENV.md b/docs/ENV.md index 019d68f..7be681d 100644 --- a/docs/ENV.md +++ b/docs/ENV.md @@ -12,8 +12,7 @@ This file centralizes runtime configuration for Textrawl server, MCP tools, CLI, | Variable | Required | Purpose | |---|---|---| -| `SUPABASE_URL` | Yes (DB features) | Supabase project URL | -| `SUPABASE_SERVICE_KEY` | Yes (DB features) | Service-role key used by server/CLI | +| `DATABASE_URL` | Yes (DB features) | Neon (or any PostgreSQL) pooled connection string | | `EMBEDDING_PROVIDER` | No | `openai` (default), `ollama`, or `google` | | `OPENAI_API_KEY` | Required for OpenAI | Embedding API key (`text-embedding-3-small`, 1536d) | | `GOOGLE_AI_API_KEY` | Required for Google | Google AI API key (`text-embedding-004`, 768d) | @@ -48,11 +47,11 @@ This file centralizes runtime configuration for Textrawl server, MCP tools, CLI, | Variable | Purpose | |---|---| +| `DATABASE_URL_UNPOOLED` | Direct Postgres connection for pg_analyze tools and migrations | | `REDIS_URL` | Shared rate limiting across instances | | `GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, `OAUTH_JWT_SECRET`, `OAUTH_SERVER_URL` | OAuth support | | `OAUTH_ALLOWED_EMAILS` | Optional OAuth email allowlist | | `CHUNKING_MODE`, `SEMANTIC_SIMILARITY_THRESHOLD` | Chunking strategy tuning | -| `DATABASE_URL` | Direct Postgres connection for pg_analyze tools | | `PG_REPORT_DIR` | Directory for analysis reports (default: `./reports/pg-analysis`) | | `INSIGHT_BATCH_THRESHOLD` | Insight scan tuning (default: `50`) | | `INSIGHT_DEBOUNCE_SECONDS` | Insight scan debounce (default: `300`) | @@ -60,17 +59,16 @@ This file centralizes runtime configuration for Textrawl server, MCP tools, CLI, ## Security notes -- Never expose `SUPABASE_SERVICE_KEY` to browser/renderer/client bundles. +- Never expose `DATABASE_URL` to browser/renderer/client bundles. - Treat `API_BEARER_TOKEN` as a secret and rotate if leaked. ## Why -- Supabase service-role keys are privileged and intended for trusted server environments only. [supabase] +- `DATABASE_URL` grants full database access and is for trusted server environments only. - MCP servers should enforce authentication/authorization at transport boundaries in deployed environments. [mcp] - OpenAI MCP integration and client connector behavior are documented in OpenAI MCP docs. [openai-mcp] ## References -[supabase]: https://supabase.com/docs/guides/api/api-keys [mcp]: https://modelcontextprotocol.io [openai-mcp]: https://platform.openai.com/docs/mcp diff --git a/src/tools/ask.ts b/src/tools/ask.ts index 69d8631..c7e83a0 100644 --- a/src/tools/ask.ts +++ b/src/tools/ask.ts @@ -44,7 +44,7 @@ export function registerAskTool(server: McpServer): void { logger.info('ask called', { question: question.slice(0, 100), scope, limit }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } if (!isEmbeddingsConfigured()) { return configError('Embeddings', 'Configure an embedding provider'); diff --git a/src/tools/briefing.ts b/src/tools/briefing.ts index 6402c5e..b817f2d 100644 --- a/src/tools/briefing.ts +++ b/src/tools/briefing.ts @@ -40,7 +40,7 @@ export function registerBriefingTool(server: McpServer): void { logger.info('daily_briefing called', { includeOnThisDay, recentDays }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { diff --git a/src/tools/conversation.ts b/src/tools/conversation.ts index f8fa710..18b3764 100644 --- a/src/tools/conversation.ts +++ b/src/tools/conversation.ts @@ -81,7 +81,7 @@ export function registerConversationTools(server: McpServer): void { }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } if (!isOpenAIConfigured()) { @@ -317,7 +317,7 @@ export function registerConversationTools(server: McpServer): void { logger.info('query_conversations called', { mode, query, sessionId, sessionKey, limit }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { @@ -631,7 +631,7 @@ export function registerConversationTools(server: McpServer): void { } if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { diff --git a/src/tools/document.ts b/src/tools/document.ts index 2d94fb5..3e9b577 100644 --- a/src/tools/document.ts +++ b/src/tools/document.ts @@ -88,7 +88,7 @@ export function registerDocumentTools(server: McpServer): void { logger.info('get_document called', { documentId, includeChunks }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { @@ -206,7 +206,7 @@ export function registerDocumentTools(server: McpServer): void { }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { @@ -292,7 +292,7 @@ export function registerDocumentTools(server: McpServer): void { logger.info('update_document called', { documentId, title, tags }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } if (title === undefined && tags === undefined) { diff --git a/src/tools/insights.ts b/src/tools/insights.ts index f6fe9e9..7e459b9 100644 --- a/src/tools/insights.ts +++ b/src/tools/insights.ts @@ -74,12 +74,12 @@ export function registerInsightTools(server: McpServer): void { logger.info('get_insights called', { status, insightType, query, limit }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } const schema = await ensureSchema(); if (!schema.ok) { - return toolError(`Insight schema not initialized: ${schema.error}`); + return configError('Insight schema', schema.error); } try { @@ -199,12 +199,12 @@ export function registerInsightTools(server: McpServer): void { logger.info('discover_connections called', { fullScan, maxChunks }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } const schema = await ensureSchema(); if (!schema.ok) { - return toolError(`Insight schema not initialized: ${schema.error}`); + return configError('Insight schema', schema.error); } if (!isOpenAIConfigured()) { @@ -281,12 +281,12 @@ export function registerInsightTools(server: McpServer): void { logger.info('dismiss_insight called', { insightId }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } const schema = await ensureSchema(); if (!schema.ok) { - return toolError(`Insight schema not initialized: ${schema.error}`); + return configError('Insight schema', schema.error); } try { diff --git a/src/tools/memory.ts b/src/tools/memory.ts index 3b848aa..fd893a9 100644 --- a/src/tools/memory.ts +++ b/src/tools/memory.ts @@ -121,7 +121,7 @@ export function registerMemoryTools(server: McpServer): void { }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } if (!isOpenAIConfigured()) { @@ -357,7 +357,7 @@ export function registerMemoryTools(server: McpServer): void { logger.info('query_memory called', { mode, query, entityName, entityTypes, limit }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { @@ -612,7 +612,7 @@ export function registerMemoryTools(server: McpServer): void { }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { @@ -716,7 +716,7 @@ export function registerMemoryTools(server: McpServer): void { } if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { @@ -800,9 +800,9 @@ export function registerMemoryTools(server: McpServer): void { ); } - // Only require Supabase if we're storing results + // Only require Postgres when storing results if (storeResults && !isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { @@ -954,7 +954,7 @@ export function registerMemoryTools(server: McpServer): void { } if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } if (!isOpenAIConfigured()) { diff --git a/src/tools/note.ts b/src/tools/note.ts index 406fba5..37dff7e 100644 --- a/src/tools/note.ts +++ b/src/tools/note.ts @@ -55,7 +55,7 @@ export function registerNoteTool(server: McpServer): void { // Check if services are configured if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } if (!isOpenAIConfigured()) { diff --git a/src/tools/timeline.ts b/src/tools/timeline.ts index bf2a2a2..a04063e 100644 --- a/src/tools/timeline.ts +++ b/src/tools/timeline.ts @@ -36,7 +36,7 @@ export function registerTimelineTool(server: McpServer): void { logger.info('timeline called', { startDate, endDate, topic, limit }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } try { diff --git a/src/tools/url.ts b/src/tools/url.ts index 07f1d4c..0088241 100644 --- a/src/tools/url.ts +++ b/src/tools/url.ts @@ -81,7 +81,7 @@ export function registerUrlTool(server: McpServer): void { logger.info('save_url called', { url: redactUrl(url), title, tags, extractMemories }); if (!isDatabaseConfigured()) { - return configError('Database', 'Set SUPABASE_URL and SUPABASE_SERVICE_KEY'); + return configError('Database', 'Set DATABASE_URL'); } if (!isEmbeddingsConfigured()) { return configError('Embeddings', 'Configure an embedding provider');