diff --git a/apps/tui/package.json b/apps/tui/package.json index 54009445..43ddb21a 100644 --- a/apps/tui/package.json +++ b/apps/tui/package.json @@ -13,13 +13,13 @@ "postpublish": "chmod +x dist/ccflare" }, "dependencies": { - "@ccflare/tui-core": "workspace:*", - "@ccflare/ui-common": "workspace:*", - "@ccflare/database": "workspace:*", - "@ccflare/core-di": "workspace:*", "@ccflare/config": "workspace:*", + "@ccflare/core-di": "workspace:*", + "@ccflare/database": "workspace:*", "@ccflare/logger": "workspace:*", "@ccflare/server": "workspace:*", + "@ccflare/tui-core": "workspace:*", + "@ccflare/ui-common": "workspace:*", "ink": "^6.0.0", "ink-select-input": "^6.0.0", "ink-spinner": "^5.0.0", diff --git a/apps/tui/src/components/MigrationProgress.tsx b/apps/tui/src/components/MigrationProgress.tsx new file mode 100644 index 00000000..2e8516c4 --- /dev/null +++ b/apps/tui/src/components/MigrationProgress.tsx @@ -0,0 +1,69 @@ +import type { MigrationProgress } from "@ccflare/database"; +import { Box, Text, useApp } from "ink"; +import Spinner from "ink-spinner"; + +interface MigrationProgressProps { + progress: MigrationProgress; +} + +// Simple progress bar component without external dependencies +function SimpleProgressBar({ percent }: { percent: number }) { + const width = 40; + const filled = Math.round(width * percent); + const empty = width - filled; + + return ( + + {"█".repeat(filled)} + {"░".repeat(empty)} + + ); +} + +export function MigrationProgressComponent({ + progress, +}: MigrationProgressProps) { + const { exit } = useApp(); + + // Exit if progress is complete + if (progress.percentage === 100) { + setTimeout(() => exit(), 500); + } + + return ( + + + + Database Migration in Progress + + + + + {progress.operation} + + + + + {progress.current.toLocaleString()} /{" "} + {progress.total.toLocaleString()} records + + + + + Progress: {progress.percentage}% + + + + + This is a one-time operation to enable full-text search. + + Please wait while we index your request data... + + ); +} diff --git a/apps/tui/src/main.ts b/apps/tui/src/main.ts index 0709a291..54dbe07a 100644 --- a/apps/tui/src/main.ts +++ b/apps/tui/src/main.ts @@ -2,7 +2,7 @@ import { Config } from "@ccflare/config"; import { CLAUDE_MODEL_IDS, NETWORK, shutdown } from "@ccflare/core"; import { container, SERVICE_KEYS } from "@ccflare/core-di"; -import { DatabaseFactory } from "@ccflare/database"; +import { DatabaseFactory, type MigrationProgress } from "@ccflare/database"; import { Logger } from "@ccflare/logger"; // Import server import startServer from "@ccflare/server"; @@ -11,6 +11,7 @@ import { parseArgs } from "@ccflare/tui-core"; import { render } from "ink"; import React from "react"; import { App } from "./App"; +import { MigrationProgressComponent } from "./components/MigrationProgress"; // Global singleton for auto-started server let runningServer: ReturnType | null = null; @@ -27,11 +28,32 @@ async function main() { container.registerInstance(SERVICE_KEYS.Config, new Config()); container.registerInstance(SERVICE_KEYS.Logger, new Logger("TUI")); - // Initialize database factory - DatabaseFactory.initialize(); + // Track migration progress + let migrationProgress: MigrationProgress | null = null; + let migrationComplete = false; + + // Initialize database factory with progress callback + DatabaseFactory.initialize(undefined, undefined, (progress) => { + migrationProgress = progress; + if (progress.percentage === 100) { + migrationComplete = true; + } + }); + + // Show migration progress if needed const dbOps = DatabaseFactory.getInstance(); container.registerInstance(SERVICE_KEYS.Database, dbOps); + // If migration is in progress, show progress UI first + if (migrationProgress && !migrationComplete) { + const { waitUntilExit } = render( + React.createElement(MigrationProgressComponent, { + progress: migrationProgress, + }), + ); + await waitUntilExit(); + } + const args = process.argv.slice(2); const parsed = parseArgs(args); diff --git a/packages/dashboard-web/src/api.ts b/packages/dashboard-web/src/api.ts index f712ec58..fba22c73 100644 --- a/packages/dashboard-web/src/api.ts +++ b/packages/dashboard-web/src/api.ts @@ -26,6 +26,47 @@ export type { RequestResponse, } from "@ccflare/types"; +// Search types +export interface SearchFilters { + accountId?: string; + method?: string; + path?: string; + statusCode?: number; + success?: boolean; + dateFrom?: string; + dateTo?: string; + model?: string; + agentUsed?: string; + limit?: number; + offset?: number; +} + +export interface SearchResult { + id: string; + timestamp: number; + method: string; + path: string; + accountUsed: string | null; + accountName: string | null; + statusCode: number | null; + success: boolean; + responseTime: number; + model: string | null; + agentUsed: string | null; + requestSnippet?: string; + responseSnippet?: string; + requestSnippets?: string[]; + responseSnippets?: string[]; + rank: number; +} + +export interface SearchResponse { + results: SearchResult[]; + total: number; + query: string; + filters: SearchFilters; +} + // Agent response interface export interface AgentsResponse { agents: Agent[]; @@ -297,6 +338,49 @@ class API extends HttpClient { throw error; } } + + async searchRequests( + query: string, + filters: SearchFilters = {}, + ): Promise { + const params = new URLSearchParams({ q: query }); + + if (filters.limit !== undefined) { + params.append("limit", filters.limit.toString()); + } + if (filters.offset !== undefined) { + params.append("offset", filters.offset.toString()); + } + if (filters.accountId) { + params.append("accountId", filters.accountId); + } + if (filters.method) { + params.append("method", filters.method); + } + if (filters.path) { + params.append("path", filters.path); + } + if (filters.statusCode !== undefined) { + params.append("statusCode", filters.statusCode.toString()); + } + if (filters.success !== undefined) { + params.append("success", filters.success.toString()); + } + if (filters.dateFrom) { + params.append("dateFrom", filters.dateFrom); + } + if (filters.dateTo) { + params.append("dateTo", filters.dateTo); + } + if (filters.model) { + params.append("model", filters.model); + } + if (filters.agentUsed) { + params.append("agentUsed", filters.agentUsed); + } + + return this.get(`/api/requests/search?${params}`); + } } export const api = new API(); diff --git a/packages/dashboard-web/src/components/RequestsTab.tsx b/packages/dashboard-web/src/components/RequestsTab.tsx index 7565e22e..a97d1000 100644 --- a/packages/dashboard-web/src/components/RequestsTab.tsx +++ b/packages/dashboard-web/src/components/RequestsTab.tsx @@ -13,13 +13,16 @@ import { Eye, Filter, Hash, + Loader2, RefreshCw, + Search, User, X, } from "lucide-react"; import { useState } from "react"; -import type { RequestPayload, RequestSummary } from "../api"; +import type { RequestPayload, RequestSummary, SearchResult } from "../api"; import { useRequests } from "../hooks/queries"; +import { useDebounceSearch } from "../hooks/useDebounceSearch"; import { useRequestStream } from "../hooks/useRequestStream"; import { CopyButton } from "./CopyButton"; import { RequestDetailsModal } from "./RequestDetailsModal"; @@ -61,6 +64,8 @@ export function RequestsTab() { const [statusCodeFilters, setStatusCodeFilters] = useState>( new Set(), ); + const [searchQuery, setSearchQuery] = useState(""); + const [_isSearchMode, setIsSearchMode] = useState(false); const { data: requestsData, @@ -72,6 +77,24 @@ export function RequestsTab() { // Enable real-time updates useRequestStream(200); + // Search functionality + const { + data: searchData, + isLoading: isSearchLoading, + isSearching, + hasSearched, + } = useDebounceSearch(searchQuery, { + accountId: accountFilter !== "all" ? accountFilter : undefined, + method: agentFilter !== "all" ? agentFilter : undefined, + statusCode: + statusCodeFilters.size === 1 + ? Number.parseInt(Array.from(statusCodeFilters)[0]) + : undefined, + dateFrom: dateFrom ? new Date(dateFrom).toISOString() : undefined, + dateTo: dateTo ? new Date(dateTo).toISOString() : undefined, + limit: 100, + }); + // Transform the data to match the expected structure const data = requestsData ? { @@ -233,6 +256,13 @@ export function RequestsTab() { setDateFrom(""); setDateTo(""); setStatusCodeFilters(new Set()); + setSearchQuery(""); + setIsSearchMode(false); + }; + + const clearSearch = () => { + setSearchQuery(""); + setIsSearchMode(false); }; const hasActiveFilters = @@ -242,6 +272,14 @@ export function RequestsTab() { dateTo || statusCodeFilters.size > 0; + const hasActiveSearch = searchQuery.trim().length > 0; + + // Determine what to display + const shouldShowSearch = hasActiveSearch && hasSearched; + const searchResults = searchData?.results || []; + const _displayData = shouldShowSearch ? null : data; // Use null when in search mode + const displayRequests = shouldShowSearch ? [] : filteredRequests; // Empty when in search mode + const decodeBase64 = (str: string | null): string => { if (!str) return "No data"; try { @@ -262,6 +300,164 @@ export function RequestsTab() { */ // copyRequest helper removed – handled inline by CopyButton + // Search Result Card Component + interface SearchResultCardProps { + result: SearchResult; + onClick: () => void; + } + + const SearchResultCard = ({ result, onClick }: SearchResultCardProps) => { + const getStatusCodeColor = (code: number) => { + if (code >= 200 && code < 300) return "text-green-600"; + if (code >= 400 && code < 500) return "text-yellow-600"; + if (code >= 500) return "text-red-600"; + return "text-gray-600"; + }; + + // Safe highlighting that only handles tags + const renderHighlightedText = (text: string) => { + // Only allow and tags for highlighting + const parts = text.split(/(<\/?mark>)/); + return parts + .map((part, index) => { + if (part === "") { + return null; // Skip opening tag + } + if (part === "") { + return null; // Skip closing tag + } + // Check if this part should be highlighted (between mark tags) + const isHighlighted = index > 0 && parts[index - 1] === ""; + if (isHighlighted) { + return ( + + {part} + + ); + } + return part; + }) + .filter(Boolean); + }; + + return ( + + + + + {new Date(result.timestamp).toLocaleTimeString()} + + {result.method} + + {result.path} + + {result.statusCode && ( + + {result.statusCode} + + )} + {result.model && ( + + {result.model} + + )} + {result.agentUsed && ( + + Agent: {result.agentUsed} + + )} + {result.accountName && ( + + via {result.accountName} + + )} + + + {result.responseTime}ms + ID: {result.id.slice(0, 8)}... + + + + {/* Search snippets */} + + {/* Request snippets */} + {result.requestSnippets && result.requestSnippets.length > 0 ? ( + + + Request matches ({result.requestSnippets.length}):{" "} + + + {result.requestSnippets.map((snippet: string, idx: number) => ( + + {renderHighlightedText(snippet)} + + ))} + + + ) : result.requestSnippet && result.requestSnippet.trim() !== "" ? ( + // Fallback for backward compatibility + + + Request:{" "} + + + {renderHighlightedText(result.requestSnippet)} + + + ) : null} + + {/* Response snippets */} + {result.responseSnippets && result.responseSnippets.length > 0 ? ( + + + Response matches ({result.responseSnippets.length}):{" "} + + + {result.responseSnippets.map((snippet: string, idx: number) => ( + + {renderHighlightedText(snippet)} + + ))} + + + ) : result.responseSnippet && result.responseSnippet.trim() !== "" ? ( + // Fallback for backward compatibility + + + Response:{" "} + + + {renderHighlightedText(result.responseSnippet)} + + + ) : null} + + + + + + View Details + + + + ); + }; + if (loading) { return ( @@ -323,6 +519,35 @@ export function RequestsTab() { + {/* Search Input */} + + + + { + setSearchQuery(e.target.value); + setIsSearchMode(e.target.value.trim().length > 0); + }} + className="pl-10 pr-10" + /> + {isSearching && ( + + )} + {hasActiveSearch && !isSearching && ( + + + + )} + + + {/* Active Filters Display */} {hasActiveFilters && ( @@ -386,10 +611,24 @@ export function RequestsTab() { )} + {hasActiveSearch && ( + + "{searchQuery.substring(0, 20)} + {searchQuery.length > 20 ? "..." : ""}" + + + + + )} - {filteredRequests.length} of {data?.requests.length || 0}{" "} - requests + {shouldShowSearch + ? `${searchResults.length} search results` + : `${filteredRequests.length} of ${data?.requests.length || 0} requests`} )} - {!data ? ( + {/* Search Results or Regular Results */} + {shouldShowSearch ? ( + /* Search Results View */ + searchResults.length === 0 ? ( + + + + {isSearchLoading ? "Searching..." : "No results found"} + + {!isSearchLoading && ( + + Try adjusting your search terms or filters + + )} + + ) : ( + + {searchResults.map((result) => ( + { + // Find the full request payload to show in modal + const fullRequest = data?.requests.find( + (r) => r.id === result.id, + ); + if (fullRequest) { + setModalRequest(fullRequest); + } + }} + /> + ))} + + ) + ) : /* Regular Results View */ + !data ? ( No requests found - ) : filteredRequests.length === 0 ? ( + ) : displayRequests.length === 0 ? ( No requests match the selected filters ) : ( - {filteredRequests.map((request) => { + {displayRequests.map((request) => { const isExpanded = expandedRequests.has(request.id); const isError = request.error || !request.meta.success; const statusCode = request.response?.status; diff --git a/packages/dashboard-web/src/hooks/useDebounceSearch.ts b/packages/dashboard-web/src/hooks/useDebounceSearch.ts new file mode 100644 index 00000000..51bdf22b --- /dev/null +++ b/packages/dashboard-web/src/hooks/useDebounceSearch.ts @@ -0,0 +1,68 @@ +import { useQuery } from "@tanstack/react-query"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import type { SearchFilters, SearchResponse } from "../api"; +import { api } from "../api"; +import { queryKeys } from "../lib/query-keys"; + +/** + * Custom hook for debounced search functionality + */ +export function useDebounceSearch( + query: string, + filters: SearchFilters = {}, + debounceMs: number = 300, +) { + const [debouncedQuery, setDebouncedQuery] = useState(query); + + // Debounce the search query + useEffect(() => { + const timer = setTimeout(() => { + setDebouncedQuery(query); + }, debounceMs); + + return () => clearTimeout(timer); + }, [query, debounceMs]); + + // Create a stable filters object for the query key + const stableFilters = useMemo(() => filters, [filters]); + + // Search function using useQuery + const searchQuery = useQuery({ + queryKey: queryKeys.search(debouncedQuery, stableFilters), + queryFn: () => api.searchRequests(debouncedQuery, stableFilters), + enabled: debouncedQuery.trim().length > 0, // Only search if query is not empty + refetchOnWindowFocus: false, + staleTime: 30000, // Consider data fresh for 30 seconds + }); + + // Manual search function for immediate search trigger + const search = useCallback( + (immediateQuery?: string, immediateFilters?: SearchFilters) => { + const queryToUse = immediateQuery ?? debouncedQuery; + const filtersToUse = immediateFilters ?? stableFilters; + + if (queryToUse.trim().length === 0) { + return Promise.resolve({ + results: [], + total: 0, + query: queryToUse, + filters: filtersToUse, + } as SearchResponse); + } + + return api.searchRequests(queryToUse, filtersToUse); + }, + [debouncedQuery, stableFilters], + ); + + return { + data: searchQuery.data, + isLoading: searchQuery.isLoading, + error: searchQuery.error, + isError: searchQuery.isError, + search, + refetch: searchQuery.refetch, + isSearching: debouncedQuery !== query || searchQuery.isFetching, + hasSearched: debouncedQuery.trim().length > 0, + }; +} diff --git a/packages/dashboard-web/src/lib/query-keys.ts b/packages/dashboard-web/src/lib/query-keys.ts index 2018fb32..b3f5d997 100644 --- a/packages/dashboard-web/src/lib/query-keys.ts +++ b/packages/dashboard-web/src/lib/query-keys.ts @@ -22,4 +22,6 @@ export const queryKeys = { logHistory: () => [...queryKeys.all, "logs", "history"] as const, defaultAgentModel: () => [...queryKeys.all, "config", "defaultAgentModel"] as const, + search: (query: string, filters?: unknown) => + [...queryKeys.all, "search", { query, filters }] as const, } as const; diff --git a/packages/database/src/database-operations.ts b/packages/database/src/database-operations.ts index 2db7aaa0..857d802c 100644 --- a/packages/database/src/database-operations.ts +++ b/packages/database/src/database-operations.ts @@ -3,7 +3,11 @@ import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import type { Disposable } from "@ccflare/core"; import type { Account, StrategyStore } from "@ccflare/types"; -import { ensureSchema, runMigrations } from "./migrations"; +import { + ensureSchema, + type MigrationProgress, + runMigrations, +} from "./migrations"; import { resolveDbPath } from "./paths"; import { AccountRepository } from "./repositories/account.repository"; import { AgentPreferenceRepository } from "./repositories/agent-preference.repository"; @@ -11,6 +15,8 @@ import { OAuthRepository } from "./repositories/oauth.repository"; import { type RequestData, RequestRepository, + type SearchFilters, + type SearchResult, } from "./repositories/request.repository"; import { StatsRepository } from "./repositories/stats.repository"; import { StrategyRepository } from "./repositories/strategy.repository"; @@ -35,7 +41,10 @@ export class DatabaseOperations implements StrategyStore, Disposable { private stats: StatsRepository; private agentPreferences: AgentPreferenceRepository; - constructor(dbPath?: string) { + constructor( + dbPath?: string, + onMigrationProgress?: (progress: MigrationProgress) => void, + ) { const resolvedPath = dbPath ?? resolveDbPath(); // Ensure the directory exists @@ -50,7 +59,7 @@ export class DatabaseOperations implements StrategyStore, Disposable { this.db.exec("PRAGMA synchronous = NORMAL"); // Better performance while maintaining safety ensureSchema(this.db); - runMigrations(this.db); + runMigrations(this.db, onMigrationProgress); // Initialize repositories this.accounts = new AccountRepository(this.db); @@ -325,6 +334,15 @@ export class DatabaseOperations implements StrategyStore, Disposable { this.agentPreferences.setBulkPreferences(agentIds, model); } + // Search operations delegated to request repository + + getSearchResults( + query: string, + filters?: SearchFilters, + ): Array { + return this.requests.getSearchResults(query, filters); + } + close(): void { // Ensure all write operations are flushed before closing this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); diff --git a/packages/database/src/factory.ts b/packages/database/src/factory.ts index 854e020e..f41c89dd 100644 --- a/packages/database/src/factory.ts +++ b/packages/database/src/factory.ts @@ -1,21 +1,27 @@ import { registerDisposable, unregisterDisposable } from "@ccflare/core"; import { DatabaseOperations, type RuntimeConfig } from "./index"; +import type { MigrationProgress } from "./migrations"; let instance: DatabaseOperations | null = null; let dbPath: string | undefined; let runtimeConfig: RuntimeConfig | undefined; +let migrationProgressCallback: + | ((progress: MigrationProgress) => void) + | undefined; export function initialize( dbPathParam?: string, runtimeConfigParam?: RuntimeConfig, + onMigrationProgress?: (progress: MigrationProgress) => void, ): void { dbPath = dbPathParam; runtimeConfig = runtimeConfigParam; + migrationProgressCallback = onMigrationProgress; } export function getInstance(): DatabaseOperations { if (!instance) { - instance = new DatabaseOperations(dbPath); + instance = new DatabaseOperations(dbPath, migrationProgressCallback); if (runtimeConfig) { instance.setRuntimeConfig(runtimeConfig); } @@ -37,9 +43,17 @@ export function reset(): void { closeAll(); } +export function createDbOps( + dbPath?: string, + onMigrationProgress?: (progress: MigrationProgress) => void, +): DatabaseOperations { + return new DatabaseOperations(dbPath, onMigrationProgress); +} + export const DatabaseFactory = { initialize, getInstance, closeAll, reset, + createDbOps, }; diff --git a/packages/database/src/index.ts b/packages/database/src/index.ts index da488b0d..2c573fdf 100644 --- a/packages/database/src/index.ts +++ b/packages/database/src/index.ts @@ -6,9 +6,14 @@ export { DatabaseOperations }; export { AsyncDbWriter } from "./async-writer"; export type { RuntimeConfig } from "./database-operations"; export { DatabaseFactory } from "./factory"; +export type { MigrationProgress } from "./migrations"; export { ensureSchema, runMigrations } from "./migrations"; export { resolveDbPath } from "./paths"; export { analyzeIndexUsage } from "./performance-indexes"; - +export type { + RequestData, + SearchFilters, + SearchResult, +} from "./repositories/request.repository"; // Re-export repository types export type { StatsRepository } from "./repositories/stats.repository"; diff --git a/packages/database/src/migrations.ts b/packages/database/src/migrations.ts index 0afb29eb..c720ca74 100644 --- a/packages/database/src/migrations.ts +++ b/packages/database/src/migrations.ts @@ -4,6 +4,13 @@ import { addPerformanceIndexes } from "./performance-indexes"; const log = new Logger("DatabaseMigrations"); +export interface MigrationProgress { + current: number; + total: number; + operation: string; + percentage: number; +} + export function ensureSchema(db: Database): void { // Create accounts table db.run(` @@ -92,7 +99,10 @@ export function ensureSchema(db: Database): void { `); } -export function runMigrations(db: Database): void { +export function runMigrations( + db: Database, + onProgress?: (progress: MigrationProgress) => void, +): void { // Ensure base schema exists first ensureSchema(db); // Check if columns exist before adding them @@ -269,4 +279,131 @@ export function runMigrations(db: Database): void { // Add performance indexes addPerformanceIndexes(db); + + // Add FTS5 table for full-text search + addFTSMigration(db, onProgress); +} + +function addFTSMigration( + db: Database, + onProgress?: (progress: MigrationProgress) => void, +): void { + // Check if FTS table already exists + const ftsExists = db + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name='request_payloads_fts'", + ) + .get(); + + if (!ftsExists) { + log.info("Creating FTS5 table for full-text search..."); + + // Create the FTS5 virtual table + db.run(` + CREATE VIRTUAL TABLE request_payloads_fts USING fts5( + id UNINDEXED, + request_body, + response_body, + tokenize='porter unicode61' + ) + `); + + // Count total records to migrate + const totalResult = db + .prepare("SELECT COUNT(*) as count FROM request_payloads") + .get() as { count: number }; + const totalRecords = totalResult.count; + + if (totalRecords > 0) { + log.info(`Migrating ${totalRecords} records to FTS index...`); + + // Report initial progress + if (onProgress && totalRecords > 100) { + onProgress({ + current: 0, + total: totalRecords, + operation: "Indexing request/response data for search", + percentage: 0, + }); + } + + // Helper function to decode base64 + const decodeBase64 = (str: string | null): string => { + if (!str || str === "[streamed]") return ""; + try { + return Buffer.from(str, "base64").toString("utf-8"); + } catch { + return str || ""; + } + }; + + // Migrate existing data in batches + const batchSize = 100; // Smaller batch size for processing + let processed = 0; + + // Prepare statements + const selectStmt = db.prepare(` + SELECT id, json + FROM request_payloads + LIMIT ?1 OFFSET ?2 + `); + + const insertStmt = db.prepare(` + INSERT INTO request_payloads_fts (id, request_body, response_body) + VALUES (?, ?, ?) + `); + + // Process in batches + while (processed < totalRecords) { + const currentBatch = Math.min(batchSize, totalRecords - processed); + const rows = selectStmt.all(currentBatch, processed) as Array<{ + id: string; + json: string; + }>; + + // Process each row + for (const row of rows) { + try { + const data = JSON.parse(row.json); + const requestBody = decodeBase64(data.request?.body); + const responseBody = decodeBase64(data.response?.body); + insertStmt.run(row.id, requestBody, responseBody); + } catch (error) { + log.debug(`Failed to process row ${row.id}:`, error); + // Insert empty strings on error + insertStmt.run(row.id, "", ""); + } + } + + processed += rows.length; + + // Report progress + if (onProgress && totalRecords > 100) { + const percentage = Math.round((processed / totalRecords) * 100); + onProgress({ + current: processed, + total: totalRecords, + operation: "Indexing request/response data for search", + percentage, + }); + } + } + + log.info("FTS migration completed successfully"); + } + + // Note: We can't create triggers that decode base64 since SQLite doesn't support custom functions in triggers + // Instead, we'll handle the decoding when we insert payloads in the RequestRepository + + // Create delete trigger + db.run(` + CREATE TRIGGER request_payloads_fts_delete + AFTER DELETE ON request_payloads + BEGIN + DELETE FROM request_payloads_fts WHERE id = old.id; + END + `); + + log.info("FTS5 table and triggers created successfully"); + } } diff --git a/packages/database/src/repositories/request.repository.ts b/packages/database/src/repositories/request.repository.ts index 0b37b5a0..d556cc05 100644 --- a/packages/database/src/repositories/request.repository.ts +++ b/packages/database/src/repositories/request.repository.ts @@ -25,6 +25,39 @@ export interface RequestData { }; } +export interface SearchFilters { + accountId?: string; + method?: string; + path?: string; + statusCode?: number; + success?: boolean; + dateFrom?: number; + dateTo?: number; + model?: string; + agentUsed?: string; + limit?: number; + offset?: number; +} + +export interface SearchResult { + id: string; + timestamp: number; + method: string; + path: string; + accountUsed: string | null; + accountName: string | null; + statusCode: number | null; + success: boolean; + responseTime: number; + model: string | null; + agentUsed: string | null; + requestSnippet?: string; // For backward compatibility + responseSnippet?: string; // For backward compatibility + requestSnippets?: string[]; // Multiple snippets + responseSnippets?: string[]; // Multiple snippets + rank: number; +} + export class RequestRepository extends BaseRepository { saveMeta( id: string, @@ -127,6 +160,39 @@ export class RequestRepository extends BaseRepository { `INSERT OR REPLACE INTO request_payloads (id, json) VALUES (?, ?)`, [id, json], ); + + // Also update FTS table with decoded content + try { + const payload = data as { + request?: { body?: string }; + response?: { body?: string }; + }; + const requestBody = this.decodeBase64Content(payload.request?.body || ""); + const responseBody = this.decodeBase64Content( + payload.response?.body || "", + ); + + // Check if FTS record exists + const exists = this.get<{ id: string }>( + `SELECT id FROM request_payloads_fts WHERE id = ?`, + [id], + ); + + if (exists) { + this.run( + `UPDATE request_payloads_fts SET request_body = ?, response_body = ? WHERE id = ?`, + [requestBody, responseBody, id], + ); + } else { + this.run( + `INSERT INTO request_payloads_fts (id, request_body, response_body) VALUES (?, ?, ?)`, + [id, requestBody, responseBody], + ); + } + } catch (error) { + // Log error but don't fail the main operation + console.error(`Failed to update FTS for ${id}:`, error); + } } getPayload(id: string): unknown | null { @@ -378,4 +444,158 @@ export class RequestRepository extends BaseRepository { successRate: row.success_rate, })); } + + /** + * Comprehensive search with filters and metadata + */ + getSearchResults( + query: string, + filters: SearchFilters = {}, + ): Array { + const { + accountId, + method, + path, + statusCode, + success, + dateFrom, + dateTo, + model, + agentUsed, + limit = 50, + offset = 0, + } = filters; + + // Build WHERE conditions + const conditions: string[] = ["request_payloads_fts MATCH ?"]; + const params: (string | number)[] = [query]; + + if (accountId) { + conditions.push("r.account_used = ?"); + params.push(accountId); + } + + if (method) { + conditions.push("r.method = ?"); + params.push(method); + } + + if (path) { + conditions.push("r.path LIKE ?"); + params.push(`%${path}%`); + } + + if (statusCode !== undefined) { + conditions.push("r.status_code = ?"); + params.push(statusCode); + } + + if (success !== undefined) { + conditions.push("r.success = ?"); + params.push(success ? 1 : 0); + } + + if (dateFrom) { + conditions.push("r.timestamp >= ?"); + params.push(dateFrom); + } + + if (dateTo) { + conditions.push("r.timestamp <= ?"); + params.push(dateTo); + } + + if (model) { + conditions.push("r.model = ?"); + params.push(model); + } + + if (agentUsed) { + conditions.push("r.agent_used = ?"); + params.push(agentUsed); + } + + params.push(limit, offset); + + const whereClause = conditions.join(" AND "); + + const results = this.query<{ + id: string; + timestamp: number; + method: string; + path: string; + account_used: string | null; + account_name: string | null; + status_code: number | null; + success: 0 | 1; + response_time_ms: number; + model: string | null; + agent_used: string | null; + rank: number; + request_snippet: string; + response_snippet: string; + }>( + ` + SELECT + r.id, + r.timestamp, + r.method, + r.path, + r.account_used, + a.name as account_name, + r.status_code, + r.success, + r.response_time_ms, + r.model, + r.agent_used, + fts.rank, + snippet(request_payloads_fts, 1, '', '', '...', 32) as request_snippet, + snippet(request_payloads_fts, 2, '', '', '...', 32) as response_snippet + FROM request_payloads_fts fts + JOIN requests r ON fts.id = r.id + LEFT JOIN accounts a ON r.account_used = a.id + WHERE ${whereClause} + ORDER BY r.timestamp DESC + LIMIT ? OFFSET ? + `, + params, + ); + + return results.map((row) => ({ + id: row.id, + timestamp: row.timestamp, + method: row.method, + path: row.path, + accountUsed: row.account_used, + accountName: row.account_name, + statusCode: row.status_code, + success: row.success === 1, + responseTime: row.response_time_ms, + model: row.model, + agentUsed: row.agent_used, + rank: row.rank, + requestSnippet: row.request_snippet, // Already decoded by snippet() + responseSnippet: row.response_snippet, // Already decoded by snippet() + })); + } + + /** + * Helper method to decode base64 content safely + */ + private decodeBase64Content(content: string): string { + if (!content || content === "[streamed]" || content === "") { + return content; + } + + try { + // Check if the content looks like base64 (and is long enough to be meaningful) + if (content.length > 10 && /^[A-Za-z0-9+/]+=*$/.test(content)) { + return Buffer.from(content, "base64").toString("utf-8"); + } + return content; + } catch { + // If decoding fails, return original content + return content; + } + } } diff --git a/packages/http-api/src/handlers/requests-search.ts b/packages/http-api/src/handlers/requests-search.ts new file mode 100644 index 00000000..15da9442 --- /dev/null +++ b/packages/http-api/src/handlers/requests-search.ts @@ -0,0 +1,311 @@ +import { validateNumber, validateString } from "@ccflare/core"; +import type { DatabaseOperations, SearchFilters } from "@ccflare/database"; +import { jsonResponse } from "@ccflare/http-common"; + +/** + * Create a requests search handler for full-text search functionality + */ +export function createRequestsSearchHandler(dbOps: DatabaseOperations) { + return async (req: Request): Promise => { + const url = new URL(req.url); + const query = url.searchParams.get("q"); + + if (!query || query.trim() === "") { + return jsonResponse( + { + error: "Search query is required", + results: [], + total: 0, + }, + 400, + ); + } + + // Parse search parameters + const limitParam = url.searchParams.get("limit"); + const offsetParam = url.searchParams.get("offset"); + const accountIdParam = url.searchParams.get("accountId"); + const methodParam = url.searchParams.get("method"); + const pathParam = url.searchParams.get("path"); + const statusCodeParam = url.searchParams.get("statusCode"); + const successParam = url.searchParams.get("success"); + const dateFromParam = url.searchParams.get("dateFrom"); + const dateToParam = url.searchParams.get("dateTo"); + const modelParam = url.searchParams.get("model"); + const agentUsedParam = url.searchParams.get("agentUsed"); + + // Validate and build filters + const filters: SearchFilters = { + limit: + validateNumber(limitParam || "50", "limit", { + min: 1, + max: 200, + integer: true, + }) || 50, + offset: + validateNumber(offsetParam || "0", "offset", { + min: 0, + integer: true, + }) || 0, + }; + + if (accountIdParam) { + filters.accountId = validateString(accountIdParam, "accountId", { + minLength: 1, + maxLength: 100, + }); + } + + if (methodParam) { + filters.method = validateString(methodParam, "method", { + minLength: 1, + maxLength: 10, + }); + } + + if (pathParam) { + filters.path = validateString(pathParam, "path", { + minLength: 1, + maxLength: 500, + }); + } + + if (statusCodeParam) { + filters.statusCode = validateNumber(statusCodeParam, "statusCode", { + min: 100, + max: 599, + integer: true, + }); + } + + if (successParam) { + filters.success = successParam === "true"; + } + + if (dateFromParam) { + const dateFrom = new Date(dateFromParam); + if (!Number.isNaN(dateFrom.getTime())) { + filters.dateFrom = dateFrom.getTime(); + } + } + + if (dateToParam) { + const dateTo = new Date(dateToParam); + if (!Number.isNaN(dateTo.getTime())) { + filters.dateTo = dateTo.getTime(); + } + } + + if (modelParam) { + filters.model = validateString(modelParam, "model", { + minLength: 1, + maxLength: 100, + }); + } + + if (agentUsedParam) { + filters.agentUsed = validateString(agentUsedParam, "agentUsed", { + minLength: 1, + maxLength: 100, + }); + } + + try { + // Process the query to handle camelCase and other patterns + let ftsQuery = query; + const isCamelCase = /[a-z][A-Z]/.test(query); + + // Escape FTS5 special characters first + const escapedQuery = query + .replace(/"/g, '""') // Escape quotes + .replace(/[*()]/g, ""); // Remove special FTS chars + + if (isCamelCase) { + // For camelCase, split it for FTS5 search + ftsQuery = escapedQuery.replace(/([a-z])([A-Z])/g, "$1 $2"); + ftsQuery = `"${ftsQuery}"`; + } else if (escapedQuery.includes(" ")) { + // For multi-word queries, use phrase search + ftsQuery = `"${escapedQuery}"`; + } else { + // Single word, non-camelCase + ftsQuery = escapedQuery; + } + + const results = dbOps.getDatabase().prepare(` + SELECT + r.id, + r.timestamp, + r.method, + r.path, + r.account_used, + a.name as account_name, + r.status_code, + r.success, + r.response_time_ms, + r.model, + r.agent_used, + fts.rank, + fts.request_body, + fts.response_body + FROM request_payloads_fts fts + JOIN requests r ON fts.id = r.id + LEFT JOIN accounts a ON r.account_used = a.id + WHERE request_payloads_fts MATCH ? + ${filters.accountId ? " AND r.account_used = ?" : ""} + ${filters.method ? " AND r.method = ?" : ""} + ${filters.path ? " AND r.path LIKE ?" : ""} + ${filters.statusCode ? " AND r.status_code = ?" : ""} + ${filters.success !== undefined ? " AND r.success = ?" : ""} + ${filters.dateFrom ? " AND r.timestamp >= ?" : ""} + ${filters.dateTo ? " AND r.timestamp <= ?" : ""} + ${filters.model ? " AND r.model = ?" : ""} + ${filters.agentUsed ? " AND r.agent_used = ?" : ""} + ORDER BY r.timestamp DESC + LIMIT ? OFFSET ? + `); + + // Build parameters array + const params: (string | number)[] = [ftsQuery]; + + if (filters.accountId) params.push(filters.accountId); + if (filters.method) params.push(filters.method); + if (filters.path) params.push(`%${filters.path}%`); + if (filters.statusCode) params.push(filters.statusCode); + if (filters.success !== undefined) params.push(filters.success ? 1 : 0); + if (filters.dateFrom) params.push(filters.dateFrom); + if (filters.dateTo) params.push(filters.dateTo); + if (filters.model) params.push(filters.model); + if (filters.agentUsed) params.push(filters.agentUsed); + + params.push(filters.limit || 50, filters.offset || 0); + + const rawResults = results.all(...params) as Array<{ + id: string; + timestamp: number; + method: string; + path: string; + account_used: string | null; + account_name: string | null; + status_code: number | null; + success: 0 | 1; + response_time_ms: number; + model: string | null; + agent_used: string | null; + rank: number; + request_body: string; + response_body: string; + }>; + + // Helper to find and extract snippets with the search term + const findMatchingSnippets = ( + text: string, + searchTerm: string, + contextChars = 50, + ) => { + if (!text) return []; + + const snippets: string[] = []; + // Escape regex special characters in search term + const escapedTerm = searchTerm.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const searchRegex = new RegExp(escapedTerm, "gi"); + let match: RegExpExecArray | null = searchRegex.exec(text); + + while (match !== null) { + const matchStart = match.index; + const matchEnd = matchStart + match[0].length; + + // Find context boundaries + let contextStart = Math.max(0, matchStart - contextChars); + let contextEnd = Math.min(text.length, matchEnd + contextChars); + + // Adjust to word boundaries + while ( + contextStart > 0 && + text[contextStart - 1] !== " " && + text[contextStart - 1] !== "\n" + ) { + contextStart--; + } + while ( + contextEnd < text.length && + text[contextEnd] !== " " && + text[contextEnd] !== "\n" + ) { + contextEnd++; + } + + // Extract snippet + let snippet = text.substring(contextStart, contextEnd); + + // Highlight the matched term + snippet = snippet.replace( + new RegExp(`(${escapedTerm})`, "gi"), + "$1", + ); + + // Add ellipsis if needed + if (contextStart > 0) snippet = `...${snippet}`; + if (contextEnd < text.length) snippet = `${snippet}...`; + + snippets.push(snippet.trim()); + + // Get next match + match = searchRegex.exec(text); + } + + return snippets; + }; + + // Transform results and extract all snippets + const searchResults = rawResults + .map((row) => { + const requestSnippets = findMatchingSnippets(row.request_body, query); + const responseSnippets = findMatchingSnippets( + row.response_body, + query, + ); + + // Only include results that actually have matches + if (requestSnippets.length === 0 && responseSnippets.length === 0) { + return null; + } + + return { + id: row.id, + timestamp: row.timestamp, + method: row.method, + path: row.path, + accountUsed: row.account_used, + accountName: row.account_name, + statusCode: row.status_code, + success: row.success === 1, + responseTime: row.response_time_ms, + model: row.model, + agentUsed: row.agent_used, + rank: row.rank, + requestSnippets, + responseSnippets, + }; + }) + .filter((result) => result !== null); + + return jsonResponse({ + results: searchResults, + total: searchResults.length, + query: query, + filters: filters, + }); + } catch (error) { + console.error("Search error:", error); + return jsonResponse( + { + error: "Search failed", + results: [], + total: 0, + }, + 500, + ); + } + }; +} diff --git a/packages/http-api/src/router.ts b/packages/http-api/src/router.ts index 22f0fc6f..28881624 100644 --- a/packages/http-api/src/router.ts +++ b/packages/http-api/src/router.ts @@ -28,6 +28,7 @@ import { createRequestsDetailHandler, createRequestsSummaryHandler, } from "./handlers/requests"; +import { createRequestsSearchHandler } from "./handlers/requests-search"; import { createRequestsStreamHandler } from "./handlers/requests-stream"; import { createStatsHandler, createStatsResetHandler } from "./handlers/stats"; import type { APIContext } from "./types"; @@ -71,6 +72,7 @@ export class APIRouter { const agentsHandler = createAgentsListHandler(dbOps); const workspacesHandler = createWorkspacesListHandler(); const requestsStreamHandler = createRequestsStreamHandler(); + const searchHandler = createRequestsSearchHandler(dbOps); // Register routes this.handlers.set("GET:/health", () => healthHandler()); @@ -134,6 +136,7 @@ export class APIRouter { return bulkHandler(req); }); this.handlers.set("GET:/api/workspaces", () => workspacesHandler()); + this.handlers.set("GET:/api/requests/search", (req) => searchHandler(req)); } /** diff --git a/packages/proxy/src/post-processor.worker.ts b/packages/proxy/src/post-processor.worker.ts index 5cc16d1e..3118b73f 100644 --- a/packages/proxy/src/post-processor.worker.ts +++ b/packages/proxy/src/post-processor.worker.ts @@ -69,6 +69,10 @@ function shouldLogRequest(path: string, status: number): boolean { if (path.startsWith("/.well-known/") && status === 404) { return false; } + // Skip logging internal API calls (dashboard, search, etc.) + if (path.startsWith("/api/")) { + return false; + } return true; } diff --git a/packages/types/src/api.ts b/packages/types/src/api.ts index cbadd560..0bb84e90 100644 --- a/packages/types/src/api.ts +++ b/packages/types/src/api.ts @@ -14,3 +14,15 @@ export interface AgentUpdatePayload { systemPrompt?: string; mode?: "all" | "edit" | "read-only" | "execution" | "custom"; } + +/** + * Search response with pagination metadata + */ +export interface SearchResponse { + results: T[]; + total: number; + page: number; + pageSize: number; + hasNext: boolean; + hasPrev: boolean; +}
+ {isSearchLoading ? "Searching..." : "No results found"} +
+ Try adjusting your search terms or filters +
No requests found
No requests match the selected filters