diff --git a/docs/design/lovable-vectorless.png b/docs/design/lovable-vectorless.png deleted file mode 100644 index 40bb7047..00000000 Binary files a/docs/design/lovable-vectorless.png and /dev/null differ diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md new file mode 100644 index 00000000..0dd02df3 --- /dev/null +++ b/docs/guide/configuration.md @@ -0,0 +1,218 @@ +# Configuration Reference + +Vectorless supports multiple configuration layers. Settings from higher-priority sources override lower ones. + +## Priority (highest → lowest) + +1. Builder/constructor parameters +2. Environment variables +3. Explicit config file (`config_path`) +4. Auto-detected config file (`vectorless.toml`, `config.toml`, `.vectorless.toml`) +5. Default values + +## Environment Variables + +| Variable | Description | Example | +|----------|-------------|---------| +| `OPENAI_API_KEY` | LLM API key | `sk-...` | +| `VECTORLESS_MODEL` | Default model | `gpt-4o-mini` | +| `VECTORLESS_ENDPOINT` | Custom API endpoint | `https://api.openai.com/v1` | +| `VECTORLESS_WORKSPACE` | Workspace directory | `./data` | + +## Configuration File + +Create `vectorless.toml` for full control: + +```bash +cp vectorless.example.toml ./vectorless.toml +``` + +### LLM Configuration + +Three separate LLM clients for different tasks: + +```toml +[llm] +api_key = "sk-your-api-key" + +# Summarization — used during indexing +[llm.summary] +model = "gpt-4o-mini" +endpoint = "https://api.openai.com/v1" +max_tokens = 200 +temperature = 0.0 + +# Retrieval — used during query analysis +[llm.retrieval] +model = "gpt-4o" +endpoint = "https://api.openai.com/v1" +max_tokens = 100 +temperature = 0.0 + +# Pilot — used for tree navigation guidance +[llm.pilot] +model = "gpt-4o-mini" +endpoint = "https://api.openai.com/v1" +max_tokens = 300 +temperature = 0.0 +``` + +### Retry and Rate Limiting + +```toml +[llm.retry] +max_attempts = 3 +initial_delay_ms = 500 +max_delay_ms = 30000 +multiplier = 2.0 +retry_on_rate_limit = true + +[llm.throttle] +max_concurrent_requests = 10 +requests_per_minute = 500 +enabled = true +``` + +### Fallback + +```toml +[llm.fallback] +enabled = true +models = ["gpt-4o-mini", "glm-4-flash"] +on_rate_limit = "retry_then_fallback" +on_timeout = "retry_then_fallback" +on_all_failed = "return_error" +``` + +### Retrieval Configuration + +```toml +[retrieval] +top_k = 3 +max_tokens = 1000 + +[retrieval.search] +top_k = 5 +beam_width = 3 +max_iterations = 10 +min_score = 0.1 + +[retrieval.sufficiency] +min_tokens = 500 +target_tokens = 2000 +max_tokens = 4000 +confidence_threshold = 0.7 + +[retrieval.cache] +max_entries = 1000 +ttl_secs = 3600 +``` + +### Strategy Configuration + +```toml +# Hybrid (BM25 + LLM) +[retrieval.strategy.hybrid] +enabled = true +pre_filter_ratio = 0.3 +bm25_weight = 0.4 +llm_weight = 0.6 + +# Cross-document search +[retrieval.strategy.cross_document] +enabled = true +max_documents = 10 +max_results_per_doc = 3 +merge_strategy = "TopK" +parallel_search = true + +# Page range (PDF) +[retrieval.strategy.page_range] +enabled = true +include_boundary_nodes = true +expand_context_pages = 0 +``` + +### Pilot Configuration + +```toml +[pilot] +mode = "Balanced" # "Conservative", "Balanced", "Aggressive" +guide_at_start = true +guide_at_backtrack = true + +[pilot.budget] +max_tokens_per_query = 2000 +max_tokens_per_call = 500 +max_calls_per_query = 5 + +[pilot.intervention] +fork_threshold = 3 +score_gap_threshold = 0.15 +low_score_threshold = 0.3 +``` + +### Multi-turn Retrieval + +```toml +[retrieval.multiturn] +enabled = true +max_sub_queries = 3 +decomposition_model = "gpt-4o-mini" +aggregation_strategy = "merge" +``` + +### Reference Following + +```toml +[retrieval.reference] +enabled = true +max_depth = 3 +max_references = 10 +follow_pages = true +follow_tables_figures = true +min_confidence = 0.5 +``` + +### Storage Configuration + +```toml +[storage] +workspace_dir = "./workspace" +cache_size = 100 +atomic_writes = true +file_lock = true +checksum_enabled = true + +[storage.compression] +enabled = false +algorithm = "gzip" +level = 6 +``` + +### Metrics + +```toml +[metrics] +enabled = true +storage_path = "./workspace/metrics" +retention_days = 30 + +[metrics.llm] +track_tokens = true +track_latency = true +track_cost = true +cost_per_1k_input_tokens = 0.00015 +cost_per_1k_output_tokens = 0.0006 + +[metrics.pilot] +track_decisions = true +track_accuracy = true +track_feedback = true + +[metrics.retrieval] +track_paths = true +track_scores = true +track_iterations = true +track_cache = true +``` diff --git a/docs/guide/python.md b/docs/guide/python.md new file mode 100644 index 00000000..f2b6c351 --- /dev/null +++ b/docs/guide/python.md @@ -0,0 +1,193 @@ +# Python Usage Guide + +## Installation + +```bash +pip install vectorless +``` + +## Configuration + +### Zero Configuration + +Set your API key and you're ready: + +```bash +export OPENAI_API_KEY="sk-..." +``` + +```python +from vectorless import Engine, IndexContext + +engine = Engine(workspace="./data") +``` + +### Custom Settings + +```python +engine = Engine( + workspace="./data", + api_key="sk-...", + model="gpt-4o", + endpoint="https://api.openai.com/v1", +) +``` + +### Configuration File + +For fine-grained control, use a config file: + +```bash +cp vectorless.example.toml ./vectorless.toml +``` + +```python +engine = Engine(config_path="./vectorless.toml") +``` + +**Configuration priority** (highest → lowest): +1. Constructor parameters (`api_key`, `model`, `endpoint`) +2. Environment variables (`OPENAI_API_KEY`, `VECTORLESS_MODEL`, etc.) +3. Explicit config file (`config_path`) +4. Auto-detected config file (`vectorless.toml`, `config.toml`) +5. Default configuration + +## Indexing + +### From File + +Supports PDF, Markdown, DOCX, HTML: + +```python +doc_id = engine.index(IndexContext.from_file("./report.pdf")) +doc_id = engine.index(IndexContext.from_file("./readme.md")) +doc_id = engine.index(IndexContext.from_file("./doc.docx")) +doc_id = engine.index(IndexContext.from_file("./page.html")) +``` + +### From Content + +Index from a string: + +```python +ctx = IndexContext.from_content( + content="# Manual\n## Chapter 1\nIntroduction...", + name="manual", + format="markdown", +) +doc_id = engine.index(ctx) +``` + +Supported formats: `"markdown"`, `"html"`, `"text"` + +### From Bytes + +Index from binary data (PDF, DOCX): + +```python +with open("./report.pdf", "rb") as f: + data = f.read() + +ctx = IndexContext.from_bytes(data, name="report", format="pdf") +doc_id = engine.index(ctx) +``` + +## Querying + +### Basic Query + +```python +result = engine.query(doc_id, "What is the total revenue?") + +print(result.content) # Retrieved content +print(result.score) # Relevance score (0.0 - 1.0) +print(result.doc_id) # Document ID +print(result.node_ids) # Matched node IDs +``` + +### Multiple Queries + +```python +questions = [ + "What are the main components?", + "How does the retrieval pipeline work?", + "What is the architecture?", +] + +for q in questions: + result = engine.query(doc_id, q) + print(f"Q: {q}") + print(f"A: {result.content}") + print(f"Score: {result.score:.2f}\n") +``` + +## Document Management + +### List Documents + +```python +for doc in engine.list_docs(): + print(f"{doc.id}: {doc.name} ({doc.format})") +``` + +### Check Existence + +```python +if engine.exists(doc_id): + print("Found") +``` + +### Remove Document + +```python +engine.remove(doc_id) +``` + +### Clear All + +```python +count = engine.clear() +print(f"Removed {count} documents") +``` + +### Get Page Content (PDF) + +```python +# Single page +content = engine.get_page_content(doc_id, "1") + +# Page range +content = engine.get_page_content(doc_id, "1-5") + +# Multiple pages +content = engine.get_page_content(doc_id, "1,3,7") +``` + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `OPENAI_API_KEY` | LLM API key | +| `VECTORLESS_MODEL` | Default model (e.g., `gpt-4o-mini`) | +| `VECTORLESS_ENDPOINT` | Custom API endpoint URL | +| `VECTORLESS_WORKSPACE` | Workspace directory | + +## Error Handling + +```python +from vectorless import VectorlessError + +try: + result = engine.query(doc_id, "question") +except VectorlessError as e: + print(f"Error: {e.message}") + print(f"Kind: {e.kind}") # "not_found", "parse", "config", "workspace", "llm", "unknown" +``` + +## Examples + +See [examples/python/](../../examples/python/) for complete examples: + +- **basic/** — Zero configuration, simplest usage +- **advanced/** — Full configuration file with all options +- **custom_config/** — Custom API key, model, and endpoint diff --git a/docs/guide/rust.md b/docs/guide/rust.md new file mode 100644 index 00000000..28ed9e17 --- /dev/null +++ b/docs/guide/rust.md @@ -0,0 +1,357 @@ +# Rust Usage Guide + +## Installation + +```toml +[dependencies] +vectorless = "0.1" +tokio = { version = "1", features = ["full"] } +``` + +## Configuration + +### Engine Builder + +```rust +use vectorless::client::{EngineBuilder, IndexContext}; + +// Zero configuration — uses OPENAI_API_KEY env var +let engine = EngineBuilder::new() + .with_workspace("./data") + .build() + .await?; +``` + +### Custom Model and API Key + +```rust +let engine = EngineBuilder::new() + .with_workspace("./data") + .with_model("gpt-4o-mini", Some("sk-your-key")) + .build() + .await?; +``` + +### OpenAI Preset + +```rust +let engine = EngineBuilder::new() + .with_workspace("./data") + .with_openai("sk-your-key") + .build() + .await?; +``` + +### Custom Endpoint + +```rust +// DeepSeek, Azure OpenAI, local LLM, etc. +let engine = EngineBuilder::new() + .with_workspace("./data") + .with_model("deepseek-chat", Some("sk-key")) + .with_endpoint("https://api.deepseek.com/v1") + .build() + .await?; +``` + +### Configuration File + +```rust +// Auto-detects vectorless.toml, config.toml, .vectorless.toml +let engine = EngineBuilder::new().build().await?; + +// Explicit config file +let engine = EngineBuilder::new() + .with_config_path("./vectorless.toml") + .build() + .await?; +``` + +### Preset Modes + +```rust +// Fast mode — keyword-based, fewer iterations +let engine = EngineBuilder::new() + .with_workspace("./data") + .fast() + .build() + .await?; + +// Precise mode — MCTS-based, more iterations +let engine = EngineBuilder::new() + .with_workspace("./data") + .precise() + .build() + .await?; +``` + +**Configuration priority** (highest → lowest): +1. Builder methods +2. Environment variables +3. Explicit config file +4. Auto-detected config file +5. Default configuration + +## Indexing + +### From File Path + +```rust +let doc_id = engine.index(IndexContext::from_path("./report.pdf")).await?; +let doc_id = engine.index(IndexContext::from_path("./readme.md")).await?; +let doc_id = engine.index(IndexContext::from_path("./doc.docx")).await?; +``` + +Format is auto-detected from file extension. + +### From String Content + +```rust +use vectorless::parser::DocumentFormat; + +let ctx = IndexContext::from_content("# Title\nContent", DocumentFormat::Markdown) + .with_name("manual"); +let doc_id = engine.index(ctx).await?; +``` + +### From Bytes + +```rust +let pdf_bytes = std::fs::read("./report.pdf")?; +let ctx = IndexContext::from_bytes(pdf_bytes, DocumentFormat::Pdf); +let doc_id = engine.index(ctx).await?; +``` + +### Index Options + +```rust +use vectorless::client::IndexOptions; +use vectorless::client::IndexMode; + +let options = IndexOptions { + mode: IndexMode::Force, // Always re-index + generate_summaries: true, + include_text: true, + generate_ids: true, + generate_description: false, +}; + +let ctx = IndexContext::from_path("./doc.md") + .with_name("My Document") + .with_options(options); +``` + +**IndexMode types:** +- `Default` — Skip if already indexed +- `Force` — Always re-index +- `Incremental` — Only re-index changed files + +## Querying + +### Basic Query + +```rust +let result = engine.query(&doc_id, "What is the total revenue?").await?; + +println!("Content: {}", result.content); +println!("Score: {}", result.score); +println!("Nodes: {:?}", result.node_ids); +println!("Is empty: {}", result.is_empty()); +``` + +### With Retrieve Options + +```rust +use vectorless::retrieval::RetrieveOptions; +use vectorless::retrieval::StrategyPreference; + +let options = RetrieveOptions::new() + .with_top_k(5) + .with_beam_width(3) + .with_max_iterations(10) + .with_min_score(0.1) + .with_strategy(StrategyPreference::Auto) + .with_sufficiency_check(true) + .with_max_tokens(4000) + .with_streaming(false); + +let result = engine.query_with_options(&doc_id, "question", &options).await?; +``` + +### Strategy Selection + +```rust +use vectorless::retrieval::StrategyPreference; + +StrategyPreference::Auto // Automatic based on query complexity +StrategyPreference::ForceKeyword // Fast, no LLM calls +StrategyPreference::ForceSemantic // Embedding-based +StrategyPreference::ForceLlm // Deep LLM reasoning +StrategyPreference::ForceHybrid // BM25 + LLM refinement +StrategyPreference::ForceCrossDocument // Multi-document search +StrategyPreference::ForcePageRange // PDF page range filtering +``` + +## Streaming + +```rust +use vectorless::retrieval::{RetrieveOptions, RetrieveEvent}; + +let options = RetrieveOptions::new().with_streaming(true); +let mut rx = engine.query_stream(&doc_id, "architecture", &options).await?; + +while let Some(event) = rx.recv().await { + match event { + RetrieveEvent::Started { query, strategy } => { + println!("Started: {} ({})", query, strategy); + } + RetrieveEvent::StageCompleted { stage, elapsed_ms } => { + println!("Stage {} done in {}ms", stage, elapsed_ms); + } + RetrieveEvent::NodeVisited { node_id, title, score } => { + println!("Visited: {} (score: {:.2})", title, score); + } + RetrieveEvent::ContentFound { node_id, title, preview, score } => { + println!("Found: {} — {} ({:.2})", title, preview, score); + } + RetrieveEvent::Backtracking { from, to, reason } => { + println!("Backtrack: {} → {} ({})", from, to, reason); + } + RetrieveEvent::SufficiencyCheck { level, tokens } => { + println!("Sufficiency: {:?}, {} tokens", level, tokens); + } + RetrieveEvent::Completed { response } => { + println!("Done: {}", response.content); + break; + } + RetrieveEvent::Error { message } => { + eprintln!("Error: {}", message); + break; + } + } +} +``` + +## Document Management + +```rust +// List documents +let docs = engine.list_documents().await?; +for doc in &docs { + println!("{}: {} ({})", doc.id, doc.name, doc.format); +} + +// Check existence +if engine.exists(&doc_id).await? { /* ... */ } + +// Remove +engine.remove(&doc_id).await?; + +// Batch remove +let ids = &["doc1", "doc2"]; +engine.batch_remove(ids).await?; + +// Clear all +let count = engine.clear().await?; +``` + +## Document Graph + +### Building a Graph + +```rust +use vectorless::document::{DocumentGraph, DocumentGraphConfig, DocumentGraphNode, WeightedKeyword}; +use vectorless::index::graph_builder::DocumentGraphBuilder; + +let config = DocumentGraphConfig { + enabled: true, + min_keyword_jaccard: 0.05, + min_shared_keywords: 2, + max_keywords_per_doc: 50, + max_edges_per_node: 20, + retrieval_boost_factor: 0.15, +}; + +let mut builder = DocumentGraphBuilder::new(config); + +builder.add_document("doc1", "Rust Guide", "md", 35, keywords(&[ + ("ownership", 0.95), ("borrowing", 0.90), ("lifetimes", 0.85), +])); +builder.add_document("doc2", "Async Rust", "md", 28, keywords(&[ + ("async", 0.95), ("lifetimes", 0.60), ("tokio", 0.90), +])); + +let graph = builder.build(); +``` + +### Exploring the Graph + +```rust +// Get neighbors (connected documents) +let neighbors = graph.get_neighbors("doc1"); +for edge in neighbors { + println!("→ {} [weight={:.3}, jaccard={:.3}]", + edge.target_doc_id, edge.weight, edge.evidence.keyword_jaccard); +} + +// Find documents by keyword +let entries = graph.find_by_keyword("lifetimes"); +for e in entries { + println!("{} (weight: {:.2})", e.doc_id, e.weight); +} + +// Stats +println!("Documents: {}", graph.node_count()); +println!("Edges: {}", graph.edge_count()); +``` + +## Custom Pilot + +```rust +use vectorless::retrieval::pilot::{Pilot, PilotDecision, SearchState, PilotConfig}; + +struct MyPilot; + +#[async_trait] +impl Pilot for MyPilot { + async fn guide(&self, state: &SearchState) -> PilotDecision { + // Custom navigation logic + PilotDecision::Continue + } +} +``` + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `OPENAI_API_KEY` | LLM API key | +| `VECTORLESS_MODEL` | Default model | +| `VECTORLESS_ENDPOINT` | Custom API endpoint URL | +| `VECTORLESS_WORKSPACE` | Workspace directory | + +## Examples + +See [examples/rust/](../../examples/rust/) for complete examples: + +| Example | Description | +|---------|-------------| +| `basic` | Core API: index, list, query | +| `advanced` | Full configuration file usage | +| `streaming` | Streaming retrieval with events | +| `document_graph` | Cross-document concept graph | +| `session` | Multi-document session operations | +| `feedback_learning` | User feedback and Pilot adaptation | +| `custom_pilot` | Custom navigation guidance | +| `reference_following` | In-document reference resolution | +| `strategy_hybrid` | BM25 + LLM hybrid retrieval | +| `strategy_cross_document` | Multi-document search | +| `strategy_page_range` | PDF page range filtering | +| `multi_format` | PDF, MD, DOCX, HTML handling | +| `events` | Event system monitoring | +| `storage_backend` | Custom storage implementations | +| `storage_compression` | Compression support | +| `storage_workspace` | Workspace and caching | +| `content_aggregation` | Content scoring and relevance | +| `batch_processing` | Batch document processing | +| `cli_tool` | Building CLI tools |