diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..2337d0f0 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +## Summary + + + +## Changes + + + +- + +## Checklist + +- [ ] Code compiles (`cargo build`) +- [ ] Tests pass (`cargo test --lib --all-features`) +- [ ] No new clippy warnings (`cargo clippy --all-features`) +- [ ] Public APIs have documentation comments +- [ ] Python bindings updated (if Rust API changed) + +## Notes + + diff --git a/Cargo.toml b/Cargo.toml index 1a626bab..cd8fca41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["rust", "python"] resolver = "2" [workspace.package] -version = "0.1.28" +version = "0.1.29" edition = "2024" authors = ["zTgx "] license = "Apache-2.0" diff --git a/README.md b/README.md index 96d73505..556ba5f3 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,11 @@ -**Vectorless** is a reasoning-native document engine designed to be the foundational layer for AI applications that need structured access to documents, with the core written in Rust. It does not use vector databases, embeddings, or similarity search. Instead, it transforms documents into hierarchical semantic trees and uses the LLM itself to navigate and retrieve — purely LLM-guided, from indexing to querying. +**Vectorless** is a reasoning-native document engine designed to be the foundational layer for AI applications that need structured access to documents, with the core written in Rust. It does not use vector databases, embeddings, or similarity search. Instead, it will reason through any of your structured documents — **PDFs, Markdown, reports, contracts** — and retrieve only what's relevant. Nothing more, nothing less. -## Why Vectorless - -Most document retrieval solutions rely on vector similarity — splitting documents into chunks, embedding them, and searching by cosine distance. This works for rough topic matching, but breaks down when you need **precision**: specific numbers, cross-section references, or multi-step reasoning across a document. - -Vectorless takes a different approach. No vectors at all. It builds a **semantic tree index** of each document — preserving the original hierarchy — and uses the LLM itself to navigate that structure. The LLM generates the tree during indexing and reasons through it during retrieval. Pure LLM guidance, end to end. +## How It Works
Vectorless Workflow @@ -48,6 +44,7 @@ async fn main() -> vectorless::Result<()> { let engine = EngineBuilder::new() .with_key("sk-...") .with_model("gpt-4o") + .with_endpoint("https://api.openai.com/v1") .build() .await?; @@ -77,7 +74,7 @@ import asyncio from vectorless import Engine, IndexContext, QueryContext async def main(): - engine = Engine(api_key="sk-...", model="gpt-4o") + engine = Engine(api_key="sk-...", model="gpt-4o", endpoint="https://api.openai.com/v1") # Index a document result = await engine.index(IndexContext.from_path("./report.pdf")) @@ -130,7 +127,7 @@ result = await engine.query( Indexed documents are stored in a workspace — there's no need to reprocess files between sessions: ```python -engine = Engine(api_key="sk-...", model="gpt-4o") +engine = Engine(api_key="sk-...", model="gpt-4o", endpoint="https://api.openai.com/v1") # List all indexed documents docs = await engine.list() diff --git a/docs/docs/api-reference.mdx b/docs/docs/api-reference.mdx new file mode 100644 index 00000000..5261afbf --- /dev/null +++ b/docs/docs/api-reference.mdx @@ -0,0 +1,17 @@ +--- +sidebar_position: 9 +title: API Reference +description: Complete API reference for Vectorless Rust crate and Python SDK. +--- + +# API Reference + +> This page is a work in progress. The full API reference will be published in a future update. + +In the meantime, you can refer to the following resources: + +- **Rust crate docs**: [docs.rs/vectorless](https://docs.rs/vectorless) — auto-generated documentation from source code +- **Python SDK docs**: Available via `help(vectorless)` in an interactive Python session +- **Source code**: [github.com/vectorlessflow/vectorless](https://github.com/vectorlessflow/vectorless) + +For usage examples, see [Quick Query](/docs/examples/quick-query), [Multi-Document](/docs/examples/multi-document), and [Batch Indexing](/docs/examples/batch-indexing). diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index 6b329f6e..b72d033d 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -111,7 +111,7 @@ const config: Config = { }, { label: 'API Reference', - href: 'https://docs.rs/vectorless', + to: '/docs/api-reference', }, ], }, @@ -133,7 +133,7 @@ const config: Config = { ], }, ], - copyright: `Copyright © ${new Date().getFullYear()} Vectorless`, + copyright: `Copyright \u00A9 ${new Date().getFullYear()} Vectorless`, }, prism: { theme: prismThemes.github, diff --git a/docs/sidebars.ts b/docs/sidebars.ts index e3ddf067..2f70bedb 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -42,6 +42,7 @@ const sidebars: SidebarsConfig = { 'sdk/rust', ], }, + 'api-reference', { type: 'category', label: 'Examples', diff --git a/docs/src/components/GitHubStar/index.tsx b/docs/src/components/GitHubStar/index.tsx index b2247874..712e8b74 100644 --- a/docs/src/components/GitHubStar/index.tsx +++ b/docs/src/components/GitHubStar/index.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from 'react'; -import { FaGithub, FaStar } from 'react-icons/fa'; +import { FaGithub } from 'react-icons/fa'; import styles from './styles.module.css'; function formatStars(count: number | null): string { @@ -40,16 +40,16 @@ export default function GitHubStar(): React.ReactElement { rel="noopener noreferrer" className={styles.githubStarButton} > - + Star {loading ? (
- ... + …
) : (
-
- {siteConfig.title} -

- No vector database. No embeddings. No similarity search.
- Retrieve by reasoning, not by math. -

-
- - Get Started - - - GitHub - -
+

+ Reason, + don't vector +

+

+ + Vectorless will reason through any of your structured documents — PDFs, Markdown, reports, contracts, + +
+ and retrieve only what's relevant. Nothing more, nothing less. +

+
+ + + Star on GitHub + +
+
+ + ); +} -
-
- - - - Python -
- r.re.source).join('|'); + const re = new RegExp(combined, 'gm'); + + const nodes: ReactNode[] = []; + let lastIdx = 0; + let m: RegExpExecArray | null; + re.lastIndex = 0; + + while ((m = re.exec(code)) !== null) { + if (m.index > lastIdx) { + nodes.push(code.slice(lastIdx, m.index)); + } + // match[1..rules.length] corresponds to each rule's capture group + for (let i = 0; i < rules.length; i++) { + const captured = m[i + 1]; + if (captured !== undefined) { + nodes.push({captured}); + break; + } + } + lastIdx = re.lastIndex; + } + if (lastIdx < code.length) { + nodes.push(code.slice(lastIdx)); + } + return nodes; +} + +// Exact code from README +const PYTHON_CODE = `import asyncio +from vectorless import Engine, IndexContext, QueryContext async def main(): - engine = Engine( - api_key="sk-...", - model="gpt-4o", - ) + engine = Engine(api_key="sk-...", model="gpt-4o", endpoint="https://api.openai.com/v1") # Index a document - result = await engine.index( - IndexContext.from_path("./report.pdf") - ) + result = await engine.index(IndexContext.from_path("./report.pdf")) doc_id = result.doc_id - # Query — LLM navigates the tree + # Query result = await engine.query( - doc_id, "What is the total revenue?" + QueryContext("What is the total revenue?").with_doc_ids([doc_id]) ) print(result.single().content) -asyncio.run(main())`} language="python"> - {({tokens, getLineProps, getTokenProps}) => ( - 
-                
-                  {tokens.map((line, i) => (
-                    

-                      {line.map((token, key) => (
-                        
-                      ))}
-                    

-                  ))}
-                
-
- )} - -
-
- - ); +asyncio.run(main())`; + +const RUST_CODE = `use vectorless::client::{EngineBuilder, IndexContext, QueryContext}; + +#[tokio::main] +async fn main() -> vectorless::Result<()> { + let engine = EngineBuilder::new() + .with_key("sk-...") + .with_model("gpt-4o") + .with_endpoint("https://api.openai.com/v1") + .build() + .await?; + + // Index a document + let result = engine.index(IndexContext::from_path("./report.pdf")).await?; + let doc_id = result.doc_id().unwrap(); + + // Query + let result = engine.query( + QueryContext::new("What is the total revenue?") + .with_doc_ids(vec![doc_id.to_string()]) + ).await?; + println!("{}", result.content); + + Ok(()) +}`; + +function PythonCode() { + const nodes = useMemo(() => highlight(PYTHON_CODE, 'python'), []); + return 
{nodes}
; +} + +function RustCode() { + const nodes = useMemo(() => highlight(RUST_CODE, 'rust'), []); + return 
{nodes}
; } -function SectionWhy() { - const items = [ - { - icon: '\u{1F9E0}', - title: 'Reasoning-Native', - desc: 'LLMs navigate hierarchical document trees with semantic understanding \u2014 not vector proximity.', - }, - { - icon: '\u{1F5C2}\u{FE0F}', - title: 'No Vector Database', - desc: 'Eliminate embedding pipelines, vector stores, and similarity search entirely. Trees are the index.', - }, - { - icon: '\u26A1', - title: 'Rust-Powered', - desc: 'Core engine in Rust with Python bindings. Arena-based trees, async I/O, and zero-copy traversal.', - }, - { - icon: '\u{1F50D}', - title: 'Multi-Algorithm Search', - desc: 'Beam search, MCTS, and greedy algorithms with LLM-guided Pilot at key decision points.', - }, - { - icon: '\u{1F4CA}', - title: 'Explainable Results', - desc: 'Full reasoning chain traces every navigation decision. Audit how and why content was retrieved.', - }, - { - icon: '\u{1F4C4}', - title: 'PDF & Markdown', - desc: 'Index PDFs and Markdown out of the box. Hierarchical structure extracted automatically.', - }, - ]; +function SectionGetStarted() { + const [activeTab, setActiveTab] = useState<'python' | 'rust'>('python'); + const [copyLabel, setCopyLabel] = useState('Copy'); + const [installLabel, setInstallLabel] = useState('Copy & install'); + + const installCmd = activeTab === 'python' ? 'pip install vectorless' : 'cargo add vectorless'; + + const handleCopy = () => { + const code = activeTab === 'python' ? PYTHON_CODE : RUST_CODE; + navigator.clipboard.writeText(code); + setCopyLabel('\u2713 Copied!'); + setTimeout(() => setCopyLabel('Copy'), 1500); + }; + + const handleInstallCopy = () => { + navigator.clipboard.writeText(installCmd); + setInstallLabel('\u2713 Copied!'); + setTimeout(() => setInstallLabel('Copy & install'), 1500); + }; return (
- Why Vectorless? + Get Started

- RAG without the baggage. + Just a few lines of code to get up and running.

-
- {items.map((item, i) => ( -
- {item.icon} - {item.title} -

{item.desc}

+
+ {/* Tabs */} +
+ + +
+ + {/* Python panel */} + {activeTab === 'python' && ( +
+
+
+ + + +
+ +
+ +
+ $ python demo.py
+ → The total revenue for fiscal year 2024 was $2.3 billion, a 15% increase YoY. + +
- ))} + )} + + {/* Rust panel */} + {activeTab === 'rust' && ( +
+
+
+ + + +
+ +
+ +
+ $ cargo run
+ → The total revenue for fiscal year 2024 was $2.3 billion, a 15% increase YoY. + +
+
+ )} + + {/* Install bar */} +
+
+ $ {installCmd} +
+ +
@@ -144,28 +240,133 @@ function SectionWhy() { } function SectionHowItWorks() { - const steps = [ - { num: '01', title: 'Index', desc: 'Parse documents into hierarchical semantic trees with LLM-generated summaries.' }, - { num: '02', title: 'Navigate', desc: 'Pilot uses LLM to navigate the tree at key forks \u2014 beam search explores multiple paths in parallel.' }, - { num: '03', title: 'Retrieve', desc: 'Evaluate sufficiency and backtrack if needed. Aggregate only the most relevant content within budget.' }, - ]; + return ( +
+
+ + How does Vectorless work? + +

+ You declare a few lines of code. We do everything else. +

+
+ How Vectorless works +
+
+
+ ); +} + +const USE_CASES = [ + { + title: 'Financial reports', + desc: 'Extract specific KPIs from 10\u2011K, annual reports, or earnings transcripts \u2014 even across fiscal years.', + query: '\u201cWhat was the net profit margin for Q3 2024?\u201d', + answer: '18.4%, up from 16.2% in Q3 2023. Source: Section 6.2, page 34.', + }, + { + title: 'Legal & contracts', + desc: 'Locate clauses, definitions, or obligations across complex agreements without missing cross\u2011references.', + query: '\u201cWhich sections define \u2018force majeure\u2019 and what are the notice requirements?\u201d', + answer: 'Section 12.3(a) + 12.3(b) \u2014 30\u2011day written notice required.', + }, + { + title: 'Technical docs', + desc: 'Navigate large API references, internal wikis, or on\u2011prem manuals with step\u2011by\u2011step reasoning.', + query: '\u201cHow to configure authentication for the WebSocket gateway?\u201d', + answer: 'See \u201cWebSocket Auth\u201d \u2192 section 4.2.1: use Authorization: Bearer .', + }, + { + title: 'Research papers', + desc: 'Cross\u2011reference findings, tables, or citations across arXiv preprints or internal literature.', + query: '\u201cWhat datasets were used for evaluation in Section 4?\u201d', + answer: 'Table 2: SQuAD, Natural Questions, and TriviaQA.', + }, + { + title: 'Cross\u2011document analysis', + desc: 'Compare metrics, definitions, or timelines across multiple reports in one query.', + query: '\u201cCompare R&D spending from 2023 vs 2024 annual reports.\u201d', + answer: '2023: $12.4M (page 9) \u00b7 2024: $15.1M (page 11) \u2192 +21.8% YoY.', + }, + { + title: 'Compliance & audit', + desc: 'Trace every retrieved statement back to its source \u2014 full explainability for regulated industries.', + query: '\u201cShow me all references to data retention policy.\u201d', + answer: 'Section 3.2 (page 8), Section 5.1 (page 14), and Appendix B.', + }, +]; + +function SectionUseCases() { + const [current, setCurrent] = useState(0); + const outerRef = useRef(null); + const trackRef = useRef(null); + const [offset, setOffset] = useState(0); + + const total = USE_CASES.length; + + const measure = useCallback(() => { + if (!outerRef.current || !trackRef.current) return; + const outerW = outerRef.current.offsetWidth; + const firstCard = trackRef.current.children[0] as HTMLElement; + if (!firstCard) return; + const cardW = firstCard.offsetWidth; + const gap = 24; // 1.5rem + const step = cardW + gap; + const newOffset = outerW / 2 - current * step - cardW / 2; + setOffset(newOffset); + }, [current]); + + useEffect(() => { + measure(); + window.addEventListener('resize', measure); + return () => window.removeEventListener('resize', measure); + }, [measure]); + + const prev = () => setCurrent(i => Math.max(0, i - 1)); + const next = () => setCurrent(i => Math.min(total - 1, i + 1)); return ( -
+
- How It Works + Use cases · precision reasoning -
- {steps.map((step, i) => ( -
-
{step.num}
-
- {step.title} -

{step.desc}

+

+ Vectorless navigates through the structure of any document to retrieve exact context. +

+
+
+ {USE_CASES.map((c, i) => ( +
+ {c.title} +

{c.desc}

+
+
Query:
+
{c.query}
+
{c.answer}
+
-
- ))} + ))} +
+
+
+ +
+ {USE_CASES.map((_, i) => ( +
+
@@ -173,28 +374,49 @@ function SectionHowItWorks() { } function SectionCTA() { + const [pipLabel, setPipLabel] = useState('Copy'); + const [cargoLabel, setCargoLabel] = useState('Copy'); + + const handlePipCopy = () => { + navigator.clipboard.writeText('pip install vectorless'); + setPipLabel('\u2713'); + setTimeout(() => setPipLabel('Copy'), 1500); + }; + + const handleCargoCopy = () => { + navigator.clipboard.writeText('cargo add vectorless'); + setCargoLabel('\u2713'); + setTimeout(() => setCargoLabel('Copy'), 1500); + }; + return (
- Start building in minutes + Start reasoning, not vectoring -

- pip install vectorless -

-
- - Read the Docs - +
- View on GitHub + + Star on GitHub +
+
+
+
$ pip install vectorless
+ +
+
+
$ cargo add vectorless
+ +
+
@@ -209,8 +431,9 @@ export default function Home(): ReactNode { description="Reasoning-native document intelligence engine. No vector database, no embeddings. Retrieve by reasoning.">
- + +
diff --git a/docs/src/theme/Navbar/index.tsx b/docs/src/theme/Navbar/index.tsx index 35abb2a6..2437b65e 100644 --- a/docs/src/theme/Navbar/index.tsx +++ b/docs/src/theme/Navbar/index.tsx @@ -26,6 +26,7 @@ export default function Navbar(): React.ReactElement { alt={logo?.alt || title} /> +
Vectorless
{leftItems.map((item, i) => )} diff --git a/docs/src/theme/Navbar/styles.module.css b/docs/src/theme/Navbar/styles.module.css index c8d5283d..9e3c803e 100644 --- a/docs/src/theme/Navbar/styles.module.css +++ b/docs/src/theme/Navbar/styles.module.css @@ -27,10 +27,17 @@ } .navbarLogo { - height: 32px; + height: 40px; width: auto; } +.logo { + font-size: 1.6rem; + font-weight: 800; + letter-spacing: -0.02em; + color: var(--primary); +} + /* Center: navigation links */ .navbarCenter { flex: 1; diff --git a/pyproject.toml b/pyproject.toml index f752a6ea..9d83bdd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "vectorless" -version = "0.1.7" +version = "0.1.8" description = "Reasoning-native document intelligence engine for AI" readme = "README.md" requires-python = ">=3.9" diff --git a/python/src/config.rs b/python/src/config.rs new file mode 100644 index 00000000..93a0552e --- /dev/null +++ b/python/src/config.rs @@ -0,0 +1,86 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Config Python wrapper. + +use pyo3::prelude::*; + +/// Advanced configuration for Engine internals. +/// +/// Create a Config to customize storage, retrieval, concurrency, +/// and other engine parameters beyond the basic builder API. +/// +/// Example: +/// +/// ```python +/// from vectorless import Config, Engine +/// +/// config = Config() +/// config.set_workspace_dir("/data/vectorless") +/// config.set_top_k(10) +/// config.set_max_concurrent_requests(20) +/// +/// engine = Engine(api_key="sk-...", model="gpt-4o", config=config) +/// ``` +#[pyclass(name = "Config")] +pub struct PyConfig { + pub(crate) inner: vectorless::Config, +} + +#[pymethods] +impl PyConfig { + /// Create a new Config with defaults. + #[new] + fn new() -> Self { + Self { + inner: vectorless::Config::default(), + } + } + + /// Set the workspace directory for persisted documents. + /// + /// Default: ~/.vectorless + fn set_workspace_dir(&mut self, dir: &str) { + self.inner.storage.workspace_dir = std::path::PathBuf::from(dir); + } + + /// Set the number of top-k results to return from queries. + /// + /// Default: 3 + fn set_top_k(&mut self, k: usize) { + self.inner.retrieval.top_k = k; + } + + /// Set the maximum concurrent LLM API calls. + /// + /// Default: 10 + fn set_max_concurrent_requests(&mut self, max: usize) { + self.inner.concurrency.max_concurrent_requests = max; + } + + /// Set the rate limit (requests per minute). + /// + /// Default: 500 + fn set_requests_per_minute(&mut self, rpm: usize) { + self.inner.concurrency.requests_per_minute = rpm; + } + + /// Set the maximum iterations for retrieval search. + fn set_max_iterations(&mut self, max: usize) { + self.inner.retrieval.search.max_iterations = max; + } + + /// Set the retrieval temperature. + /// + /// Default: 0.0 + fn set_temperature(&mut self, temp: f32) { + self.inner.retrieval.temperature = temp; + } + + /// Enable or disable metrics collection. + /// + /// Default: True + fn set_metrics_enabled(&mut self, enabled: bool) { + self.inner.metrics.enabled = enabled; + } +} diff --git a/python/src/context.rs b/python/src/context.rs new file mode 100644 index 00000000..3eedc6f9 --- /dev/null +++ b/python/src/context.rs @@ -0,0 +1,288 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! IndexContext, QueryContext, and IndexOptions Python wrappers. + +use pyo3::prelude::*; + +use ::vectorless::client::{DocumentFormat, IndexContext, IndexMode, IndexOptions, QueryContext}; + +use super::error::VectorlessError; + +/// Parse format string to DocumentFormat. +fn parse_format(format: &str) -> PyResult { + match format.to_lowercase().as_str() { + "markdown" | "md" => Ok(DocumentFormat::Markdown), + "pdf" => Ok(DocumentFormat::Pdf), + _ => Err(PyErr::from(VectorlessError::new( + format!("Unknown format: {}. Supported: markdown, pdf", format), + "config", + ))), + } +} + +// ============================================================ +// IndexOptions +// ============================================================ + +/// Options for controlling indexing behavior. +/// +/// Args: +/// mode: Indexing mode - "default", "force", or "incremental". +/// generate_summaries: Whether to generate summaries. Default: True. +/// generate_description: Whether to generate document description. Default: False. +/// include_text: Whether to include node text in the tree. Default: True. +/// generate_ids: Whether to generate node IDs. Default: True. +/// enable_synonym_expansion: Whether to expand keywords with LLM-generated +/// synonyms during indexing. Improves recall for differently-worded queries. +/// Default: False. +#[pyclass(name = "IndexOptions", skip_from_py_object)] +#[derive(Clone)] +pub struct PyIndexOptions { + pub(crate) inner: IndexOptions, +} + +#[pymethods] +impl PyIndexOptions { + #[new] + #[pyo3(signature = (mode="default", generate_summaries=true, generate_description=false, include_text=true, generate_ids=true, enable_synonym_expansion=false))] + fn new( + mode: &str, + generate_summaries: bool, + generate_description: bool, + include_text: bool, + generate_ids: bool, + enable_synonym_expansion: bool, + ) -> PyResult { + let mut opts = IndexOptions::new(); + match mode { + "default" => {} + "force" => opts = opts.with_mode(IndexMode::Force), + "incremental" => opts = opts.with_mode(IndexMode::Incremental), + _ => { + return Err(PyErr::from(VectorlessError::new( + format!( + "Unknown mode: {}. Supported: default, force, incremental", + mode + ), + "config", + ))); + } + } + opts.generate_summaries = generate_summaries; + opts.generate_description = generate_description; + opts.include_text = include_text; + opts.generate_ids = generate_ids; + opts.enable_synonym_expansion = enable_synonym_expansion; + Ok(Self { inner: opts }) + } + + fn __repr__(&self) -> String { + format!( + "IndexOptions(mode='{}', generate_summaries={}, generate_description={}, include_text={}, generate_ids={}, enable_synonym_expansion={})", + match self.inner.mode { + IndexMode::Default => "default", + IndexMode::Force => "force", + IndexMode::Incremental => "incremental", + }, + self.inner.generate_summaries, + self.inner.generate_description, + self.inner.include_text, + self.inner.generate_ids, + self.inner.enable_synonym_expansion, + ) + } +} + +// ============================================================ +// IndexContext +// ============================================================ + +/// Context for indexing a document. +/// +/// Create using the static methods: +/// +/// ```python +/// from vectorless import IndexContext +/// +/// # Single file +/// ctx = IndexContext.from_path("./document.pdf") +/// +/// # Multiple files +/// ctx = IndexContext.from_paths(["./a.pdf", "./b.md"]) +/// +/// # Directory +/// ctx = IndexContext.from_dir("./docs/") +/// +/// # From text +/// ctx = IndexContext.from_content("# Title\\nContent...", "markdown").with_name("doc") +/// +/// # From bytes +/// ctx = IndexContext.from_bytes(data, "pdf").with_name("doc") +/// ``` +#[pyclass(name = "IndexContext")] +pub struct PyIndexContext { + pub(crate) inner: IndexContext, +} + +#[pymethods] +impl PyIndexContext { + /// Create an IndexContext from a single file path. + #[staticmethod] + fn from_path(path: String) -> Self { + Self { + inner: IndexContext::from_path(&path), + } + } + + /// Create an IndexContext from multiple file paths. + #[staticmethod] + fn from_paths(paths: Vec) -> Self { + Self { + inner: IndexContext::from_paths(&paths), + } + } + + /// Create an IndexContext from all supported files in a directory. + /// + /// Args: + /// path: Directory path to scan. + /// recursive: If True, scan subdirectories recursively. Default: False. + #[staticmethod] + #[pyo3(signature = (path, recursive=false))] + fn from_dir(path: String, recursive: bool) -> Self { + let inner = IndexContext::from_dir(&path, recursive); + Self { inner } + } + + /// Create an IndexContext from text content. + #[staticmethod] + #[pyo3(signature = (content, format="markdown"))] + fn from_content(content: String, format: &str) -> PyResult { + let doc_format = parse_format(format)?; + let ctx = IndexContext::from_content(&content, doc_format); + Ok(Self { inner: ctx }) + } + + /// Create an IndexContext from binary data. + #[staticmethod] + fn from_bytes(data: Vec, format: &str) -> PyResult { + let doc_format = parse_format(format)?; + let ctx = IndexContext::from_bytes(data, doc_format); + Ok(Self { inner: ctx }) + } + + /// Set the document name (single-source only). + fn with_name(&self, name: String) -> Self { + let ctx = self.inner.clone().with_name(&name); + Self { inner: ctx } + } + + /// Apply indexing options. + fn with_options(&self, options: &PyIndexOptions) -> Self { + let ctx = self.inner.clone().with_options(options.inner.clone()); + Self { inner: ctx } + } + + /// Set indexing mode. + fn with_mode(&self, mode: &str) -> PyResult { + let m = match mode { + "default" => IndexMode::Default, + "force" => IndexMode::Force, + "incremental" => IndexMode::Incremental, + _ => { + return Err(PyErr::from(VectorlessError::new( + format!( + "Unknown mode: {}. Supported: default, force, incremental", + mode + ), + "config", + ))); + } + }; + let ctx = self.inner.clone().with_mode(m); + Ok(Self { inner: ctx }) + } + + /// Number of document sources. + fn __len__(&self) -> usize { + self.inner.len() + } + + /// Whether no sources are present. + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn __repr__(&self) -> String { + format!("IndexContext(sources={})", self.inner.len()) + } +} + +// ============================================================ +// QueryContext +// ============================================================ + +/// Context for a query operation. +/// +/// ```python +/// from vectorless import QueryContext +/// +/// # Query specific documents +/// ctx = QueryContext("What is the total revenue?").with_doc_ids([doc_id]) +/// +/// # Query multiple documents +/// ctx = QueryContext("What is the architecture?").with_doc_ids(["doc-1", "doc-2"]) +/// +/// # Query entire workspace +/// ctx = QueryContext("Explain the algorithm") +/// ``` +#[pyclass(name = "QueryContext")] +pub struct PyQueryContext { + pub(crate) inner: QueryContext, +} + +#[pymethods] +impl PyQueryContext { + /// Create a new query context (defaults to workspace scope). + #[new] + fn new(query: String) -> Self { + Self { + inner: QueryContext::new(&query), + } + } + + /// Set scope to specific documents. + fn with_doc_ids(&self, doc_ids: Vec) -> Self { + let ctx = self.inner.clone().with_doc_ids(doc_ids); + Self { inner: ctx } + } + + /// Set scope to entire workspace. + fn with_workspace(&self) -> Self { + let ctx = self.inner.clone().with_workspace(); + Self { inner: ctx } + } + + /// Set the maximum tokens for the result content. + fn with_max_tokens(&self, tokens: usize) -> Self { + let ctx = self.inner.clone().with_max_tokens(tokens); + Self { inner: ctx } + } + + /// Set whether to include the reasoning chain. + fn with_include_reasoning(&self, include: bool) -> Self { + let ctx = self.inner.clone().with_include_reasoning(include); + Self { inner: ctx } + } + + /// Set the maximum tree traversal depth. + fn with_depth_limit(&self, depth: usize) -> Self { + let ctx = self.inner.clone().with_depth_limit(depth); + Self { inner: ctx } + } + + fn __repr__(&self) -> String { + "QueryContext(...)".to_string() + } +} diff --git a/python/src/document.rs b/python/src/document.rs new file mode 100644 index 00000000..eee70c0e --- /dev/null +++ b/python/src/document.rs @@ -0,0 +1,59 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! DocumentInfo Python wrapper. + +use pyo3::prelude::*; + +use ::vectorless::client::DocumentInfo; + +/// Information about an indexed document. +#[pyclass(name = "DocumentInfo")] +pub struct PyDocumentInfo { + pub(crate) inner: DocumentInfo, +} + +#[pymethods] +impl PyDocumentInfo { + #[getter] + fn id(&self) -> &str { + &self.inner.id + } + + #[getter] + fn name(&self) -> &str { + &self.inner.name + } + + #[getter] + fn format(&self) -> &str { + &self.inner.format + } + + #[getter] + fn description(&self) -> Option<&str> { + self.inner.description.as_deref() + } + + #[getter] + fn source_path(&self) -> Option<&str> { + self.inner.source_path.as_deref() + } + + #[getter] + fn page_count(&self) -> Option { + self.inner.page_count + } + + #[getter] + fn line_count(&self) -> Option { + self.inner.line_count + } + + fn __repr__(&self) -> String { + format!( + "DocumentInfo(id='{}', name='{}', format='{}')", + self.inner.id, self.inner.name, self.inner.format + ) + } +} diff --git a/python/src/engine.rs b/python/src/engine.rs new file mode 100644 index 00000000..8f7dc015 --- /dev/null +++ b/python/src/engine.rs @@ -0,0 +1,242 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Engine Python wrapper and async helpers. + +use pyo3::prelude::*; +use pyo3_async_runtimes::tokio::future_into_py; +use std::sync::Arc; +use tokio::runtime::Runtime; + +use ::vectorless::client::{Engine, EngineBuilder, IndexContext, QueryContext}; + +use super::config::PyConfig; +use super::context::{PyIndexContext, PyQueryContext}; +use super::document::PyDocumentInfo; +use super::error::VectorlessError; +use super::error::to_py_err; +use super::graph::PyDocumentGraph; +use super::metrics::PyMetricsReport; +use super::results::{PyIndexResult, PyQueryResult}; + +// ============================================================ +// Engine async helpers (named functions to avoid FnOnce HRTB issue) +// ============================================================ + +async fn run_index(engine: Arc, ctx: IndexContext) -> PyResult { + let result = engine.index(ctx).await.map_err(to_py_err)?; + Ok(PyIndexResult { inner: result }) +} + +async fn run_query(engine: Arc, ctx: QueryContext) -> PyResult { + let result = engine.query(ctx).await.map_err(to_py_err)?; + Ok(PyQueryResult { inner: result }) +} + +async fn run_list(engine: Arc) -> PyResult> { + let docs = engine.list().await.map_err(to_py_err)?; + Ok(docs + .into_iter() + .map(|d| PyDocumentInfo { inner: d }) + .collect()) +} + +async fn run_remove(engine: Arc, doc_id: String) -> PyResult { + engine.remove(&doc_id).await.map_err(to_py_err) +} + +async fn run_clear(engine: Arc) -> PyResult { + engine.clear().await.map_err(to_py_err) +} + +async fn run_exists(engine: Arc, doc_id: String) -> PyResult { + engine.exists(&doc_id).await.map_err(to_py_err) +} + +async fn run_get_graph(engine: Arc) -> PyResult> { + let graph = engine.get_graph().await.map_err(to_py_err)?; + Ok(graph.map(|g| PyDocumentGraph { inner: g })) +} + +fn run_metrics_report(engine: Arc) -> PyMetricsReport { + PyMetricsReport { + inner: engine.metrics_report(), + } +} + +// ============================================================ +// Engine +// ============================================================ + +/// The main vectorless engine. +/// +/// `api_key` and `model` are **required**. +/// +/// ```python +/// from vectorless import Engine, IndexContext, QueryContext +/// +/// engine = Engine( +/// api_key="sk-...", +/// model="gpt-4o", +/// ) +/// +/// # Index +/// result = await engine.index(IndexContext.from_path("./report.pdf")) +/// doc_id = result.doc_id +/// +/// # Query +/// answer = await engine.query(QueryContext("What is the revenue?").with_doc_ids([doc_id])) +/// print(answer.single().content) +/// ``` +#[pyclass(name = "Engine")] +pub struct PyEngine { + inner: Arc, +} + +#[pymethods] +impl PyEngine { + /// Create a new Engine. + /// + /// Args: + /// api_key: **Required**. LLM API key. + /// model: **Required**. LLM model name. + /// endpoint: Optional API endpoint. + /// config: Optional Config for advanced tuning. + /// + /// Raises: + /// VectorlessError: If engine creation fails. + #[new] + #[pyo3(signature = (api_key=None, model=None, endpoint=None, config=None))] + fn new( + api_key: Option, + model: Option, + endpoint: Option, + config: Option>, + ) -> PyResult { + let rt = Runtime::new().map_err(|e| { + PyErr::from(VectorlessError::new( + format!("Failed to create tokio runtime: {}", e), + "config", + )) + })?; + + let rust_config = config.map(|c| c.inner.clone()); + + let engine = rt.block_on(async { + let mut builder = EngineBuilder::new(); + + if let Some(config) = rust_config { + builder = builder.with_config(config); + } + + if let Some(m) = &model { + builder = builder.with_model(m); + } + if let Some(e) = &endpoint { + builder = builder.with_endpoint(e); + } + if let Some(key) = api_key { + builder = builder.with_key(key); + } + + builder.build().await + }); + + let engine = engine.map_err(|e| { + PyErr::from(VectorlessError::new( + format!("Failed to create engine: {}", e), + "config", + )) + })?; + + Ok(Self { + inner: Arc::new(engine), + }) + } + + /// Index a document. + /// + /// Args: + /// ctx: IndexContext created from from_path, from_paths, from_dir, etc. + /// + /// Returns: + /// IndexResult with doc_id and items. + /// + /// Raises: + /// VectorlessError: If indexing fails. + fn index<'py>(&self, py: Python<'py>, ctx: &PyIndexContext) -> PyResult> { + let engine = Arc::clone(&self.inner); + let index_ctx = ctx.inner.clone(); + future_into_py(py, run_index(engine, index_ctx)) + } + + /// Query indexed documents. + /// + /// Args: + /// ctx: QueryContext with query text and scope. + /// + /// Returns: + /// QueryResult with answer and score. + /// + /// Raises: + /// VectorlessError: If query fails. + fn query<'py>(&self, py: Python<'py>, ctx: &PyQueryContext) -> PyResult> { + let engine = Arc::clone(&self.inner); + let query_ctx = ctx.inner.clone(); + future_into_py(py, run_query(engine, query_ctx)) + } + + /// List all indexed documents. + /// + /// Returns: + /// List of DocumentInfo objects. + fn list<'py>(&self, py: Python<'py>) -> PyResult> { + let engine = Arc::clone(&self.inner); + future_into_py(py, run_list(engine)) + } + + /// Remove a document by ID. + /// + /// Returns: + /// True if removed, False if not found. + fn remove<'py>(&self, py: Python<'py>, doc_id: String) -> PyResult> { + let engine = Arc::clone(&self.inner); + future_into_py(py, run_remove(engine, doc_id)) + } + + /// Remove all indexed documents. + /// + /// Returns: + /// Number of documents removed. + fn clear<'py>(&self, py: Python<'py>) -> PyResult> { + let engine = Arc::clone(&self.inner); + future_into_py(py, run_clear(engine)) + } + + /// Check if a document exists. + fn exists<'py>(&self, py: Python<'py>, doc_id: String) -> PyResult> { + let engine = Arc::clone(&self.inner); + future_into_py(py, run_exists(engine, doc_id)) + } + + /// Get the cross-document relationship graph. + /// + /// Returns: + /// DocumentGraph if any documents exist, else None. + fn get_graph<'py>(&self, py: Python<'py>) -> PyResult> { + let engine = Arc::clone(&self.inner); + future_into_py(py, run_get_graph(engine)) + } + + /// Generate a complete metrics report. + /// + /// Returns: + /// MetricsReport with LLM, Pilot, and Retrieval metrics. + fn metrics_report(&self) -> PyMetricsReport { + run_metrics_report(Arc::clone(&self.inner)) + } + + fn __repr__(&self) -> String { + "Engine(...)".to_string() + } +} diff --git a/python/src/error.rs b/python/src/error.rs new file mode 100644 index 00000000..d128ce5a --- /dev/null +++ b/python/src/error.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Python exception types and error conversion. + +use pyo3::exceptions::PyException; +use pyo3::prelude::*; + +use ::vectorless::error::Error as RustError; + +/// Python exception for vectorless errors. +#[pyclass(extends = PyException, subclass)] +pub struct VectorlessError { + message: String, + kind: String, +} + +#[pymethods] +impl VectorlessError { + #[new] + fn new_py(message: String, kind: String) -> Self { + Self { message, kind } + } + + #[getter] + fn message(&self) -> &str { + &self.message + } + + #[getter] + fn kind(&self) -> &str { + &self.kind + } + + fn __str__(&self) -> &str { + &self.message + } + + fn __repr__(&self) -> String { + format!("VectorlessError('{}', kind='{}')", self.message, self.kind) + } +} + +impl VectorlessError { + pub fn new(message: String, kind: &str) -> Self { + Self { + message, + kind: kind.to_string(), + } + } +} + +impl From for PyErr { + fn from(err: VectorlessError) -> PyErr { + PyErr::new::((err.message, err.kind)) + } +} + +/// Convert vectorless errors to Python exceptions. +pub fn to_py_err(e: RustError) -> PyErr { + let message = e.to_string(); + let kind = match &e { + RustError::DocumentNotFound(_) => "not_found", + RustError::Parse(_) => "parse", + RustError::Config(_) => "config", + RustError::Workspace(_) => "workspace", + RustError::Llm(_) => "llm", + _ => "unknown", + }; + VectorlessError::new(message, kind).into() +} diff --git a/python/src/graph.rs b/python/src/graph.rs new file mode 100644 index 00000000..1aacd47f --- /dev/null +++ b/python/src/graph.rs @@ -0,0 +1,212 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! DocumentGraph Python wrappers. + +use pyo3::prelude::*; + +use ::vectorless::graph::{ + DocumentGraph, DocumentGraphNode, EdgeEvidence, GraphEdge, WeightedKeyword, +}; + +/// A keyword with weight from document analysis. +#[pyclass(name = "WeightedKeyword")] +pub struct PyWeightedKeyword { + pub(crate) inner: WeightedKeyword, +} + +#[pymethods] +impl PyWeightedKeyword { + #[getter] + fn keyword(&self) -> &str { + &self.inner.keyword + } + + #[getter] + fn weight(&self) -> f32 { + self.inner.weight + } + + fn __repr__(&self) -> String { + format!( + "WeightedKeyword('{}', weight={:.2})", + self.inner.keyword, self.inner.weight + ) + } +} + +/// Evidence for a cross-document connection. +#[pyclass(name = "EdgeEvidence")] +pub struct PyEdgeEvidence { + pub(crate) inner: EdgeEvidence, +} + +#[pymethods] +impl PyEdgeEvidence { + /// Number of shared keywords. + #[getter] + fn shared_keyword_count(&self) -> usize { + self.inner.shared_keyword_count + } + + /// Jaccard similarity of keyword sets. + #[getter] + fn keyword_jaccard(&self) -> f32 { + self.inner.keyword_jaccard + } + + /// Shared keywords with weights. + #[getter] + fn shared_keywords(&self) -> Vec<(String, f32, f32)> { + self.inner + .shared_keywords + .iter() + .map(|sk| (sk.keyword.clone(), sk.source_weight, sk.target_weight)) + .collect() + } + + fn __repr__(&self) -> String { + format!( + "EdgeEvidence(shared={}, jaccard={:.2})", + self.inner.shared_keyword_count, self.inner.keyword_jaccard + ) + } +} + +/// An edge representing a relationship between two documents. +#[pyclass(name = "GraphEdge")] +pub struct PyGraphEdge { + pub(crate) inner: GraphEdge, +} + +#[pymethods] +impl PyGraphEdge { + /// Target document ID. + #[getter] + fn target_doc_id(&self) -> &str { + &self.inner.target_doc_id + } + + /// Edge weight (connection strength). + #[getter] + fn weight(&self) -> f32 { + self.inner.weight + } + + /// Evidence for this connection. + #[getter] + fn evidence(&self) -> PyEdgeEvidence { + PyEdgeEvidence { + inner: self.inner.evidence.clone(), + } + } + + fn __repr__(&self) -> String { + format!( + "GraphEdge(target='{}', weight={:.2})", + self.inner.target_doc_id, self.inner.weight + ) + } +} + +/// A node in the document graph representing an indexed document. +#[pyclass(name = "DocumentGraphNode")] +pub struct PyDocumentGraphNode { + pub(crate) inner: DocumentGraphNode, +} + +#[pymethods] +impl PyDocumentGraphNode { + #[getter] + fn doc_id(&self) -> &str { + &self.inner.doc_id + } + + #[getter] + fn title(&self) -> &str { + &self.inner.title + } + + #[getter] + fn format(&self) -> &str { + &self.inner.format + } + + #[getter] + fn node_count(&self) -> usize { + self.inner.node_count + } + + /// Top keywords extracted from the document. + #[getter] + fn top_keywords(&self) -> Vec { + self.inner + .top_keywords + .iter() + .map(|kw| PyWeightedKeyword { inner: kw.clone() }) + .collect() + } + + fn __repr__(&self) -> String { + format!( + "DocumentGraphNode(doc_id='{}', title='{}')", + self.inner.doc_id, self.inner.title + ) + } +} + +/// Cross-document relationship graph. +/// +/// Automatically rebuilt after indexing. Connects documents +/// that share keywords via Jaccard similarity. +#[pyclass(name = "DocumentGraph")] +pub struct PyDocumentGraph { + pub(crate) inner: DocumentGraph, +} + +#[pymethods] +impl PyDocumentGraph { + /// Number of document nodes. + fn node_count(&self) -> usize { + self.inner.node_count() + } + + /// Number of relationship edges. + fn edge_count(&self) -> usize { + self.inner.edge_count() + } + + /// Get a document node by ID. + fn get_node(&self, doc_id: String) -> Option { + self.inner + .get_node(&doc_id) + .map(|n| PyDocumentGraphNode { inner: n.clone() }) + } + + /// Get all document IDs in the graph. + fn doc_ids(&self) -> Vec { + self.inner.doc_ids().map(|s| s.to_string()).collect() + } + + /// Get edges (neighbors) for a document. + fn get_neighbors(&self, doc_id: String) -> Vec { + self.inner + .get_neighbors(&doc_id) + .iter() + .map(|e| PyGraphEdge { inner: e.clone() }) + .collect() + } + + /// Whether the graph is empty. + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn __repr__(&self) -> String { + format!( + "DocumentGraph(nodes={}, edges={})", + self.inner.node_count(), + self.inner.edge_count() + ) + } +} diff --git a/python/src/lib.rs b/python/src/lib.rs index f6ff36ee..ebee59cf 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -3,1566 +3,30 @@ //! Python bindings for vectorless. -use pyo3::exceptions::PyException; use pyo3::prelude::*; -use pyo3_async_runtimes::tokio::future_into_py; -use std::sync::Arc; -use tokio::runtime::Runtime; -use ::vectorless::client::{ - DocumentFormat, DocumentInfo, Engine, EngineBuilder, FailedItem, IndexContext, IndexItem, - IndexMode, IndexOptions, IndexResult, QueryContext, QueryResult, QueryResultItem, +mod config; +mod context; +mod document; +mod engine; +mod error; +mod graph; +mod metrics; +mod results; + +use config::PyConfig; +use context::{PyIndexContext, PyIndexOptions, PyQueryContext}; +use document::PyDocumentInfo; +use engine::PyEngine; +use error::VectorlessError; +use graph::{PyDocumentGraph, PyDocumentGraphNode, PyEdgeEvidence, PyGraphEdge, PyWeightedKeyword}; +use metrics::{ + PyLlmMetricsReport, PyMetricsReport, PyPilotMetricsReport, PyRetrievalMetricsReport, }; -use ::vectorless::error::Error as RustError; -use ::vectorless::metrics::IndexMetrics; -use ::vectorless::metrics::{ - LlmMetricsReport, MetricsReport, PilotMetricsReport, RetrievalMetricsReport, +use results::{ + PyFailedItem, PyIndexItem, PyIndexMetrics, PyIndexResult, PyQueryResult, PyQueryResultItem, }; -// ============================================================ -// Error Types -// ============================================================ - -/// Python exception for vectorless errors. -#[pyclass(extends = PyException, subclass)] -pub struct VectorlessError { - message: String, - kind: String, -} - -#[pymethods] -impl VectorlessError { - #[new] - fn new_py(message: String, kind: String) -> Self { - Self { message, kind } - } - - #[getter] - fn message(&self) -> &str { - &self.message - } - - #[getter] - fn kind(&self) -> &str { - &self.kind - } - - fn __str__(&self) -> &str { - &self.message - } - - fn __repr__(&self) -> String { - format!("VectorlessError('{}', kind='{}')", self.message, self.kind) - } -} - -impl VectorlessError { - fn new(message: String, kind: &str) -> Self { - Self { - message, - kind: kind.to_string(), - } - } -} - -impl From for PyErr { - fn from(err: VectorlessError) -> PyErr { - PyErr::new::((err.message, err.kind)) - } -} - -/// Convert vectorless errors to Python exceptions. -fn to_py_err(e: RustError) -> PyErr { - let message = e.to_string(); - let kind = match &e { - RustError::DocumentNotFound(_) => "not_found", - RustError::Parse(_) => "parse", - RustError::Config(_) => "config", - RustError::Workspace(_) => "workspace", - RustError::Llm(_) => "llm", - _ => "unknown", - }; - VectorlessError::new(message, kind).into() -} - -/// Parse format string to DocumentFormat. -fn parse_format(format: &str) -> PyResult { - match format.to_lowercase().as_str() { - "markdown" | "md" => Ok(DocumentFormat::Markdown), - "pdf" => Ok(DocumentFormat::Pdf), - _ => Err(PyErr::from(VectorlessError::new( - format!("Unknown format: {}. Supported: markdown, pdf", format), - "config", - ))), - } -} - -// ============================================================ -// IndexOptions -// ============================================================ - -/// Options for controlling indexing behavior. -/// -/// Args: -/// mode: Indexing mode - "default", "force", or "incremental". -/// generate_summaries: Whether to generate summaries. Default: True. -/// generate_description: Whether to generate document description. Default: False. -/// include_text: Whether to include node text in the tree. Default: True. -/// generate_ids: Whether to generate node IDs. Default: True. -/// enable_synonym_expansion: Whether to expand keywords with LLM-generated -/// synonyms during indexing. Improves recall for differently-worded queries. -/// Default: False. -#[pyclass(name = "IndexOptions", skip_from_py_object)] -#[derive(Clone)] -pub struct PyIndexOptions { - inner: IndexOptions, -} - -#[pymethods] -impl PyIndexOptions { - #[new] - #[pyo3(signature = (mode="default", generate_summaries=true, generate_description=false, include_text=true, generate_ids=true, enable_synonym_expansion=false))] - fn new( - mode: &str, - generate_summaries: bool, - generate_description: bool, - include_text: bool, - generate_ids: bool, - enable_synonym_expansion: bool, - ) -> PyResult { - let mut opts = IndexOptions::new(); - match mode { - "default" => {} - "force" => opts = opts.with_mode(IndexMode::Force), - "incremental" => opts = opts.with_mode(IndexMode::Incremental), - _ => { - return Err(PyErr::from(VectorlessError::new( - format!( - "Unknown mode: {}. Supported: default, force, incremental", - mode - ), - "config", - ))); - } - } - opts.generate_summaries = generate_summaries; - opts.generate_description = generate_description; - opts.include_text = include_text; - opts.generate_ids = generate_ids; - opts.enable_synonym_expansion = enable_synonym_expansion; - Ok(Self { inner: opts }) - } - - fn __repr__(&self) -> String { - format!( - "IndexOptions(mode='{}', generate_summaries={}, generate_description={}, include_text={}, generate_ids={}, enable_synonym_expansion={})", - match self.inner.mode { - IndexMode::Default => "default", - IndexMode::Force => "force", - IndexMode::Incremental => "incremental", - }, - self.inner.generate_summaries, - self.inner.generate_description, - self.inner.include_text, - self.inner.generate_ids, - self.inner.enable_synonym_expansion, - ) - } -} - -// ============================================================ -// IndexContext -// ============================================================ - -/// Context for indexing a document. -/// -/// Create using the static methods: -/// -/// ```python -/// from vectorless import IndexContext -/// -/// # Single file -/// ctx = IndexContext.from_path("./document.pdf") -/// -/// # Multiple files -/// ctx = IndexContext.from_paths(["./a.pdf", "./b.md"]) -/// -/// # Directory -/// ctx = IndexContext.from_dir("./docs/") -/// -/// # From text -/// ctx = IndexContext.from_content("# Title\\nContent...", "markdown").with_name("doc") -/// -/// # From bytes -/// ctx = IndexContext.from_bytes(data, "pdf").with_name("doc") -/// ``` -#[pyclass(name = "IndexContext")] -pub struct PyIndexContext { - inner: IndexContext, -} - -#[pymethods] -impl PyIndexContext { - /// Create an IndexContext from a single file path. - #[staticmethod] - fn from_path(path: String) -> Self { - Self { - inner: IndexContext::from_path(&path), - } - } - - /// Create an IndexContext from multiple file paths. - #[staticmethod] - fn from_paths(paths: Vec) -> Self { - Self { - inner: IndexContext::from_paths(&paths), - } - } - - /// Create an IndexContext from all supported files in a directory. - /// - /// Args: - /// path: Directory path to scan. - /// recursive: If True, scan subdirectories recursively. Default: False. - #[staticmethod] - #[pyo3(signature = (path, recursive=false))] - fn from_dir(path: String, recursive: bool) -> Self { - let inner = IndexContext::from_dir(&path, recursive); - Self { inner } - } - - /// Create an IndexContext from text content. - #[staticmethod] - #[pyo3(signature = (content, format="markdown"))] - fn from_content(content: String, format: &str) -> PyResult { - let doc_format = parse_format(format)?; - let ctx = IndexContext::from_content(&content, doc_format); - Ok(Self { inner: ctx }) - } - - /// Create an IndexContext from binary data. - #[staticmethod] - fn from_bytes(data: Vec, format: &str) -> PyResult { - let doc_format = parse_format(format)?; - let ctx = IndexContext::from_bytes(data, doc_format); - Ok(Self { inner: ctx }) - } - - /// Set the document name (single-source only). - fn with_name(&self, name: String) -> Self { - let ctx = self.inner.clone().with_name(&name); - Self { inner: ctx } - } - - /// Apply indexing options. - fn with_options(&self, options: &PyIndexOptions) -> Self { - let ctx = self.inner.clone().with_options(options.inner.clone()); - Self { inner: ctx } - } - - /// Set indexing mode. - fn with_mode(&self, mode: &str) -> PyResult { - let m = match mode { - "default" => IndexMode::Default, - "force" => IndexMode::Force, - "incremental" => IndexMode::Incremental, - _ => { - return Err(PyErr::from(VectorlessError::new( - format!( - "Unknown mode: {}. Supported: default, force, incremental", - mode - ), - "config", - ))); - } - }; - let ctx = self.inner.clone().with_mode(m); - Ok(Self { inner: ctx }) - } - - /// Number of document sources. - fn __len__(&self) -> usize { - self.inner.len() - } - - /// Whether no sources are present. - fn is_empty(&self) -> bool { - self.inner.is_empty() - } - - fn __repr__(&self) -> String { - format!("IndexContext(sources={})", self.inner.len()) - } -} - -// ============================================================ -// QueryContext -// ============================================================ - -/// Context for a query operation. -/// -/// ```python -/// from vectorless import QueryContext -/// -/// # Query specific documents -/// ctx = QueryContext("What is the total revenue?").with_doc_ids([doc_id]) -/// -/// # Query multiple documents -/// ctx = QueryContext("What is the architecture?").with_doc_ids(["doc-1", "doc-2"]) -/// -/// # Query entire workspace -/// ctx = QueryContext("Explain the algorithm") -/// ``` -#[pyclass(name = "QueryContext")] -pub struct PyQueryContext { - inner: QueryContext, -} - -#[pymethods] -impl PyQueryContext { - /// Create a new query context (defaults to workspace scope). - #[new] - fn new(query: String) -> Self { - Self { - inner: QueryContext::new(&query), - } - } - - /// Set scope to specific documents. - fn with_doc_ids(&self, doc_ids: Vec) -> Self { - let ctx = self.inner.clone().with_doc_ids(doc_ids); - Self { inner: ctx } - } - - /// Set scope to entire workspace. - fn with_workspace(&self) -> Self { - let ctx = self.inner.clone().with_workspace(); - Self { inner: ctx } - } - - /// Set the maximum tokens for the result content. - fn with_max_tokens(&self, tokens: usize) -> Self { - let ctx = self.inner.clone().with_max_tokens(tokens); - Self { inner: ctx } - } - - /// Set whether to include the reasoning chain. - fn with_include_reasoning(&self, include: bool) -> Self { - let ctx = self.inner.clone().with_include_reasoning(include); - Self { inner: ctx } - } - - /// Set the maximum tree traversal depth. - fn with_depth_limit(&self, depth: usize) -> Self { - let ctx = self.inner.clone().with_depth_limit(depth); - Self { inner: ctx } - } - - fn __repr__(&self) -> String { - "QueryContext(...)".to_string() - } -} - -// ============================================================ -// QueryResultItem -// ============================================================ - -/// A single document's query result. -#[pyclass(name = "QueryResultItem")] -pub struct PyQueryResultItem { - inner: QueryResultItem, -} - -#[pymethods] -impl PyQueryResultItem { - /// The document ID. - #[getter] - fn doc_id(&self) -> &str { - &self.inner.doc_id - } - - /// The retrieved content. - #[getter] - fn content(&self) -> &str { - &self.inner.content - } - - /// Relevance score (0.0 to 1.0). - #[getter] - fn score(&self) -> f32 { - self.inner.score - } - - /// Node IDs that matched. - #[getter] - fn node_ids(&self) -> Vec { - self.inner.node_ids.clone() - } - - fn __repr__(&self) -> String { - format!( - "QueryResultItem(doc_id='{}', score={:.2}, content_len={})", - self.inner.doc_id, - self.inner.score, - self.inner.content.len() - ) - } -} - -// ============================================================ -// FailedItem -// ============================================================ - -/// A failed item in a batch operation. -#[pyclass(name = "FailedItem")] -pub struct PyFailedItem { - inner: FailedItem, -} - -#[pymethods] -impl PyFailedItem { - /// Source description. - #[getter] - fn source(&self) -> &str { - &self.inner.source - } - - /// Error message. - #[getter] - fn error(&self) -> &str { - &self.inner.error - } - - fn __repr__(&self) -> String { - format!( - "FailedItem(source='{}', error='{}')", - self.inner.source, self.inner.error - ) - } -} - -// ============================================================ -// QueryResult -// ============================================================ - -/// Result of a document query. -#[pyclass(name = "QueryResult")] -pub struct PyQueryResult { - inner: QueryResult, -} - -#[pymethods] -impl PyQueryResult { - /// Result items (one per document). - #[getter] - fn items(&self) -> Vec { - self.inner - .items - .iter() - .map(|i| PyQueryResultItem { inner: i.clone() }) - .collect() - } - - /// Get the first (single-doc) result item. - fn single(&self) -> Option { - self.inner - .single() - .map(|i| PyQueryResultItem { inner: i.clone() }) - } - - /// Number of result items. - fn __len__(&self) -> usize { - self.inner.len() - } - - /// Whether any documents failed. - fn has_failures(&self) -> bool { - self.inner.has_failures() - } - - /// Failed items. - #[getter] - fn failed(&self) -> Vec { - self.inner - .failed - .iter() - .map(|f| PyFailedItem { inner: f.clone() }) - .collect() - } - - fn __repr__(&self) -> String { - format!( - "QueryResult(items={}, failed={})", - self.inner.len(), - self.inner.failed.len() - ) - } -} - -// ============================================================ -// IndexMetrics -// ============================================================ - -/// Indexing pipeline metrics. -#[pyclass(name = "IndexMetrics")] -pub struct PyIndexMetrics { - inner: IndexMetrics, -} - -#[pymethods] -impl PyIndexMetrics { - /// Total indexing time (ms). - #[getter] - fn total_time_ms(&self) -> u64 { - self.inner.total_time_ms() - } - - /// Parse stage duration (ms). - #[getter] - fn parse_time_ms(&self) -> u64 { - self.inner.parse_time_ms - } - - /// Build stage duration (ms). - #[getter] - fn build_time_ms(&self) -> u64 { - self.inner.build_time_ms - } - - /// Enhance (summary) stage duration (ms). - #[getter] - fn enhance_time_ms(&self) -> u64 { - self.inner.enhance_time_ms - } - - /// Number of nodes processed. - #[getter] - fn nodes_processed(&self) -> usize { - self.inner.nodes_processed - } - - /// Number of summaries successfully generated. - #[getter] - fn summaries_generated(&self) -> usize { - self.inner.summaries_generated - } - - /// Number of summaries that failed to generate. - #[getter] - fn summaries_failed(&self) -> usize { - self.inner.summaries_failed - } - - /// Number of LLM calls made. - #[getter] - fn llm_calls(&self) -> usize { - self.inner.llm_calls - } - - /// Total tokens generated by LLM. - #[getter] - fn total_tokens_generated(&self) -> usize { - self.inner.total_tokens_generated - } - - /// Number of topics in reasoning index. - #[getter] - fn topics_indexed(&self) -> usize { - self.inner.topics_indexed - } - - /// Number of keywords in reasoning index. - #[getter] - fn keywords_indexed(&self) -> usize { - self.inner.keywords_indexed - } - - fn __repr__(&self) -> String { - format!( - "IndexMetrics(total={}ms, summaries={}, failed={}, llm_calls={})", - self.inner.total_time_ms(), - self.inner.summaries_generated, - self.inner.summaries_failed, - self.inner.llm_calls, - ) - } -} - -// ============================================================ -// Runtime Metrics Reports -// ============================================================ - -/// LLM usage metrics report. -#[pyclass(name = "LlmMetricsReport")] -pub struct PyLlmMetricsReport { - inner: LlmMetricsReport, -} - -#[pymethods] -impl PyLlmMetricsReport { - /// Total number of LLM calls. - #[getter] - fn total_calls(&self) -> u64 { - self.inner.total_calls - } - - /// Number of successful calls. - #[getter] - fn successful_calls(&self) -> u64 { - self.inner.successful_calls - } - - /// Number of failed calls. - #[getter] - fn failed_calls(&self) -> u64 { - self.inner.failed_calls - } - - /// Success rate (0.0 - 1.0). - #[getter] - fn success_rate(&self) -> f64 { - self.inner.success_rate - } - - /// Total input tokens. - #[getter] - fn total_input_tokens(&self) -> u64 { - self.inner.total_input_tokens - } - - /// Total output tokens. - #[getter] - fn total_output_tokens(&self) -> u64 { - self.inner.total_output_tokens - } - - /// Total tokens (input + output). - #[getter] - fn total_tokens(&self) -> u64 { - self.inner.total_tokens - } - - /// Average latency per call in milliseconds. - #[getter] - fn avg_latency_ms(&self) -> f64 { - self.inner.avg_latency_ms - } - - /// Total latency in milliseconds. - #[getter] - fn total_latency_ms(&self) -> u64 { - self.inner.total_latency_ms - } - - /// Estimated cost in USD. - #[getter] - fn estimated_cost_usd(&self) -> f64 { - self.inner.estimated_cost_usd - } - - /// Number of rate limit errors. - #[getter] - fn rate_limit_errors(&self) -> u64 { - self.inner.rate_limit_errors - } - - /// Number of timeout errors. - #[getter] - fn timeout_errors(&self) -> u64 { - self.inner.timeout_errors - } - - /// Number of fallback triggers. - #[getter] - fn fallback_triggers(&self) -> u64 { - self.inner.fallback_triggers - } - - fn __repr__(&self) -> String { - format!( - "LlmMetricsReport(calls={}, tokens={}, cost=${:.4})", - self.inner.total_calls, self.inner.total_tokens, self.inner.estimated_cost_usd, - ) - } -} - -/// Pilot decision metrics report. -#[pyclass(name = "PilotMetricsReport")] -pub struct PyPilotMetricsReport { - inner: PilotMetricsReport, -} - -#[pymethods] -impl PyPilotMetricsReport { - /// Total number of Pilot decisions. - #[getter] - fn total_decisions(&self) -> u64 { - self.inner.total_decisions - } - - /// Number of start guidance calls. - #[getter] - fn start_guidance_calls(&self) -> u64 { - self.inner.start_guidance_calls - } - - /// Number of fork decisions. - #[getter] - fn fork_decisions(&self) -> u64 { - self.inner.fork_decisions - } - - /// Number of backtrack calls. - #[getter] - fn backtrack_calls(&self) -> u64 { - self.inner.backtrack_calls - } - - /// Number of evaluate calls. - #[getter] - fn evaluate_calls(&self) -> u64 { - self.inner.evaluate_calls - } - - /// Decision accuracy based on feedback (0.0 - 1.0). - #[getter] - fn accuracy(&self) -> f64 { - self.inner.accuracy - } - - /// Number of correct decisions. - #[getter] - fn correct_decisions(&self) -> u64 { - self.inner.correct_decisions - } - - /// Number of incorrect decisions. - #[getter] - fn incorrect_decisions(&self) -> u64 { - self.inner.incorrect_decisions - } - - /// Average confidence across all decisions. - #[getter] - fn avg_confidence(&self) -> f64 { - self.inner.avg_confidence - } - - /// Number of LLM calls made by Pilot. - #[getter] - fn llm_calls(&self) -> u64 { - self.inner.llm_calls - } - - /// Number of interventions. - #[getter] - fn interventions(&self) -> u64 { - self.inner.interventions - } - - /// Number of skipped interventions. - #[getter] - fn skipped_interventions(&self) -> u64 { - self.inner.skipped_interventions - } - - /// Number of budget exhausted events. - #[getter] - fn budget_exhausted(&self) -> u64 { - self.inner.budget_exhausted - } - - /// Number of algorithm fallbacks. - #[getter] - fn algorithm_fallbacks(&self) -> u64 { - self.inner.algorithm_fallbacks - } - - fn __repr__(&self) -> String { - format!( - "PilotMetricsReport(decisions={}, accuracy={:.2}, avg_confidence={:.2})", - self.inner.total_decisions, self.inner.accuracy, self.inner.avg_confidence, - ) - } -} - -/// Retrieval operation metrics report. -#[pyclass(name = "RetrievalMetricsReport")] -pub struct PyRetrievalMetricsReport { - inner: RetrievalMetricsReport, -} - -#[pymethods] -impl PyRetrievalMetricsReport { - /// Total number of queries. - #[getter] - fn total_queries(&self) -> u64 { - self.inner.total_queries - } - - /// Total number of search iterations. - #[getter] - fn total_iterations(&self) -> u64 { - self.inner.total_iterations - } - - /// Average iterations per query. - #[getter] - fn avg_iterations(&self) -> f64 { - self.inner.avg_iterations - } - - /// Total nodes visited. - #[getter] - fn nodes_visited(&self) -> u64 { - self.inner.nodes_visited - } - - /// Total paths found. - #[getter] - fn paths_found(&self) -> u64 { - self.inner.paths_found - } - - /// Average path length. - #[getter] - fn avg_path_length(&self) -> f64 { - self.inner.avg_path_length - } - - /// Average path score (0.0 - 1.0). - #[getter] - fn avg_path_score(&self) -> f64 { - self.inner.avg_path_score - } - - /// Number of high-score paths (>= 0.5). - #[getter] - fn high_score_paths(&self) -> u64 { - self.inner.high_score_paths - } - - /// Number of low-score paths (< 0.3). - #[getter] - fn low_score_paths(&self) -> u64 { - self.inner.low_score_paths - } - - /// Number of cache hits. - #[getter] - fn cache_hits(&self) -> u64 { - self.inner.cache_hits - } - - /// Number of cache misses. - #[getter] - fn cache_misses(&self) -> u64 { - self.inner.cache_misses - } - - /// Cache hit rate (0.0 - 1.0). - #[getter] - fn cache_hit_rate(&self) -> f64 { - self.inner.cache_hit_rate - } - - /// Total latency in milliseconds. - #[getter] - fn total_latency_ms(&self) -> u64 { - self.inner.total_latency_ms - } - - /// Average latency per query in milliseconds. - #[getter] - fn avg_latency_ms(&self) -> f64 { - self.inner.avg_latency_ms - } - - /// Number of backtracks. - #[getter] - fn backtracks(&self) -> u64 { - self.inner.backtracks - } - - /// Number of sufficiency checks. - #[getter] - fn sufficiency_checks(&self) -> u64 { - self.inner.sufficiency_checks - } - - /// Sufficiency rate (0.0 - 1.0). - #[getter] - fn sufficiency_rate(&self) -> f64 { - self.inner.sufficiency_rate - } - - fn __repr__(&self) -> String { - format!( - "RetrievalMetricsReport(queries={}, avg_score={:.2}, cache_hit={:.1}%)", - self.inner.total_queries, - self.inner.avg_path_score, - self.inner.cache_hit_rate * 100.0, - ) - } -} - -/// Complete metrics report combining all subsystem metrics. -#[pyclass(name = "MetricsReport")] -pub struct PyMetricsReport { - inner: MetricsReport, -} - -#[pymethods] -impl PyMetricsReport { - /// LLM metrics. - #[getter] - fn llm(&self) -> PyLlmMetricsReport { - PyLlmMetricsReport { - inner: self.inner.llm.clone(), - } - } - - /// Pilot metrics. - #[getter] - fn pilot(&self) -> PyPilotMetricsReport { - PyPilotMetricsReport { - inner: self.inner.pilot.clone(), - } - } - - /// Retrieval metrics. - #[getter] - fn retrieval(&self) -> PyRetrievalMetricsReport { - PyRetrievalMetricsReport { - inner: self.inner.retrieval.clone(), - } - } - - /// Total estimated cost in USD. - fn total_cost_usd(&self) -> f64 { - self.inner.total_cost_usd() - } - - /// Overall success rate (0.0 - 1.0). - fn overall_success_rate(&self) -> f64 { - self.inner.overall_success_rate() - } - - fn __repr__(&self) -> String { - format!( - "MetricsReport(llm_calls={}, cost=${:.4}, queries={})", - self.inner.llm.total_calls, - self.inner.total_cost_usd(), - self.inner.retrieval.total_queries, - ) - } -} - -// ============================================================ -// IndexItem / IndexResult -// ============================================================ - -/// A single indexed document item. -#[pyclass(name = "IndexItem")] -pub struct PyIndexItem { - inner: IndexItem, -} - -#[pymethods] -impl PyIndexItem { - #[getter] - fn doc_id(&self) -> &str { - &self.inner.doc_id - } - - #[getter] - fn name(&self) -> &str { - &self.inner.name - } - - #[getter] - fn format(&self) -> String { - format!("{:?}", self.inner.format).to_lowercase() - } - - #[getter] - fn description(&self) -> Option<&str> { - self.inner.description.as_deref() - } - - #[getter] - fn source_path(&self) -> Option<&str> { - self.inner.source_path.as_deref() - } - - #[getter] - fn page_count(&self) -> Option { - self.inner.page_count - } - - /// Indexing pipeline metrics (timing, LLM usage, etc.). - #[getter] - fn metrics(&self) -> Option { - self.inner - .metrics - .as_ref() - .map(|m| PyIndexMetrics { inner: m.clone() }) - } - - fn __repr__(&self) -> String { - format!( - "IndexItem(doc_id='{}', name='{}')", - self.inner.doc_id, self.inner.name - ) - } -} - -/// Result of a document indexing operation. -#[pyclass(name = "IndexResult")] -pub struct PyIndexResult { - inner: IndexResult, -} - -#[pymethods] -impl PyIndexResult { - /// The document ID (convenience for single-document indexing). - #[getter] - fn doc_id(&self) -> Option { - self.inner.doc_id().map(|s| s.to_string()) - } - - /// All indexed items. - #[getter] - fn items(&self) -> Vec { - self.inner - .items - .iter() - .map(|i| PyIndexItem { inner: i.clone() }) - .collect() - } - - /// Failed items. - #[getter] - fn failed(&self) -> Vec { - self.inner - .failed - .iter() - .map(|f| PyFailedItem { inner: f.clone() }) - .collect() - } - - /// Whether any items failed. - fn has_failures(&self) -> bool { - self.inner.has_failures() - } - - /// Total number of items (successful + failed). - fn total(&self) -> usize { - self.inner.total() - } - - fn __len__(&self) -> usize { - self.inner.len() - } - - fn __repr__(&self) -> String { - format!( - "IndexResult(doc_id={:?}, count={}, failed={})", - self.inner.doc_id(), - self.inner.items.len(), - self.inner.failed.len() - ) - } -} - -// ============================================================ -// DocumentInfo -// ============================================================ - -/// Information about an indexed document. -#[pyclass(name = "DocumentInfo")] -pub struct PyDocumentInfo { - inner: DocumentInfo, -} - -#[pymethods] -impl PyDocumentInfo { - #[getter] - fn id(&self) -> &str { - &self.inner.id - } - - #[getter] - fn name(&self) -> &str { - &self.inner.name - } - - #[getter] - fn format(&self) -> &str { - &self.inner.format - } - - #[getter] - fn description(&self) -> Option<&str> { - self.inner.description.as_deref() - } - - #[getter] - fn source_path(&self) -> Option<&str> { - self.inner.source_path.as_deref() - } - - #[getter] - fn page_count(&self) -> Option { - self.inner.page_count - } - - #[getter] - fn line_count(&self) -> Option { - self.inner.line_count - } - - fn __repr__(&self) -> String { - format!( - "DocumentInfo(id='{}', name='{}', format='{}')", - self.inner.id, self.inner.name, self.inner.format - ) - } -} - -// ============================================================ -// DocumentGraph types -// ============================================================ - -use ::vectorless::graph::{ - DocumentGraph, DocumentGraphNode, EdgeEvidence, GraphEdge, WeightedKeyword, -}; - -/// A keyword with weight from document analysis. -#[pyclass(name = "WeightedKeyword")] -pub struct PyWeightedKeyword { - inner: WeightedKeyword, -} - -#[pymethods] -impl PyWeightedKeyword { - #[getter] - fn keyword(&self) -> &str { - &self.inner.keyword - } - - #[getter] - fn weight(&self) -> f32 { - self.inner.weight - } - - fn __repr__(&self) -> String { - format!( - "WeightedKeyword('{}', weight={:.2})", - self.inner.keyword, self.inner.weight - ) - } -} - -/// Evidence for a cross-document connection. -#[pyclass(name = "EdgeEvidence")] -pub struct PyEdgeEvidence { - inner: EdgeEvidence, -} - -#[pymethods] -impl PyEdgeEvidence { - /// Number of shared keywords. - #[getter] - fn shared_keyword_count(&self) -> usize { - self.inner.shared_keyword_count - } - - /// Jaccard similarity of keyword sets. - #[getter] - fn keyword_jaccard(&self) -> f32 { - self.inner.keyword_jaccard - } - - /// Shared keywords with weights. - #[getter] - fn shared_keywords(&self) -> Vec<(String, f32, f32)> { - self.inner - .shared_keywords - .iter() - .map(|sk| (sk.keyword.clone(), sk.source_weight, sk.target_weight)) - .collect() - } - - fn __repr__(&self) -> String { - format!( - "EdgeEvidence(shared={}, jaccard={:.2})", - self.inner.shared_keyword_count, self.inner.keyword_jaccard - ) - } -} - -/// An edge representing a relationship between two documents. -#[pyclass(name = "GraphEdge")] -pub struct PyGraphEdge { - inner: GraphEdge, -} - -#[pymethods] -impl PyGraphEdge { - /// Target document ID. - #[getter] - fn target_doc_id(&self) -> &str { - &self.inner.target_doc_id - } - - /// Edge weight (connection strength). - #[getter] - fn weight(&self) -> f32 { - self.inner.weight - } - - /// Evidence for this connection. - #[getter] - fn evidence(&self) -> PyEdgeEvidence { - PyEdgeEvidence { - inner: self.inner.evidence.clone(), - } - } - - fn __repr__(&self) -> String { - format!( - "GraphEdge(target='{}', weight={:.2})", - self.inner.target_doc_id, self.inner.weight - ) - } -} - -/// A node in the document graph representing an indexed document. -#[pyclass(name = "DocumentGraphNode")] -pub struct PyDocumentGraphNode { - inner: DocumentGraphNode, -} - -#[pymethods] -impl PyDocumentGraphNode { - #[getter] - fn doc_id(&self) -> &str { - &self.inner.doc_id - } - - #[getter] - fn title(&self) -> &str { - &self.inner.title - } - - #[getter] - fn format(&self) -> &str { - &self.inner.format - } - - #[getter] - fn node_count(&self) -> usize { - self.inner.node_count - } - - /// Top keywords extracted from the document. - #[getter] - fn top_keywords(&self) -> Vec { - self.inner - .top_keywords - .iter() - .map(|kw| PyWeightedKeyword { inner: kw.clone() }) - .collect() - } - - fn __repr__(&self) -> String { - format!( - "DocumentGraphNode(doc_id='{}', title='{}')", - self.inner.doc_id, self.inner.title - ) - } -} - -/// Cross-document relationship graph. -/// -/// Automatically rebuilt after indexing. Connects documents -/// that share keywords via Jaccard similarity. -#[pyclass(name = "DocumentGraph")] -pub struct PyDocumentGraph { - inner: DocumentGraph, -} - -#[pymethods] -impl PyDocumentGraph { - /// Number of document nodes. - fn node_count(&self) -> usize { - self.inner.node_count() - } - - /// Number of relationship edges. - fn edge_count(&self) -> usize { - self.inner.edge_count() - } - - /// Get a document node by ID. - fn get_node(&self, doc_id: String) -> Option { - self.inner - .get_node(&doc_id) - .map(|n| PyDocumentGraphNode { inner: n.clone() }) - } - - /// Get all document IDs in the graph. - fn doc_ids(&self) -> Vec { - self.inner.doc_ids().map(|s| s.to_string()).collect() - } - - /// Get edges (neighbors) for a document. - fn get_neighbors(&self, doc_id: String) -> Vec { - self.inner - .get_neighbors(&doc_id) - .iter() - .map(|e| PyGraphEdge { inner: e.clone() }) - .collect() - } - - /// Whether the graph is empty. - fn is_empty(&self) -> bool { - self.inner.is_empty() - } - - fn __repr__(&self) -> String { - format!( - "DocumentGraph(nodes={}, edges={})", - self.inner.node_count(), - self.inner.edge_count() - ) - } -} - -// ============================================================ -// Engine async helpers (named functions to avoid FnOnce HRTB issue) -// ============================================================ - -async fn run_index(engine: Arc, ctx: IndexContext) -> PyResult { - let result = engine.index(ctx).await.map_err(to_py_err)?; - Ok(PyIndexResult { inner: result }) -} - -async fn run_query(engine: Arc, ctx: QueryContext) -> PyResult { - let result = engine.query(ctx).await.map_err(to_py_err)?; - Ok(PyQueryResult { inner: result }) -} - -async fn run_list(engine: Arc) -> PyResult> { - let docs = engine.list().await.map_err(to_py_err)?; - Ok(docs - .into_iter() - .map(|d| PyDocumentInfo { inner: d }) - .collect()) -} - -async fn run_remove(engine: Arc, doc_id: String) -> PyResult { - engine.remove(&doc_id).await.map_err(to_py_err) -} - -async fn run_clear(engine: Arc) -> PyResult { - engine.clear().await.map_err(to_py_err) -} - -async fn run_exists(engine: Arc, doc_id: String) -> PyResult { - engine.exists(&doc_id).await.map_err(to_py_err) -} - -async fn run_get_graph(engine: Arc) -> PyResult> { - let graph = engine.get_graph().await.map_err(to_py_err)?; - Ok(graph.map(|g| PyDocumentGraph { inner: g })) -} - -fn run_metrics_report(engine: Arc) -> PyMetricsReport { - PyMetricsReport { - inner: engine.metrics_report(), - } -} - -// ============================================================ -// Engine -// ============================================================ - -/// The main vectorless engine. -/// -/// `api_key` and `model` are **required**. -/// -/// ```python -/// from vectorless import Engine, IndexContext, QueryContext -/// -/// engine = Engine( -/// api_key="sk-...", -/// model="gpt-4o", -/// ) -/// -/// # Index -/// result = await engine.index(IndexContext.from_path("./report.pdf")) -/// doc_id = result.doc_id -/// -/// # Query -/// answer = await engine.query(QueryContext("What is the revenue?").with_doc_ids([doc_id])) -/// print(answer.single().content) -/// ``` -#[pyclass(name = "Engine")] -pub struct PyEngine { - inner: Arc, -} - -#[pymethods] -impl PyEngine { - /// Create a new Engine. - /// - /// Args: - /// config_path: Path to configuration file (optional). - /// api_key: **Required**. LLM API key. - /// model: **Required**. LLM model name. - /// endpoint: Optional API endpoint. - /// - /// Raises: - /// VectorlessError: If engine creation fails. - #[new] - #[pyo3(signature = (config_path=None, api_key=None, model=None, endpoint=None))] - fn new( - config_path: Option, - api_key: Option, - model: Option, - endpoint: Option, - ) -> PyResult { - let rt = Runtime::new().map_err(|e| { - PyErr::from(VectorlessError::new( - format!("Failed to create tokio runtime: {}", e), - "config", - )) - })?; - - let engine = rt.block_on(async { - let mut builder = EngineBuilder::new(); - - if let Some(path) = &config_path { - builder = builder.with_config_path(path); - } - if let Some(m) = &model { - builder = builder.with_model(m); - } - if let Some(e) = &endpoint { - builder = builder.with_endpoint(e); - } - if let Some(key) = api_key { - builder = builder.with_key(key); - } - - builder.build().await - }); - - let engine = engine.map_err(|e| { - PyErr::from(VectorlessError::new( - format!("Failed to create engine: {}", e), - "config", - )) - })?; - - Ok(Self { - inner: Arc::new(engine), - }) - } - - /// Index a document. - /// - /// Args: - /// ctx: IndexContext created from from_path, from_paths, from_dir, etc. - /// - /// Returns: - /// IndexResult with doc_id and items. - /// - /// Raises: - /// VectorlessError: If indexing fails. - fn index<'py>(&self, py: Python<'py>, ctx: &PyIndexContext) -> PyResult> { - let engine = Arc::clone(&self.inner); - let index_ctx = ctx.inner.clone(); - future_into_py(py, run_index(engine, index_ctx)) - } - - /// Query indexed documents. - /// - /// Args: - /// ctx: QueryContext with query text and scope. - /// - /// Returns: - /// QueryResult with answer and score. - /// - /// Raises: - /// VectorlessError: If query fails. - fn query<'py>(&self, py: Python<'py>, ctx: &PyQueryContext) -> PyResult> { - let engine = Arc::clone(&self.inner); - let query_ctx = ctx.inner.clone(); - future_into_py(py, run_query(engine, query_ctx)) - } - - /// List all indexed documents. - /// - /// Returns: - /// List of DocumentInfo objects. - fn list<'py>(&self, py: Python<'py>) -> PyResult> { - let engine = Arc::clone(&self.inner); - future_into_py(py, run_list(engine)) - } - - /// Remove a document by ID. - /// - /// Returns: - /// True if removed, False if not found. - fn remove<'py>(&self, py: Python<'py>, doc_id: String) -> PyResult> { - let engine = Arc::clone(&self.inner); - future_into_py(py, run_remove(engine, doc_id)) - } - - /// Remove all indexed documents. - /// - /// Returns: - /// Number of documents removed. - fn clear<'py>(&self, py: Python<'py>) -> PyResult> { - let engine = Arc::clone(&self.inner); - future_into_py(py, run_clear(engine)) - } - - /// Check if a document exists. - fn exists<'py>(&self, py: Python<'py>, doc_id: String) -> PyResult> { - let engine = Arc::clone(&self.inner); - future_into_py(py, run_exists(engine, doc_id)) - } - - /// Get the cross-document relationship graph. - /// - /// Returns: - /// DocumentGraph if any documents exist, else None. - fn get_graph<'py>(&self, py: Python<'py>) -> PyResult> { - let engine = Arc::clone(&self.inner); - future_into_py(py, run_get_graph(engine)) - } - - /// Generate a complete metrics report. - /// - /// Returns: - /// MetricsReport with LLM, Pilot, and Retrieval metrics. - fn metrics_report(&self) -> PyMetricsReport { - run_metrics_report(Arc::clone(&self.inner)) - } - - fn __repr__(&self) -> String { - "Engine(...)".to_string() - } -} - -// ============================================================ -// Module Definition -// ============================================================ - /// Vectorless - Reasoning-native document intelligence engine. /// /// ```python @@ -1595,6 +59,7 @@ fn _vectorless(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add("__version__", env!("CARGO_PKG_VERSION"))?; diff --git a/python/src/metrics.rs b/python/src/metrics.rs new file mode 100644 index 00000000..669511cb --- /dev/null +++ b/python/src/metrics.rs @@ -0,0 +1,376 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Metrics report Python wrappers. + +use pyo3::prelude::*; + +use ::vectorless::metrics::{ + LlmMetricsReport, MetricsReport, PilotMetricsReport, RetrievalMetricsReport, +}; + +/// LLM usage metrics report. +#[pyclass(name = "LlmMetricsReport")] +pub struct PyLlmMetricsReport { + pub(crate) inner: LlmMetricsReport, +} + +#[pymethods] +impl PyLlmMetricsReport { + /// Total number of LLM calls. + #[getter] + fn total_calls(&self) -> u64 { + self.inner.total_calls + } + + /// Number of successful calls. + #[getter] + fn successful_calls(&self) -> u64 { + self.inner.successful_calls + } + + /// Number of failed calls. + #[getter] + fn failed_calls(&self) -> u64 { + self.inner.failed_calls + } + + /// Success rate (0.0 - 1.0). + #[getter] + fn success_rate(&self) -> f64 { + self.inner.success_rate + } + + /// Total input tokens. + #[getter] + fn total_input_tokens(&self) -> u64 { + self.inner.total_input_tokens + } + + /// Total output tokens. + #[getter] + fn total_output_tokens(&self) -> u64 { + self.inner.total_output_tokens + } + + /// Total tokens (input + output). + #[getter] + fn total_tokens(&self) -> u64 { + self.inner.total_tokens + } + + /// Average latency per call in milliseconds. + #[getter] + fn avg_latency_ms(&self) -> f64 { + self.inner.avg_latency_ms + } + + /// Total latency in milliseconds. + #[getter] + fn total_latency_ms(&self) -> u64 { + self.inner.total_latency_ms + } + + /// Estimated cost in USD. + #[getter] + fn estimated_cost_usd(&self) -> f64 { + self.inner.estimated_cost_usd + } + + /// Number of rate limit errors. + #[getter] + fn rate_limit_errors(&self) -> u64 { + self.inner.rate_limit_errors + } + + /// Number of timeout errors. + #[getter] + fn timeout_errors(&self) -> u64 { + self.inner.timeout_errors + } + + /// Number of fallback triggers. + #[getter] + fn fallback_triggers(&self) -> u64 { + self.inner.fallback_triggers + } + + fn __repr__(&self) -> String { + format!( + "LlmMetricsReport(calls={}, tokens={}, cost=${:.4})", + self.inner.total_calls, self.inner.total_tokens, self.inner.estimated_cost_usd, + ) + } +} + +/// Pilot decision metrics report. +#[pyclass(name = "PilotMetricsReport")] +pub struct PyPilotMetricsReport { + pub(crate) inner: PilotMetricsReport, +} + +#[pymethods] +impl PyPilotMetricsReport { + /// Total number of Pilot decisions. + #[getter] + fn total_decisions(&self) -> u64 { + self.inner.total_decisions + } + + /// Number of start guidance calls. + #[getter] + fn start_guidance_calls(&self) -> u64 { + self.inner.start_guidance_calls + } + + /// Number of fork decisions. + #[getter] + fn fork_decisions(&self) -> u64 { + self.inner.fork_decisions + } + + /// Number of backtrack calls. + #[getter] + fn backtrack_calls(&self) -> u64 { + self.inner.backtrack_calls + } + + /// Number of evaluate calls. + #[getter] + fn evaluate_calls(&self) -> u64 { + self.inner.evaluate_calls + } + + /// Decision accuracy based on feedback (0.0 - 1.0). + #[getter] + fn accuracy(&self) -> f64 { + self.inner.accuracy + } + + /// Number of correct decisions. + #[getter] + fn correct_decisions(&self) -> u64 { + self.inner.correct_decisions + } + + /// Number of incorrect decisions. + #[getter] + fn incorrect_decisions(&self) -> u64 { + self.inner.incorrect_decisions + } + + /// Average confidence across all decisions. + #[getter] + fn avg_confidence(&self) -> f64 { + self.inner.avg_confidence + } + + /// Number of LLM calls made by Pilot. + #[getter] + fn llm_calls(&self) -> u64 { + self.inner.llm_calls + } + + /// Number of interventions. + #[getter] + fn interventions(&self) -> u64 { + self.inner.interventions + } + + /// Number of skipped interventions. + #[getter] + fn skipped_interventions(&self) -> u64 { + self.inner.skipped_interventions + } + + /// Number of budget exhausted events. + #[getter] + fn budget_exhausted(&self) -> u64 { + self.inner.budget_exhausted + } + + /// Number of algorithm fallbacks. + #[getter] + fn algorithm_fallbacks(&self) -> u64 { + self.inner.algorithm_fallbacks + } + + fn __repr__(&self) -> String { + format!( + "PilotMetricsReport(decisions={}, accuracy={:.2}, avg_confidence={:.2})", + self.inner.total_decisions, self.inner.accuracy, self.inner.avg_confidence, + ) + } +} + +/// Retrieval operation metrics report. +#[pyclass(name = "RetrievalMetricsReport")] +pub struct PyRetrievalMetricsReport { + pub(crate) inner: RetrievalMetricsReport, +} + +#[pymethods] +impl PyRetrievalMetricsReport { + /// Total number of queries. + #[getter] + fn total_queries(&self) -> u64 { + self.inner.total_queries + } + + /// Total number of search iterations. + #[getter] + fn total_iterations(&self) -> u64 { + self.inner.total_iterations + } + + /// Average iterations per query. + #[getter] + fn avg_iterations(&self) -> f64 { + self.inner.avg_iterations + } + + /// Total nodes visited. + #[getter] + fn nodes_visited(&self) -> u64 { + self.inner.nodes_visited + } + + /// Total paths found. + #[getter] + fn paths_found(&self) -> u64 { + self.inner.paths_found + } + + /// Average path length. + #[getter] + fn avg_path_length(&self) -> f64 { + self.inner.avg_path_length + } + + /// Average path score (0.0 - 1.0). + #[getter] + fn avg_path_score(&self) -> f64 { + self.inner.avg_path_score + } + + /// Number of high-score paths (>= 0.5). + #[getter] + fn high_score_paths(&self) -> u64 { + self.inner.high_score_paths + } + + /// Number of low-score paths (< 0.3). + #[getter] + fn low_score_paths(&self) -> u64 { + self.inner.low_score_paths + } + + /// Number of cache hits. + #[getter] + fn cache_hits(&self) -> u64 { + self.inner.cache_hits + } + + /// Number of cache misses. + #[getter] + fn cache_misses(&self) -> u64 { + self.inner.cache_misses + } + + /// Cache hit rate (0.0 - 1.0). + #[getter] + fn cache_hit_rate(&self) -> f64 { + self.inner.cache_hit_rate + } + + /// Total latency in milliseconds. + #[getter] + fn total_latency_ms(&self) -> u64 { + self.inner.total_latency_ms + } + + /// Average latency per query in milliseconds. + #[getter] + fn avg_latency_ms(&self) -> f64 { + self.inner.avg_latency_ms + } + + /// Number of backtracks. + #[getter] + fn backtracks(&self) -> u64 { + self.inner.backtracks + } + + /// Number of sufficiency checks. + #[getter] + fn sufficiency_checks(&self) -> u64 { + self.inner.sufficiency_checks + } + + /// Sufficiency rate (0.0 - 1.0). + #[getter] + fn sufficiency_rate(&self) -> f64 { + self.inner.sufficiency_rate + } + + fn __repr__(&self) -> String { + format!( + "RetrievalMetricsReport(queries={}, avg_score={:.2}, cache_hit={:.1}%)", + self.inner.total_queries, + self.inner.avg_path_score, + self.inner.cache_hit_rate * 100.0, + ) + } +} + +/// Complete metrics report combining all subsystem metrics. +#[pyclass(name = "MetricsReport")] +pub struct PyMetricsReport { + pub(crate) inner: MetricsReport, +} + +#[pymethods] +impl PyMetricsReport { + /// LLM metrics. + #[getter] + fn llm(&self) -> PyLlmMetricsReport { + PyLlmMetricsReport { + inner: self.inner.llm.clone(), + } + } + + /// Pilot metrics. + #[getter] + fn pilot(&self) -> PyPilotMetricsReport { + PyPilotMetricsReport { + inner: self.inner.pilot.clone(), + } + } + + /// Retrieval metrics. + #[getter] + fn retrieval(&self) -> PyRetrievalMetricsReport { + PyRetrievalMetricsReport { + inner: self.inner.retrieval.clone(), + } + } + + /// Total estimated cost in USD. + fn total_cost_usd(&self) -> f64 { + self.inner.total_cost_usd() + } + + /// Overall success rate (0.0 - 1.0). + fn overall_success_rate(&self) -> f64 { + self.inner.overall_success_rate() + } + + fn __repr__(&self) -> String { + format!( + "MetricsReport(llm_calls={}, cost=${:.4}, queries={})", + self.inner.llm.total_calls, + self.inner.total_cost_usd(), + self.inner.retrieval.total_queries, + ) + } +} diff --git a/python/src/results.rs b/python/src/results.rs new file mode 100644 index 00000000..fe780a4c --- /dev/null +++ b/python/src/results.rs @@ -0,0 +1,351 @@ +// Copyright (c) 2026 vectorless developers +// SPDX-License-Identifier: Apache-2.0 + +//! Query and index result Python wrappers. + +use pyo3::prelude::*; + +use ::vectorless::client::{FailedItem, IndexItem, IndexResult, QueryResult, QueryResultItem}; +use ::vectorless::metrics::IndexMetrics; + +// ============================================================ +// QueryResultItem +// ============================================================ + +/// A single document's query result. +#[pyclass(name = "QueryResultItem")] +pub struct PyQueryResultItem { + pub(crate) inner: QueryResultItem, +} + +#[pymethods] +impl PyQueryResultItem { + /// The document ID. + #[getter] + fn doc_id(&self) -> &str { + &self.inner.doc_id + } + + /// The retrieved content. + #[getter] + fn content(&self) -> &str { + &self.inner.content + } + + /// Relevance score (0.0 to 1.0). + #[getter] + fn score(&self) -> f32 { + self.inner.score + } + + /// Node IDs that matched. + #[getter] + fn node_ids(&self) -> Vec { + self.inner.node_ids.clone() + } + + fn __repr__(&self) -> String { + format!( + "QueryResultItem(doc_id='{}', score={:.2}, content_len={})", + self.inner.doc_id, + self.inner.score, + self.inner.content.len() + ) + } +} + +// ============================================================ +// FailedItem +// ============================================================ + +/// A failed item in a batch operation. +#[pyclass(name = "FailedItem")] +pub struct PyFailedItem { + pub(crate) inner: FailedItem, +} + +#[pymethods] +impl PyFailedItem { + /// Source description. + #[getter] + fn source(&self) -> &str { + &self.inner.source + } + + /// Error message. + #[getter] + fn error(&self) -> &str { + &self.inner.error + } + + fn __repr__(&self) -> String { + format!( + "FailedItem(source='{}', error='{}')", + self.inner.source, self.inner.error + ) + } +} + +// ============================================================ +// QueryResult +// ============================================================ + +/// Result of a document query. +#[pyclass(name = "QueryResult")] +pub struct PyQueryResult { + pub(crate) inner: QueryResult, +} + +#[pymethods] +impl PyQueryResult { + /// Result items (one per document). + #[getter] + fn items(&self) -> Vec { + self.inner + .items + .iter() + .map(|i| PyQueryResultItem { inner: i.clone() }) + .collect() + } + + /// Get the first (single-doc) result item. + fn single(&self) -> Option { + self.inner + .single() + .map(|i| PyQueryResultItem { inner: i.clone() }) + } + + /// Number of result items. + fn __len__(&self) -> usize { + self.inner.len() + } + + /// Whether any documents failed. + fn has_failures(&self) -> bool { + self.inner.has_failures() + } + + /// Failed items. + #[getter] + fn failed(&self) -> Vec { + self.inner + .failed + .iter() + .map(|f| PyFailedItem { inner: f.clone() }) + .collect() + } + + fn __repr__(&self) -> String { + format!( + "QueryResult(items={}, failed={})", + self.inner.len(), + self.inner.failed.len() + ) + } +} + +// ============================================================ +// IndexMetrics +// ============================================================ + +/// Indexing pipeline metrics. +#[pyclass(name = "IndexMetrics")] +pub struct PyIndexMetrics { + pub(crate) inner: IndexMetrics, +} + +#[pymethods] +impl PyIndexMetrics { + /// Total indexing time (ms). + #[getter] + fn total_time_ms(&self) -> u64 { + self.inner.total_time_ms() + } + + /// Parse stage duration (ms). + #[getter] + fn parse_time_ms(&self) -> u64 { + self.inner.parse_time_ms + } + + /// Build stage duration (ms). + #[getter] + fn build_time_ms(&self) -> u64 { + self.inner.build_time_ms + } + + /// Enhance (summary) stage duration (ms). + #[getter] + fn enhance_time_ms(&self) -> u64 { + self.inner.enhance_time_ms + } + + /// Number of nodes processed. + #[getter] + fn nodes_processed(&self) -> usize { + self.inner.nodes_processed + } + + /// Number of summaries successfully generated. + #[getter] + fn summaries_generated(&self) -> usize { + self.inner.summaries_generated + } + + /// Number of summaries that failed to generate. + #[getter] + fn summaries_failed(&self) -> usize { + self.inner.summaries_failed + } + + /// Number of LLM calls made. + #[getter] + fn llm_calls(&self) -> usize { + self.inner.llm_calls + } + + /// Total tokens generated by LLM. + #[getter] + fn total_tokens_generated(&self) -> usize { + self.inner.total_tokens_generated + } + + /// Number of topics in reasoning index. + #[getter] + fn topics_indexed(&self) -> usize { + self.inner.topics_indexed + } + + /// Number of keywords in reasoning index. + #[getter] + fn keywords_indexed(&self) -> usize { + self.inner.keywords_indexed + } + + fn __repr__(&self) -> String { + format!( + "IndexMetrics(total={}ms, summaries={}, failed={}, llm_calls={})", + self.inner.total_time_ms(), + self.inner.summaries_generated, + self.inner.summaries_failed, + self.inner.llm_calls, + ) + } +} + +// ============================================================ +// IndexItem / IndexResult +// ============================================================ + +/// A single indexed document item. +#[pyclass(name = "IndexItem")] +pub struct PyIndexItem { + pub(crate) inner: IndexItem, +} + +#[pymethods] +impl PyIndexItem { + #[getter] + fn doc_id(&self) -> &str { + &self.inner.doc_id + } + + #[getter] + fn name(&self) -> &str { + &self.inner.name + } + + #[getter] + fn format(&self) -> String { + format!("{:?}", self.inner.format).to_lowercase() + } + + #[getter] + fn description(&self) -> Option<&str> { + self.inner.description.as_deref() + } + + #[getter] + fn source_path(&self) -> Option<&str> { + self.inner.source_path.as_deref() + } + + #[getter] + fn page_count(&self) -> Option { + self.inner.page_count + } + + /// Indexing pipeline metrics (timing, LLM usage, etc.). + #[getter] + fn metrics(&self) -> Option { + self.inner + .metrics + .as_ref() + .map(|m| PyIndexMetrics { inner: m.clone() }) + } + + fn __repr__(&self) -> String { + format!( + "IndexItem(doc_id='{}', name='{}')", + self.inner.doc_id, self.inner.name + ) + } +} + +/// Result of a document indexing operation. +#[pyclass(name = "IndexResult")] +pub struct PyIndexResult { + pub(crate) inner: IndexResult, +} + +#[pymethods] +impl PyIndexResult { + /// The document ID (convenience for single-document indexing). + #[getter] + fn doc_id(&self) -> Option { + self.inner.doc_id().map(|s| s.to_string()) + } + + /// All indexed items. + #[getter] + fn items(&self) -> Vec { + self.inner + .items + .iter() + .map(|i| PyIndexItem { inner: i.clone() }) + .collect() + } + + /// Failed items. + #[getter] + fn failed(&self) -> Vec { + self.inner + .failed + .iter() + .map(|f| PyFailedItem { inner: f.clone() }) + .collect() + } + + /// Whether any items failed. + fn has_failures(&self) -> bool { + self.inner.has_failures() + } + + /// Total number of items (successful + failed). + fn total(&self) -> usize { + self.inner.total() + } + + fn __len__(&self) -> usize { + self.inner.len() + } + + fn __repr__(&self) -> String { + format!( + "IndexResult(doc_id={:?}, count={}, failed={})", + self.inner.doc_id(), + self.inner.items.len(), + self.inner.failed.len() + ) + } +} diff --git a/rust/examples/advanced.rs b/rust/examples/advanced.rs deleted file mode 100644 index fa14e931..00000000 --- a/rust/examples/advanced.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2026 vectorless developers -// SPDX-License-Identifier: Apache-2.0 - -//! Advanced usage example - Full Configuration. -//! -//! This example demonstrates how to use a configuration file -//! for advanced use cases where you need fine-grained control. -//! -//! # Usage -//! -//! ```bash -//! # Using environment variables for LLM config (overrides config file): -//! LLM_API_KEY=sk-xxx LLM_MODEL=gpt-4o cargo run --example advanced -//! -//! # Or with defaults (using config file): -//! cargo run --example advanced -//! ``` - -use vectorless::{EngineBuilder, IndexContext, QueryContext}; - -#[tokio::main] -async fn main() -> vectorless::Result<()> { - // Initialize tracing for debug output (set RUST_LOG=debug to see more) - tracing_subscriber::fmt::init(); - - println!("=== Vectorless Advanced Example (Config File) ===\n"); - - // Load all settings from the specified config file. - // The config file must include api_key and model. - // If environment variables are set, they override the config file values. - let mut builder = EngineBuilder::new().with_config_path("./config.toml"); - - // Override config with env vars if present - if let Ok(api_key) = std::env::var("LLM_API_KEY") { - builder = builder.with_key(&api_key); - } - if let Ok(model) = std::env::var("LLM_MODEL") { - builder = builder.with_model(&model); - } - if let Ok(endpoint) = std::env::var("LLM_ENDPOINT") { - builder = builder.with_endpoint(&endpoint); - } - - let client = builder - .build() - .await - .map_err(|e: vectorless::BuildError| vectorless::Error::Config(e.to_string()))?; - - println!("Client created with config file\n"); - - // Index a document - let result = client.index(IndexContext::from_path("./README.md")).await?; - let doc_id = result.doc_id().unwrap().to_string(); - println!("Indexed: {}\n", doc_id); - - // Query - let result = client - .query( - QueryContext::new("What features does Vectorless provide?") - .with_doc_ids(vec![doc_id.clone()]), - ) - .await?; - println!("Query: What features does Vectorless provide?"); - if let Some(item) = result.single() { - println!("Score: {:.2}", item.score); - if !item.content.is_empty() { - let preview: String = item.content.chars().take(200).collect(); - println!("Result: {}...\n", preview); - } - } - - // Cleanup - client.remove(&doc_id).await?; - println!("Cleaned up"); - - println!("\n=== Done ==="); - Ok(()) -} diff --git a/rust/src/client/builder.rs b/rust/src/client/builder.rs index 7fea2913..b0c035cf 100644 --- a/rust/src/client/builder.rs +++ b/rust/src/client/builder.rs @@ -5,89 +5,37 @@ //! //! This module provides [`EngineBuilder`] for configuring and building //! [`Engine`] instances with sensible defaults. -//! -//! # Configuration -//! -//! `api_key` and `model` are **required**. `endpoint` is optional -//! (defaults to the model provider's standard endpoint). -//! -//! Configuration sources (later overrides earlier): -//! 1. Default configuration -//! 2. Config file (via `with_config_path`) -//! 3. Builder methods (`with_key`, `with_model`, etc.) — highest priority -//! -//! # Examples -//! -//! ```rust,no_run -//! use vectorless::client::EngineBuilder; -//! -//! # #[tokio::main] -//! # async fn main() -> Result<(), vectorless::BuildError> { -//! let engine = EngineBuilder::new() -//! .with_key("sk-...") -//! .with_model("gpt-4o") -//! .build() -//! .await?; -//! # Ok(()) -//! # } -//! ``` -//! -//! ## With Custom Endpoint -//! -//! ```rust,no_run -//! use vectorless::client::EngineBuilder; -//! -//! # #[tokio::main] -//! # async fn main() -> Result<(), vectorless::BuildError> { -//! let engine = EngineBuilder::new() -//! .with_key("sk-...") -//! .with_model("deepseek-chat") -//! .with_endpoint("https://api.deepseek.com/v1") -//! .build() -//! .await?; -//! # Ok(()) -//! # } -//! ``` - -use crate::config::{Config, ConfigLoader, RetrievalConfig}; -use crate::memo::MemoStore; -use crate::retrieval::PipelineRetriever; -use crate::storage::Workspace; - -use super::engine::Engine; -use crate::events::EventEmitter; + +use crate::{ + client::engine::Engine, config::Config, events::EventEmitter, retrieval::PipelineRetriever, + storage::Workspace, +}; /// Builder for creating a [`Engine`] client. /// -/// `api_key` and `model` are required and must be set via builder methods -/// or provided through a config file. +/// `api_key`, `model` and `endpoint` are **required**. /// /// # Example /// /// ```rust,no_run /// use vectorless::client::EngineBuilder; /// -/// # #[tokio::main] -/// # async fn main() -> Result<(), vectorless::BuildError> { -/// let client = EngineBuilder::new() -/// .with_key("sk-...") -/// .with_model("gpt-4o") -/// .build() -/// .await?; -/// # Ok(()) -/// # } +/// #[tokio::main] +/// async fn main() -> Result<(), vectorless::BuildError> { +/// let client = EngineBuilder::new() +/// .with_key("sk-...") +/// .with_model("gpt-4o") +/// .with_endpoint("https://api.xxx.com/v1") +/// .build() +/// .await?; +/// Ok(()) +/// } /// ``` #[derive(Debug)] pub struct EngineBuilder { - /// Configuration file path. - config_path: Option, - - /// Custom configuration. + /// Custom configuration for advanced tuning. config: Option, - /// Custom retrieval config. - retrieval_config: Option, - /// Event emitter. events: Option, @@ -99,18 +47,6 @@ pub struct EngineBuilder { /// LLM endpoint URL (override). endpoint: Option, - - /// Top-K for retrieval (override). - top_k: Option, - - /// Fast mode flag. - fast_mode: bool, - - /// Precise mode flag. - precise_mode: bool, - - /// Memo store for caching LLM decisions. - memo_store: Option, } impl EngineBuilder { @@ -118,17 +54,11 @@ impl EngineBuilder { #[must_use] pub fn new() -> Self { Self { - config_path: None, config: None, - retrieval_config: None, events: None, api_key: None, model: None, endpoint: None, - top_k: None, - fast_mode: false, - precise_mode: false, - memo_store: None, } } @@ -136,31 +66,17 @@ impl EngineBuilder { // Basic Configuration // ============================================================ - /// Set the configuration file path. - /// - /// The file must be a valid TOML configuration. No auto-detection is performed. - #[must_use] - pub fn with_config_path(mut self, path: impl Into) -> Self { - self.config_path = Some(path.into()); - self - } - - /// Set a custom configuration object. + /// Set a custom configuration. /// - /// This overrides any config file settings. + /// When provided, this replaces the default [`Config`] entirely. + /// Builder methods (`with_key`, `with_model`, `with_endpoint`) + /// will still override the corresponding fields on top of this config. #[must_use] pub fn with_config(mut self, config: Config) -> Self { self.config = Some(config); self } - /// Set custom retrieval configuration. - #[must_use] - pub fn with_retrieval_config(mut self, config: RetrievalConfig) -> Self { - self.retrieval_config = Some(config); - self - } - /// Set the event emitter for callbacks. #[must_use] pub fn with_events(mut self, events: EventEmitter) -> Self { @@ -168,40 +84,6 @@ impl EngineBuilder { self } - /// Set a memo store for caching LLM decisions. - /// - /// When enabled, the pilot will cache navigation decisions based on - /// context fingerprints, avoiding redundant API calls for similar - /// navigation scenarios. - /// - /// # Example - /// - /// ```rust,no_run - /// use vectorless::client::EngineBuilder; - /// use vectorless::memo::MemoStore; - /// use chrono::Duration; - /// - /// # #[tokio::main] - /// # async fn main() -> Result<(), vectorless::BuildError> { - /// let memo_store = MemoStore::new() - /// .with_ttl(Duration::days(7)) - /// .with_model("gpt-4o"); - /// - /// let engine = EngineBuilder::new() - /// .with_key("sk-...") - /// .with_model("gpt-4o") - /// .with_memo_store(memo_store) - /// .build() - /// .await?; - /// # Ok(()) - /// # } - /// ``` - #[must_use] - pub fn with_memo_store(mut self, store: MemoStore) -> Self { - self.memo_store = Some(store); - self - } - // ============================================================ // LLM Configuration // ============================================================ @@ -281,45 +163,6 @@ impl EngineBuilder { // Retrieval Configuration // ============================================================ - /// Set the number of results to return from queries. - /// - /// Default is 5. Higher values return more context but cost more tokens. - #[must_use] - pub fn with_top_k(mut self, k: usize) -> Self { - self.top_k = Some(k); - self - } - - // ============================================================ - // Preset Configurations - // ============================================================ - - /// Enable fast mode for quicker but less thorough retrieval. - /// - /// Fast mode uses: - /// - Keyword-based retrieval (no LLM calls) - /// - Lower beam width / MCTS simulations - /// - Lazy summary generation - #[must_use] - pub fn fast(mut self) -> Self { - self.fast_mode = true; - self.precise_mode = false; - self - } - - /// Enable precise mode for higher quality retrieval. - /// - /// Precise mode uses: - /// - MCTS-based retrieval - /// - Higher simulation count - /// - Full summary generation - #[must_use] - pub fn precise(mut self) -> Self { - self.precise_mode = true; - self.fast_mode = false; - self - } - /// Build the Engine client. /// /// `api_key` and `model` must be provided via builder methods or config file. @@ -347,22 +190,8 @@ impl EngineBuilder { /// # } /// ``` pub async fn build(self) -> Result { - // Load or create configuration - let mut config = if let Some(config) = self.config { - config - } else if let Some(path) = self.config_path { - ConfigLoader::new() - .file(&path) - .load() - .map_err(|e| BuildError::Config(e.to_string()))? - } else { - Config::default() - }; - - // Apply builder overrides to retrieval config - if let Some(retrieval_config) = self.retrieval_config { - config.retrieval = retrieval_config; - } + // Load user-provided or default configuration + let mut config = self.config.unwrap_or_default(); // Apply individual overrides to LlmPoolConfig (primary) + legacy config (compat) if let Some(api_key) = self.api_key { @@ -392,18 +221,6 @@ impl EngineBuilder { config.retrieval.endpoint = endpoint.clone(); config.summary.endpoint = endpoint; } - if let Some(top_k) = self.top_k { - config.retrieval.top_k = top_k; - } - - // Apply preset modes - if self.fast_mode { - config.retrieval.search.max_iterations = 5; - } - if self.precise_mode { - config.retrieval.search.max_iterations = 100; - } - // Validate required settings let resolved_key = config .llm @@ -423,6 +240,9 @@ impl EngineBuilder { if retrieval_model.is_empty() { return Err(BuildError::MissingModel); } + if config.llm.endpoint.is_none() { + return Err(BuildError::MissingEndpoint); + } // Open workspace from config let workspace = Workspace::new(&config.storage.workspace_dir) @@ -456,15 +276,6 @@ impl EngineBuilder { retriever.with_content_config(retrieval_config.content.to_aggregator_config()); } - // Add memo store if provided or create default - if let Some(memo_store) = self.memo_store { - retriever = retriever.with_memo_store(memo_store); - } else { - // Create default memo store with model from config - let memo_store = MemoStore::new().with_model(retrieval_model).with_version(1); - retriever = retriever.with_memo_store(memo_store); - } - // Build engine let events = self.events.unwrap_or_default(); Engine::with_components(config, workspace, retriever, indexer, events) @@ -482,10 +293,6 @@ impl Default for EngineBuilder { /// Error during client build. #[derive(Debug, thiserror::Error)] pub enum BuildError { - /// Configuration error. - #[error("Configuration error: {0}")] - Config(String), - /// Workspace error. #[error("Workspace error: {0}")] Workspace(String), @@ -498,6 +305,12 @@ pub enum BuildError { #[error("Missing model: call .with_model(\"gpt-4o\") or set model in config file")] MissingModel, + /// Missing endpoint URL. + #[error( + "Missing endpoint: call .with_endpoint(\"https://api.xxx.com/v1\") or set endpoint in config" + )] + MissingEndpoint, + /// Other error. #[error("{0}")] Other(String), @@ -507,13 +320,6 @@ pub enum BuildError { mod tests { use super::*; - #[test] - fn test_builder_defaults() { - let builder = EngineBuilder::new(); - assert!(!builder.fast_mode); - assert!(!builder.precise_mode); - } - #[test] fn test_builder_with_key() { let builder = EngineBuilder::new().with_key("sk-test-key"); @@ -537,27 +343,4 @@ mod tests { assert_eq!(builder.model, Some("gpt-4o-mini".to_string())); assert_eq!(builder.api_key, Some("sk-test".to_string())); } - - #[test] - fn test_builder_fast_mode() { - let builder = EngineBuilder::new().fast(); - - assert!(builder.fast_mode); - assert!(!builder.precise_mode); - } - - #[test] - fn test_builder_precise_mode() { - let builder = EngineBuilder::new().precise(); - - assert!(builder.precise_mode); - assert!(!builder.fast_mode); - } - - #[test] - fn test_builder_top_k() { - let builder = EngineBuilder::new().with_top_k(10); - - assert_eq!(builder.top_k, Some(10)); - } } diff --git a/rust/src/client/engine.rs b/rust/src/client/engine.rs index 5c909c36..6cc4e207 100644 --- a/rust/src/client/engine.rs +++ b/rust/src/client/engine.rs @@ -19,6 +19,7 @@ //! let engine = EngineBuilder::new() //! .with_key("sk-...") //! .with_model("gpt-4o") +//! .with_endpoint("https://api.openai.com/v1") //! .build() //! .await?; //! @@ -36,28 +37,33 @@ //! # } //! ``` -use std::collections::HashMap; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use futures::StreamExt; use tracing::info; -use crate::config::Config; -use crate::error::Result; -use crate::index::PipelineOptions; -use crate::index::incremental::{self, IndexAction}; -use crate::metrics::MetricsHub; -use crate::retrieval::{PipelineRetriever, RetrieveEventReceiver}; -use crate::storage::{PersistedDocument, Workspace}; -use crate::{DocumentTree, Error}; - -use super::index_context::{IndexContext, IndexSource}; -use super::indexer::IndexerClient; -use super::query_context::{QueryContext, QueryScope}; -use super::retriever::RetrieverClient; -use super::types::{DocumentInfo, FailedItem, IndexItem, IndexMode, IndexResult, QueryResult}; -use super::workspace::WorkspaceClient; -use crate::events::EventEmitter; +use crate::{ + DocumentTree, Error, + config::Config, + error::Result, + events::EventEmitter, + index::{ + PipelineOptions, + incremental::{self, IndexAction}, + }, + metrics::MetricsHub, + retrieval::{PipelineRetriever, RetrieveEventReceiver}, + storage::{PersistedDocument, Workspace}, +}; + +use super::{ + index_context::{IndexContext, IndexSource}, + indexer::IndexerClient, + query_context::{QueryContext, QueryScope}, + retriever::RetrieverClient, + types::{DocumentInfo, FailedItem, IndexItem, IndexMode, IndexResult, QueryResult}, + workspace::WorkspaceClient, +}; /// The main Engine client. /// diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index 0a347826..f6d26927 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -11,9 +11,9 @@ mod merge; mod types; mod validator; -pub(crate) use loader::ConfigLoader; +pub use types::Config; pub(crate) use types::{ - CacheConfig, CompressionAlgorithm, ConcurrencyConfig, Config, FallbackBehavior, FallbackConfig, + CacheConfig, CompressionAlgorithm, ConcurrencyConfig, FallbackBehavior, FallbackConfig, IndexerConfig, LlmClientConfig, LlmConfig, LlmMetricsConfig, LlmPoolConfig, MetricsConfig, OnAllFailedBehavior, PilotMetricsConfig, RetrievalConfig, RetrievalMetricsConfig, SufficiencyConfig, SummaryConfig, diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 7541d900..26dcceae 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -4,11 +4,9 @@ //! # Vectorless //! -//! An ultra-performant reasoning-native document intelligence engine for AI. -//! -//! It transforms documents into rich semantic trees and uses LLMs to -//! intelligently traverse the hierarchy — retrieving the most relevant content -//! through structural reasoning and deep contextual understanding. +//! A document engine for AI. It transforms documents into hierarchical semantic +//! trees and uses the LLM itself to navigate and retrieve — purely LLM-guided, +//! from indexing to querying. No vector databases, no embeddings, no similarity search. //! //! ## Quick Start //! @@ -36,7 +34,8 @@ //! ``` pub mod client; -mod config; +pub mod config; +pub use config::Config; pub mod document; pub mod error; pub mod events;