Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
dialoguer = "0.12"
console = "0.15"
ctrlc = "3"
tracing-appender = "0.2.4"
log = "0.4.29"

[dev-dependencies]
tempfile = "3"
Expand Down
10 changes: 10 additions & 0 deletions src/cli/commands.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ pub struct Cli {
/// Server port to use
#[arg(short = 'p', long, global = true, env = "VGREP_PORT")]
port: Option<u16>,

/// Log level (error, warn, info, debug, trace)
#[arg(long, global = true, env = "VGREP_LOG_LEVEL")]
log_level: Option<String>,

/// Log file path
#[arg(long, global = true, env = "VGREP_LOG_FILE")]
log_file: Option<PathBuf>,
}

#[derive(Subcommand)]
Expand Down Expand Up @@ -242,6 +250,8 @@ enum ConfigKey {

impl Cli {
pub fn run(self) -> Result<()> {
crate::logging::init_logging(self.log_level.clone(), self.log_file.clone());

let mut config = Config::load()?;

// Apply global overrides
Expand Down
14 changes: 14 additions & 0 deletions src/core/indexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use indicatif::{ProgressBar, ProgressStyle};
use sha2::{Digest, Sha256};
use std::fs;
use std::path::{Path, PathBuf};
use tracing::{info, warn};

use super::db::Database;
use super::embeddings::EmbeddingEngine;
Expand Down Expand Up @@ -45,6 +46,7 @@ impl Indexer {
pub fn index_directory(&self, path: &Path, force: bool) -> Result<()> {
let abs_path = fs::canonicalize(path).context("Failed to resolve path")?;

info!("Scanning directory: {}", abs_path.display());
println!(
" {}Scanning: {}",
ui::FOLDER,
Expand All @@ -64,14 +66,17 @@ impl Indexer {
.collect();

if files.is_empty() {
warn!("No files to index in {}", abs_path.display());
println!(" {}No files to index.", ui::WARN);
return Ok(());
}

info!("Found {} files to index", files.len());
println!(" {}Found {} files", ui::FILE, style(files.len()).cyan());
println!();

// Phase 1: Collect all chunks from all files
info!("Phase 1: Reading files");
println!(" {}Phase 1: Reading files...", style("→").dim());

let pb = ProgressBar::new(files.len() as u64);
Expand Down Expand Up @@ -108,6 +113,7 @@ impl Indexer {
}

pb.finish_and_clear();
info!("Read {} files, {} chunks", pending_files.len(), total_chunks);
println!(
" {}Read {} files, {} chunks",
ui::CHECK,
Expand All @@ -117,11 +123,13 @@ impl Indexer {

if pending_files.is_empty() {
println!();
info!("All files up to date ({} skipped)", skipped);
println!(" {}All files up to date ({} skipped)", ui::CHECK, skipped);
return Ok(());
}

// Phase 2: Generate embeddings for all chunks at once
info!("Phase 2: Generating embeddings");
println!(" {}Phase 2: Generating embeddings...", style("→").dim());

let all_chunks: Vec<&str> = pending_files
Expand All @@ -139,13 +147,15 @@ impl Indexer {
let all_embeddings = self.engine.embed_batch(&all_chunks)?;

pb.finish_and_clear();
info!("Generated {} embeddings", all_embeddings.len());
println!(
" {}Generated {} embeddings",
ui::CHECK,
all_embeddings.len()
);

// Phase 3: Store in database
info!("Phase 3: Storing in database");
println!(" {}Phase 3: Storing in database...", style("→").dim());

let pb = ProgressBar::new(pending_files.len() as u64);
Expand Down Expand Up @@ -181,9 +191,11 @@ impl Indexer {
}

pb.finish_and_clear();
info!("Stored {} files", indexed);
println!(" {}Stored {} files", ui::CHECK, indexed);

println!();
info!("Indexing complete. Files: {}, Skipped: {}, Chunks: {}", indexed, skipped, total_chunks);
println!(" {}Indexing complete!", ui::SPARKLES);
println!(
" {} {} indexed, {} skipped",
Expand Down Expand Up @@ -558,9 +570,11 @@ impl ServerIndexer {
}

pb.finish_and_clear();
info!("Stored {} files", indexed);
println!(" {}Stored {} files", ui::CHECK, indexed);

println!();
info!("Indexing complete. Files: {}, Skipped: {}, Chunks: {}", indexed, skipped, total_chunks);
println!(" {}Indexing complete!", ui::SPARKLES);
println!(
" {} {} indexed, {} skipped",
Expand Down
1 change: 1 addition & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ pub mod core;
pub mod server;
pub mod ui;
pub mod watcher;
pub mod logging;

pub use config::Config;
pub use core::{Database, EmbeddingEngine, Indexer, SearchEngine, ServerIndexer};
Expand Down
44 changes: 44 additions & 0 deletions src/logging.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
use std::path::PathBuf;
use std::str::FromStr;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt::format::FmtSpan;
use tracing_subscriber::{fmt, EnvFilter, Layer};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;

pub fn init_logging(log_level: Option<String>, log_file: Option<PathBuf>) {
let env_filter = EnvFilter::try_from_default_env()
.or_else(|_| EnvFilter::try_new(log_level.unwrap_or_else(|| "info".to_string())))
.unwrap_or_else(|_| EnvFilter::new("info"));

let stdout_layer = fmt::layer()
.with_target(false)
.with_thread_ids(false)
.with_file(false)
.with_line_number(false)
.with_level(false)
.compact()
.with_filter(env_filter.clone());

let registry = tracing_subscriber::registry();

if let Some(path) = log_file {
let file_appender = tracing_appender::rolling::never(
path.parent().unwrap_or(&PathBuf::from(".")),
path.file_name().unwrap_or(&std::ffi::OsString::from("vgrep.log")),
);
let file_layer = fmt::layer()
.with_ansi(false)
.with_file(true)
.with_line_number(true)
.with_thread_ids(true)
.with_target(true)
.with_span_events(FmtSpan::CLOSE)
.with_writer(file_appender)
.with_filter(env_filter);

registry.with(stdout_layer).with(file_layer).init();
} else {
registry.with(stdout_layer).init();
}
}
9 changes: 9 additions & 0 deletions src/server/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use tower_http::cors::{Any, CorsLayer};
use tracing::{error, info};

use crate::config::Config;
use crate::core::{Database, EmbeddingEngine};
Expand Down Expand Up @@ -93,8 +94,10 @@ pub async fn run_server(config: &Config, host: &str, port: u16) -> Result<()> {

crate::ui::print_banner();

info!("Loading embedding model...");
println!(" {}Loading embedding model...", crate::ui::BRAIN);
let engine = EmbeddingEngine::new(&config)?;
info!("Model loaded successfully");
println!(" {}Model loaded successfully!", crate::ui::CHECK);
println!();

Expand Down Expand Up @@ -123,6 +126,7 @@ pub async fn run_server(config: &Config, host: &str, port: u16) -> Result<()> {
crate::ui::print_server_banner(host, port);

let listener = tokio::net::TcpListener::bind(addr).await?;
info!("Server listening on {}:{}", host, port);
axum::serve(listener, app).await?;

Ok(())
Expand All @@ -146,6 +150,7 @@ async fn status(State(state): State<SharedState>) -> impl IntoResponse {
let db = match Database::new(&state.config.db_path().unwrap_or_default()) {
Ok(db) => db,
Err(e) => {
error!("Failed to open database: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
Expand All @@ -159,6 +164,7 @@ async fn status(State(state): State<SharedState>) -> impl IntoResponse {
let stats = match db.get_stats() {
Ok(stats) => stats,
Err(e) => {
error!("Failed to get stats: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
Expand Down Expand Up @@ -216,6 +222,7 @@ async fn search(
match engine.embed(&req.query) {
Ok(emb) => emb,
Err(e) => {
error!("Failed to generate embedding: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
Expand All @@ -231,6 +238,7 @@ async fn search(
let db = match Database::new(&state.config.db_path().unwrap_or_default()) {
Ok(db) => db,
Err(e) => {
error!("Failed to open database: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
Expand All @@ -244,6 +252,7 @@ async fn search(
let candidates = match db.search_similar(&query_embedding, &abs_path, req.max_results * 3) {
Ok(c) => c,
Err(e) => {
error!("Search failed: {}", e);
return (
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
Expand Down
7 changes: 7 additions & 0 deletions src/watcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::channel;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tracing::{info, warn};

use crate::config::Config;
use crate::core::{Database, Indexer, ServerIndexer};
Expand Down Expand Up @@ -65,6 +66,7 @@ impl FileWatcher {
.expect("Error setting Ctrl+C handler");

// Initial index
info!("Starting initial indexing");
println!(" {} Initial indexing...", style(">>").dim());
self.index_all()?;

Expand All @@ -85,6 +87,7 @@ impl FileWatcher {

watcher.watch(&abs_path, RecursiveMode::Recursive)?;

info!("Watching for changes in {}", abs_path.display());
println!(" {} Watching for changes...", style("[~]").cyan());
println!();

Expand Down Expand Up @@ -125,6 +128,7 @@ impl FileWatcher {
}
}

info!("Watcher stopped");
println!();
println!(" {} Watcher stopped", style("[x]").yellow());
println!();
Expand Down Expand Up @@ -302,6 +306,7 @@ impl FileWatcher {
// File was deleted
if let Some(entry) = db.get_file_by_path(path)? {
db.delete_file(entry.id)?;
info!("File removed: {}", filename);
println!(
" {} {} {}",
style("[-]").red(),
Expand Down Expand Up @@ -332,6 +337,7 @@ impl FileWatcher {
Mode::Server => {
let client = Client::new(&self.config.server_host, self.config.server_port);
self.index_file_server(&db, &client, path, &content)?;
info!("File indexed: {}", filename);
println!(
" {} {} {}",
style("[+]").green(),
Expand All @@ -340,6 +346,7 @@ impl FileWatcher {
);
}
Mode::Local => {
info!("File modified (pending): {}", filename);
println!(
" {} {} {} {}",
style("[~]").yellow(),
Expand Down
Loading