Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
007fb77
feat: Add Docker CI/CD for multi-arch builds
EconoBen Feb 9, 2026
00a8357
docs: Add Docker deployment guide with OAuth headless and NAS setup
EconoBen Feb 9, 2026
847b462
fix: Address roborev review findings
EconoBen Feb 9, 2026
e76324f
fix: Address remaining roborev findings
EconoBen Feb 10, 2026
b212472
fix: Pin Docker base images by digest for reproducibility
EconoBen Feb 10, 2026
99c3b58
feat(api): Add token upload endpoint for headless OAuth
EconoBen Feb 10, 2026
bef1d3f
test: add tests for token upload handler and path sanitization
EconoBen Feb 10, 2026
fba8335
docs: add token export workflow to docker.md
EconoBen Feb 10, 2026
becd432
feat: friction reduction for NAS/Docker deployment
EconoBen Feb 11, 2026
11027c0
feat: enhance setup wizard with full NAS deployment automation
EconoBen Feb 11, 2026
b0c6c5a
feat: add port configuration to setup wizard
EconoBen Feb 11, 2026
c1195be
fix: allow serve to start without scheduled accounts
EconoBen Feb 11, 2026
a21d254
feat: auto-add account to remote config on token export
EconoBen Feb 11, 2026
28061fd
refactor: simplify setup wizard OAuth prompt
EconoBen Feb 11, 2026
bb82da9
fix: address PR review security and build issues
EconoBen Feb 12, 2026
986e189
feat: add transparent remote CLI access for NAS deployment
EconoBen Feb 12, 2026
f4925f0
fix(api): use robust datetime parsing for all message queries
EconoBen Feb 12, 2026
63f3267
fix: email validation, allow_insecure persistence, pagination panic
wesm Feb 13, 2026
37861fc
test: add coverage for remote store, setup bundle, and config Save
wesm Feb 13, 2026
2ebb60a
refactor: extract tokenExporter for testable export-token logic
wesm Feb 13, 2026
337d33a
fix(docs): correct Docker guide OAuth flow and sync commands
wesm Feb 13, 2026
3a0e01f
fix: honor config allow_insecure in export-token
wesm Feb 13, 2026
0906b8c
docs: replace reverse proxy section with Tailscale guidance
wesm Feb 13, 2026
ea9160e
docs: remove docker.md, consolidated into msgvault-docs site
wesm Feb 13, 2026
624fb9e
fix: remove duplicate scanMessageRowsFTS and validate remote URL host
wesm Feb 13, 2026
d26aa4f
fix(api): protect config access with mutex, register accounts with sc…
wesm Feb 13, 2026
4ce449d
fix: remove misleading cors_max_age default comment
wesm Feb 13, 2026
378a264
fix: skip Unix permission tests on Windows
wesm Feb 13, 2026
1aa9ec9
fix: create sensitive files with secure mode atomically
wesm Feb 13, 2026
82f03d8
docs: add inline rationale for accepted design decisions
wesm Feb 13, 2026
1623853
fix: copy existing client_secret.json into NAS bundle
wesm Feb 13, 2026
b2c076b
test: cover existing-secrets fallback in NAS bundle creation
wesm Feb 13, 2026
7a01b32
fix(api): rollback in-memory config on save failure
wesm Feb 13, 2026
2ea0050
test: cover save-failure rollback and fix misleading test name
wesm Feb 13, 2026
933c4bb
feat(api): add aggregate endpoints for TUI remote support
EconoBen Feb 16, 2026
368d110
feat(remote): add remote query engine for TUI support
EconoBen Feb 16, 2026
3ab27fa
feat(tui): add remote mode support
EconoBen Feb 16, 2026
c6f2414
docs: update documentation for remote TUI support and command accuracy
EconoBen Feb 16, 2026
3b6d06c
docs: update query package DESIGN.md to reflect current implementation
EconoBen Feb 16, 2026
56a2d4f
fix: address roborev findings for API handlers and remote engine
EconoBen Feb 16, 2026
65527eb
test: add test for invalid view_type in fast search endpoint
EconoBen Feb 16, 2026
0e12e62
Merge upstream/main into feat/tui-remote-support
EconoBen Feb 16, 2026
001f757
fix(lint): simplify TextTerms append in remote engine
EconoBen Feb 16, 2026
3cd30ee
chore: update roborev guidelines for remote engine design decisions
EconoBen Feb 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .roborev.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ HTTP remote defaults, plaintext key display in interactive CLI,
enabled=true override on account creation, and page-aligned pagination
are documented design decisions — see code comments at each site.

Remote engine query string reconstruction in buildSearchQueryString is
intentionally simplified — phrase quoting edge cases are acceptable since
the search parser on the server re-parses the query. Empty search queries
sending q= is expected; the server returns empty results gracefully.
TimeGranularity defaults to "month" when unspecified, which is correct.

This is a single-user personal tool with no privilege separation, no
setuid, no shared directories, and no multi-tenant access. Do not flag
symlink-following, local file overwrites, or similar CWE patterns that
Expand Down
4 changes: 4 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,14 @@ make lint # Run linter
# TUI and analytics
./msgvault tui # Launch TUI
./msgvault tui --account you@gmail.com # Filter by account
./msgvault tui --local # Force local (override remote config)
./msgvault build-cache # Build Parquet cache
./msgvault build-cache --full-rebuild # Full rebuild
./msgvault stats # Show archive stats

# Daemon mode (NAS/server deployment)
./msgvault serve # Start HTTP API + scheduled syncs

# Maintenance
./msgvault repair-encoding # Fix UTF-8 encoding issues
```
Expand Down
31 changes: 30 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,18 @@ msgvault tui
| `add-account EMAIL` | Authorize a Gmail account (use `--headless` for servers) |
| `sync-full EMAIL` | Full sync (`--limit N`, `--after`/`--before` for date ranges) |
| `sync EMAIL` | Sync only new/changed messages |
| `tui` | Launch the interactive TUI (`--account` to filter) |
| `tui` | Launch the interactive TUI (`--account` to filter, `--local` to force local) |
| `search QUERY` | Search messages (`--json` for machine output) |
| `show-message ID` | View full message details (`--json` for machine output) |
| `mcp` | Start the MCP server for AI assistant integration |
| `serve` | Run daemon with scheduled sync and HTTP API for remote TUI |
| `stats` | Show archive statistics |
| `list-accounts` | List synced email accounts |
| `verify EMAIL` | Verify archive integrity against Gmail |
| `export-eml` | Export a message as `.eml` |
| `build-cache` | Rebuild the Parquet analytics cache |
| `update` | Update msgvault to the latest version |
| `setup` | Interactive first-run configuration wizard |
| `repair-encoding` | Fix UTF-8 encoding issues |
| `list-senders` / `list-domains` / `list-labels` | Explore metadata |

Expand All @@ -111,6 +116,30 @@ See the [Configuration Guide](https://msgvault.io/configuration/) for all option

msgvault includes an MCP server that lets AI assistants search, analyze, and read your archived messages. Connect it to Claude Desktop or any MCP-capable agent and query your full message history conversationally. See the [MCP documentation](https://msgvault.io/usage/chat/) for setup instructions.

## Daemon Mode (NAS/Server)

Run msgvault as a long-running daemon for scheduled syncs and remote access:

```bash
msgvault serve
```

Configure scheduled syncs in `config.toml`:

```toml
[[accounts]]
email = "you@gmail.com"
schedule = "0 2 * * *" # 2am daily (cron)
enabled = true

[server]
api_port = 8080
bind_addr = "0.0.0.0"
api_key = "your-secret-key"
```

The TUI can connect to a remote server by configuring `[remote].url`. Use `--local` to force local database when remote is configured. See [docs/api.md](docs/api.md) for the HTTP API reference.

## Documentation

- [Setup Guide](https://msgvault.io/guides/oauth-setup/): OAuth, first sync, headless servers
Expand Down
21 changes: 19 additions & 2 deletions cmd/msgvault/cmd/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ Fetches only changes since the last sync using the Gmail History API. Much
faster than a full sync. Requires a prior full sync.

```bash
msgvault sync-incremental user@gmail.com
msgvault sync user@gmail.com
```

If Gmail's history has expired (~7 days), it will suggest running a full sync.
Expand Down Expand Up @@ -260,8 +260,25 @@ msgvault tui

# Filter by account
msgvault tui --account user@gmail.com

# Force local database (override remote config)
msgvault tui --local
```

### Remote mode

When `[remote].url` is configured in `config.toml`, the TUI connects to a remote
msgvault server instead of the local database. This is useful for accessing an
archive on a NAS or server from another machine.

```toml
[remote]
url = "http://nas.local:8080"
api_key = "your-api-key"
```

In remote mode, deletion staging and attachment export are disabled for safety.

### TUI keybindings

| Key | Action |
Expand Down Expand Up @@ -291,7 +308,7 @@ msgvault tui --account user@gmail.com
2. **Search**: `msgvault search <query> --json` — find relevant messages.
3. **Read details**: `msgvault show-message <id> --json` — get full message content.
4. **Analyze**: `list-senders`, `list-domains`, `list-labels` with `--json` for patterns.
5. **Sync new mail**: `msgvault sync-incremental user@gmail.com` if archive is stale.
5. **Sync new mail**: `msgvault sync user@gmail.com` if archive is stale.

## Tips

Expand Down
20 changes: 19 additions & 1 deletion cmd/msgvault/cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/wesm/msgvault/internal/api"
"github.com/wesm/msgvault/internal/gmail"
"github.com/wesm/msgvault/internal/oauth"
"github.com/wesm/msgvault/internal/query"
"github.com/wesm/msgvault/internal/scheduler"
"github.com/wesm/msgvault/internal/store"
"github.com/wesm/msgvault/internal/sync"
Expand Down Expand Up @@ -82,6 +83,17 @@ func runServe(cmd *cobra.Command, args []string) error {
return fmt.Errorf("init schema: %w", err)
}

// Create query engine for TUI aggregate support
analyticsDir := cfg.AnalyticsDir()
engine, err := query.NewDuckDBEngine(analyticsDir, dbPath, nil)
if err != nil {
logger.Warn("query engine not available - aggregate endpoints will return 503", "error", err)
// Continue without engine - basic endpoints still work
}
if engine != nil {
defer engine.Close()
}

// Create OAuth manager
oauthMgr, err := oauth.NewManager(cfg.OAuth.ClientSecrets, cfg.TokensDir(), logger)
if err != nil {
Expand Down Expand Up @@ -122,7 +134,13 @@ func runServe(cmd *cobra.Command, args []string) error {
schedAdapter := &schedulerAdapter{scheduler: sched}

// Create and start API server
apiServer := api.NewServer(cfg, storeAdapter, schedAdapter, logger)
apiServer := api.NewServerWithOptions(api.ServerOptions{
Config: cfg,
Store: storeAdapter,
Engine: engine,
Scheduler: schedAdapter,
Logger: logger,
})

// Start API server in goroutine
serverErr := make(chan error, 1)
Expand Down
144 changes: 87 additions & 57 deletions cmd/msgvault/cmd/tui.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ import (
tea "github.com/charmbracelet/bubbletea"
"github.com/spf13/cobra"
"github.com/wesm/msgvault/internal/query"
"github.com/wesm/msgvault/internal/remote"
"github.com/wesm/msgvault/internal/store"
"github.com/wesm/msgvault/internal/tui"
)

var forceSQL bool
var skipCacheBuild bool
var noSQLiteScanner bool
var forceLocalTUI bool

var tuiCmd = &cobra.Command{
Use: "tui",
Expand Down Expand Up @@ -48,76 +50,103 @@ Selection & Deletion:

Performance:
For large archives (100k+ messages), the TUI uses Parquet files for fast
aggregation queries. Run 'msgvault-sync build-parquet' to generate them.
Use --force-sql to bypass Parquet and query SQLite directly (slow).`,
aggregation queries. Run 'msgvault build-cache' to generate them.
Use --force-sql to bypass Parquet and query SQLite directly (slow).

Remote Mode:
When [remote].url is configured, the TUI connects to a remote msgvault server.
Use --local to force local database. Deletion and export are disabled in remote mode.`,
RunE: func(cmd *cobra.Command, args []string) error {
// Open database
dbPath := cfg.DatabaseDSN()
s, err := store.Open(dbPath)
if err != nil {
return fmt.Errorf("open database: %w", err)
}
defer s.Close()
var engine query.Engine
var isRemote bool

// Check for remote mode (unless --local flag is set)
if cfg.Remote.URL != "" && !forceLocalTUI {
// Remote mode - connect to remote msgvault server
remoteCfg := remote.Config{
URL: cfg.Remote.URL,
APIKey: cfg.Remote.APIKey,
AllowInsecure: cfg.Remote.AllowInsecure,
}
remoteEngine, err := remote.NewEngine(remoteCfg)
if err != nil {
return fmt.Errorf("connect to remote: %w", err)
}
defer remoteEngine.Close()
engine = remoteEngine
isRemote = true
fmt.Printf("Connected to remote: %s\n", cfg.Remote.URL)
} else {
// Local mode - use local database
dbPath := cfg.DatabaseDSN()
s, err := store.Open(dbPath)
if err != nil {
return fmt.Errorf("open database: %w", err)
}
defer s.Close()

// Ensure schema is up to date
if err := s.InitSchema(); err != nil {
return fmt.Errorf("init schema: %w", err)
}
// Ensure schema is up to date
if err := s.InitSchema(); err != nil {
return fmt.Errorf("init schema: %w", err)
}

// Build FTS index in background — TUI uses DuckDB/Parquet for
// aggregates and only needs FTS for deep search (Tab to switch).
if s.NeedsFTSBackfill() {
go func() {
_, _ = s.BackfillFTS(nil)
}()
}
// Build FTS index in background — TUI uses DuckDB/Parquet for
// aggregates and only needs FTS for deep search (Tab to switch).
if s.NeedsFTSBackfill() {
go func() {
_, _ = s.BackfillFTS(nil)
}()
}

analyticsDir := cfg.AnalyticsDir()
analyticsDir := cfg.AnalyticsDir()

// Check if cache needs to be built/updated (unless forcing SQL or skipping)
if !forceSQL && !skipCacheBuild {
needsBuild, reason := cacheNeedsBuild(dbPath, analyticsDir)
if needsBuild {
fmt.Printf("Building analytics cache (%s)...\n", reason)
result, err := buildCache(dbPath, analyticsDir, true)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: Failed to build cache: %v\n", err)
fmt.Fprintf(os.Stderr, "Falling back to SQLite (may be slow for large archives)\n")
} else if !result.Skipped {
fmt.Printf("Cached %d messages for fast queries.\n", result.ExportedCount)
}
}
}

// Check if cache needs to be built/updated (unless forcing SQL or skipping)
if !forceSQL && !skipCacheBuild {
needsBuild, reason := cacheNeedsBuild(dbPath, analyticsDir)
if needsBuild {
fmt.Printf("Building analytics cache (%s)...\n", reason)
result, err := buildCache(dbPath, analyticsDir, true)
// Determine query engine to use
if !forceSQL && query.HasCompleteParquetData(analyticsDir) {
// Use DuckDB for fast Parquet queries
var duckOpts query.DuckDBOptions
if noSQLiteScanner {
duckOpts.DisableSQLiteScanner = true
}
duckEngine, err := query.NewDuckDBEngine(analyticsDir, dbPath, s.DB(), duckOpts)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: Failed to build cache: %v\n", err)
fmt.Fprintf(os.Stderr, "Warning: Failed to open Parquet engine: %v\n", err)
fmt.Fprintf(os.Stderr, "Falling back to SQLite (may be slow for large archives)\n")
} else if !result.Skipped {
fmt.Printf("Cached %d messages for fast queries.\n", result.ExportedCount)
engine = query.NewSQLiteEngine(s.DB())
} else {
engine = duckEngine
defer duckEngine.Close()
}
}
}

// Determine query engine to use
var engine query.Engine

if !forceSQL && query.HasCompleteParquetData(analyticsDir) {
// Use DuckDB for fast Parquet queries
var duckOpts query.DuckDBOptions
if noSQLiteScanner {
duckOpts.DisableSQLiteScanner = true
}
duckEngine, err := query.NewDuckDBEngine(analyticsDir, dbPath, s.DB(), duckOpts)
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: Failed to open Parquet engine: %v\n", err)
fmt.Fprintf(os.Stderr, "Falling back to SQLite (may be slow for large archives)\n")
engine = query.NewSQLiteEngine(s.DB())
} else {
engine = duckEngine
defer duckEngine.Close()
}
} else {
// Use SQLite directly
if !forceSQL {
fmt.Fprintf(os.Stderr, "Note: No cache data available, using SQLite (slow for large archives)\n")
fmt.Fprintf(os.Stderr, "Run 'msgvault build-cache' to enable fast queries.\n")
// Use SQLite directly
if !forceSQL {
fmt.Fprintf(os.Stderr, "Note: No cache data available, using SQLite (slow for large archives)\n")
fmt.Fprintf(os.Stderr, "Run 'msgvault build-cache' to enable fast queries.\n")
}
engine = query.NewSQLiteEngine(s.DB())
}
engine = query.NewSQLiteEngine(s.DB())
}

// Create and run TUI
model := tui.New(engine, tui.Options{DataDir: cfg.Data.DataDir, Version: Version})
model := tui.New(engine, tui.Options{
DataDir: cfg.Data.DataDir,
Version: Version,
IsRemote: isRemote,
})
p := tea.NewProgram(model, tea.WithAltScreen())

if _, err := p.Run(); err != nil {
Expand Down Expand Up @@ -204,5 +233,6 @@ func init() {
tuiCmd.Flags().BoolVar(&forceSQL, "force-sql", false, "Force SQLite queries instead of Parquet (slow for large archives)")
tuiCmd.Flags().BoolVar(&skipCacheBuild, "no-cache-build", false, "Skip automatic cache build/update")
tuiCmd.Flags().BoolVar(&noSQLiteScanner, "no-sqlite-scanner", false, "Disable DuckDB sqlite_scanner extension (use direct SQLite fallback)")
tuiCmd.Flags().BoolVar(&forceLocalTUI, "local", false, "Force local database (override remote config)")
_ = tuiCmd.Flags().MarkHidden("no-sqlite-scanner")
}
Loading