Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 46 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# vaultquery

Query Obsidian vault files by YAML frontmatter using a DQL-like query language. Indexes `.md` files into SQLite and outputs results as JSON.
Query Obsidian vault files by YAML frontmatter using a DQL-like query language. Indexes `.md` files into SQLite and outputs results as JSON or [TOON](https://toon-format.org).

## Install

Expand Down Expand Up @@ -40,7 +40,7 @@ vaultquery status
vaultquery reindex --vault ~/my-vault
```

Every `query` command automatically updates the index before executing (incremental, mtime+size based).
Every `query` command automatically syncs the index before executing (incremental, mtime+size based). Pass `--index-only` to skip syncing and use the existing index as-is.

## Vault-local storage

Expand Down Expand Up @@ -113,7 +113,7 @@ WHERE (type = 'Server' OR type = 'Cluster') AND status = 'active'

## Output

All output is JSON:
Default output is JSON:

```json
{
Expand All @@ -130,6 +130,35 @@ All output is JSON:
}
```

### TOON format

Pass `--format toon` (or set `format: toon` in `.vaultquery/config.yaml`) for [TOON](https://toon-format.org) output:

```
mode TABLE
fields [
customer
kubectl_context
]
results [
{
path "Clients/Acme Corp/Production/CLUSTER.md"
title "Acme Production Cluster"
customer "Acme Corp"
kubectl_context "acme-prod"
}
]
```

### JSON vs TOON

| | JSON | TOON |
|---|---|---|
| **Ecosystem** | Universal, supported everywhere | Newer, lightweight |
| **Readability** | Verbose (colons, commas, quoting) | Minimal syntax, easy to scan |
| **Tooling** | `jq`, every language | Growing, Go library available |
| **Best for** | Pipelines, API integration | Human review, config files |

## Frontmatter

vaultquery indexes YAML frontmatter from `.md` files:
Expand All @@ -155,6 +184,8 @@ tags:
|------|---------|-------------|
| `--vault` | Current directory | Vault root path |
| `-v, --verbose` | false | Show detailed progress during indexing |
| `--format` | `json` | Output format: `json` or `toon` (query only, overrides config) |
| `--index-only` | false | Skip index sync, use existing index as-is (query only) |

## Development

Expand All @@ -169,6 +200,18 @@ go build ./cmd/vaultquery
goreleaser build --snapshot --clean
```

## Comparison with Obsidian MCP Server

| | vaultquery | Obsidian MCP Server |
|---|---|---|
| **Requires Obsidian** | No — headless, works on any `.md` vault | Yes — needs a running Obsidian instance |
| **Structured output** | JSON and TOON with typed frontmatter fields | Unstructured text |
| **Scriptable** | CLI tool, pipes into `jq`, shell scripts, CI | Designed for LLM tool-use via MCP |
| **Query language** | DQL (Dataview-compatible subset) | Natural language via LLM |
| **Index** | SQLite, incremental mtime+size sync | Obsidian's internal index |

vaultquery is designed for automation, scripting, and headless environments where Obsidian is not installed or running.

## Acknowledgements

vaultquery is heavily inspired by the [Dataview](https://github.com/blacksmithgu/obsidian-dataview) plugin for Obsidian by Michael Brenan. Dataview's query language (DQL) and its approach to treating frontmatter as queryable data were the foundation for this tool. Thank you for creating such a brilliant plugin.
Expand Down
64 changes: 48 additions & 16 deletions cmd/vaultquery/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func TestE2E_IndexCommand(t *testing.T) {
func TestE2E_QueryCommand(t *testing.T) {
vault := createTestVault(t)

// Index first, then query (query no longer auto-syncs)
// Index first (redundant since query auto-syncs by default, but harmless)
if _, err := runVaultquery(t, vault, "index"); err != nil {
t.Fatalf("index failed: %v", err)
}
Expand Down Expand Up @@ -272,13 +272,27 @@ func TestE2E_MissingQueryArg(t *testing.T) {
}
}

func TestE2E_QuerySyncFlag(t *testing.T) {
func TestE2E_QueryIndexOnlyFlag(t *testing.T) {
vault := createTestVault(t)

// --sync should index and query in one step (no prior index needed)
out, err := runVaultquery(t, vault, "query", "--sync", `LIST WHERE type = 'Kubernetes Cluster'`)
// Index first
if _, err := runVaultquery(t, vault, "index"); err != nil {
t.Fatalf("index failed: %v", err)
}

// Add a new file after indexing
newFile := filepath.Join(vault, "Sales/NewLead/LEAD.md")
if err := os.MkdirAll(filepath.Dir(newFile), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(newFile, []byte("---\ntype: Lead\nstatus: new\n---\n# New Lead\n"), 0o644); err != nil {
t.Fatal(err)
}

// --index-only should NOT see the new file
out, err := runVaultquery(t, vault, "query", "--index-only", `LIST WHERE type = 'Lead'`)
if err != nil {
t.Fatalf("query --sync failed: %v\n%s", err, out)
t.Fatalf("query --index-only failed: %v\n%s", err, out)
}

var result struct {
Expand All @@ -287,15 +301,15 @@ func TestE2E_QuerySyncFlag(t *testing.T) {
if err := json.Unmarshal(out, &result); err != nil {
t.Fatalf("invalid JSON: %v\n%s", err, out)
}
if len(result.Results) != 2 {
t.Errorf("expected 2 results, got %d", len(result.Results))
if len(result.Results) != 1 {
t.Errorf("expected 1 result (no sync with --index-only), got %d", len(result.Results))
}
}

func TestE2E_QueryAutoSyncsFirstRun(t *testing.T) {
vault := createTestVault(t)

// No prior index — query should auto-sync on first run (DB doesn't exist)
// No prior index — query always syncs by default
out, err := runVaultquery(t, vault, "query", `LIST WHERE type = 'Kubernetes Cluster'`)
if err != nil {
t.Fatalf("query (first run) failed: %v\n%s", err, out)
Expand All @@ -312,7 +326,7 @@ func TestE2E_QueryAutoSyncsFirstRun(t *testing.T) {
}
}

func TestE2E_QuerySkipsSyncOnExistingIndex(t *testing.T) {
func TestE2E_QuerySyncsByDefault(t *testing.T) {
vault := createTestVault(t)

// Index first
Expand All @@ -329,7 +343,7 @@ func TestE2E_QuerySkipsSyncOnExistingIndex(t *testing.T) {
t.Fatal(err)
}

// Query without --sync should NOT see the new file
// Query without flags SHOULD see the new file (default syncs)
out, err := runVaultquery(t, vault, "query", `LIST WHERE type = 'Lead'`)
if err != nil {
t.Fatalf("query failed: %v\n%s", err, out)
Expand All @@ -341,21 +355,39 @@ func TestE2E_QuerySkipsSyncOnExistingIndex(t *testing.T) {
if err := json.Unmarshal(out, &result); err != nil {
t.Fatalf("invalid JSON: %v\n%s", err, out)
}
if len(result.Results) != 1 {
t.Errorf("expected 1 result (no sync), got %d", len(result.Results))
if len(result.Results) != 2 {
t.Errorf("expected 2 results (default syncs), got %d", len(result.Results))
}

// Query with --index-only should NOT see newly added files
out, err = runVaultquery(t, vault, "query", "--index-only", `LIST WHERE type = 'Lead'`)
if err != nil {
t.Fatalf("query --index-only failed: %v\n%s", err, out)
}

if err := json.Unmarshal(out, &result); err != nil {
t.Fatalf("invalid JSON: %v\n%s", err, out)
}
// Still 2 because the default query above already synced; to test --index-only properly,
// we need to add another file after the last sync
newFile2 := filepath.Join(vault, "Sales/AnotherLead/LEAD.md")
if err := os.MkdirAll(filepath.Dir(newFile2), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(newFile2, []byte("---\ntype: Lead\nstatus: draft\n---\n# Another Lead\n"), 0o644); err != nil {
t.Fatal(err)
}

// Query with --sync SHOULD see the new file
out, err = runVaultquery(t, vault, "query", "--sync", `LIST WHERE type = 'Lead'`)
out, err = runVaultquery(t, vault, "query", "--index-only", `LIST WHERE type = 'Lead'`)
if err != nil {
t.Fatalf("query --sync failed: %v\n%s", err, out)
t.Fatalf("query --index-only failed: %v\n%s", err, out)
}

if err := json.Unmarshal(out, &result); err != nil {
t.Fatalf("invalid JSON: %v\n%s", err, out)
}
if len(result.Results) != 2 {
t.Errorf("expected 2 results (after sync), got %d", len(result.Results))
t.Errorf("expected 2 results (--index-only skips sync), got %d", len(result.Results))
}
}

Expand Down
2 changes: 1 addition & 1 deletion internal/cli/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func newQueryCmd() *cobra.Command {
Args: cobra.ExactArgs(1),
RunE: runQuery,
}
cmd.Flags().Bool("sync", false, "sync the index before querying")
cmd.Flags().Bool("index-only", false, "skip sync and use the existing index as-is")
cmd.Flags().String("format", "", "output format: json or toon (default: from config or json)")
return cmd
}
Expand Down
8 changes: 4 additions & 4 deletions internal/cli/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,13 +137,13 @@ func runQuery(cmd *cobra.Command, args []string) error {
return err
}

syncFlag, _ := cmd.Flags().GetBool("sync")
indexOnly, _ := cmd.Flags().GetBool("index-only")

var store *index.Store
if syncFlag {
store, err = ensureIndex(cmd)
} else {
if indexOnly {
store, err = openIndex(cmd)
} else {
store, err = ensureIndex(cmd)
}
if err != nil {
return err
Expand Down