Smarter symbol-level reindexing, ReFRAG memory server alignment, git commit lineage, dev-remote perms + index speed improvements + worktree deup

.env.example

@@ -7,6 +7,12 @@ QDRANT_URL=http://localhost:6333
 # Multi-repo: Each subdirectory gets its own collection
 MULTI_REPO_MODE=0
 
+# Logical repo reuse (experimental): 0=disabled (default), 1=enable logical_repo_id-based
+# collection reuse across git worktrees / clones. When enabled, indexer, watcher, and
+# upload service will try to reuse a canonical collection per logical repository and
+# use (repo_id + repo_rel_path) for skip-unchanged across worktrees.
+#LOGICAL_REPO_REUSE=0
+
 # Single unified collection for seamless cross-repo search (default: "codebase")
 # Leave unset or use "codebase" for unified search across all your code
 COLLECTION_NAME=codebase
@@ -153,6 +159,8 @@ QDRANT_TIMEOUT=20
 MEMORY_AUTODETECT=1
 MEMORY_COLLECTION_TTL_SECS=300
 
+# Smarter re-indexing for symbol cache, reuse embeddings and reduce decoder/pseudo tags to re-index
+SMART_SYMBOL_REINDEXING=0
 
 # Watcher-safe defaults (recommended)
 # Applied to watcher via compose; uncomment to apply globally.
@@ -166,3 +174,14 @@ MEMORY_COLLECTION_TTL_SECS=300
 # INDEX_UPSERT_BACKOFF=0.5
 # Debounce file events to coalesce bursts
 # WATCH_DEBOUNCE_SECS=1.5
+
+# Remote upload git history (used by upload clients)
+# Max number of commits to include per bundle (0 disables git history)
+# REMOTE_UPLOAD_GIT_MAX_COMMITS=500
+# Optional git log since filter, e.g. '6 months ago' or '2024-01-01'
+# REMOTE_UPLOAD_GIT_SINCE=
+# Enable commit lineage goals for indexing
+REFRAG_COMMIT_DESCRIBE=1
+
+STRICT_MEMORY_RESTORE=0
+

.gitignore

@@ -17,7 +17,8 @@ __pycache__/
 paper.md
 /semantic-search
 /codebase-index-cli
-/.codebase
+/.codebase/*
+!/.codebase/.gitkeep
 scripts/.codebase/cache.json
 scripts/.codebase/state.json
 tests/.codebase/cache.json
@@ -29,4 +30,5 @@ tests/.codebase/state.json
 CLAUDE.md
 .qodo/.cursor/rules
 /.augment
-/dev-workspace
+/dev-workspace/*
+!/dev-workspace/.gitkeep

README.md

@@ -1,6 +1,6 @@
 [![CI](https://github.com/m1rl0k/Context-Engine/actions/workflows/ci.yml/badge.svg)](https://github.com/m1rl0k/Context-Engine/actions/workflows/ci.yml)
 
-**Documentation:** README · [Configuration](docs/CONFIGURATION.md) · [IDE Clients](docs/IDE_CLIENTS.md) · [MCP API](docs/MCP_API.md) · [ctx CLI](docs/CTX_CLI.md) · [Memory Guide](docs/MEMORY_GUIDE.md) · [Architecture](docs/ARCHITECTURE.md) · [Multi-Repo](docs/MULTI_REPO_COLLECTIONS.md) · [Kubernetes](deploy/kubernetes/README.md) · [VS Code Extension](docs/vscode-extension.md) · [Troubleshooting](docs/TROUBLESHOOTING.md) · [Development](docs/DEVELOPMENT.md)
+**Documentation:** [Getting Started](docs/GETTING_STARTED.md) · README · [Configuration](docs/CONFIGURATION.md) · [IDE Clients](docs/IDE_CLIENTS.md) · [MCP API](docs/MCP_API.md) · [ctx CLI](docs/CTX_CLI.md) · [Memory Guide](docs/MEMORY_GUIDE.md) · [Architecture](docs/ARCHITECTURE.md) · [Multi-Repo](docs/MULTI_REPO_COLLECTIONS.md) · [Kubernetes](deploy/kubernetes/README.md) · [VS Code Extension](docs/vscode-extension.md) · [Troubleshooting](docs/TROUBLESHOOTING.md) · [Development](docs/DEVELOPMENT.md)
 
 ---
 
@@ -38,13 +38,19 @@ Context-Engine is a plug-and-play MCP retrieval stack that unifies code indexing
 | OpenAI Codex | RMCP | TOML config |
 | Augment | SSE | Simple JSON configs |
 | AmpCode | SSE | Simple URL for SSE endpoints |
-| Claude Code CLI | SSE | Simple JSON configs |
+| Claude Code CLI | SSE / HTTP (RMCP) | Simple JSON configs via .mcp.json |
 
 > **See [docs/IDE_CLIENTS.md](docs/IDE_CLIENTS.md) for detailed configuration examples.**
 
 
 ## Getting Started
 
+If you're a VS Code user trying Context-Engine locally, start with the low-friction dev-remote + extension guide:
+
+- **[docs/GETTING_STARTED.md](docs/GETTING_STARTED.md)**
+
+The options below describe the `docker compose` + CLI workflows.
+
 ### Option 1: Deploy & Connect (Recommended)
 
 Deploy Context-Engine once, connect any IDE. No need to clone this repo into your project.

ctx-hook-simple.sh

@@ -105,16 +105,17 @@ fi
 
 # Read all settings from ctx_config.json
 if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then
-    CTX_COLLECTION=$(grep -o '"default_collection"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"default_collection"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' )
-    REFRAG_RUNTIME=$(grep -o '"refrag_runtime"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"refrag_runtime"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' || echo "glm")
-    GLM_API_KEY=$(grep -o '"glm_api_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"glm_api_key"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' )
-    GLM_API_BASE=$(grep -o '"glm_api_base"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"glm_api_base"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/')
-    GLM_MODEL=$(grep -o '"glm_model"[[:space:]]*:[[:space:]]*"[^\"]*"' "$CONFIG_FILE" | sed 's/.*"glm_model"[[:space:]]*:[[:space:]]*"\([^\"]*\)".*/\1/' || echo "glm-4.6")
-    CTX_DEFAULT_MODE=$(grep -o '"default_mode"[[:space:]]*:[[:space:]]*"[^\"]*"' "$CONFIG_FILE" | sed 's/.*"default_mode"[[:space:]]*:[[:space:]]*"\([^\"]*\)".*/\1/')
-    CTX_REQUIRE_CONTEXT=$(grep -o '"require_context"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"require_context"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/')
-    CTX_RELEVANCE_GATE=$(grep -o '"relevance_gate_enabled"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"relevance_gate_enabled"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/')
-    CTX_MIN_RELEVANCE=$(grep -o '"min_relevance"[[:space:]]*:[[:space:]]*[0-9.][0-9.]*' "$CONFIG_FILE" | sed 's/.*"min_relevance"[[:space:]]*:[[:space:]]*\([0-9.][0-9.]*\).*/\1/')
-    CTX_REWRITE_MAX_TOKENS=$(grep -o '"rewrite_max_tokens"[[:space:]]*:[[:space:]]*[0-9][0-9]*' "$CONFIG_FILE" | sed 's/.*"rewrite_max_tokens"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/')
+	CTX_COLLECTION=$(grep -o '"default_collection"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"default_collection"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' )
+	REFRAG_RUNTIME=$(grep -o '"refrag_runtime"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"refrag_runtime"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' || echo "glm")
+	GLM_API_KEY=$(grep -o '"glm_api_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"glm_api_key"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/' )
+	GLM_API_BASE=$(grep -o '"glm_api_base"[[:space:]]*:[[:space:]]*"[^"]*"' "$CONFIG_FILE" | sed 's/.*"glm_api_base"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/')
+	GLM_MODEL=$(grep -o '"glm_model"[[:space:]]*:[[:space:]]*"[^\"]*"' "$CONFIG_FILE" | sed 's/.*"glm_model"[[:space:]]*:[[:space:]]*"\([^\"]*\)".*/\1/' || echo "glm-4.6")
+	CTX_DEFAULT_MODE=$(grep -o '"default_mode"[[:space:]]*:[[:space:]]*"[^\"]*"' "$CONFIG_FILE" | sed 's/.*"default_mode"[[:space:]]*:[[:space:]]*"\([^\"]*\)".*/\1/')
+	CTX_REQUIRE_CONTEXT=$(grep -o '"require_context"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"require_context"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/')
+	CTX_RELEVANCE_GATE=$(grep -o '"relevance_gate_enabled"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"relevance_gate_enabled"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/')
+	CTX_MIN_RELEVANCE=$(grep -o '"min_relevance"[[:space:]]*:[[:space:]]*[0-9.][0-9.]*' "$CONFIG_FILE" | sed 's/.*"min_relevance"[[:space:]]*:[[:space:]]*\([0-9.][0-9.]*\).*/\1/')
+	CTX_REWRITE_MAX_TOKENS=$(grep -o '"rewrite_max_tokens"[[:space:]]*:[[:space:]]*[0-9][0-9]*' "$CONFIG_FILE" | sed 's/.*"rewrite_max_tokens"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/')
+	CTX_SURFACE_COLLECTION_CFG=$(grep -o '"surface_qdrant_collection_hint"[[:space:]]*:[[:space:]]*\(true\|false\)' "$CONFIG_FILE" | sed 's/.*"surface_qdrant_collection_hint"[[:space:]]*:[[:space:]]*\(true\|false\).*/\1/')
 fi
 
 # Set defaults if not found in config
@@ -129,6 +130,25 @@ CTX_RELEVANCE_GATE=${CTX_RELEVANCE_GATE:-false}
 CTX_MIN_RELEVANCE=${CTX_MIN_RELEVANCE:-0.1}
 CTX_REWRITE_MAX_TOKENS=${CTX_REWRITE_MAX_TOKENS:-320}
 
+# Normalize surface_qdrant_collection_hint from config (true/false) into 1/0
+CFG_HINT=""
+if [ -n "$CTX_SURFACE_COLLECTION_CFG" ]; then
+	if [ "$CTX_SURFACE_COLLECTION_CFG" = "true" ]; then
+		CFG_HINT="1"
+	elif [ "$CTX_SURFACE_COLLECTION_CFG" = "false" ]; then
+		CFG_HINT="0"
+	fi
+fi
+
+# Precedence: explicit env override > ctx_config flag > auto-on when collection known
+if [ -n "${CTX_SURFACE_COLLECTION_HINT:-}" ]; then
+	:
+elif [ -n "$CFG_HINT" ]; then
+	CTX_SURFACE_COLLECTION_HINT="$CFG_HINT"
+elif [ -n "$CTX_COLLECTION" ]; then
+	CTX_SURFACE_COLLECTION_HINT="1"
+fi
+
 # Export GLM/context environment variables from config
 export REFRAG_RUNTIME GLM_API_KEY GLM_API_BASE GLM_MODEL CTX_REQUIRE_CONTEXT CTX_RELEVANCE_GATE CTX_MIN_RELEVANCE CTX_REWRITE_MAX_TOKENS
 

deploy/kubernetes/indexer-services.yaml

@@ -47,6 +47,10 @@ spec:
             configMapKeyRef:
               name: context-engine-config
               key: EMBEDDING_MODEL
+        - name: HF_HOME
+          value: /work/models/hf-cache
+        - name: XDG_CACHE_HOME
+          value: /work/models/hf-cache
         - name: WATCH_ROOT
           value: /work
         - name: QDRANT_TIMEOUT
@@ -74,6 +78,10 @@ spec:
             configMapKeyRef:
               name: context-engine-config
               key: WATCH_DEBOUNCE_SECS
+        - name: HF_HOME
+          value: /work/models/hf-cache
+        - name: XDG_CACHE_HOME
+          value: /work/models/hf-cache
         resources:
           requests:
             memory: 512Mi
@@ -86,6 +94,8 @@ spec:
           mountPath: /work
         - name: metadata-volume
           mountPath: /work/.codebase
+        - name: models-volume
+          mountPath: /work/models
         envFrom:
         - configMapRef:
             name: context-engine-config
@@ -96,6 +106,9 @@ spec:
       - name: metadata-volume
         persistentVolumeClaim:
           claimName: code-metadata-pvc
+      - name: models-volume
+        persistentVolumeClaim:
+          claimName: code-models-pvc
 ---
 apiVersion: batch/v1
 kind: Job
@@ -142,6 +155,10 @@ spec:
             configMapKeyRef:
               name: context-engine-config
               key: EMBEDDING_MODEL
+        - name: HF_HOME
+          value: /work/models/hf-cache
+        - name: XDG_CACHE_HOME
+          value: /work/models/hf-cache
         resources:
           requests:
             memory: 1Gi
@@ -154,6 +171,8 @@ spec:
           mountPath: /work
         - name: metadata-volume
           mountPath: /work/.codebase
+        - name: models-volume
+          mountPath: /work/models
         envFrom:
         - configMapRef:
             name: context-engine-config
@@ -164,6 +183,9 @@ spec:
       - name: metadata-volume
         persistentVolumeClaim:
           claimName: code-metadata-pvc
+      - name: models-volume
+        persistentVolumeClaim:
+          claimName: code-models-pvc
 ---
 apiVersion: batch/v1
 kind: Job

deploy/kubernetes/mcp-http.yaml

@@ -53,6 +53,10 @@ spec:
             configMapKeyRef:
               name: context-engine-config
               key: EMBEDDING_MODEL
+        - name: HF_HOME
+          value: /work/models/hf-cache
+        - name: XDG_CACHE_HOME
+          value: /work/models/hf-cache
         - name: EMBEDDING_PROVIDER
           valueFrom:
             configMapKeyRef:
@@ -216,6 +220,12 @@ spec:
             configMapKeyRef:
               name: context-engine-config
               key: EMBEDDING_MODEL
+        - name: HF_HOME
+          value: /work/models/hf-cache
+        - name: XDG_CACHE_HOME
+          value: /work/models/hf-cache
+        - name: HF_HUB_CACHE
+          value: /work/models/hf-cache/huggingface
         - name: INDEX_MICRO_CHUNKS
           valueFrom:
             configMapKeyRef:
@@ -277,6 +287,8 @@ spec:
           mountPath: /work
         - name: codebase-volume
           mountPath: /work/.codebase
+        - name: models-volume
+          mountPath: /work/models
         livenessProbe:
           httpGet:
             path: /readyz
@@ -303,6 +315,9 @@ spec:
       - name: codebase-volume
         persistentVolumeClaim:
           claimName: code-metadata-pvc
+      - name: models-volume
+        persistentVolumeClaim:
+          claimName: code-models-pvc
 ---
 apiVersion: v1
 kind: Service

deploy/kubernetes/mcp-indexer.yaml

@@ -67,6 +67,12 @@ spec:
             configMapKeyRef:
               name: context-engine-config
               key: EMBEDDING_MODEL
+        - name: HF_HOME
+          value: /work/models/hf-cache
+        - name: XDG_CACHE_HOME
+          value: /work/models/hf-cache
+        - name: HF_HUB_CACHE
+          value: /work/models/hf-cache/huggingface
         resources:
           requests:
             memory: 512Mi
@@ -79,6 +85,8 @@ spec:
           mountPath: /work
         - name: codebase-volume
           mountPath: /work/.codebase
+        - name: models-volume
+          mountPath: /work/models
         livenessProbe:
           httpGet:
             path: /readyz
@@ -105,6 +113,9 @@ spec:
       - name: codebase-volume
         persistentVolumeClaim:
           claimName: code-metadata-pvc
+      - name: models-volume
+        persistentVolumeClaim:
+          claimName: code-models-pvc
 ---
 apiVersion: v1
 kind: Service

deploy/kubernetes/qdrant.yaml

@@ -47,12 +47,6 @@ spec:
         volumeMounts:
         - name: qdrant-storage
           mountPath: /qdrant/storage
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: http
-          initialDelaySeconds: 30
-          periodSeconds: 10
         readinessProbe:
           httpGet:
             path: /readyz

docker-compose.dev-remote.yml

@@ -34,7 +34,7 @@ services:
       - FASTMCP_PORT=${FASTMCP_PORT}
       - QDRANT_URL=${QDRANT_URL}
       - COLLECTION_NAME=${COLLECTION_NAME}
-      - PATH_EMIT_MODE=container
+      - PATH_EMIT_MODE=auto
       - HF_HOME=/work/.cache/huggingface
       - TRANSFORMERS_CACHE=/work/.cache/huggingface
       - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface
@@ -82,7 +82,7 @@ services:
       - GLM_MODEL=${GLM_MODEL:-glm-4.6}
       - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080}
       - COLLECTION_NAME=${COLLECTION_NAME}
-      - PATH_EMIT_MODE=container
+      - PATH_EMIT_MODE=auto
       - HF_HOME=/tmp/huggingface
       - HF_HUB_CACHE=/tmp/huggingface/hub
       - TRANSFORMERS_CACHE=/tmp/huggingface/transformers
@@ -121,7 +121,7 @@ services:
       - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT}
       - QDRANT_URL=${QDRANT_URL}
       - COLLECTION_NAME=${COLLECTION_NAME}
-      - PATH_EMIT_MODE=container
+      - PATH_EMIT_MODE=auto
       - HF_HOME=/work/.cache/huggingface
       - TRANSFORMERS_CACHE=/work/.cache/huggingface
       - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface
@@ -170,7 +170,7 @@ services:
       - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080}
       - FASTMCP_HEALTH_PORT=18001
       - COLLECTION_NAME=${COLLECTION_NAME}
-      - PATH_EMIT_MODE=container
+      - PATH_EMIT_MODE=auto
       - HF_HOME=/tmp/huggingface
       - HF_HUB_CACHE=/tmp/huggingface/hub
       - TRANSFORMERS_CACHE=/tmp/huggingface/transformers
@@ -215,6 +215,7 @@ services:
       context: .
       dockerfile: Dockerfile.indexer
     container_name: indexer-dev-remote
+    user: "1000:1000"
     depends_on:
       - qdrant
     env_file:

docs/ARCHITECTURE.md

@@ -1,6 +1,6 @@
 # Context Engine Architecture
 
-**Documentation:** [README](../README.md) · [Configuration](CONFIGURATION.md) · [IDE Clients](IDE_CLIENTS.md) · [MCP API](MCP_API.md) · [ctx CLI](CTX_CLI.md) · [Memory Guide](MEMORY_GUIDE.md) · [Architecture](ARCHITECTURE.md) · [Multi-Repo](MULTI_REPO_COLLECTIONS.md) · [Kubernetes](../deploy/kubernetes/README.md) · [VS Code Extension](vscode-extension.md) · [Troubleshooting](TROUBLESHOOTING.md) · [Development](DEVELOPMENT.md)
+**Documentation:** [README](../README.md) · [Getting Started](GETTING_STARTED.md) · [Configuration](CONFIGURATION.md) · [IDE Clients](IDE_CLIENTS.md) · [MCP API](MCP_API.md) · [ctx CLI](CTX_CLI.md) · [Memory Guide](MEMORY_GUIDE.md) · [Architecture](ARCHITECTURE.md) · [Multi-Repo](MULTI_REPO_COLLECTIONS.md) · [Kubernetes](../deploy/kubernetes/README.md) · [VS Code Extension](vscode-extension.md) · [Troubleshooting](TROUBLESHOOTING.md) · [Development](DEVELOPMENT.md)
 
 ---