From 246359c8b92cbd82c3569837b36315149c37d2ed Mon Sep 17 00:00:00 2001 From: Reese Date: Thu, 27 Nov 2025 23:04:45 +0000 Subject: [PATCH] K8:Refactors and configures indexer deployments Updates indexer configurations and deployment settings. This includes: - Adds new configuration options for index upsert operations. - Removes the `readOnly: true` restriction from volume mounts, enabling write access where needed. - Removes unused configuration parameters. - Updates the persistent volume claim used by the upload service. - Improves the environment variable parsing script to better handle commented-out default values, allowing template-style .env files to configure the application, while avoiding commented defaults overriding explicit values or breaking on values with inline comments. --- deploy/kubernetes/configmap.yaml | 8 +++-- deploy/kubernetes/indexer-services.yaml | 3 -- deploy/kubernetes/mcp-http.yaml | 5 ---- deploy/kubernetes/mcp-indexer.yaml | 5 ---- deploy/kubernetes/upload-service.yaml | 2 +- scripts/sync_env_to_k8s.py | 39 ++++++++++++++++++++++++- 6 files changed, 45 insertions(+), 17 deletions(-) diff --git a/deploy/kubernetes/configmap.yaml b/deploy/kubernetes/configmap.yaml index b3b9b8f0..34fe12c5 100644 --- a/deploy/kubernetes/configmap.yaml +++ b/deploy/kubernetes/configmap.yaml @@ -37,6 +37,9 @@ data: INDEX_CHUNK_OVERLAP: '10' INDEX_MICRO_CHUNKS: '0' INDEX_SEMANTIC_CHUNKS: '1' + INDEX_UPSERT_BACKOFF: '0.5' + INDEX_UPSERT_BATCH: '128' + INDEX_UPSERT_RETRIES: '5' LLAMACPP_EXTRA_ARGS: '' LLAMACPP_GPU_LAYERS: '32' LLAMACPP_GPU_SPLIT: '' @@ -47,7 +50,7 @@ data: LLM_EXPAND_MAX: '0' LLM_EXPAND_MODEL: phi3:mini LLM_PROVIDER: ollama - MAX_CHANGED_SYMBOLS_RATIO: '0.6 # If >60% of symbols changed, do full reprocessing' + MAX_CHANGED_SYMBOLS_RATIO: '0.6' MAX_EMBED_CACHE: '16384' MAX_MICRO_CHUNKS_PER_FILE: '500' MCP_INDEXER_URL: http://localhost:8003/mcp @@ -68,12 +71,13 @@ data: MULTI_REPO_MODE: '1' OLLAMA_HOST: http://host.docker.internal:11434 PRF_ENABLED: '1' + QDRANT_API_KEY: '' QDRANT_TIMEOUT: '20' QDRANT_URL: http://qdrant:6333 REFRAG_CANDIDATES: '200' REFRAG_COMMIT_DESCRIBE: '1' REFRAG_DECODER: '1' - REFRAG_DECODER_MODE: 'prompt # prompt|soft' + REFRAG_DECODER_MODE: prompt REFRAG_ENCODER_MODEL: BAAI/bge-base-en-v1.5 REFRAG_GATE_FIRST: '1' REFRAG_MODE: '1' diff --git a/deploy/kubernetes/indexer-services.yaml b/deploy/kubernetes/indexer-services.yaml index f176e615..851de662 100644 --- a/deploy/kubernetes/indexer-services.yaml +++ b/deploy/kubernetes/indexer-services.yaml @@ -80,7 +80,6 @@ spec: volumeMounts: - name: work-volume mountPath: /work - readOnly: true - name: metadata-volume mountPath: /work/.codebase envFrom: @@ -145,7 +144,6 @@ spec: volumeMounts: - name: work-volume mountPath: /work - readOnly: true - name: metadata-volume mountPath: /work/.codebase envFrom: @@ -205,7 +203,6 @@ spec: volumeMounts: - name: work-volume mountPath: /work - readOnly: true - name: metadata-volume mountPath: /work/.codebase envFrom: diff --git a/deploy/kubernetes/mcp-http.yaml b/deploy/kubernetes/mcp-http.yaml index 19c2894f..f5a48b20 100644 --- a/deploy/kubernetes/mcp-http.yaml +++ b/deploy/kubernetes/mcp-http.yaml @@ -265,11 +265,6 @@ spec: configMapKeyRef: name: context-engine-config key: MEMORY_MCP_URL - - name: CTX_MULTI_COLLECTION - valueFrom: - configMapKeyRef: - name: context-engine-config - key: CTX_MULTI_COLLECTION - name: FASTMCP_HOST valueFrom: configMapKeyRef: diff --git a/deploy/kubernetes/mcp-indexer.yaml b/deploy/kubernetes/mcp-indexer.yaml index 39089e2b..c07a622b 100644 --- a/deploy/kubernetes/mcp-indexer.yaml +++ b/deploy/kubernetes/mcp-indexer.yaml @@ -63,11 +63,6 @@ spec: configMapKeyRef: name: context-engine-config key: EMBEDDING_MODEL - - name: CTX_MULTI_COLLECTION - valueFrom: - configMapKeyRef: - name: context-engine-config - key: CTX_MULTI_COLLECTION resources: requests: memory: 512Mi diff --git a/deploy/kubernetes/upload-service.yaml b/deploy/kubernetes/upload-service.yaml index db827dc3..a5c5cd89 100644 --- a/deploy/kubernetes/upload-service.yaml +++ b/deploy/kubernetes/upload-service.yaml @@ -104,7 +104,7 @@ spec: volumes: - name: work-volume persistentVolumeClaim: - claimName: upload-work-pvc + claimName: code-repos-pvc - name: codebase-volume persistentVolumeClaim: claimName: upload-codebase-pvc diff --git a/scripts/sync_env_to_k8s.py b/scripts/sync_env_to_k8s.py index 4e0c3ef1..46cd18fa 100644 --- a/scripts/sync_env_to_k8s.py +++ b/scripts/sync_env_to_k8s.py @@ -39,8 +39,37 @@ def parse_env_file(env_path: Path) -> dict: with env_path.open("r", encoding="utf-8") as f: for raw_line in f: line = raw_line.strip() - if not line or line.startswith("#"): + if not line: continue + + commented = False + if line.startswith("#"): + # Support commented defaults of the form "# KEY=VALUE" so that + # template-style .env files can still drive configmap.yaml. + stripped = line.lstrip("#").strip() + if "=" not in stripped: + # Pure comment, skip. + continue + + key_part, value_part = stripped.split("=", 1) + key_candidate = key_part.strip() + + # Only treat as a default if the key looks like an env var: + # UPPERCASE letters, digits, and underscores, with no spaces/colons. + if ( + not key_candidate + or " " in key_candidate + or ":" in key_candidate + or not key_candidate.replace("_", "").isalnum() + or not key_candidate.isupper() + ): + # Looks like prose (e.g. "Repository mode: 0=...") rather than + # an environment variable. Leave it as a comment. + continue + + line = f"{key_candidate}={value_part.strip()}" + commented = True + if "=" not in line: continue key, value = line.split("=", 1) @@ -48,6 +77,14 @@ def parse_env_file(env_path: Path) -> dict: value = value.strip() if not key: continue + # Do not let commented defaults override explicit KEY=VALUE entries + # that appeared earlier in the file. + if commented and key in data: + continue + # Strip inline shell-style comments from unquoted values, so that + # "VALUE # comment" becomes just "VALUE". + if "#" in value and not (value.startswith("\"") or value.startswith("'")): + value = value.split("#", 1)[0].rstrip() # Strip single/double quotes if the whole value is quoted if (value.startswith("\"") and value.endswith("\"")) or ( value.startswith("'") and value.endswith("'")