From 8087d8ca4a48cf52def21e4f86fd30340bdf2f13 Mon Sep 17 00:00:00 2001 From: John Donalson Date: Sun, 23 Nov 2025 11:32:35 -0500 Subject: [PATCH 1/3] add auto installer --- scripts/standalone_upload_client.py | 44 ++-- scripts/upload_service.py | 3 + .../context-engine-uploader/extension.js | 245 +++++++++++++++++- .../context-engine-uploader/package.json | 13 +- 4 files changed, 270 insertions(+), 35 deletions(-) diff --git a/scripts/standalone_upload_client.py b/scripts/standalone_upload_client.py index dfc63c8b..64ffcf2b 100644 --- a/scripts/standalone_upload_client.py +++ b/scripts/standalone_upload_client.py @@ -550,7 +550,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, "source_absolute_path": str(source_path.resolve()), "size_bytes": stat.st_size, "content_hash": content_hash, - "file_hash": f"sha1:{idx.hash_id(content.decode('utf-8', errors='ignore'), dest_rel_path, 1, len(content.splitlines()))}", + "file_hash": f"sha1:{hash_id(content.decode('utf-8', errors='ignore'), dest_rel_path, 1, len(content.splitlines()))}", "modified_time": datetime.fromtimestamp(stat.st_mtime).isoformat(), "language": language } @@ -890,24 +890,29 @@ def watch_loop(self, interval: int = 5): logger.info(f"[watch] File monitoring stopped by user") def get_all_code_files(self) -> List[Path]: - """Get all code files in the workspace.""" - all_files = [] + """Get all code files in the workspace, excluding heavy/third-party dirs.""" + files: List[Path] = [] try: workspace_path = Path(self.workspace_path) + # Gather files by extension mapping for ext in CODE_EXTS: - all_files.extend(workspace_path.rglob(f"*{ext}")) - - # Filter out directories and hidden files - all_files = [ - f for f in all_files - if f.is_file() - and not any(part.startswith('.') for part in f.parts) - and '.context-engine' not in str(f) - ] + pattern = f"*{ext}" if str(ext).startswith('.') else str(ext) + files.extend(workspace_path.rglob(pattern)) + + # Exclude hidden and heavy directories + EXCLUDED_DIRS = { + "node_modules", "vendor", "dist", "build", "target", "out", + ".git", ".hg", ".svn", ".vscode", ".idea", ".venv", "venv", "__pycache__", + ".pytest_cache", ".mypy_cache", ".cache", ".context-engine", ".context-engine-uploader", ".codebase" + } + def is_excluded(p: Path) -> bool: + return any(part in EXCLUDED_DIRS or str(part).startswith('.') for part in p.parts) + + files = [f for f in files if f.is_file() and not is_excluded(f)] except Exception as e: logger.error(f"[watch] Error scanning files: {e}") - return all_files + return files def process_and_upload_changes(self, changed_paths: List[Path]) -> bool: """ @@ -1226,16 +1231,9 @@ def main(): logger.info("Scanning repository for files...") workspace_path = Path(config['workspace_path']) - # Find all files in the repository - all_files = [] - for file_path in workspace_path.rglob('*'): - if file_path.is_file() and not file_path.name.startswith('.'): - rel_path = file_path.relative_to(workspace_path) - # Skip .codebase directory and other metadata - if not str(rel_path).startswith('.codebase'): - all_files.append(file_path) - - logger.info(f"Found {len(all_files)} files to upload") + # Find code files in the repository (exclude hidden and heavy dirs) + all_files = client.get_all_code_files() + logger.info(f"Found {len(all_files)} code files to upload") if not all_files: logger.warning("No files found to upload") diff --git a/scripts/upload_service.py b/scripts/upload_service.py index 0b5c1589..8300d67b 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -362,8 +362,10 @@ async def upload_delta_bundle( ): """Upload and process delta bundle.""" start_time = datetime.now() + client_host = request.client.host if hasattr(request, 'client') and request.client else 'unknown' try: + logger.info(f"[upload_service] Begin processing upload for workspace={workspace_path} from {client_host}") # Validate workspace path workspace = Path(workspace_path) if not workspace.is_absolute(): @@ -460,6 +462,7 @@ async def upload_delta_bundle( # Calculate processing time processing_time = (datetime.now() - start_time).total_seconds() * 1000 + logger.info(f"[upload_service] Completed bundle {bundle_id} seq={sequence_number} ops={operations_count} in {int(processing_time)}ms") return UploadResponse( success=True, diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index ee9d525b..65f634c8 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -7,19 +7,36 @@ let watchProcess; let forceProcess; let extensionRoot; let statusBarItem; +let logsTerminal; let statusMode = 'idle'; +let workspaceWatcher; +let watchedTargetPath; +let indexedWatchDisposables = []; +let globalStoragePath; +let pythonOverridePath; const REQUIRED_PYTHON_MODULES = ['requests', 'urllib3', 'charset_normalizer']; const DEFAULT_CONTAINER_ROOT = '/work'; function activate(context) { outputChannel = vscode.window.createOutputChannel('Context Engine Upload'); context.subscriptions.push(outputChannel); extensionRoot = context.extensionPath; + globalStoragePath = context.globalStorageUri && context.globalStorageUri.fsPath ? context.globalStorageUri.fsPath : undefined; + try { + const venvPy = resolvePrivateVenvPython(); + if (venvPy) { + pythonOverridePath = venvPy; + log(`Detected existing private venv interpreter: ${venvPy}`); + } + } catch (_) {} statusBarItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 100); statusBarItem.command = 'contextEngineUploader.indexCodebase'; context.subscriptions.push(statusBarItem); statusBarItem.show(); setStatusBarState('idle'); updateStatusBarTooltip(); + + // Ensure an output channel is visible early for user feedback + if (outputChannel) { outputChannel.show(true); } const startDisposable = vscode.commands.registerCommand('contextEngineUploader.start', () => { runSequence('auto').catch(error => log(`Start failed: ${error instanceof Error ? error.message : String(error)}`)); }); @@ -31,8 +48,20 @@ function activate(context) { }); const indexDisposable = vscode.commands.registerCommand('contextEngineUploader.indexCodebase', () => { vscode.window.showInformationMessage('Context Engine indexing started.'); + if (outputChannel) { outputChannel.show(true); } + try { + const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); + if (cfg.get('autoTailUploadLogs', true)) { + openUploadServiceLogsTerminal(); + } + } catch (e) { + log(`Auto-tail logs failed: ${e && e.message ? e.message : String(e)}`); + } runSequence('force').catch(error => log(`Index failed: ${error instanceof Error ? error.message : String(error)}`)); }); + const showLogsDisposable = vscode.commands.registerCommand('contextEngineUploader.showUploadServiceLogs', () => { + try { openUploadServiceLogsTerminal(); } catch (e) { log(`Show logs failed: ${e && e.message ? e.message : String(e)}`); } + }); const configDisposable = vscode.workspace.onDidChangeConfiguration(event => { if (event.affectsConfiguration('contextEngineUploader') && watchProcess) { runSequence('auto').catch(error => log(`Auto-restart failed: ${error instanceof Error ? error.message : String(error)}`)); @@ -44,7 +73,7 @@ function activate(context) { const workspaceDisposable = vscode.workspace.onDidChangeWorkspaceFolders(() => { ensureTargetPathConfigured(); }); - context.subscriptions.push(startDisposable, stopDisposable, restartDisposable, indexDisposable, configDisposable, workspaceDisposable); + context.subscriptions.push(startDisposable, stopDisposable, restartDisposable, indexDisposable, showLogsDisposable, configDisposable, workspaceDisposable); const config = vscode.workspace.getConfiguration('contextEngineUploader'); ensureTargetPathConfigured(); if (config.get('runOnStartup')) { @@ -61,13 +90,20 @@ async function runSequence(mode = 'auto') { setStatusBarState('idle'); return; } + // Re-resolve options in case ensurePythonDependencies switched to a private venv interpreter + const reoptions = resolveOptions(); + if (reoptions) { + Object.assign(options, reoptions); + } await stopProcesses(); const needsForce = mode === 'force' || needsForceSync(options.targetPath); if (needsForce) { setStatusBarState('indexing'); + if (outputChannel) { outputChannel.show(true); } const code = await runOnce(options); if (code === 0) { - startWatch(options); + setStatusBarState('indexed'); + ensureIndexedWatcher(options.targetPath); } else { setStatusBarState('idle'); } @@ -77,7 +113,10 @@ async function runSequence(mode = 'auto') { } function resolveOptions() { const config = vscode.workspace.getConfiguration('contextEngineUploader'); - const pythonPath = (config.get('pythonPath') || 'python3').trim(); + let pythonPath = (config.get('pythonPath') || 'python3').trim(); + if (pythonOverridePath && fs.existsSync(pythonOverridePath)) { + pythonPath = pythonOverridePath; + } const endpoint = (config.get('endpoint') || '').trim(); const targetPath = getTargetPath(config); const interval = config.get('intervalSeconds') || 5; @@ -90,7 +129,12 @@ function resolveOptions() { if (configuredScriptDir) { candidates.push(configuredScriptDir); } + // Prefer packaged script; also try workspace ./scripts fallback for dev candidates.push(extensionRoot); + const wsRoot = getWorkspaceFolderPath(); + if (wsRoot) { + candidates.push(path.join(wsRoot, 'scripts')); + } candidates.push(path.join(extensionRoot, '..', 'out')); let workingDirectory; let scriptPath; @@ -209,6 +253,32 @@ function needsForceSync(targetPath) { } } async function ensurePythonDependencies(pythonPath) { + // Probe current interpreter; if modules missing, offer to create a private venv and install deps + const ok = await checkPythonDeps(pythonPath); + if (ok) return true; + const choice = await vscode.window.showErrorMessage( + 'Context Engine Uploader: missing Python modules. Create isolated environment and auto-install?', + 'Auto-install to private venv', + 'Cancel' + ); + if (choice !== 'Auto-install to private venv') { + return false; + } + const created = await ensurePrivateVenv(); + if (!created) return false; + const venvPython = resolvePrivateVenvPython(); + if (!venvPython) { + vscode.window.showErrorMessage('Context Engine Uploader: failed to locate private venv python.'); + return false; + } + const installed = await installDepsInto(venvPython); + if (!installed) return false; + pythonOverridePath = venvPython; + log(`Using private venv interpreter: ${pythonOverridePath}`); + return await checkPythonDeps(pythonOverridePath); +} + +async function checkPythonDeps(pythonPath) { const missing = []; let pythonError; for (const moduleName of REQUIRED_PYTHON_MODULES) { @@ -228,17 +298,103 @@ async function ensurePythonDependencies(pythonPath) { return false; } if (missing.length) { - const installCommand = `${pythonPath} -m pip install ${REQUIRED_PYTHON_MODULES.join(' ')}`; - log(`Missing Python modules: ${missing.join(', ')}. Run: ${installCommand}`); - const action = await vscode.window.showErrorMessage(`Context Engine Uploader: missing Python modules (${missing.join(', ')}).`, 'Copy install command'); - if (action === 'Copy install command') { - await vscode.env.clipboard.writeText(installCommand); - vscode.window.showInformationMessage('Pip install command copied to clipboard.'); - } + log(`Missing Python modules for ${pythonPath}: ${missing.join(', ')}`); return false; } return true; } + +function venvRootDir() { + // Prefer workspace storage; fallback to extension storage + try { + const ws = getWorkspaceFolderPath(); + const base = ws && fs.existsSync(ws) ? path.join(ws, '.vscode', '.context-engine-uploader') + : (globalStoragePath || path.join(extensionRoot, '.storage')); + if (!fs.existsSync(base)) fs.mkdirSync(base, { recursive: true }); + return base; + } catch (e) { + return extensionRoot; + } +} + +function privateVenvPath() { + return path.join(venvRootDir(), 'py-venv'); +} + +function resolvePrivateVenvPython() { + const venvPath = privateVenvPath(); + const bin = process.platform === 'win32' ? path.join(venvPath, 'Scripts', 'python.exe') : path.join(venvPath, 'bin', 'python'); + return fs.existsSync(bin) ? bin : undefined; +} + +async function ensurePrivateVenv() { + try { + const python = resolvePrivateVenvPython(); + if (python) { + log('Private venv already exists.'); + return true; + } + const venvPath = privateVenvPath(); + const basePy = await detectSystemPython(); + if (!basePy) { + vscode.window.showErrorMessage('Context Engine Uploader: no Python interpreter found to bootstrap venv.'); + return false; + } + log(`Creating private venv at ${venvPath} using ${basePy}`); + const res = spawnSync(basePy, ['-m', 'venv', venvPath], { encoding: 'utf8' }); + if (res.status !== 0) { + log(`venv creation failed: ${res.stderr || res.stdout}`); + vscode.window.showErrorMessage('Context Engine Uploader: failed to create private venv.'); + return false; + } + return true; + } catch (e) { + log(`ensurePrivateVenv error: ${e && e.message ? e.message : String(e)}`); + return false; + } +} + +async function installDepsInto(pythonBin) { + try { + log(`Installing Python deps into private venv via ${pythonBin}`); + const args = ['-m', 'pip', 'install', ...REQUIRED_PYTHON_MODULES]; + const res = spawnSync(pythonBin, args, { encoding: 'utf8' }); + if (res.status !== 0) { + log(`pip install failed: ${res.stderr || res.stdout}`); + vscode.window.showErrorMessage('Context Engine Uploader: pip install failed. See Output for details.'); + return false; + } + return true; + } catch (e) { + log(`installDepsInto error: ${e && e.message ? e.message : String(e)}`); + return false; + } +} + +async function detectSystemPython() { + // Try configured pythonPath, then common names + const candidates = []; + try { + const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); + const configured = (cfg.get('pythonPath') || '').trim(); + if (configured) candidates.push(configured); + } catch {} + if (process.platform === 'win32') { + candidates.push('py', 'python3', 'python'); + } else { + candidates.push('python3', 'python'); + // Add common Homebrew path on Apple Silicon + candidates.push('/opt/homebrew/bin/python3'); + } + for (const cmd of candidates) { + const probe = spawnSync(cmd, ['-c', 'import sys; print(sys.executable)'], { encoding: 'utf8' }); + if (!probe.error && probe.status === 0) { + const p = (probe.stdout || '').trim(); + if (p) return p; + } + } + return undefined; +} function setStatusBarState(mode) { if (!statusBarItem) { return; @@ -247,6 +403,9 @@ function setStatusBarState(mode) { if (mode === 'indexing') { statusBarItem.text = '$(sync~spin) Indexing...'; statusBarItem.color = undefined; + } else if (mode === 'indexed') { + statusBarItem.text = '$(check) Indexed'; + statusBarItem.color = new vscode.ThemeColor('charts.green'); } else if (mode === 'watch') { statusBarItem.text = '$(sync) Watching'; statusBarItem.color = new vscode.ThemeColor('charts.purple'); @@ -281,10 +440,12 @@ function runOnce(options) { }); } function startWatch(options) { + disposeIndexedWatcher(); const args = buildArgs(options, 'watch'); const child = spawn(options.pythonPath, args, { cwd: options.workingDirectory, env: buildChildEnv(options) }); watchProcess = child; attachOutput(child, 'watch'); + if (outputChannel) { outputChannel.show(true); } setStatusBarState('watch'); child.on('close', code => { log(`Watch exited with code ${code}`); @@ -336,6 +497,69 @@ function attachOutput(child, label) { }); } } +function openUploadServiceLogsTerminal() { + try { + const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); + const wsPath = getWorkspaceFolderPath() || (cfg.get('targetPath') || ''); + const cwd = (wsPath && typeof wsPath === 'string' && fs.existsSync(wsPath)) ? wsPath : undefined; + if (!logsTerminal || logsTerminal.exitStatus) { + logsTerminal = vscode.window.createTerminal({ name: 'Context Engine Upload Logs', cwd: cwd ? vscode.Uri.file(cwd) : undefined }); + } + logsTerminal.show(true); + logsTerminal.sendText('docker compose logs -f upload_service', true); + } catch (e) { + log(`Unable to open logs terminal: ${e && e.message ? e.message : String(e)}`); + } +} + +function ensureIndexedWatcher(targetPath) { + try { + disposeIndexedWatcher(); + watchedTargetPath = targetPath; + let pattern; + if (targetPath && fs.existsSync(targetPath)) { + pattern = new vscode.RelativePattern(targetPath, '**/*'); + } else { + const folder = getWorkspaceFolderPath(); + if (folder && fs.existsSync(folder)) { + pattern = new vscode.RelativePattern(folder, '**/*'); + } else { + pattern = '**/*'; + } + } + workspaceWatcher = vscode.workspace.createFileSystemWatcher(pattern, false, false, false); + const flipToIdle = () => { + if (statusMode === 'indexed') { + setStatusBarState('idle'); + } + }; + indexedWatchDisposables.push(workspaceWatcher); + indexedWatchDisposables.push(workspaceWatcher.onDidCreate(flipToIdle)); + indexedWatchDisposables.push(workspaceWatcher.onDidChange(flipToIdle)); + indexedWatchDisposables.push(workspaceWatcher.onDidDelete(flipToIdle)); + indexedWatchDisposables.push(vscode.workspace.onDidChangeTextDocument(() => flipToIdle())); + log('Indexed watcher armed; any file change will return status bar to "Index Codebase".'); + } catch (e) { + log(`Failed to arm indexed watcher: ${e && e.message ? e.message : String(e)}`); + } +} + +function disposeIndexedWatcher() { + try { + for (const d of indexedWatchDisposables) { + try { if (d && typeof d.dispose === 'function') d.dispose(); } catch (_) {} + } + indexedWatchDisposables = []; + if (workspaceWatcher && typeof workspaceWatcher.dispose === 'function') { + workspaceWatcher.dispose(); + } + workspaceWatcher = undefined; + watchedTargetPath = undefined; + } catch (e) { + // ignore + } +} + async function stopProcesses() { await Promise.all([terminateProcess(forceProcess, 'force'), terminateProcess(watchProcess, 'watch')]); if (!forceProcess && !watchProcess && statusMode !== 'indexing') { @@ -433,6 +657,7 @@ function buildChildEnv(options) { return env; } function deactivate() { + disposeIndexedWatcher(); return stopProcesses(); } module.exports = { diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index 4e4c2ac2..0ffbf5a1 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -2,7 +2,7 @@ "name": "context-engine-uploader", "displayName": "Context Engine Uploader", "description": "Runs the Context-Engine remote upload client with a force sync on startup followed by watch mode. Requires Python with pip install requests urllib3 charset_normalizer.", - "version": "0.1.1", + "version": "0.1.6", "publisher": "context-engine", "engines": { "vscode": "^1.85.0" @@ -35,6 +35,10 @@ { "command": "contextEngineUploader.indexCodebase", "title": "Context Engine Uploader: Index Codebase" + }, + { + "command": "contextEngineUploader.showUploadServiceLogs", + "title": "Context Engine Uploader: Show Upload Service Logs" } ], "configuration": { @@ -54,7 +58,7 @@ "contextEngineUploader.scriptWorkingDirectory": { "type": "string", "default": "", - "description": "Optional override for the folder that contains standalone_upload_client.py. Defaults to the extension install directory." + "description": "Optional override for the folder that contains standalone_upload_client.py. Defaults to the extension install directory or the workspace ./scripts folder." }, "contextEngineUploader.targetPath": { "type": "string", @@ -98,6 +102,11 @@ "default": "/work", "description": "Container path that mirrors the host root on the upload server." }, + "contextEngineUploader.autoTailUploadLogs": { + "type": "boolean", + "default": true, + "description": "Automatically open a terminal and tail 'docker compose logs -f upload_service' when indexing starts." + }, "contextEngineUploader.bundleSizeLimitMB": { "type": "number", "default": 100, From 770e75576446fa4f49b372aa930de49f93cb417f Mon Sep 17 00:00:00 2001 From: John Donalson Date: Sun, 23 Nov 2025 11:45:42 -0500 Subject: [PATCH 2/3] make default localhost --- .gitignore | 1 + scripts/standalone_upload_client.py | 2 +- vscode-extension/context-engine-uploader/package.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 04674eee..0ddd7594 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ tests/.codebase/state.json CLAUDE.md .qodo/.cursor/rules /.augment +/dev-workspace diff --git a/scripts/standalone_upload_client.py b/scripts/standalone_upload_client.py index 64ffcf2b..8aa6cf41 100644 --- a/scripts/standalone_upload_client.py +++ b/scripts/standalone_upload_client.py @@ -576,7 +576,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, "previous_hash": f"sha1:{previous_hash}" if previous_hash else None, "file_hash": None, "modified_time": datetime.now().isoformat(), - "language": idx.CODE_EXTS.get(path.suffix.lower(), "unknown") + "language": CODE_EXTS.get(path.suffix.lower(), "unknown") } operations.append(operation) diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index 0ffbf5a1..d723d837 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -67,7 +67,7 @@ }, "contextEngineUploader.endpoint": { "type": "string", - "default": "http://mcp.speramus.id:8004", + "default": "http://localhost:8004", "description": "Endpoint URL for the remote upload client." }, "contextEngineUploader.intervalSeconds": { From dc0c4d902515d9b13dc94891d54b0fc3521c2298 Mon Sep 17 00:00:00 2001 From: John Donalson Date: Sun, 23 Nov 2025 12:09:36 -0500 Subject: [PATCH 3/3] add watching, tweaks --- scripts/standalone_upload_client.py | 33 ++++++++++++----- scripts/upload_service.py | 36 ++++++++++++++---- .../context-engine-uploader/extension.js | 37 ++++++++++++++----- .../context-engine-uploader/package.json | 7 +++- 4 files changed, 86 insertions(+), 27 deletions(-) diff --git a/scripts/standalone_upload_client.py b/scripts/standalone_upload_client.py index 8aa6cf41..41c14a06 100644 --- a/scripts/standalone_upload_client.py +++ b/scripts/standalone_upload_client.py @@ -477,6 +477,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, operations.append(operation) file_hashes[rel_path] = f"sha1:{file_hash}" total_size += stat.st_size + set_cached_file_hash(str(path.resolve()), file_hash, self.repo_name) except Exception as e: print(f"[bundle_create] Error processing created file {path}: {e}") @@ -516,6 +517,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, operations.append(operation) file_hashes[rel_path] = f"sha1:{file_hash}" total_size += stat.st_size + set_cached_file_hash(str(path.resolve()), file_hash, self.repo_name) except Exception as e: print(f"[bundle_create] Error processing updated file {path}: {e}") @@ -557,6 +559,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, operations.append(operation) file_hashes[dest_rel_path] = f"sha1:{file_hash}" total_size += stat.st_size + set_cached_file_hash(str(dest_path.resolve()), file_hash, self.repo_name) except Exception as e: print(f"[bundle_create] Error processing moved file {source_path} -> {dest_path}: {e}") @@ -894,21 +897,33 @@ def get_all_code_files(self) -> List[Path]: files: List[Path] = [] try: workspace_path = Path(self.workspace_path) - # Gather files by extension mapping - for ext in CODE_EXTS: - pattern = f"*{ext}" if str(ext).startswith('.') else str(ext) - files.extend(workspace_path.rglob(pattern)) + if not workspace_path.exists(): + return files - # Exclude hidden and heavy directories + # Single walk with early pruning and set-based matching to reduce IO + ext_suffixes = {str(ext).lower() for ext in CODE_EXTS if str(ext).startswith('.')} + name_matches = {str(ext) for ext in CODE_EXTS if not str(ext).startswith('.')} EXCLUDED_DIRS = { - "node_modules", "vendor", "dist", "build", "target", "out", + "node_modules", "vendor", "dist", "build", "target", "out", "dev-workspace", ".git", ".hg", ".svn", ".vscode", ".idea", ".venv", "venv", "__pycache__", ".pytest_cache", ".mypy_cache", ".cache", ".context-engine", ".context-engine-uploader", ".codebase" } - def is_excluded(p: Path) -> bool: - return any(part in EXCLUDED_DIRS or str(part).startswith('.') for part in p.parts) - files = [f for f in files if f.is_file() and not is_excluded(f)] + seen = set() + for root, dirnames, filenames in os.walk(workspace_path): + # Prune heavy/hidden directories before descending + dirnames[:] = [d for d in dirnames if d not in EXCLUDED_DIRS and not d.startswith('.')] + + for filename in filenames: + if filename.startswith('.'): + continue + candidate = Path(root) / filename + suffix = candidate.suffix.lower() + if filename in name_matches or suffix in ext_suffixes: + resolved = candidate.resolve() + if resolved not in seen: + seen.add(resolved) + files.append(candidate) except Exception as e: logger.error(f"[watch] Error scanning files: {e}") diff --git a/scripts/upload_service.py b/scripts/upload_service.py index 8300d67b..1391f532 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -372,14 +372,13 @@ async def upload_delta_bundle( workspace = Path(WORK_DIR) / workspace workspace_path = str(workspace.resolve()) + repo_name = _extract_repo_name_from_path(workspace_path) if _extract_repo_name_from_path else None + if not repo_name: + repo_name = Path(workspace_path).name # Get collection name if not collection_name: if get_collection_name: - repo_name = _extract_repo_name_from_path(workspace_path) if _extract_repo_name_from_path else None - # Fallback to directory name if repo detection fails - if not repo_name: - repo_name = Path(workspace_path).name collection_name = get_collection_name(repo_name) else: collection_name = DEFAULT_COLLECTION @@ -410,9 +409,32 @@ async def upload_delta_bundle( with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as temp_file: bundle_path = Path(temp_file.name) - # Stream upload to file - content = await bundle.read() - bundle_path.write_bytes(content) + max_bytes = MAX_BUNDLE_SIZE_MB * 1024 * 1024 + if bundle.size and bundle.size > max_bytes: + raise HTTPException( + status_code=413, + detail=f"Bundle too large. Max size: {MAX_BUNDLE_SIZE_MB}MB" + ) + + # Stream upload to file while enforcing size + total = 0 + chunk_size = 1024 * 1024 + while True: + chunk = await bundle.read(chunk_size) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + try: + temp_file.close() + bundle_path.unlink(missing_ok=True) + except Exception: + pass + raise HTTPException( + status_code=413, + detail=f"Bundle too large. Max size: {MAX_BUNDLE_SIZE_MB}MB" + ) + temp_file.write(chunk) try: # Validate bundle format diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index 65f634c8..2fdab666 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -8,6 +8,7 @@ let forceProcess; let extensionRoot; let statusBarItem; let logsTerminal; +let logTailActive = false; let statusMode = 'idle'; let workspaceWatcher; let watchedTargetPath; @@ -73,7 +74,13 @@ function activate(context) { const workspaceDisposable = vscode.workspace.onDidChangeWorkspaceFolders(() => { ensureTargetPathConfigured(); }); - context.subscriptions.push(startDisposable, stopDisposable, restartDisposable, indexDisposable, showLogsDisposable, configDisposable, workspaceDisposable); + const terminalCloseDisposable = vscode.window.onDidCloseTerminal(term => { + if (term === logsTerminal) { + logsTerminal = undefined; + logTailActive = false; + } + }); + context.subscriptions.push(startDisposable, stopDisposable, restartDisposable, indexDisposable, showLogsDisposable, configDisposable, workspaceDisposable, terminalCloseDisposable); const config = vscode.workspace.getConfiguration('contextEngineUploader'); ensureTargetPathConfigured(); if (config.get('runOnStartup')) { @@ -104,6 +111,9 @@ async function runSequence(mode = 'auto') { if (code === 0) { setStatusBarState('indexed'); ensureIndexedWatcher(options.targetPath); + if (options.startWatchAfterForce) { + startWatch(options); + } } else { setStatusBarState('idle'); } @@ -124,6 +134,7 @@ function resolveOptions() { const extraWatchArgs = config.get('extraWatchArgs') || []; const hostRootOverride = (config.get('hostRoot') || '').trim(); const containerRoot = (config.get('containerRoot') || DEFAULT_CONTAINER_ROOT).trim() || DEFAULT_CONTAINER_ROOT; + const startWatchAfterForce = config.get('startWatchAfterForce', true); const configuredScriptDir = (config.get('scriptWorkingDirectory') || '').trim(); const candidates = []; if (configuredScriptDir) { @@ -180,7 +191,8 @@ function resolveOptions() { extraForceArgs, extraWatchArgs, hostRoot, - containerRoot + containerRoot, + startWatchAfterForce }; } function getTargetPath(config) { @@ -195,10 +207,8 @@ function getTargetPath(config) { updateStatusBarTooltip(); return undefined; } - targetPath = folderPath; - saveTargetPath(config, targetPath); - updateStatusBarTooltip(targetPath); - return targetPath; + updateStatusBarTooltip(folderPath); + return folderPath; } function saveTargetPath(config, targetPath) { const hasWorkspace = vscode.workspace.workspaceFolders && vscode.workspace.workspaceFolders.length; @@ -226,7 +236,6 @@ function ensureTargetPathConfigured() { updateStatusBarTooltip(); return; } - saveTargetPath(config, folderPath); updateStatusBarTooltip(folderPath); } function updateStatusBarTooltip(targetPath) { @@ -407,7 +416,7 @@ function setStatusBarState(mode) { statusBarItem.text = '$(check) Indexed'; statusBarItem.color = new vscode.ThemeColor('charts.green'); } else if (mode === 'watch') { - statusBarItem.text = '$(sync) Watching'; + statusBarItem.text = '$(sync) Watching (Click Force Index)'; statusBarItem.color = new vscode.ThemeColor('charts.purple'); } else { statusBarItem.text = '$(sync) Index Codebase'; @@ -502,11 +511,19 @@ function openUploadServiceLogsTerminal() { const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); const wsPath = getWorkspaceFolderPath() || (cfg.get('targetPath') || ''); const cwd = (wsPath && typeof wsPath === 'string' && fs.existsSync(wsPath)) ? wsPath : undefined; - if (!logsTerminal || logsTerminal.exitStatus) { + if (logsTerminal && logsTerminal.exitStatus) { + logsTerminal = undefined; + logTailActive = false; + } + if (!logsTerminal) { logsTerminal = vscode.window.createTerminal({ name: 'Context Engine Upload Logs', cwd: cwd ? vscode.Uri.file(cwd) : undefined }); + logTailActive = false; } logsTerminal.show(true); - logsTerminal.sendText('docker compose logs -f upload_service', true); + if (!logTailActive) { + logsTerminal.sendText('docker compose logs -f upload_service', true); + logTailActive = true; + } } catch (e) { log(`Unable to open logs terminal: ${e && e.message ? e.message : String(e)}`); } diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index d723d837..27f8f431 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -2,7 +2,7 @@ "name": "context-engine-uploader", "displayName": "Context Engine Uploader", "description": "Runs the Context-Engine remote upload client with a force sync on startup followed by watch mode. Requires Python with pip install requests urllib3 charset_normalizer.", - "version": "0.1.6", + "version": "0.1.11", "publisher": "context-engine", "engines": { "vscode": "^1.85.0" @@ -111,6 +111,11 @@ "type": "number", "default": 100, "description": "Maximum upload bundle size enforced by the Context Engine server (MB)." + }, + "contextEngineUploader.startWatchAfterForce": { + "type": "boolean", + "default": true, + "description": "After a force index completes, automatically launch watch mode." } } }