diff --git a/stac_server/main.py b/stac_server/main.py index bd5a0f6..b9cbaab 100644 --- a/stac_server/main.py +++ b/stac_server/main.py @@ -1,17 +1,22 @@ from .key_ordering import dataset_key_orders +import base64 import json import logging import os +import subprocess +import sys +from io import BytesIO, StringIO from pathlib import Path from typing import Mapping import yaml from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, HTMLResponse +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates +from pydantic import BaseModel from qubed import Qube from qubed.formatters import node_tree_to_html @@ -57,6 +62,43 @@ allow_headers=["*"], ) + +@app.on_event("startup") +async def startup_event(): + """Install required packages on startup.""" + required_packages = [ + "covjsonkit", + "earthkit-plots", + "xarray", + "matplotlib", + "numpy", + ] + logger.info("Checking and installing required packages on startup...") + + for package in required_packages: + try: + # Try to import to check if already installed + __import__(package.replace("-", "_")) + logger.info(f"{package} is already installed") + except ImportError: + logger.info(f"Installing {package}...") + try: + result = subprocess.run( + [sys.executable, "-m", "pip", "install", package], + capture_output=True, + text=True, + timeout=120, + ) + if result.returncode == 0: + logger.info(f"Successfully installed {package}") + else: + logger.warning(f"Failed to install {package}: {result.stderr}") + except Exception as e: + logger.warning(f"Error installing {package}: {e}") + + logger.info("Package installation check complete") + + app.mount( "/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static" ) @@ -473,3 +515,184 @@ async def get_STAC( } return stac_collection + + +# Pydantic models for notebook execution +class ExecuteRequest(BaseModel): + code: str + data: dict | None = None + + +class InstallPackageRequest(BaseModel): + packages: str # Space or comma-separated package names + + +@app.post("/api/v2/execute") +async def execute_code(request: ExecuteRequest): + """ + Execute Python code on the server with optional data context. + Allows installation of any Python package, including C extensions. + Captures matplotlib figures and returns them as base64 images. + """ + try: + # Create a namespace with the data available + namespace = {} + if request.data: + namespace["polytope_data"] = request.data + + # Capture stdout and stderr + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = StringIO() + sys.stderr = StringIO() + + images = [] + + try: + # Set matplotlib to non-interactive backend before execution + try: + import matplotlib + + matplotlib.use("Agg") # Non-interactive backend + except ImportError: + pass + + # Execute the code + exec(request.code, namespace) + + # Capture matplotlib figures if any were created + try: + import matplotlib.pyplot as plt + + figures = [plt.figure(num) for num in plt.get_fignums()] + + for fig in figures: + # Save figure to bytes + buf = BytesIO() + fig.savefig(buf, format="png", dpi=150, bbox_inches="tight") + buf.seek(0) + + # Convert to base64 + img_base64 = base64.b64encode(buf.read()).decode("utf-8") + images.append(img_base64) + + # Close the figure + plt.close(fig) + except ImportError: + # matplotlib not available, skip figure capture + pass + except Exception as fig_error: + # Log but don't fail if figure capture fails + sys.stderr.write(f"\nWarning: Could not capture figures: {fig_error}\n") + + # Get the output + stdout_output = sys.stdout.getvalue() + stderr_output = sys.stderr.getvalue() + + return JSONResponse( + { + "success": True, + "stdout": stdout_output, + "stderr": stderr_output, + "images": images, + } + ) + finally: + # Restore stdout and stderr + sys.stdout = old_stdout + sys.stderr = old_stderr + + except Exception as e: + return JSONResponse( + { + "success": False, + "error": str(e), + "error_type": type(e).__name__, + }, + status_code=400, + ) + + +@app.post("/api/v2/install_packages") +async def install_packages(request: InstallPackageRequest): + """ + Install Python packages using pip in the server environment. + """ + try: + # Split packages by space or comma + packages = [ + pkg.strip() + for pkg in request.packages.replace(",", " ").split() + if pkg.strip() + ] + + if not packages: + return JSONResponse( + { + "success": False, + "error": "No packages specified", + }, + status_code=400, + ) + + results = [] + for package in packages: + try: + # Run pip install + result = subprocess.run( + [sys.executable, "-m", "pip", "install", package], + capture_output=True, + text=True, + timeout=120, # 2 minute timeout per package + ) + + if result.returncode == 0: + results.append( + { + "package": package, + "success": True, + "message": f"Successfully installed {package}", + } + ) + else: + results.append( + { + "package": package, + "success": False, + "error": result.stderr, + } + ) + except subprocess.TimeoutExpired: + results.append( + { + "package": package, + "success": False, + "error": "Installation timed out after 120 seconds", + } + ) + except Exception as e: + results.append( + { + "package": package, + "success": False, + "error": str(e), + } + ) + + all_success = all(r["success"] for r in results) + + return JSONResponse( + { + "success": all_success, + "results": results, + } + ) + + except Exception as e: + return JSONResponse( + { + "success": False, + "error": str(e), + }, + status_code=500, + ) diff --git a/stac_server/static/app.js b/stac_server/static/app.js index 23899c8..4c336d3 100644 --- a/stac_server/static/app.js +++ b/stac_server/static/app.js @@ -910,6 +910,9 @@ async function queryPolytope() { polytopeStatus.textContent = `Successfully submitted ${result.total} request(s). ${result.successful} succeeded, ${result.failed} failed.`; polytopeStatus.className = 'polytope-status success'; + // Store results globally for notebook access + window.polytopeResults = result.results; + // Display detailed results if (result.results && result.results.length > 0) { polytopeResults.innerHTML = result.results.map((res, idx) => ` @@ -925,9 +928,17 @@ async function queryPolytope() { ${res.message ? `
${res.message}
` : ''} ${res.success && res.json_data ? ` - +
+ + +
` : ''} `).join(''); @@ -943,6 +954,17 @@ async function queryPolytope() { } }); }); + + // Add event listeners to notebook buttons + document.querySelectorAll('.open-notebook-btn').forEach(btn => { + btn.addEventListener('click', (e) => { + const idx = parseInt(e.target.closest('.open-notebook-btn').getAttribute('data-request-idx')); + const resultData = result.results[idx]; + if (resultData && resultData.json_data) { + openInNotebook(resultData.json_data, idx); + } + }); + }); } polytopeBtnText.textContent = 'Query Complete'; @@ -959,6 +981,288 @@ async function queryPolytope() { } } +// ============================================ +// JupyterLite Notebook Integration +// ============================================ + +let codeEditor = null; +let currentNotebookData = null; + +// Server-side execution - no Pyodide initialization needed +// Python code runs on the server with full package support + +function initCodeEditor() { + if (codeEditor) { + return codeEditor; + } + + const editorElement = document.getElementById('code-editor'); + codeEditor = CodeMirror(editorElement, { + value: getDefaultNotebookCode(), + mode: 'python', + theme: 'monokai', + lineNumbers: true, + indentUnit: 4, + tabSize: 4, + indentWithTabs: false, + lineWrapping: true, + }); + + return codeEditor; +} + +function getDefaultNotebookCode() { + return `# Polytope Data Visualization - Request 1 +# The data is available in the 'polytope_data' variable + +import json +import numpy as np +import covjsonkit +import earthkit.plots + +from covjsonkit.api import Covjsonkit + +decoder = Covjsonkit().decode(polytope_data) + +ds = decoder.to_xarray() + +print(ds) + +# Handle missing/masked values +if '2t' in ds: + data = ds['2t'] + # Replace NaN with a fill value or drop them + data_filled = data.where(~np.isnan(data), drop=True) + + chart = earthkit.plots.Map(domain="Germany") + chart.point_cloud( + data_filled, + x="longitude", + y="latitude", + auto_style=True + ) + + chart.coastlines() + chart.borders() + chart.gridlines() + + chart.title("{variable_name} (number={number})") + + chart.legend() +else: + print("Variable '2t' not found in dataset") + print("Available variables:", list(ds.data_vars)) + +# chart.show() # Not needed - figure is captured automatically +`; +} + +async function openInNotebook(jsonData, requestIdx) { + const notebookSection = document.getElementById('notebook-section'); + + // Show notebook section + notebookSection.style.display = 'block'; + notebookSection.scrollIntoView({ behavior: 'smooth', block: 'start' }); + + // Store data globally + currentNotebookData = jsonData; + window.currentNotebookRequestIdx = requestIdx; + + // Initialize code editor if not already done + if (!codeEditor) { + initCodeEditor(); + } + + // Update the default code with the request index + const defaultCode = `# Polytope Data Visualization - Request ${requestIdx + 1} +# The data is available in the 'polytope_data' variable + +import json +import numpy as np +import covjsonkit +import earthkit.plots + +from covjsonkit.api import Covjsonkit + +decoder = Covjsonkit().decode(polytope_data) + +ds = decoder.to_xarray() + +print(ds) + +# Handle missing/masked values +if '2t' in ds: + data = ds['2t'] + # Replace NaN with a fill value or drop them + data_filled = data.where(~np.isnan(data), drop=True) + + chart = earthkit.plots.Map(domain="Germany") + chart.point_cloud( + data_filled, + x="longitude", + y="latitude", + auto_style=True + ) + + chart.coastlines() + chart.borders() + chart.gridlines() + + chart.title("{variable_name} (number={number})") + + chart.legend() +else: + print("Variable '2t' not found in dataset") + print("Available variables:", list(ds.data_vars)) + +# chart.show() # Not needed - figure is captured automatically +`; + + codeEditor.setValue(defaultCode); +} + +async function runPythonCode() { + const runBtn = document.getElementById('run-code-btn'); + const runBtnText = document.getElementById('run-code-text'); + const outputDiv = document.getElementById('notebook-output'); + const outputContent = document.getElementById('output-content'); + const outputImages = document.getElementById('output-images'); + const loadingDiv = document.getElementById('notebook-loading-exec'); + + if (!currentNotebookData) { + outputContent.textContent = 'Error: No data available. Please query Polytope first.'; + outputDiv.style.display = 'block'; + return; + } + + // Disable button and show loading + runBtn.disabled = true; + runBtnText.textContent = 'Executing...'; + loadingDiv.style.display = 'flex'; + outputDiv.style.display = 'none'; + + try { + // Get code from editor + const code = codeEditor.getValue(); + + // Send code to server for execution + const response = await fetch('/api/v2/execute', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + code: code, + data: currentNotebookData + }) + }); + + const result = await response.json(); + + if (result.success) { + // Display output from stdout and stderr + let output = result.stdout || ''; + if (result.stderr) { + output += '\n\nErrors/Warnings:\n' + result.stderr; + } + outputContent.textContent = output || '(No output)'; + + // Display images if any + outputImages.innerHTML = ''; + if (result.images && result.images.length > 0) { + result.images.forEach((imgBase64, idx) => { + const img = document.createElement('img'); + img.src = `data:image/png;base64,${imgBase64}`; + img.alt = `Plot ${idx + 1}`; + img.className = 'output-image'; + outputImages.appendChild(img); + }); + } + } else { + // Display error + outputContent.textContent = `Error (${result.error_type || 'Error'}): ${result.error}`; + outputImages.innerHTML = ''; + } + + outputDiv.style.display = 'block'; + loadingDiv.style.display = 'none'; + runBtnText.textContent = 'Run Code'; + runBtn.disabled = false; + + } catch (error) { + console.error('Python execution error:', error); + outputContent.textContent = `Error communicating with server: ${error.message}`; + outputImages.innerHTML = ''; + outputDiv.style.display = 'block'; + loadingDiv.style.display = 'none'; + runBtnText.textContent = 'Run Code'; + runBtn.disabled = false; + } +} + +function resetCode() { + if (codeEditor) { + const requestIdx = window.currentNotebookRequestIdx || 0; + const defaultCode = `# Polytope Data Visualization - Request ${requestIdx + 1} +# The data is available in the 'polytope_data' variable + +import json +import numpy as np +import covjsonkit +import earthkit.plots + +from covjsonkit.api import Covjsonkit + +decoder = Covjsonkit().decode(polytope_data) + +ds = decoder.to_xarray() + +print(ds) + +# Handle missing/masked values +if '2t' in ds: + data = ds['2t'] + # Replace NaN with a fill value or drop them + data_filled = data.where(~np.isnan(data), drop=True) + + chart = earthkit.plots.Map(domain="Germany") + chart.point_cloud( + data_filled, + x="longitude", + y="latitude", + auto_style=True + ) + + chart.coastlines() + chart.borders() + chart.gridlines() + + chart.title("{variable_name} (number={number})") + + chart.legend() +else: + print("Variable '2t' not found in dataset") + print("Available variables:", list(ds.data_vars)) + +# chart.show() # Not needed - figure is captured automatically +`; + codeEditor.setValue(defaultCode); + } + + // Clear output + const outputDiv = document.getElementById('notebook-output'); + outputDiv.style.display = 'none'; +} + +function closeNotebook() { + const notebookSection = document.getElementById('notebook-section'); + notebookSection.style.display = 'none'; + + // Clear output + const outputDiv = document.getElementById('notebook-output'); + outputDiv.style.display = 'none'; +} + // Call initializeViewer on page load initializeViewer(); @@ -974,4 +1278,22 @@ document.addEventListener("DOMContentLoaded", () => { if (polytopeBtn) { polytopeBtn.addEventListener('click', queryPolytope); } + + // Add event listener for close notebook button + const closeNotebookBtn = document.getElementById('close-notebook-btn'); + if (closeNotebookBtn) { + closeNotebookBtn.addEventListener('click', closeNotebook); + } + + // Add event listener for run code button + const runCodeBtn = document.getElementById('run-code-btn'); + if (runCodeBtn) { + runCodeBtn.addEventListener('click', runPythonCode); + } + + // Add event listener for reset code button + const resetCodeBtn = document.getElementById('reset-code-btn'); + if (resetCodeBtn) { + resetCodeBtn.addEventListener('click', resetCode); + } }); diff --git a/stac_server/static/styles.css b/stac_server/static/styles.css index 31f2c14..7d33642 100644 --- a/stac_server/static/styles.css +++ b/stac_server/static/styles.css @@ -514,7 +514,9 @@ body { flex: 1; padding: 2rem; overflow-y: auto; + overflow-x: hidden; background: var(--bg-secondary); + max-width: 100%; } .detail-section { @@ -1396,3 +1398,352 @@ canvas { .download-json-btn:active { transform: translateY(0); } +/* ============================================ + JupyterLite Notebook Section + ============================================ */ + +.notebook-section { + background: var(--bg-primary); + padding: 2rem; + border-radius: var(--radius-lg); + border: 1px solid var(--border-light); + margin-bottom: 2rem; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); +} + +.notebook-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 1rem; +} + +.notebook-title { + font-size: 1.3rem; + font-weight: 600; + color: var(--text-primary); + display: flex; + align-items: center; + gap: 0.75rem; + margin: 0; +} + +.notebook-title svg { + color: var(--primary-color); +} + +.close-notebook-btn { + background: transparent; + border: 1px solid var(--border-light); + color: var(--text-secondary); + padding: 0.5rem; + border-radius: var(--radius-sm); + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s ease; +} + +.close-notebook-btn:hover { + background: var(--bg-secondary); + color: var(--text-primary); + border-color: var(--primary-color); +} + +.notebook-description { + color: var(--text-secondary); + margin-bottom: 1.5rem; + line-height: 1.6; +} + +.notebook-description code { + background: var(--bg-secondary); + padding: 0.2rem 0.5rem; + border-radius: 3px; + font-size: 0.9em; + color: var(--primary-color); + font-family: 'Monaco', 'Courier New', monospace; +} + +.notebook-controls { + display: flex; + gap: 0.75rem; + margin-bottom: 1rem; +} + +.run-code-btn, +.reset-code-btn { + padding: 0.6rem 1.2rem; + border: none; + border-radius: var(--radius-md); + cursor: pointer; + font-size: 0.9rem; + font-weight: 500; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s ease; +} + +.run-code-btn { + background: linear-gradient(135deg, var(--primary-color), var(--secondary-color)); + color: white; +} + +.run-code-btn:hover { + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(0, 115, 230, 0.4); +} + +.run-code-btn:disabled { + opacity: 0.6; + cursor: not-allowed; + transform: none; +} + +.reset-code-btn { + background: var(--bg-secondary); + color: var(--text-primary); + border: 1px solid var(--border-light); +} + +.reset-code-btn:hover { + background: var(--bg-tertiary); + border-color: var(--primary-color); +} + +.package-installer { + background: var(--bg-secondary); + border: 1px solid var(--border-light); + border-radius: var(--radius-md); + padding: 1rem; + margin-bottom: 1rem; +} + +.package-installer-title { + font-size: 0.95rem; + font-weight: 600; + color: var(--text-primary); + margin: 0 0 0.5rem 0; + display: flex; + align-items: center; + gap: 0.5rem; +} + +.package-installer-title svg { + color: var(--primary-color); +} + +.package-installer-hint { + font-size: 0.85rem; + color: var(--text-secondary); + margin: 0 0 0.75rem 0; + line-height: 1.4; +} + +.package-installer-controls { + display: flex; + gap: 0.5rem; + align-items: stretch; +} + +.package-input { + flex: 1; + padding: 0.6rem; + border: 1px solid var(--border-light); + border-radius: var(--radius-md); + background: var(--bg-primary); + color: var(--text-primary); + font-size: 0.9rem; + font-family: 'Monaco', 'Courier New', monospace; + transition: border-color 0.2s ease; +} + +.package-input:focus { + outline: none; + border-color: var(--primary-color); + box-shadow: 0 0 0 3px rgba(0, 115, 230, 0.1); +} + +.package-input::placeholder { + color: var(--text-secondary); + opacity: 0.6; +} + +.install-package-btn { + padding: 0.6rem 1.2rem; + background: linear-gradient(135deg, var(--primary-color), var(--secondary-color)); + color: white; + border: none; + border-radius: var(--radius-md); + cursor: pointer; + font-size: 0.9rem; + font-weight: 500; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s ease; + white-space: nowrap; +} + +.install-package-btn:hover { + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(0, 115, 230, 0.4); +} + +.install-package-btn:disabled { + opacity: 0.6; + cursor: not-allowed; + transform: none; +} + +.package-install-status { + margin-top: 0.75rem; + padding: 0.75rem; + border-radius: var(--radius-sm); + font-size: 0.85rem; + font-family: 'Monaco', 'Courier New', monospace; + line-height: 1.5; +} + +.package-install-status.loading { + background: rgba(0, 115, 230, 0.1); + border: 1px solid rgba(0, 115, 230, 0.2); + color: var(--primary-color); +} + +.package-install-status.success { + background: rgba(40, 167, 69, 0.1); + border: 1px solid rgba(40, 167, 69, 0.2); + color: #28a745; +} + +.package-install-status.error { + background: rgba(220, 53, 69, 0.1); + border: 1px solid rgba(220, 53, 69, 0.2); + color: #dc3545; +} + +.code-editor { + border: 1px solid var(--border-light); + border-radius: var(--radius-md); + overflow: hidden; + margin-bottom: 1rem; +} + +.CodeMirror { + height: 400px; + font-size: 14px; + font-family: 'Monaco', 'Courier New', monospace; +} + +.notebook-output { + background: var(--bg-secondary); + border: 1px solid var(--border-light); + border-radius: var(--radius-md); + padding: 1rem; + margin-top: 1rem; +} + +.output-title { + font-size: 0.9rem; + font-weight: 600; + color: var(--text-primary); + margin: 0 0 0.75rem 0; +} + +.output-content { + margin: 0; + padding: 0.75rem; + background: #1e1e1e; + color: #d4d4d4; + border-radius: var(--radius-sm); + font-family: 'Monaco', 'Courier New', monospace; + font-size: 0.85rem; + line-height: 1.5; + overflow-x: auto; + white-space: pre-wrap; + word-wrap: break-word; +} + +.output-images { + display: flex; + flex-direction: column; + gap: 1rem; + margin-top: 1rem; +} + +.output-image { + max-width: 100%; + height: auto; + border-radius: var(--radius-md); + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); + background: white; + display: block; +} + +.notebook-loading-exec { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 1rem; + background: var(--bg-secondary); + border-radius: var(--radius-md); + color: var(--text-secondary); + margin-top: 1rem; +} + +.loading-spinner-small { + width: 20px; + height: 20px; + border: 3px solid var(--border-light); + border-top: 3px solid var(--primary-color); + border-radius: 50%; + animation: spin 1s linear infinite; +} + +.loading-spinner { + width: 40px; + height: 40px; + border: 4px solid var(--border-light); + border-top: 4px solid var(--primary-color); + border-radius: 50%; + animation: spin 1s linear infinite; + margin-bottom: 1rem; +} + +@keyframes spin { + 0% { transform: rotate(0deg); } + 100% { transform: rotate(360deg); } +} + +.open-notebook-btn { + margin-top: 0.5rem; + padding: 0.5rem 1rem; + background: linear-gradient(135deg, var(--primary-color), var(--secondary-color)); + color: white; + border: none; + border-radius: var(--radius-md); + cursor: pointer; + font-size: 0.9rem; + font-weight: 500; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s ease; +} + +.open-notebook-btn:hover { + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(0, 115, 230, 0.4); +} + +.open-notebook-btn:active { + transform: translateY(0); +} + +.open-notebook-btn svg { + width: 16px; + height: 16px; +} diff --git a/stac_server/templates/index.html b/stac_server/templates/index.html index 8aeb9a8..2dab2f6 100644 --- a/stac_server/templates/index.html +++ b/stac_server/templates/index.html @@ -25,6 +25,12 @@ + + + + + +