feat(backend/shared): upgrade Copilot SDK, image processing, and agent capabilities

apps/backend/package.json

@@ -50,10 +50,10 @@
   },
   "dependencies": {
     "@fastify/cors": "^10.0.2",
-    "@github/copilot-sdk": "^0.1.0",
+    "@github/copilot-sdk": "^0.1.28",
     "better-sqlite3": "^11.7.0",
     "drizzle-orm": "^0.38.4",
-    "fastify": "^5.2.1",
+    "fastify": "^5.7.4",
     "pino": "^9.6.0",
     "pino-pretty": "^13.0.0",
     "sharp": "^0.34.5",

apps/backend/src/native/host.ts

@@ -254,10 +254,14 @@ class NativeMessagingHost {
           break;
         }
 
-        // Process message asynchronously
-        this.handleMessage(message).catch((err) => {
-          this.log(`Error handling message: ${err}`);
-        });
+        // Process message asynchronously but catch synchronous errors from the promise initialization
+        try {
+          this.handleMessage(message).catch((err) => {
+            this.log(`Error handling message: ${err}`);
+          });
+        } catch (handleErr) {
+          this.log(`Error initializing message handler: ${handleErr}`);
+        }
       } catch (error) {
         this.log(`Error reading message: ${error}`);
         break;

apps/backend/src/routes/chat.ts

@@ -112,20 +112,33 @@ const fullContextSchema = z.object({
   }),
 }).passthrough();
 
-// Schema for image payload
+// Schema for image payload (inline data URL - legacy / small images)
 const imagePayloadSchema = z.object({
   id: z.string(),
   dataUrl: z.string(),
   mimeType: z.enum(['image/png', 'image/jpeg', 'image/webp']),
   source: z.enum(['screenshot', 'paste', 'drop']),
 });
 
+// Schema for pre-uploaded image reference (from /api/images/upload endpoint)
+const preUploadedImageSchema = z.object({
+  id: z.string(),
+  fullImagePath: z.string(),
+  mimeType: z.string(),
+  thumbnailUrl: z.string().optional(),
+  fullImageUrl: z.string().optional(),
+  dimensions: z.object({ width: z.number(), height: z.number() }).optional(),
+  fileSize: z.number().optional(),
+});
+
 const sendMessageSchema = z.object({
   prompt: z.string().min(1),
   context: simpleContextSchema.optional(),
   fullContext: fullContextSchema.optional(),
   useContextAwareMode: z.boolean().optional(),
   images: z.array(imagePayloadSchema).max(5).optional(),
+  /** Pre-uploaded image references (already processed on disk) */
+  preUploadedImages: z.array(preUploadedImageSchema).max(5).optional(),
 });
 
 export async function chatRoutes(fastify: FastifyInstance) {
@@ -269,6 +282,7 @@ export async function chatRoutes(fastify: FastifyInstance) {
       // Process images if provided
       let processedImagesRaw: ProcessedImage[] = [];
       let processedImages: ImageAttachment[] = [];
+      let copilotAttachments: Array<{ type: 'file'; path: string; displayName: string }> = [];
       // Build backend URL with port for image serving
       const host = request.headers.host || `${request.hostname}:3847`;
       const backendUrl = `http://${host}`;
@@ -288,9 +302,36 @@ export async function chatRoutes(fastify: FastifyInstance) {
       );
 
       // Process images after we have the message ID
-      if (body.images && body.images.length > 0) {
+      // Prefer pre-uploaded images (already on disk) over inline data URLs
+      if (body.preUploadedImages && body.preUploadedImages.length > 0) {
+        // Images were already processed via /api/images/upload endpoint
+        console.log(`[ChatRoute] Using ${body.preUploadedImages.length} pre-uploaded images`);
+        copilotAttachments = body.preUploadedImages.map((img, index) => ({
+          type: 'file' as const,
+          path: img.fullImagePath,
+          displayName: `image_${index + 1}.${(img.mimeType || 'image/jpeg').split('/')[1] || 'jpg'}`,
+        }));
+
+        // Build metadata for message storage
+        processedImages = body.preUploadedImages.map(img => ({
+          id: img.id,
+          source: 'screenshot' as const,
+          mimeType: (img.mimeType || 'image/jpeg') as any,
+          dimensions: img.dimensions || { width: 0, height: 0 },
+          fileSize: img.fileSize || 0,
+          timestamp: new Date().toISOString(),
+          thumbnailUrl: img.thumbnailUrl,
+          fullImageUrl: img.fullImageUrl,
+        }));
+
+        fastify.sessionService.updateMessageMetadata(userMessage.id, {
+          ...userMessage.metadata,
+          images: processedImages,
+        });
+      } else if (body.images && body.images.length > 0) {
+        // Inline image upload (legacy path for small images)
         try {
-          console.log(`[ChatRoute] Processing ${body.images.length} images for message ${userMessage.id}`);
+          console.log(`[ChatRoute] Processing ${body.images.length} inline images for message ${userMessage.id}`);
           processedImagesRaw = await processMessageImages(
             sessionId,
             userMessage.id,
@@ -305,19 +346,18 @@ export async function chatRoutes(fastify: FastifyInstance) {
             images: processedImages,
           });
 
+          copilotAttachments = processedImagesRaw.map((img, index) => ({
+            type: 'file' as const,
+            path: img.fullImagePath,
+            displayName: `image_${index + 1}.${img.mimeType.split('/')[1] || 'jpg'}`,
+          }));
+
           console.log(`[ChatRoute] Processed ${processedImages.length} images successfully`);
         } catch (err) {
           console.error('[ChatRoute] Failed to process images:', err);
           // Continue without images - don't fail the message
         }
       }
-
-      // Build attachments for Copilot SDK from processed images
-      const copilotAttachments = processedImagesRaw.map((img, index) => ({
-        type: 'file' as const,
-        path: img.fullImagePath,
-        displayName: `image_${index + 1}.${img.mimeType.split('/')[1] || 'jpg'}`,
-      }));
 
       if (copilotAttachments.length > 0) {
         console.log(`[ChatRoute] Built ${copilotAttachments.length} attachments for Copilot:`, copilotAttachments);
@@ -475,6 +515,12 @@ export async function chatRoutes(fastify: FastifyInstance) {
           cleanupTimers();
 
           if (!streamEnded) {
+            if (!reply.raw.headersSent) {
+              reply.raw.setHeader('Content-Type', 'text/event-stream');
+              reply.raw.setHeader('Cache-Control', 'no-cache');
+              reply.raw.setHeader('Connection', 'keep-alive');
+              reply.raw.setHeader('Access-Control-Allow-Origin', '*');
+            }
             sendSSE({
               type: 'error',
               data: {
@@ -516,6 +562,12 @@ export async function chatRoutes(fastify: FastifyInstance) {
       }
 
       // Send error via SSE
+      if (!reply.raw.headersSent) {
+        reply.raw.setHeader('Content-Type', 'text/event-stream');
+        reply.raw.setHeader('Cache-Control', 'no-cache');
+        reply.raw.setHeader('Connection', 'keep-alive');
+        reply.raw.setHeader('Access-Control-Allow-Origin', '*');
+      }
       const errorEvent: StreamEvent = {
         type: 'error',
         data: { error: error instanceof Error ? error.message : 'Unknown error' },

apps/backend/src/routes/images.ts

@@ -2,16 +2,24 @@
  * Images Route
  * 
  * Serves thumbnail and full images stored on disk.
+ * Also provides an upload endpoint for pre-uploading images before chat.
+ * 
  * Routes:
- *   GET /api/images/:sessionId/:messageId/thumb_:index.jpg - Thumbnail
- *   GET /api/images/:sessionId/:messageId/image_:index.:ext - Full image
+ *   GET  /api/images/:sessionId/:messageId/thumb_:index.jpg  - Thumbnail
+ *   GET  /api/images/:sessionId/:messageId/image_:index.:ext  - Full image
+ *   POST /api/images/upload/:sessionId/:messageId             - Pre-upload images
  */
 
 import type { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
+import { z } from 'zod';
 import fs from 'node:fs';
 import path from 'node:path';
 import { IMAGES_DIR } from '../lib/paths.js';
-import { getThumbnailFilePath } from '../services/thumbnail-service.js';
+import {
+  getThumbnailFilePath,
+  processMessageImages,
+  toImageAttachments,
+} from '../services/thumbnail-service.js';
 
 interface ImageParams {
   sessionId: string;
@@ -28,7 +36,89 @@ const MIME_TYPES: Record<string, string> = {
   'gif': 'image/gif',
 };
 
+// Schema for identifying a single image in an upload payload
+const uploadImageSchema = z.object({
+  id: z.string(),
+  dataUrl: z.string(),
+  mimeType: z.enum(['image/png', 'image/jpeg', 'image/webp']),
+  source: z.enum(['screenshot', 'paste', 'drop']),
+});
+
+const uploadBodySchema = z.object({
+  images: z.array(uploadImageSchema).min(1).max(5),
+});
+
 export async function imagesRoutes(fastify: FastifyInstance) {
+  /**
+   * POST /api/images/upload/:sessionId/:messageId
+   * Pre-upload images before sending the chat message.
+   * Returns processed image info (thumbnailUrl, fullImageUrl, fullImagePath).
+   */
+  fastify.post<{
+    Params: { sessionId: string; messageId: string };
+    Body: z.infer<typeof uploadBodySchema>;
+  }>(
+    '/upload/:sessionId/:messageId',
+    async (request, reply) => {
+      try {
+        const { sessionId, messageId } = request.params;
+        const body = uploadBodySchema.parse(request.body);
+
+        // Build backend URL for image serving
+        const host = request.headers.host || `${request.hostname}:3847`;
+        const backendUrl = `http://${host}`;
+
+        console.log(`[ImagesRoute] Pre-uploading ${body.images.length} images for ${sessionId}/${messageId}`);
+
+        const processedImagesRaw = await processMessageImages(
+          sessionId,
+          messageId,
+          body.images,
+          backendUrl
+        );
+
+        const processedImages = toImageAttachments(processedImagesRaw);
+
+        // Return the processed image metadata + absolute paths for Copilot SDK attachments
+        const responseImages = processedImagesRaw.map((img, i) => ({
+          id: img.id,
+          thumbnailUrl: img.thumbnailUrl,
+          fullImageUrl: img.fullImageUrl,
+          fullImagePath: img.fullImagePath,
+          mimeType: img.mimeType,
+          dimensions: img.dimensions,
+          fileSize: img.fileSize,
+        }));
+
+        console.log(`[ImagesRoute] Pre-upload complete: ${responseImages.length} images processed`);
+
+        return reply.send({
+          success: true,
+          data: { images: responseImages },
+        });
+      } catch (error) {
+        if (error instanceof z.ZodError) {
+          return reply.code(400).send({
+            success: false,
+            error: {
+              code: 'VALIDATION_ERROR',
+              message: 'Invalid upload body',
+              details: error.errors,
+            },
+          });
+        }
+        console.error('[ImagesRoute] Upload failed:', error);
+        return reply.code(500).send({
+          success: false,
+          error: {
+            code: 'UPLOAD_ERROR',
+            message: error instanceof Error ? error.message : 'Failed to process images',
+          },
+        });
+      }
+    }
+  );
+
   /**
    * GET /api/images/:sessionId/:messageId/:filename
    * Serves thumbnail or full image

apps/backend/src/server.ts

@@ -30,6 +30,8 @@ function truncate(str: string | undefined | null, maxLen = 500): string {
 
 export async function createServer() {
   const fastify = Fastify({
+    // Allow large payloads for image uploads (data URLs can be 10-30MB for full-page screenshots)
+    bodyLimit: 50 * 1024 * 1024, // 50MB
     logger: {
       level: DEBUG_MODE ? 'debug' : 'info',
       transport: {

apps/backend/src/services/copilot.service.ts

@@ -1,4 +1,4 @@
-import { CopilotClient, type SessionEvent, type Tool as CopilotTool } from '@github/copilot-sdk';
+import { CopilotClient, type SessionEvent, type Tool as CopilotTool, approveAll } from '@github/copilot-sdk';
 import { SessionService } from './session.service.js';
 import { getAgentConfig, SESSION_TYPE_CONFIGS } from '@devmentorai/shared';
 import type {
@@ -360,6 +360,7 @@ export class CopilotService {
       systemMessage: systemPrompt ? { content: systemPrompt } : undefined,
       tools,
       mcpServers,
+      onPermissionRequest: approveAll,
     });
 
     this.sessions.set(sessionId, { sessionId, session, type });
@@ -394,7 +395,7 @@ export class CopilotService {
 
     // First, try to resume existing session
     try {
-      const session = await this.client.resumeSession(sessionId);
+      const session = await this.client.resumeSession(sessionId, { onPermissionRequest: approveAll });
       // Try to get type from DB, fallback to 'general'
       const dbSession = this.sessionService.getSession(sessionId);
       this.sessions.set(sessionId, { sessionId, session, type: dbSession?.type || 'general' });

apps/backend/src/services/thumbnail-service.ts

@@ -28,6 +28,13 @@ const THUMBNAIL_CONFIG = {
   format: 'jpeg' as const,
 };
 
+/** Full image generation settings */
+const FULL_IMAGE_CONFIG = {
+  maxDimension: 2000,
+  quality: 80,
+  format: 'jpeg' as const,
+};
+
 export interface ImageInput {
   id: string;
   dataUrl: string;
@@ -88,6 +95,19 @@ async function generateThumbnail(buffer: Buffer): Promise<Buffer> {
     .toBuffer();
 }
 
+/**
+ * Generate a compressed full-size image from an image buffer
+ */
+async function generateFullImage(buffer: Buffer): Promise<Buffer> {
+  return sharp(buffer)
+    .resize(FULL_IMAGE_CONFIG.maxDimension, FULL_IMAGE_CONFIG.maxDimension, {
+      fit: 'inside',
+      withoutEnlargement: true,
+    })
+    .jpeg({ quality: FULL_IMAGE_CONFIG.quality })
+    .toBuffer();
+}
+
 /**
  * Get the file extension for a MIME type
  */
@@ -156,10 +176,12 @@ export async function processMessageImages(
       fs.writeFileSync(thumbnailAbsPath, thumbnailBuffer);
 
       // Save FULL IMAGE to disk (for Copilot SDK attachments and lightbox view)
-      const extension = getExtensionForMimeType(mimeType);
+      // Resize and compress to avoid huge payloads crashing the backend
+      const fullImageBuffer = await generateFullImage(buffer);
+      const extension = 'jpg'; // Always jpeg as per FULL_IMAGE_CONFIG
       const fullImageAbsPath = getFullImagePath(sessionId, messageId, index, extension);
-      fs.writeFileSync(fullImageAbsPath, buffer);
-      console.log(`[ThumbnailService] Saved full image to ${fullImageAbsPath}`);
+      fs.writeFileSync(fullImageAbsPath, fullImageBuffer);
+      console.log(`[ThumbnailService] Saved compressed full image to ${fullImageAbsPath}`);
 
       // Generate relative paths for DB storage (from DATA_DIR)
       const thumbnailRelativePath = toRelativePath(thumbnailAbsPath);