Add Gemini scanner service for vinyl record metadata extraction

apps/backend/app.ts

@@ -7,6 +7,7 @@ import { dj_route } from './routes/djs.route.js';
 import { flowsheet_route } from './routes/flowsheet.route.js';
 
 import { library_route } from './routes/library.route.js';
+import { scanner_route } from './routes/scanner.route.js';
 import { schedule_route } from './routes/schedule.route.js';
 import { events_route } from './routes/events.route.js';
 import { request_line_route } from './routes/requestLine.route.js';
@@ -36,6 +37,9 @@ const swaggerDoc = parse_yaml(swaggerContent);
 app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDoc));
 
 // Business logic routes
+// Scanner route must be registered before the general library route
+// because /library/scan is a more specific prefix than /library
+app.use('/library/scan', scanner_route);
 app.use('/library', library_route);
 
 app.use('/flowsheet', flowsheet_route);

apps/backend/controllers/scanner.controller.ts

@@ -0,0 +1,98 @@
+/**
+ * Scanner controller for vinyl record image scanning and UPC lookup.
+ */
+
+import { RequestHandler } from 'express';
+import { processImages } from '../services/scanner/processor.js';
+import { ScanContext } from '../services/scanner/types.js';
+import { DiscogsService } from '../services/discogs/discogs.service.js';
+
+/**
+ * POST /library/scan
+ *
+ * Accepts multipart form data with vinyl record images and optional context.
+ * Uses Gemini to extract metadata and attempts catalog matching.
+ *
+ * Form fields:
+ * - images: up to 5 JPEG files (via multer)
+ * - photo_types: JSON string array or comma-separated list of photo type labels
+ * - catalog_item_id: optional known catalog item ID
+ * - sticker_text: optional text from library sticker
+ * - detected_upc: optional UPC from barcode scanner
+ * - artist_name: optional known artist name
+ * - album_title: optional known album title
+ */
+export const scanImages: RequestHandler = async (req, res, next) => {
+  try {
+    const files = req.files as Express.Multer.File[] | undefined;
+    if (!files || files.length === 0) {
+      res.status(400).json({ status: 400, message: 'No images provided' });
+      return;
+    }
+
+    // Parse photo_types from form field
+    let photoTypes: string[] = [];
+    const rawPhotoTypes = req.body.photo_types;
+    if (rawPhotoTypes) {
+      if (typeof rawPhotoTypes === 'string') {
+        try {
+          photoTypes = JSON.parse(rawPhotoTypes);
+        } catch {
+          // Fall back to comma-separated parsing
+          photoTypes = rawPhotoTypes.split(',').map((s: string) => s.trim());
+        }
+      } else if (Array.isArray(rawPhotoTypes)) {
+        photoTypes = rawPhotoTypes;
+      }
+    }
+
+    // Build scan context from optional form fields
+    const context: ScanContext = {};
+    if (req.body.catalog_item_id) {
+      context.catalogItemId = parseInt(req.body.catalog_item_id, 10);
+    }
+    if (req.body.sticker_text) {
+      context.stickerText = req.body.sticker_text;
+    }
+    if (req.body.detected_upc) {
+      context.detectedUPC = req.body.detected_upc;
+    }
+    if (req.body.artist_name) {
+      context.artistName = req.body.artist_name;
+    }
+    if (req.body.album_title) {
+      context.albumTitle = req.body.album_title;
+    }
+
+    const images = files.map((file) => file.buffer);
+    const result = await processImages(images, photoTypes, context);
+
+    res.status(200).json(result);
+  } catch (error) {
+    console.error('Error scanning images:', error);
+    next(error);
+  }
+};
+
+/**
+ * POST /library/scan/upc-lookup
+ *
+ * Looks up a UPC barcode on Discogs to find release information.
+ *
+ * Body: { upc: string }
+ */
+export const upcLookup: RequestHandler = async (req, res, next) => {
+  const { upc } = req.body;
+  if (!upc || typeof upc !== 'string') {
+    res.status(400).json({ status: 400, message: 'Missing or invalid parameter: upc' });
+    return;
+  }
+
+  try {
+    const results = await DiscogsService.searchByBarcode(upc);
+    res.status(200).json(results);
+  } catch (error) {
+    console.error('Error looking up UPC:', error);
+    next(error);
+  }
+};

apps/backend/package.json

@@ -15,6 +15,8 @@
   "author": "AyBruno",
   "license": "MIT",
   "dependencies": {
+    "@google/generative-ai": "^0.24.1",
+    "@types/multer": "^2.0.0",
     "@wxyc/authentication": "*",
     "@wxyc/database": "*",
     "async-mutex": "^0.5.0",
@@ -26,6 +28,7 @@
     "groq-sdk": "^0.5.0",
     "jose": "^6.1.3",
     "lru-cache": "^10.2.0",
+    "multer": "^2.1.0",
     "node-fetch": "^3.3.2",
     "node-ssh": "^13.2.1",
     "postgres": "^3.4.4",

apps/backend/routes/scanner.route.ts

@@ -0,0 +1,24 @@
+/**
+ * Scanner routes for vinyl record image scanning and UPC lookup.
+ */
+
+import { requirePermissions } from '@wxyc/authentication';
+import { Router } from 'express';
+import multer from 'multer';
+import * as scannerController from '../controllers/scanner.controller.js';
+
+export const scanner_route = Router();
+
+const upload = multer({
+  storage: multer.memoryStorage(),
+  limits: { fileSize: 10 * 1024 * 1024 },
+});
+
+scanner_route.post(
+  '/',
+  requirePermissions({ catalog: ['write'] }),
+  upload.array('images', 5),
+  scannerController.scanImages
+);
+
+scanner_route.post('/upc-lookup', requirePermissions({ catalog: ['read'] }), scannerController.upcLookup);

apps/backend/services/discogs/discogs.service.ts

@@ -357,12 +357,20 @@ class DiscogsServiceClass {
     }
   }
 
+  /**
+   * Search for releases by UPC/EAN barcode.
+   */
+  async searchByBarcode(barcode: string): Promise<DiscogsSearchResponse> {
+    const response = await this.search({ barcode, type: 'release' });
+    return response;
+  }
+
   /**
    * Build search params using Discogs-specific fields.
    */
   private buildSearchParams(request: DiscogsSearchRequest, limit: number): Record<string, string | number> {
     const params: Record<string, string | number> = {
-      type: 'release',
+      type: request.type || 'release',
       per_page: limit,
     };
 
@@ -374,6 +382,9 @@ class DiscogsServiceClass {
     } else if (request.track) {
       params.release_title = request.track;
     }
+    if (request.barcode) {
+      params.barcode = request.barcode;
+    }
 
     return params;
   }

apps/backend/services/requestLine/types.ts

@@ -271,6 +271,8 @@ export interface DiscogsSearchRequest {
   artist?: string;
   album?: string;
   track?: string;
+  barcode?: string;
+  type?: string;
 }
 
 /**

apps/backend/services/scanner/gemini.service.ts

@@ -0,0 +1,138 @@
+/**
+ * Gemini AI Service for vinyl record image extraction.
+ *
+ * Uses Google's Gemini Flash model to analyze photos of vinyl records
+ * and extract metadata (label, catalog number, UPC, DJ reviews).
+ *
+ * Follows the singleton pattern from parser.service.ts.
+ */
+
+import { GoogleGenerativeAI } from '@google/generative-ai';
+import { ScanContext, ScanExtraction, ExtractionField } from './types.js';
+import { SCANNER_SYSTEM_PROMPT, buildUserPrompt } from './prompts.js';
+
+/**
+ * Gemini client singleton.
+ */
+let _geminiClient: GoogleGenerativeAI | null = null;
+
+/**
+ * Get or create the Gemini client.
+ */
+function getGeminiClient(): GoogleGenerativeAI {
+  if (!_geminiClient) {
+    const apiKey = process.env.GEMINI_API_KEY;
+    if (!apiKey) {
+      throw new Error('GEMINI_API_KEY is not configured');
+    }
+    _geminiClient = new GoogleGenerativeAI(apiKey);
+  }
+  return _geminiClient;
+}
+
+/**
+ * Reset the Gemini client (useful for testing).
+ */
+export function resetGeminiClient(): void {
+  _geminiClient = null;
+}
+
+/**
+ * Raw response shape from Gemini extraction.
+ */
+interface RawExtractionResponse {
+  label_name?: { value: string; confidence: number };
+  catalog_number?: { value: string; confidence: number };
+  review_text?: { value: string; confidence: number };
+  upc?: { value: string; confidence: number };
+}
+
+/**
+ * Parse a raw field into an ExtractionField, validating structure.
+ */
+function parseField(raw: { value: string; confidence: number } | undefined): ExtractionField | undefined {
+  if (!raw || typeof raw.value !== 'string' || typeof raw.confidence !== 'number') {
+    return undefined;
+  }
+  return {
+    value: raw.value,
+    confidence: Math.max(0, Math.min(1, raw.confidence)),
+  };
+}
+
+/**
+ * Extract metadata from vinyl record images using Gemini Flash.
+ *
+ * @param images - JPEG image buffers to analyze
+ * @param photoTypes - Descriptive labels for each image (e.g., "front_cover", "center_label")
+ * @param context - Optional context about the known album
+ * @returns Extracted metadata fields with confidence scores
+ * @throws Error if Gemini API fails or returns invalid response
+ */
+export async function extractFromImages(
+  images: Buffer[],
+  photoTypes: string[],
+  context: ScanContext
+): Promise<ScanExtraction> {
+  const client = getGeminiClient();
+  const model = client.getGenerativeModel({ model: 'gemini-2.0-flash' });
+
+  console.log(`[Scanner] Extracting metadata from ${images.length} image(s)`);
+
+  const userPrompt = buildUserPrompt(photoTypes, context);
+
+  // Build multimodal content parts: images as inline base64 + text prompt
+  const imageParts = images.map((buffer) => ({
+    inlineData: {
+      mimeType: 'image/jpeg',
+      data: buffer.toString('base64'),
+    },
+  }));
+
+  try {
+    const result = await model.generateContent({
+      contents: [
+        {
+          role: 'user',
+          parts: [{ text: SCANNER_SYSTEM_PROMPT }, ...imageParts, { text: userPrompt }],
+        },
+      ],
+      generationConfig: {
+        responseMimeType: 'application/json',
+        temperature: 0.1,
+      },
+    });
+
+    const response = result.response;
+    const content = response.text();
+    if (!content) {
+      throw new Error('Empty response from Gemini');
+    }
+
+    const parsed: RawExtractionResponse = JSON.parse(content);
+    console.log(`[Scanner] Raw extraction response:`, JSON.stringify(parsed));
+
+    const extraction: ScanExtraction = {};
+
+    const labelName = parseField(parsed.label_name);
+    if (labelName) extraction.labelName = labelName;
+
+    const catalogNumber = parseField(parsed.catalog_number);
+    if (catalogNumber) extraction.catalogNumber = catalogNumber;
+
+    const reviewText = parseField(parsed.review_text);
+    if (reviewText) extraction.reviewText = reviewText;
+
+    const upc = parseField(parsed.upc);
+    if (upc) extraction.upc = upc;
+
+    return extraction;
+  } catch (error) {
+    if (error instanceof SyntaxError) {
+      console.error(`[Scanner] Failed to parse JSON response:`, error);
+      throw new Error(`Invalid JSON response from Gemini: ${error.message}`);
+    }
+    console.error(`[Scanner] Error extracting from images:`, error);
+    throw error;
+  }
+}