VLM integration via Ollama added

.env

@@ -37,3 +37,4 @@ AWS_S3_BUCKET_NAME=
 #SERVER_TIMEOUT=120000
 #SERVER_HEADERS_TIMEOUT=60000
 #SERVER_KEEP_ALIVE_TIMEOUT=5000
+OLLAMA_BASE_URL=http://localhost:11434

.github/workflows/workflow.yml

@@ -33,6 +33,9 @@ jobs:
       - name: Run acceptance tests
         run: npm run test:acceptance
 
+      - name: Wait for services to be ready with migrations
+        run: sleep 30
+
       - name: Run e2e tests
         run: npm run test:e2e
 

docker-compose.yml

@@ -16,10 +16,13 @@ services:
       APP_FRONTEND_URL: ${APP_FRONTEND_URL}
       BODY_PARSER_JSON_LIMIT: ${BODY_PARSER_JSON_LIMIT}
       ELASTIC_URL: ${ELASTIC_URL}
+      OLLAMA_BASE_URL: http://host.docker.internal:11434
     ports:
       - "${APP_PORT}:3000"
     expose:
       - "${APP_PORT}"
+    extra_hosts:
+      - host.docker.internal:host-gateway
     depends_on:
       postgres:
         condition: service_healthy

prisma/Dockerfile

@@ -12,6 +12,4 @@ RUN npm ci --verbose
 RUN chmod +x /app/wait-for-it.sh
 RUN chmod +x /app/entrypoint.sh
 
-ENTRYPOINT ["/app/entrypoint.sh"]
-
-CMD ["sh"]
+ENTRYPOINT ["/app/entrypoint.sh"]

prisma/migrations/20251209181002_add_vlm_support/migration.sql

@@ -0,0 +1,6 @@
+-- AlterEnum
+ALTER TYPE "ImageComparison" ADD VALUE 'vlm';
+
+-- AlterTable
+ALTER TABLE "TestRun" ADD COLUMN "vlmDescription" TEXT;
+

prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
-provider = "postgresql"
+# It should be added in your version-control system (e.g., Git)
+provider = "postgresql"

prisma/schema.prisma

@@ -69,6 +69,7 @@ model TestRun {
   baselineBranchName    String?
   ignoreAreas           String         @default("[]")
   tempIgnoreAreas       String         @default("[]")
+  vlmDescription        String?
   baseline              Baseline?
   build                 Build          @relation(fields: [buildId], references: [id])
   project               Project?       @relation(fields: [projectId], references: [id])
@@ -138,6 +139,7 @@ enum ImageComparison {
   pixelmatch
   lookSame
   odiff
+  vlm
 }
 
 enum Role {

src/_data_/index.ts

@@ -96,6 +96,7 @@ export const generateTestRun = (testRun?: Partial<TestRun>): TestRun => {
     baselineBranchName: 'master',
     branchName: 'develop',
     merge: false,
+    vlmDescription: null,
     ...testRun,
   };
 };

src/compare/compare.module.ts

@@ -3,10 +3,14 @@ import { CompareService } from './compare.service';
 import { LookSameService } from './libs/looks-same/looks-same.service';
 import { OdiffService } from './libs/odiff/odiff.service';
 import { PixelmatchService } from './libs/pixelmatch/pixelmatch.service';
+import { VlmService } from './libs/vlm/vlm.service';
+import { OllamaController } from './libs/vlm/ollama.controller';
+import { OllamaService } from './libs/vlm/ollama.service';
 import { StaticModule } from '../static/static.module';
 
 @Module({
-  providers: [CompareService, PixelmatchService, LookSameService, OdiffService],
+  controllers: [OllamaController],
+  providers: [CompareService, PixelmatchService, LookSameService, OdiffService, VlmService, OllamaService],
   imports: [StaticModule],
   exports: [CompareService],
 })

src/compare/compare.service.spec.ts

@@ -1,9 +1,12 @@
 import { Test, TestingModule } from '@nestjs/testing';
+import { ConfigService } from '@nestjs/config';
 import { PrismaService } from '../prisma/prisma.service';
 import { CompareService } from './compare.service';
 import { LookSameService } from './libs/looks-same/looks-same.service';
 import { OdiffService } from './libs/odiff/odiff.service';
 import { PixelmatchService } from './libs/pixelmatch/pixelmatch.service';
+import { VlmService } from './libs/vlm/vlm.service';
+import { OllamaService } from './libs/vlm/ollama.service';
 import { StaticModule } from '../static/static.module';
 import { ImageComparison } from '@prisma/client';
 import * as utils from '../static/utils';
@@ -16,7 +19,21 @@ describe('CompareService', () => {
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [CompareService, OdiffService, PixelmatchService, LookSameService, PrismaService],
+      providers: [
+        CompareService,
+        OdiffService,
+        PixelmatchService,
+        LookSameService,
+        VlmService,
+        OllamaService,
+        PrismaService,
+        {
+          provide: ConfigService,
+          useValue: {
+            getOrThrow: jest.fn().mockReturnValue('http://localhost:11434'),
+          },
+        },
+      ],
       imports: [StaticModule],
     }).compile();
 

src/compare/compare.service.ts

@@ -7,6 +7,7 @@ import { PrismaService } from '../prisma/prisma.service';
 import { DiffResult } from '../test-runs/diffResult';
 import { LookSameService } from './libs/looks-same/looks-same.service';
 import { OdiffService } from './libs/odiff/odiff.service';
+import { VlmService } from './libs/vlm/vlm.service';
 import { isHddStaticServiceConfigured } from '../static/utils';
 
 @Injectable()
@@ -17,6 +18,7 @@ export class CompareService {
     private readonly pixelmatchService: PixelmatchService,
     private readonly lookSameService: LookSameService,
     private readonly odiffService: OdiffService,
+    private readonly vlmService: VlmService,
     private readonly prismaService: PrismaService
   ) {}
 
@@ -44,6 +46,9 @@ export class CompareService {
 
         return this.odiffService;
       }
+      case ImageComparison.vlm: {
+        return this.vlmService;
+      }
       default: {
         this.logger.warn(`Unknown ImageComparison value: ${imageComparison}. Falling back to pixelmatch.`);
         return this.pixelmatchService;

src/compare/libs/image-comparator.interface.ts

@@ -3,8 +3,9 @@ import { ImageCompareInput } from './ImageCompareInput';
 import { LooksSameConfig } from './looks-same/looks-same.types';
 import { OdiffConfig } from './odiff/odiff.types';
 import { PixelmatchConfig } from './pixelmatch/pixelmatch.types';
+import { VlmConfig } from './vlm/vlm.types';
 
-export type ImageCompareConfig = PixelmatchConfig | LooksSameConfig | OdiffConfig;
+export type ImageCompareConfig = PixelmatchConfig | LooksSameConfig | OdiffConfig | VlmConfig;
 
 export interface ImageComparator {
   getDiff(data: ImageCompareInput, config: ImageCompareConfig): Promise<DiffResult>;

src/compare/libs/vlm/README.md

@@ -0,0 +1,96 @@
+# VLM (Vision Language Model) Image Comparison
+
+AI-powered semantic image comparison using Vision Language Models via Ollama.
+
+## Quick Start
+
+### 1. Install & Start Ollama
+
+```bash
+# Install (macOS)
+brew install ollama
+
+# Start Ollama
+ollama serve
+```
+
+### 2. Download a Model
+
+```bash
+# Recommended for accuracy
+ollama pull llava:7b
+
+# Or for speed (smaller, less accurate)
+ollama pull moondream
+```
+
+### 3. Configure Backend
+
+Add to `.env`:
+```bash
+OLLAMA_BASE_URL=http://localhost:11434
+```
+
+### 4. Use VLM in Project
+
+Set project's image comparison to `vlm` with config:
+```json
+{
+  "model": "llava:7b",
+  "temperature": 0.1
+}
+```
+
+Optional custom prompt (replaces default system prompt):
+```json
+{
+  "model": "llava:7b",
+  "prompt": "Focus on button colors and text changes",
+  "temperature": 0.1
+}
+```
+
+**Note:** The `prompt` field replaces the entire system prompt. If omitted, a default system prompt is used that focuses on semantic differences while ignoring rendering artifacts.
+
+## Recommended Models
+
+| Model | Size | Speed | Accuracy | Best For |
+|-------|------|-------|----------|----------|
+| `llava:7b` | 4.7GB | ⚡⚡ | ⭐⭐⭐ | **Recommended** - best balance (minimal) |
+| `qwen3-vl:8b` | ~8GB | ⚡⚡ | ⭐⭐⭐ | Minimal model option |
+| `gemma3:latest` | ~ | ⚡⚡ | ⭐⭐⭐ | Minimal model option |
+| `llava:13b` | 8GB | ⚡ | ⭐⭐⭐⭐ | Best accuracy |
+| `moondream` | 1.7GB | ⚡⚡⚡ | ⭐⭐ | Fast, may hallucinate |
+| `minicpm-v` | 5.5GB | ⚡⚡ | ⭐⭐⭐ | Good alternative |
+
+## Configuration
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `model` | string | `llava:7b` | Ollama vision model name |
+| `prompt` | string | System prompt (see below) | Custom prompt for image comparison |
+| `temperature` | number | `0.1` | Lower = more consistent results (0.0-1.0) |
+
+## How It Works
+
+1. VLM analyzes both images semantically
+2. Returns JSON with `{"identical": true/false, "description": "..."}` 
+3. `identical: true` = images match (pass), `identical: false` = differences found (fail)
+4. Ignores technical differences (anti-aliasing, shadows, 1-2px shifts)
+5. Provides description of differences found
+
+### Default System Prompt
+
+The default prompt instructs the model to:
+- **CHECK** for: data changes, missing/added elements, state changes, structural differences
+- **IGNORE**: rendering artifacts, anti-aliasing, shadows, minor pixel shifts
+
+## API Endpoints
+
+```bash
+# List available models
+GET /ollama/models
+
+# Compare two images (for testing)
+POST /ollama/compare?model=llava:7b&prompt=<prompt>&temperature=0.1
+```

src/compare/libs/vlm/ollama.controller.ts

@@ -0,0 +1,59 @@
+import {
+  Controller,
+  Get,
+  Post,
+  Query,
+  HttpException,
+  HttpStatus,
+  UseInterceptors,
+  UploadedFiles,
+} from '@nestjs/common';
+import { FilesInterceptor } from '@nestjs/platform-express';
+import { ApiTags, ApiConsumes, ApiBody } from '@nestjs/swagger';
+import { OllamaService } from './ollama.service';
+
+@ApiTags('Ollama')
+@Controller('ollama')
+export class OllamaController {
+  constructor(private readonly ollamaService: OllamaService) {}
+
+  @Get('models')
+  async listModels() {
+    return { models: await this.ollamaService.listModels() };
+  }
+
+  @Post('compare')
+  @ApiConsumes('multipart/form-data')
+  @ApiBody({
+    schema: {
+      type: 'object',
+      required: ['images'],
+      properties: {
+        images: {
+          type: 'array',
+          items: { type: 'string', format: 'binary' },
+          description: 'Two images to compare (baseline and comparison)',
+        },
+      },
+    },
+  })
+  @UseInterceptors(FilesInterceptor('images', 2))
+  async compareImages(
+    @UploadedFiles() files: Express.Multer.File[],
+    @Query('model') model: string,
+    @Query('prompt') prompt: string,
+    @Query('temperature') temperature: string
+  ) {
+    if (files?.length !== 2) {
+      throw new HttpException('Two images required', HttpStatus.BAD_REQUEST);
+    }
+
+    return this.ollamaService.generate({
+      model,
+      prompt,
+      format: 'json',
+      images: files.map((f) => f.buffer.toString('base64')),
+      options: { temperature: Number(temperature) },
+    });
+  }
+}