Feat/drop support of face recoginition library

.github/workflows/release.yml

@@ -16,7 +16,6 @@ env:
   IMAGE_PREFIX: ${{ github.repository_owner }}/edit-mind
 
 jobs:
-
   build-and-push:
     runs-on: ubuntu-latest
     permissions:
@@ -25,7 +24,7 @@ jobs:
       packages: write
       artifact-metadata: write
       attestations: write
-        
+
     strategy:
       matrix:
         service:
@@ -36,23 +35,26 @@ jobs:
             dockerfile: docker/Dockerfile.web
             context: .
         platform:
+          - linux/amd64
           - linux/arm64
 
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: "node-cleanup"
+      - name: Maximize build space
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf /usr/local/lib/android
           sudo rm -rf /opt/ghc
           sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/.ghcup
           sudo rm -rf /usr/share/swift
           sudo rm -rf /usr/local/share/boost
+          sudo apt-get autoremove -y
           sudo apt-get clean
-          docker system prune -af --volumes
+          sudo docker system prune -a -f
           df -h
+
+      - name: Checkout code
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -73,42 +75,56 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Generate platform tag
-        id: platform-tag
+      - name: Generate platform suffix
+        id: platform
         run: |
-          PLATFORM_TAG=$(echo "${{ matrix.platform }}" | sed 's/\//-/g')
-          echo "tag=${PLATFORM_TAG}" >> $GITHUB_OUTPUT
+          echo "suffix=$(echo ${{ matrix.platform }} | sed 's/\//-/g')" >> $GITHUB_OUTPUT
 
       - name: Extract metadata (tags, labels)
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.service.name }}
           tags: |
-            type=raw,value=latest-${{ steps.platform-tag.outputs.tag }},enable={{is_default_branch}}
-            type=semver,pattern={{version}}-${{ steps.platform-tag.outputs.tag }}
-            type=semver,pattern={{major}}.{{minor}}-${{ steps.platform-tag.outputs.tag }}
-            type=sha,prefix={{branch}}-${{ steps.platform-tag.outputs.tag }}-
-      - name: Build and push Docker image
+            type=ref,event=branch,suffix=-${{ steps.platform.outputs.suffix }}
+            type=semver,pattern={{version}},suffix=-${{ steps.platform.outputs.suffix }}
+            type=semver,pattern={{major}}.{{minor}},suffix=-${{ steps.platform.outputs.suffix }}
+
+      - name: Set up env files
+        run: |
+          cp .env.example .env
+          cp .env.system.example .env.system
+
+      - name: Start ChromaDB (testing)
+        run: docker compose up -d chroma
+
+      - name: Build and Test
         uses: docker/build-push-action@v6
+        with:
+          context: ${{ matrix.service.context }}
+          file: ${{ matrix.service.dockerfile }}
+          target: testing
+          platforms: ${{ matrix.platform }}
+          push: false 
+          cache-from: type=gha,scope=${{ matrix.service.name }}-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.service.name }}-${{ matrix.platform }}
+
+      - name: Build and push Docker image
         id: push
-        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
         with:
           context: ${{ matrix.service.context }}
           file: ${{ matrix.service.dockerfile }}
           target: production
-          platforms: linux/arm64
+          platforms: ${{ matrix.platform }}
           build-args: |
             NODE_VERSION=22.20.0
             PNPM_VERSION=10.20.0
           push: ${{ github.event_name != 'pull_request' }}
-          load: false
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: |
-            type=gha,scope=${{ matrix.service.name }}-${{ matrix.platform }}
-            type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.service.name }}:buildcache-${{ matrix.platform }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.service.name }}-${{ matrix.platform }}
+          cache-from: type=gha,scope=${{ matrix.service }}-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.service }}-${{ matrix.platform }}
           sbom: false
 
       - name: Generate artifact attestation
@@ -119,12 +135,42 @@ jobs:
           subject-digest: ${{ steps.push.outputs.digest }}
           push-to-registry: true
 
-  create-release:
+  create-manifest:
     needs: build-and-push
     runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
     permissions:
-      contents: write
+      packages: write
+    strategy:
+      matrix:
+        service:
+          - name: background-jobs
+          - name: web
+    steps:
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest
+        run: |
+          TAG="${{ github.ref_name }}"
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            TAG="main"
+          fi
+
+          docker buildx imagetools create -t ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.service.name }}:${TAG} \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.service.name }}:${TAG}-linux-amd64 \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-${{ matrix.service.name }}:${TAG}-linux-arm64
+
+  create-release:
+    needs: create-manifest
+    runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/')
+    permissions:
+      contents: write
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -135,7 +181,6 @@ jobs:
           tag_name: ${{ github.ref_name }}
           name: Release ${{ github.ref_name }}
           draft: true
-          prerelease: false
           files: |
             docker-compose.yml
             .env.example

.gitignore

@@ -31,3 +31,5 @@ test-results/*
 .env.dev
 docker/.env.system
 .env.system
+!python/.faces
+*.pkl

apps/background-jobs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "background-jobs",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "private": true,
   "type": "module",
   "scripts": {

apps/background-jobs/src/index.ts

@@ -5,12 +5,13 @@ import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'
 import { ExpressAdapter } from '@bull-board/express'
 import foldersRoute from './routes/folders'
 import stitcherRoute from './routes/stitcher'
+import faceRoute from './routes/face'
 import { config } from './config'
-import { faceMatcherQueue, immichImporterQueue, videoQueue, videoStitcherQueue } from './queue'
+import { immichImporterQueue, videoQueue, videoStitcherQueue } from './queue'
 import './jobs/videoIndexer'
-import './jobs/faceMatcher'
 import './jobs/ImmichImporter'
 import './jobs/videoStitcher'
+import './jobs/faceLabelling'
 
 import { pythonService } from '@shared/services/pythonService'
 import { initializeWatchers } from './watcher'
@@ -28,7 +29,6 @@ if (process.env.NODE_ENV === 'development') {
   createBullBoard({
     queues: [
       new BullMQAdapter(videoQueue),
-      new BullMQAdapter(faceMatcherQueue),
       new BullMQAdapter(immichImporterQueue),
       new BullMQAdapter(videoStitcherQueue),
     ],
@@ -40,6 +40,7 @@ if (process.env.NODE_ENV === 'development') {
 
 app.use('/folders', foldersRoute)
 app.use('/stitcher', stitcherRoute)
+app.use('/face', faceRoute)
 
 app.get('/health', (_req, res) => res.json({ status: 'ok' }))
 

apps/background-jobs/src/jobs/ImmichImporter.ts

@@ -1,10 +1,8 @@
 import { decryptApiKey } from '@shared/services/encryption'
 import { Worker, Job } from 'bullmq'
-import { connection } from '../queue'
+import { connection } from '../services/redis'
 import { ImmichImporterJobData } from '@shared/types/immich'
 import { getAllImmichFaces } from '@shared/services/immich'
-import { reindexFaces } from '@shared/utils/faces'
-import { pythonService } from '@shared/services/pythonService'
 import { prisma } from 'src/services/db'
 
 async function processImmichImporterJob(job: Job<ImmichImporterJobData>) {
@@ -14,9 +12,7 @@ async function processImmichImporterJob(job: Job<ImmichImporterJobData>) {
     })
     if (!integration) throw new Error('Integration not found')
     const apiKey = decryptApiKey(integration.immichApiKey)
-    const facesFiles = await getAllImmichFaces({ baseUrl: integration.immichBaseUrl, apiKey })
-    if (!pythonService.isServiceRunning) await pythonService.start()
-    if (job.id) await reindexFaces(facesFiles, job.id)
+    await getAllImmichFaces({ baseUrl: integration.immichBaseUrl, apiKey })
   } catch (error) {
     console.error(error)
   }

apps/background-jobs/src/jobs/faceLabelling.ts

@@ -0,0 +1,130 @@
+import { Worker, Job } from 'bullmq'
+import { connection } from '../services/redis'
+import { logger } from '@shared/services/logger'
+import { FaceLabellingJobData } from '@shared/types/face'
+import type { FaceDetectionData } from '@shared/types/unknownFace'
+import { promises as fs } from 'fs'
+import { existsSync } from 'fs'
+import path from 'path'
+import type { Scene } from '@shared/types/scene'
+import { FACES_DIR, PROCESSED_VIDEOS_DIR, UNKNOWN_FACES_DIR } from '@shared/constants'
+import { getByVideoSource, updateMetadata } from '@shared/services/vectorDb'
+
+async function processFaceLabellingJob(job: Job<FaceLabellingJobData>) {
+  const { faces, name } = job.data
+  logger.info({ jobId: job.id }, 'Starting Face labelling job')
+
+  const personDir = path.join(FACES_DIR, name)
+  if (!existsSync(personDir)) {
+    await fs.mkdir(personDir, { recursive: true })
+  }
+
+  for (const face of faces) {
+    try {
+      const jsonPath = path.join(UNKNOWN_FACES_DIR, face.jsonFile)
+
+      if (existsSync(jsonPath)) {
+        let faceData: FaceDetectionData
+        try {
+          faceData = JSON.parse(await fs.readFile(jsonPath, 'utf8'))
+        } catch {
+          continue
+        }
+
+        const imageFile = faceData.image_file
+        const srcImagePath = path.join(UNKNOWN_FACES_DIR, imageFile)
+        const destImagePath = path.join(personDir, imageFile)
+
+        const scenes = await getByVideoSource(faceData.video_path)
+        const sortedAppearances = faceData.all_appearances?.sort((a, b) => a.frame_index - b.frame_index)
+
+        if (sortedAppearances && scenes && scenes.length > 0) {
+          const firstAppearance = sortedAppearances[0]
+          const lastAppearance = sortedAppearances[sortedAppearances.length - 1]
+
+          for (const scene of scenes) {
+            // Check if the face appears at any point during the scene
+            const overlapsScene =
+              firstAppearance.timestamp_seconds <= scene.endTime && lastAppearance.timestamp_seconds >= scene.startTime
+
+            if (!overlapsScene) continue
+
+            if (scene.faces.includes(face.faceId)) {
+              scene.faces = scene.faces.map((f) => (f === face.faceId ? name : f))
+            }
+
+            if (scene.facesData) {
+              scene.facesData = scene.facesData.map((f) =>
+                f.name === face.faceId ? { ...f, name, confidence: 100 } : f
+              )
+            }
+
+            await updateMetadata(scene)
+          }
+
+          const videoDir = path.join(PROCESSED_VIDEOS_DIR, path.basename(faceData.video_path))
+          const scenesJsonPath = path.join(videoDir, 'scenes.json')
+
+          if (existsSync(scenesJsonPath)) {
+            const fileScenes: Scene[] = JSON.parse(await fs.readFile(scenesJsonPath, 'utf8'))
+            let modified = false
+
+            for (const scene of fileScenes) {
+              const inRange =
+                scene.startTime <= faceData.last_appearance?.timestamp_seconds &&
+                scene.endTime >= faceData.first_appearance?.timestamp_seconds
+
+              if (!inRange) continue
+
+              let sceneModified = false
+
+              if (scene.faces.includes(face.faceId)) {
+                scene.faces = scene.faces.map((f) => (f === face.faceId ? name : f))
+                sceneModified = true
+              }
+
+              if (scene.facesData) {
+                const hadFace = scene.facesData.some((f) => f.name === face.faceId)
+                scene.facesData = scene.facesData.map((f) =>
+                  f.name === face.faceId ? { ...f, name, confidence: 100 } : f
+                )
+                if (hadFace) sceneModified = true
+              }
+
+              if (sceneModified) modified = true
+            }
+
+            if (modified) {
+              await fs.writeFile(scenesJsonPath, JSON.stringify(fileScenes, null, 2), 'utf8')
+            }
+          }
+        }
+        if (existsSync(srcImagePath)) {
+          await fs.copyFile(srcImagePath, destImagePath)
+          await fs.unlink(srcImagePath)
+        }
+        try {
+          await fs.unlink(jsonPath)
+        } catch (error) {
+          logger.warn(error)
+        }
+      } else {
+        try {
+          const imageFile = face.jsonFile.replace('.json', '.jpg')
+          const srcImagePath = path.join(UNKNOWN_FACES_DIR, imageFile)
+          await fs.unlink(srcImagePath)
+        } catch (error) {
+          logger.warn(error)
+        }
+      }
+    } catch (err) {
+      logger.error('Label error: ' + err)
+    }
+  }
+  logger.info({ jobId: job.id }, 'Face labelling job completed')
+}
+
+export const faceLabellingWorker = new Worker('face-labelling', processFaceLabellingJob, {
+  connection,
+  concurrency: 3,
+})