feat(data): add HEIC/HEIF image support and normalize MIME types (#1127)

Because

- The codebase lacked support for modern HEIC/HEIF image formats which
are widely used on mobile devices
- Users were getting "audio format audio/x-m4a is not supported" errors
despite M4A being supported due to non-standard MIME type variants

This commit

- **Adds comprehensive HEIC/HEIF image format support** using
[libheif](https://github.com/strukturag/libheif) library with full
encode/decode capabilities
- **Implements MIME type normalization** to handle equivalent formats
(e.g., `audio/x-m4a` → `audio/mp4`, `video/mov` → `video/quicktime`)
- **Updates Docker configurations** (both production and development)
with libheif dependencies
- **Adds comprehensive test coverage** for HEIC/HEIF processing and MIME
type normalization

Author: pinglin

.env

@@ -27,5 +27,5 @@ ONNX_MODEL_FOLDER_PATH=${PWD}/pkg/component/resources/onnx
 # that will be rolled back on cleanup, but there might be cases where this
 # isn't possible (e.g. lock tests). We want to keep these queries isolated from
 # the main database.
-TEST_DBHOST=localhost
+TEST_DBHOST=pg_sql
 TEST_DBNAME=pipeline_test

.github/workflows/coverage.yml

@@ -122,13 +122,13 @@ jobs:
 
       - name: Generate coverage report
         env:
-          CFG_DATABASE_HOST: localhost
-          CFG_DATABASE_NAME: pipeline_test
+          TEST_DBHOST: localhost
+          TEST_DBNAME: pipeline_test
           CFG_DATABASE_USERNAME: postgres
           CFG_DATABASE_PASSWORD: password
           CFG_DATABASE_PORT: 5432
         run: |
-          make coverage DBTEST=true OCR=true ONNX=true
+          make coverage DBTEST=true OCR=true ONNX=true DOCKER_NETWORK=host
 
       - name: Upload coverage report
         uses: codecov/codecov-action@v2

.github/workflows/golangci-lint.yml

@@ -25,10 +25,10 @@ jobs:
         with:
           go-version: ${{ env.GOLANG_VERSION }}
           cache: false
-      - name: Install sorx
+      - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y libsoxr-dev
+          sudo apt-get install -y libsoxr-dev libheif-dev libde265-dev libx265-dev libaom-dev libdav1d-dev
       - name: Install onnxruntime library and headers
         run: |
           export ONNXRUNTIME_ROOT_PATH=$GITHUB_WORKSPACE/onnxruntime

Dockerfile

@@ -13,6 +13,12 @@ RUN apt-get update && apt-get install -y \
     libleptonica-dev \
     libtesseract-dev \
     libsoxr-dev \
+    libheif-dev \
+    libde265-dev \
+    libx265-dev \
+    libaom-dev \
+    libdav1d-dev \
+    pkg-config \
     wget \
     jq \
     && rm -rf /var/lib/apt/lists/*
@@ -110,6 +116,7 @@ RUN apt update && \
     libtesseract-dev \
     libreoffice \
     libsoxr-dev \
+    libheif1 \
     chromium \
     qpdf && \
     rm -rf /var/lib/apt/lists/*

Dockerfile.dev

@@ -28,6 +28,13 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     libtesseract-dev \
     libreoffice \
     libsoxr-dev \
+    libde265-dev \
+    libx265-dev \
+    libaom-dev \
+    libdav1d-dev \
+    libheif1 \
+    libheif-dev \
+    pkg-config \
     chromium \
     qpdf \
     jq

Makefile

@@ -8,6 +8,9 @@ export
 
 GOTEST_FLAGS := CFG_DATABASE_HOST=${TEST_DBHOST} CFG_DATABASE_NAME=${TEST_DBNAME}
 
+# Default network for local development, can be overridden
+DOCKER_NETWORK ?= instill-network
+
 #============================================================================
 
 .PHONY: dev
@@ -70,7 +73,14 @@ go-gen: ## Generate codes
 
 .PHONY: dbtest-pre
 dbtest-pre:
-	@${GOTEST_FLAGS} go run ./cmd/migration
+	@docker run --rm \
+		-v $(PWD):/${SERVICE_NAME} \
+		--user $(id -u):$(id -g) \
+		-e GOTEST_FLAGS="${GOTEST_FLAGS}" \
+		--network ${DOCKER_NETWORK} \
+		--entrypoint= \
+		instill/${SERVICE_NAME}:dev \
+			go run ./cmd/migration
 
 .PHONY: coverage
 coverage: ## Generate coverage report
@@ -79,6 +89,7 @@ coverage: ## Generate coverage report
 		-v $(PWD):/${SERVICE_NAME} \
 		--user $(id -u):$(id -g) \
 		-e GOTEST_FLAGS="${GOTEST_FLAGS}" \
+		--network ${DOCKER_NETWORK} \
 		--entrypoint= \
 		instill/${SERVICE_NAME}:dev \
 			go test -v -race ${GOTEST_TAGS} -coverpkg=./... -coverprofile=coverage.out -covermode=atomic -timeout 30m ./...

pkg/data/audio.go

@@ -61,13 +61,15 @@ func NewAudioFromURL(ctx context.Context, binaryFetcher external.BinaryFetcher,
 }
 
 func createAudioData(b []byte, contentType, filename string, isUnified bool) (*audioData, error) {
-	finalContentType := contentType
+	// Normalize MIME type first
+	normalizedContentType := normalizeMIMEType(contentType)
+	finalContentType := normalizedContentType
 
 	// If the audio should be unified, convert it to OGG (the internal unified audio format)
 	if isUnified {
-		if contentType != OGG {
+		if normalizedContentType != OGG {
 			var err error
-			b, err = convertAudio(b, contentType, OGG)
+			b, err = convertAudio(b, normalizedContentType, OGG)
 			if err != nil {
 				return nil, err
 			}

pkg/data/audio_test.go

@@ -28,6 +28,7 @@ func TestNewAudioFromBytes(t *testing.T) {
 		{"Valid AAC audio", "small_sample.aac", "audio/aac", 1.0},
 		{"Valid FLAC audio", "small_sample.flac", "audio/flac", 1.0},
 		{"Valid M4A audio", "small_sample.m4a", "audio/mp4", 1.0},
+		{"Valid M4A audio (non-standard MIME)", "small_sample.m4a", "audio/x-m4a", 1.0},
 		{"Valid WMA audio", "small_sample.wma", "audio/x-ms-wma", 1.0},
 		{"Valid AIFF audio", "small_sample.aiff", "audio/aiff", 1.0},
 		{"Invalid file type", "sample_640_426.png", "", 0.0},
@@ -104,6 +105,7 @@ func TestAudioProperties(t *testing.T) {
 		{"AAC audio", "small_sample.aac", "audio/aac", 1.0},
 		{"FLAC audio", "small_sample.flac", "audio/flac", 1.0},
 		{"M4A audio", "small_sample.m4a", "audio/mp4", 1.0},
+		{"M4A audio (non-standard MIME)", "small_sample.m4a", "audio/x-m4a", 1.0},
 		{"WMA audio", "small_sample.wma", "audio/x-ms-wma", 1.0},
 		{"AIFF audio", "small_sample.aiff", "audio/aiff", 1.0},
 	}
@@ -191,6 +193,7 @@ func TestNewAudioFromBytesUnified(t *testing.T) {
 		{"AAC as unified", "small_sample.aac", "audio/aac", 1.0},
 		{"FLAC as unified", "small_sample.flac", "audio/flac", 1.0},
 		{"M4A as unified", "small_sample.m4a", "audio/mp4", 1.0},
+		{"M4A as unified (non-standard MIME)", "small_sample.m4a", "audio/x-m4a", 1.0},
 		{"WMA as unified", "small_sample.wma", "audio/x-ms-wma", 1.0},
 		{"AIFF as unified", "small_sample.aiff", "audio/aiff", 1.0},
 	}
@@ -206,10 +209,15 @@ func TestNewAudioFromBytesUnified(t *testing.T) {
 			c.Assert(audio.ContentType().String(), qt.Equals, "audio/ogg")
 			c.Assert(audio.Duration().Float64(), qt.CmpEquals(cmpopts.EquateApprox(0, 0.1)), tc.duration)
 
-			// Test as non-unified (should preserve original format)
+			// Test as non-unified (should preserve original format, but normalized)
 			audioOriginal, err := NewAudioFromBytes(audioBytes, tc.contentType, tc.filename, false)
 			c.Assert(err, qt.IsNil)
-			c.Assert(audioOriginal.ContentType().String(), qt.Equals, tc.contentType)
+			expectedContentType := tc.contentType
+			// Handle MIME type normalization for non-standard types
+			if tc.contentType == "audio/x-m4a" {
+				expectedContentType = "audio/mp4"
+			}
+			c.Assert(audioOriginal.ContentType().String(), qt.Equals, expectedContentType)
 			c.Assert(audioOriginal.Duration().Float64(), qt.CmpEquals(cmpopts.EquateApprox(0, 0.1)), tc.duration)
 		})
 	}
@@ -306,3 +314,54 @@ func TestAllSupportedAudioFormats(t *testing.T) {
 		})
 	}
 }
+
+func TestAudioMIMETypeNormalization(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	// Test that audio/x-m4a is properly normalized to audio/mp4
+	c.Run("audio/x-m4a normalization", func(c *qt.C) {
+		audioBytes, err := os.ReadFile("testdata/small_sample.m4a")
+		c.Assert(err, qt.IsNil)
+
+		// Create audio with non-standard MIME type
+		audioXM4A, err := NewAudioFromBytes(audioBytes, "audio/x-m4a", "test.m4a", false)
+		c.Assert(err, qt.IsNil)
+
+		// Create audio with standard MIME type
+		audioMP4, err := NewAudioFromBytes(audioBytes, "audio/mp4", "test.m4a", false)
+		c.Assert(err, qt.IsNil)
+
+		// Both should have the same normalized content type
+		c.Assert(audioXM4A.ContentType().String(), qt.Equals, "audio/mp4")
+		c.Assert(audioMP4.ContentType().String(), qt.Equals, "audio/mp4")
+		c.Assert(audioXM4A.ContentType().String(), qt.Equals, audioMP4.ContentType().String())
+
+		// Both should have the same duration and properties
+		c.Assert(audioXM4A.Duration().Float64(), qt.CmpEquals(cmpopts.EquateApprox(0, 0.1)), audioMP4.Duration().Float64())
+		c.Assert(audioXM4A.SampleRate().Integer(), qt.Equals, audioMP4.SampleRate().Integer())
+	})
+
+	// Test that audio/mp3 is properly normalized to audio/mpeg
+	c.Run("audio/mp3 normalization", func(c *qt.C) {
+		audioBytes, err := os.ReadFile("testdata/small_sample.mp3")
+		c.Assert(err, qt.IsNil)
+
+		// Create audio with non-standard MIME type
+		audioMP3, err := NewAudioFromBytes(audioBytes, "audio/mp3", "test.mp3", false)
+		c.Assert(err, qt.IsNil)
+
+		// Create audio with standard MIME type
+		audioMPEG, err := NewAudioFromBytes(audioBytes, "audio/mpeg", "test.mp3", false)
+		c.Assert(err, qt.IsNil)
+
+		// Both should have the same normalized content type
+		c.Assert(audioMP3.ContentType().String(), qt.Equals, "audio/mpeg")
+		c.Assert(audioMPEG.ContentType().String(), qt.Equals, "audio/mpeg")
+		c.Assert(audioMP3.ContentType().String(), qt.Equals, audioMPEG.ContentType().String())
+
+		// Both should have the same duration and properties
+		c.Assert(audioMP3.Duration().Float64(), qt.CmpEquals(cmpopts.EquateApprox(0, 0.1)), audioMPEG.Duration().Float64())
+		c.Assert(audioMP3.SampleRate().Integer(), qt.Equals, audioMPEG.SampleRate().Integer())
+	})
+}