feat(firestore-multimodal-genai) added support for response_mime_type

firestore-multimodal-genai/extension.yaml

@@ -268,6 +268,22 @@ params:
     required: false
     immutable: false
 
+  - param: RESPONSE_MIME_TYPE
+    label: Output response mimetype of the generated candidate text.
+    description: >-
+      Supported mimetype:
+      - text/plain: (default) Text output.
+      - application/json: JSON response in the candidates.
+      The model needs to be prompted to output the appropriate response type, otherwise the behavior is undefined.
+    type: select
+    options:
+      - label: JSON
+        value: application/json
+      - label: Text
+        value: text/plain
+    default: text/plain
+    required: false
+
   - param: CANDIDATE_COUNT
     label: Candidate count
     description: >-

firestore-multimodal-genai/functions/package.json

@@ -14,7 +14,7 @@
     "@google-ai/generativelanguage": "^2.3.0",
     "@google-cloud/aiplatform": "^3.19.0",
     "@google-cloud/vertexai": "^1.1.0",
-    "@google/generative-ai": "^0.8.0",
+    "@google/generative-ai": "^0.11.4",
     "@types/jest": "^29.5.12",
     "@types/mustache": "^4.2.2",
     "firebase-admin": "^12.1.0",

firestore-multimodal-genai/functions/src/config.ts

@@ -126,4 +126,5 @@ export default {
   safetySettings: getSafetySettings(),
   bucketName: process.env.BUCKET_NAME || defaultBucketName,
   imageField: process.env.IMAGE_FIELD,
+  responseMimeType: process.env.RESPONSE_MIME_TYPE || 'text/plain',
 };

firestore-multimodal-genai/functions/src/generative-client/generative_ai.ts

@@ -66,6 +66,7 @@ export class GeminiGenerativeClient extends GenerativeClient<
           temperature: options.temperature,
           candidateCount: options.candidateCount,
           maxOutputTokens: options.maxOutputTokens,
+          responseMimeType: config.responseMimeType,
         },
         safetySettings: options.safetySettings,
       });