Stop redundantly encoding binary data as base64 when sending to Google genai SDK (#2962)

Author: DouweM

pydantic_ai_slim/pydantic_ai/models/google.py

@@ -51,13 +51,15 @@
 try:
     from google.genai import Client
     from google.genai.types import (
+        BlobDict,
         CodeExecutionResult,
         CodeExecutionResultDict,
         ContentDict,
         ContentUnionDict,
         CountTokensConfigDict,
         ExecutableCode,
         ExecutableCodeDict,
+        FileDataDict,
         FinishReason as GoogleFinishReason,
         FunctionCallDict,
         FunctionCallingConfigDict,
@@ -79,6 +81,7 @@
         ToolDict,
         ToolListUnionDict,
         UrlContextDict,
+        VideoMetadataDict,
     )
 
     from ..providers.google import GoogleProvider
@@ -525,31 +528,33 @@ async def _map_user_prompt(self, part: UserPromptPart) -> list[PartDict]:
                 if isinstance(item, str):
                     content.append({'text': item})
                 elif isinstance(item, BinaryContent):
-                    # NOTE: The type from Google GenAI is incorrect, it should be `str`, not `bytes`.
-                    base64_encoded = base64.b64encode(item.data).decode('utf-8')
-                    inline_data_dict = {'inline_data': {'data': base64_encoded, 'mime_type': item.media_type}}
+                    inline_data_dict: BlobDict = {'data': item.data, 'mime_type': item.media_type}
+                    part_dict: PartDict = {'inline_data': inline_data_dict}
                     if item.vendor_metadata:
-                        inline_data_dict['video_metadata'] = item.vendor_metadata
-                    content.append(inline_data_dict)  # type: ignore
+                        part_dict['video_metadata'] = cast(VideoMetadataDict, item.vendor_metadata)
+                    content.append(part_dict)
                 elif isinstance(item, VideoUrl) and item.is_youtube:
-                    file_data_dict = {'file_data': {'file_uri': item.url, 'mime_type': item.media_type}}
+                    file_data_dict: FileDataDict = {'file_uri': item.url, 'mime_type': item.media_type}
+                    part_dict: PartDict = {'file_data': file_data_dict}
                     if item.vendor_metadata:  # pragma: no branch
-                        file_data_dict['video_metadata'] = item.vendor_metadata
-                    content.append(file_data_dict)  # type: ignore
+                        part_dict['video_metadata'] = cast(VideoMetadataDict, item.vendor_metadata)
+                    content.append(part_dict)
                 elif isinstance(item, FileUrl):
                     if item.force_download or (
                         # google-gla does not support passing file urls directly, except for youtube videos
                         # (see above) and files uploaded to the file API (which cannot be downloaded anyway)
                         self.system == 'google-gla'
                         and not item.url.startswith(r'https://generativelanguage.googleapis.com/v1beta/files')
                     ):
-                        downloaded_item = await download_item(item, data_format='base64')
-                        inline_data = {'data': downloaded_item['data'], 'mime_type': downloaded_item['data_type']}
-                        content.append({'inline_data': inline_data})  # type: ignore
+                        downloaded_item = await download_item(item, data_format='bytes')
+                        inline_data: BlobDict = {
+                            'data': downloaded_item['data'],
+                            'mime_type': downloaded_item['data_type'],
+                        }
+                        content.append({'inline_data': inline_data})
                     else:
-                        content.append(
-                            {'file_data': {'file_uri': item.url, 'mime_type': item.media_type}}
-                        )  # pragma: lax no cover
+                        file_data_dict: FileDataDict = {'file_uri': item.url, 'mime_type': item.media_type}
+                        content.append({'file_data': file_data_dict})  # pragma: lax no cover
                 else:
                     assert_never(item)
         return content
@@ -827,7 +832,7 @@ def _metadata_as_usage(response: GenerateContentResponse) -> usage.RequestUsage:
         if not metadata_details:
             continue
         for detail in metadata_details:
-            if not detail.modality or not detail.token_count:  # pragma: no cover
+            if not detail.modality or not detail.token_count:
                 continue
             details[f'{detail.modality.lower()}_{prefix}_tokens'] = detail.token_count
             if detail.modality != 'AUDIO':

tests/models/cassettes/test_google/test_google_model_document_url_input.yaml

@@ -251,12 +251,10 @@ interactions:
     headers:
       accept-ranges:
       - bytes
-      age:
-      - '264068'
       alt-svc:
       - h3=":443"; ma=86400
       cache-control:
-      - public, max-age=604800, s-maxage=604800
+      - max-age=21600
       connection:
       - keep-alive
       content-length:
@@ -268,7 +266,7 @@ interactions:
       etag:
       - '"33d0-438b181451e00"'
       expires:
-      - Tue, 24 Jun 2025 13:27:15 GMT
+      - Fri, 19 Sep 2025 22:42:26 GMT
       last-modified:
       - Mon, 27 Aug 2007 17:15:36 GMT
       strict-transport-security:
@@ -312,11 +310,11 @@ interactions:
       alt-svc:
       - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
       content-length:
-      - '776'
+      - '772'
       content-type:
       - application/json; charset=UTF-8
       server-timing:
-      - gfet4t7; dur=1228
+      - gfet4t7; dur=1306
       transfer-encoding:
       - chunked
       vary:
@@ -325,27 +323,27 @@ interactions:
       - Referer
     parsed_body:
       candidates:
-      - avgLogprobs: -0.28572704394658405
+      - avgLogprobs: -0.2349079738963734
         content:
           parts:
           - text: |
-              The document appears to be a "Dummy PDF file".
+              The document appears to be a dummy PDF file.
           role: model
         finishReason: STOP
       modelVersion: gemini-2.0-flash
-      responseId: 4FpeaJWYOLq3nvgP0vasuQk
+      responseId: 8ofNaO_lJsO1qtsP0OzFsQQ
       usageMetadata:
-        candidatesTokenCount: 12
+        candidatesTokenCount: 11
         candidatesTokensDetails:
         - modality: TEXT
-          tokenCount: 12
+          tokenCount: 11
         promptTokenCount: 1305
         promptTokensDetails:
-        - modality: TEXT
-          tokenCount: 15
         - modality: DOCUMENT
           tokenCount: 1290
-        totalTokenCount: 1317
+        - modality: TEXT
+          tokenCount: 15
+        totalTokenCount: 1316
     status:
       code: 200
       message: OK

tests/models/cassettes/test_google/test_google_model_image_as_binary_content_input.yaml

@@ -33,11 +33,11 @@ interactions:
       alt-svc:
       - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
       content-length:
-      - '756'
+      - '754'
       content-type:
       - application/json; charset=UTF-8
       server-timing:
-      - gfet4t7; dur=2659
+      - gfet4t7; dur=1900
       transfer-encoding:
       - chunked
       vary:
@@ -46,25 +46,25 @@ interactions:
       - Referer
     parsed_body:
       candidates:
-      - avgLogprobs: -0.005608726706769731
+      - avgLogprobs: -0.00857612325085534
         content:
           parts:
           - text: The fruit in the image is a kiwi.
           role: model
         finishReason: STOP
       modelVersion: gemini-2.0-flash
-      responseId: 2VpeaPm3DaHp1PIPwK-EmAM
+      responseId: 4YfNaLXqOsKVmtkPqqehuAQ
       usageMetadata:
         candidatesTokenCount: 9
         candidatesTokensDetails:
         - modality: TEXT
           tokenCount: 9
         promptTokenCount: 3367
         promptTokensDetails:
-        - modality: TEXT
-          tokenCount: 13
         - modality: IMAGE
           tokenCount: 3354
+        - modality: TEXT
+          tokenCount: 13
         totalTokenCount: 3376
     status:
       code: 200

tests/models/cassettes/test_google/test_google_model_image_url_input.yaml

@@ -580,7 +580,7 @@ interactions:
       access-control-allow-origin:
       - '*'
       age:
-      - '1386476'
+      - '1500997'
       cache-control:
       - public, max-age=31536000
       connection:
@@ -632,11 +632,11 @@ interactions:
       alt-svc:
       - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
       content-length:
-      - '740'
+      - '738'
       content-type:
       - application/json; charset=UTF-8
       server-timing:
-      - gfet4t7; dur=1424
+      - gfet4t7; dur=867
       transfer-encoding:
       - chunked
       vary:
@@ -645,27 +645,26 @@ interactions:
       - Referer
     parsed_body:
       candidates:
-      - avgLogprobs: -0.1821905771891276
+      - avgLogprobs: -0.18855420351028443
         content:
           parts:
-          - text: |
-              That is a potato.
+          - text: That is a potato.
           role: model
         finishReason: STOP
       modelVersion: gemini-2.0-flash
-      responseId: 3VpeaPexBLq3nvgP0vasuQk
+      responseId: 64fNaIPHJNulqtsPx8OZsQQ
       usageMetadata:
-        candidatesTokenCount: 6
+        candidatesTokenCount: 5
         candidatesTokensDetails:
         - modality: TEXT
-          tokenCount: 6
+          tokenCount: 5
         promptTokenCount: 1817
         promptTokensDetails:
         - modality: TEXT
           tokenCount: 11
         - modality: IMAGE
           tokenCount: 1806
-        totalTokenCount: 1823
+        totalTokenCount: 1822
     status:
       code: 200
       message: OK

tests/models/cassettes/test_google/test_google_model_text_as_binary_content_input.yaml

@@ -33,11 +33,11 @@ interactions:
       alt-svc:
       - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
       content-length:
-      - '712'
+      - '710'
       content-type:
       - application/json; charset=UTF-8
       server-timing:
-      - gfet4t7; dur=459
+      - gfet4t7; dur=571
       transfer-encoding:
       - chunked
       vary:
@@ -46,15 +46,15 @@ interactions:
       - Referer
     parsed_body:
       candidates:
-      - avgLogprobs: -0.014047189553578695
+      - avgLogprobs: -0.2041482448577881
         content:
           parts:
           - text: |
               The main content of the document is that it is a test document.
           role: model
         finishReason: STOP
       modelVersion: gemini-2.0-flash
-      responseId: 41peaPz5EtOvnvgPgYfPiQY
+      responseId: 9ofNaNqNKNWDmtkPs-nsqAU
       usageMetadata:
         candidatesTokenCount: 15
         candidatesTokensDetails:

tests/models/cassettes/test_google/test_google_model_text_document_url_input.yaml

@@ -49,9 +49,11 @@ interactions:
       etag:
       - W/"61efea10-a0e"
       expires:
-      - Fri, 04 Jul 2025 08:48:34 GMT
+      - Fri, 26 Sep 2025 16:42:28 GMT
       last-modified:
       - Tue, 25 Jan 2022 12:16:16 GMT
+      strict-transport-security:
+      - max-age=15552000; includeSubDomains
       transfer-encoding:
       - chunked
       vary:
@@ -93,11 +95,11 @@ interactions:
       alt-svc:
       - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
       content-length:
-      - '1189'
+      - '985'
       content-type:
       - application/json; charset=UTF-8
       server-timing:
-      - gfet4t7; dur=802
+      - gfet4t7; dur=888
       transfer-encoding:
       - chunked
       vary:
@@ -106,25 +108,25 @@ interactions:
       - Referer
     parsed_body:
       candidates:
-      - avgLogprobs: -0.6026712397939151
+      - avgLogprobs: -0.5004191543116714
         content:
           parts:
           - text: |
-              The main content of the document is an example of a TXT file, specifically providing information about the placeholder names "John Doe" (and related variations) used for unidentified or anonymous individuals, particularly in legal contexts in the United States and Canada. It also explains alternative names used in other countries and some additional context and examples of when "John Doe" might be used. The document also includes attribution to Wikipedia for the example content and a link to the license under which it is shared.
+              The main content of the TXT file is an explanation of the placeholder name "John Doe" (and related variations) and its usage in legal contexts, popular culture, and other situations where the identity of a person is unknown or needs to be withheld. The document also includes the purpose of the file and other file type information.
           role: model
         finishReason: STOP
       modelVersion: gemini-2.0-flash
-      responseId: 4lpeaLX9EYzj1PIP0MPrsAg
+      responseId: 9YfNaLGGDuOmqtsPoLXu4AQ
       usageMetadata:
-        candidatesTokenCount: 97
+        candidatesTokenCount: 66
         candidatesTokensDetails:
         - modality: TEXT
-          tokenCount: 97
+          tokenCount: 66
         promptTokenCount: 614
         promptTokensDetails:
         - modality: TEXT
           tokenCount: 614
-        totalTokenCount: 711
+        totalTokenCount: 680
     status:
       code: 200
       message: OK