From dec09908824bcac8f6ca97836255485b0025dbb4 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 24 Jun 2025 10:26:23 -0700 Subject: [PATCH 01/22] Add imagen editing options like inpainting and outpainting --- .../com/google/firebase/ai/ImagenModel.kt | 86 +++++++++++++++++-- .../com/google/firebase/ai/common/Request.kt | 45 +++++++++- .../google/firebase/ai/type/ImagenEditMode.kt | 10 +++ .../firebase/ai/type/ImagenEditingConfig.kt | 68 +++++++++++++++ .../firebase/ai/type/ImagenInlineImage.kt | 18 ++++ 5 files changed, 216 insertions(+), 11 deletions(-) create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index 4d88d09b1e1..a20354ba9f0 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -22,6 +22,7 @@ import com.google.firebase.ai.common.AppCheckHeaderProvider import com.google.firebase.ai.common.ContentBlockedException import com.google.firebase.ai.common.GenerateImageRequest import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.ImagenEditingConfig import com.google.firebase.ai.type.ImagenGenerationConfig import com.google.firebase.ai.type.ImagenGenerationResponse import com.google.firebase.ai.type.ImagenInlineImage @@ -75,30 +76,97 @@ internal constructor( public suspend fun generateImages(prompt: String): ImagenGenerationResponse = try { controller - .generateImage(constructRequest(prompt, null, generationConfig)) + .generateImage(constructGenerationRequest(prompt, null, generationConfig)) .validate() .toPublicInline() } catch (e: Throwable) { throw FirebaseAIException.from(e) } - private fun constructRequest( + public suspend fun editImage( prompt: String, - gcsUri: String?, - config: ImagenGenerationConfig?, + config: ImagenEditingConfig + ): ImagenGenerationResponse = + try { + controller + .generateImage(constructEditRequest(prompt, null, config)) + .validate() + .toPublicInline() + } catch (e: Throwable) { + throw FirebaseAIException.from(e) + } + + private fun constructGenerationRequest( + prompt: String, + gcsUri: String? = null, + generationConfig: ImagenGenerationConfig? = null, ): GenerateImageRequest { return GenerateImageRequest( listOf(GenerateImageRequest.ImagenPrompt(prompt)), GenerateImageRequest.ImagenParameters( - sampleCount = config?.numberOfImages ?: 1, + sampleCount = generationConfig?.numberOfImages ?: 1, + includeRaiReason = true, + addWatermark = this.generationConfig?.addWatermark, + personGeneration = safetySettings?.personFilterLevel?.internalVal, + negativePrompt = generationConfig?.negativePrompt, + safetySetting = safetySettings?.safetyFilterLevel?.internalVal, + storageUri = gcsUri, + aspectRatio = generationConfig?.aspectRatio?.internalVal, + imageOutputOptions = this.generationConfig?.imageFormat?.toInternal(), + editMode = null, + editConfig = null + ), + ) + } + + private fun constructEditRequest( + prompt: String, + gcsUri: String? = null, + editConfig: ImagenEditingConfig, + ): GenerateImageRequest { + return GenerateImageRequest( + listOf( + GenerateImageRequest.ImagenPrompt( + prompt = prompt, + referenceImages = + buildList { + add( + GenerateImageRequest.ReferenceImage( + referenceType = GenerateImageRequest.ReferenceType.RAW, + referenceId = 1, + referenceImage = editConfig.image.toInternal(), + maskImageConfig = null + ) + ) + if (editConfig.mask != null) { + add( + GenerateImageRequest.ReferenceImage( + referenceType = GenerateImageRequest.ReferenceType.MASK, + referenceId = 2, + referenceImage = editConfig.mask.toInternal(), + maskImageConfig = + GenerateImageRequest.MaskImageConfig( + maskMode = GenerateImageRequest.MaskMode.USER_PROVIDED, + dilation = editConfig.maskDilation + ) + ) + ) + } + } + ) + ), + GenerateImageRequest.ImagenParameters( + sampleCount = generationConfig?.numberOfImages ?: 1, includeRaiReason = true, - addWatermark = generationConfig?.addWatermark, + addWatermark = this.generationConfig?.addWatermark, personGeneration = safetySettings?.personFilterLevel?.internalVal, - negativePrompt = config?.negativePrompt, + negativePrompt = generationConfig?.negativePrompt, safetySetting = safetySettings?.safetyFilterLevel?.internalVal, storageUri = gcsUri, - aspectRatio = config?.aspectRatio?.internalVal, - imageOutputOptions = generationConfig?.imageFormat?.toInternal(), + aspectRatio = generationConfig?.aspectRatio?.internalVal, + imageOutputOptions = this.generationConfig?.imageFormat?.toInternal(), + editMode = editConfig.editMode.value, + editConfig = editConfig.toInternal() ), ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index ebc3db7f282..0dabb8fef5e 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -21,7 +21,9 @@ import com.google.firebase.ai.common.util.fullModelName import com.google.firebase.ai.common.util.trimmedModelName import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.GenerationConfig +import com.google.firebase.ai.type.ImagenEditingConfig import com.google.firebase.ai.type.ImagenImageFormat +import com.google.firebase.ai.type.ImagenInlineImage import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.SafetySetting import com.google.firebase.ai.type.Tool @@ -75,17 +77,23 @@ internal data class CountTokensRequest( } @Serializable +@PublicPreviewAPI internal data class GenerateImageRequest( val instances: List, val parameters: ImagenParameters, ) : Request { - @Serializable internal data class ImagenPrompt(val prompt: String) + @Serializable + internal data class ImagenPrompt( + val prompt: String? = null, + val image: ImagenInlineImage.Internal? = null, + val referenceImages: List? = null + ) @OptIn(PublicPreviewAPI::class) @Serializable internal data class ImagenParameters( val sampleCount: Int, - val includeRaiReason: Boolean, + val includeRaiReason: Boolean?, val storageUri: String?, val negativePrompt: String?, val aspectRatio: String?, @@ -93,5 +101,38 @@ internal data class GenerateImageRequest( val personGeneration: String?, val addWatermark: Boolean?, val imageOutputOptions: ImagenImageFormat.Internal?, + val editMode: String?, + val editConfig: ImagenEditingConfig.Internal?, + ) + + @Serializable + internal enum class ReferenceType { + @SerialName("REFERENCE_TYPE_UNSPECIFIED") UNSPECIFIED, + @SerialName("REFERENCE_TYPE_RAW") RAW, + @SerialName("REFERENCE_TYPE_MASK") MASK, + @SerialName("REFERENCE_TYPE_CONTROL") CONTROL, + @SerialName("REFERENCE_TYPE_STYLE") STYLE, + @SerialName("REFERENCE_TYPE_SUBJECT") SUBJECT, + @SerialName("REFERENCE_TYPE_MASKED_SUBJECT") MASKED_SUBJECT, + @SerialName("REFERENCE_TYPE_PRODUCT") PRODUCT + } + + @Serializable + internal enum class MaskMode { + @SerialName("MASK_MODE_DEFAULT") DEFAULT, + @SerialName("MASK_MODE_USER_PROVIDED") USER_PROVIDED, + @SerialName("MASK_MODE_BACKGROUND") BACKGROUND, + @SerialName("MASK_MODE_FOREGROUND") FOREGROUND, + @SerialName("MASK_MODE_SEMANTIC") SEMANTIC + } + + @Serializable internal data class MaskImageConfig(val maskMode: MaskMode, val dilation: Double?) + + @Serializable + internal data class ReferenceImage( + val referenceType: ReferenceType, + val referenceId: Int, + val referenceImage: ImagenInlineImage.Internal, + val maskImageConfig: MaskImageConfig? ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt new file mode 100644 index 00000000000..339ac440ea1 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt @@ -0,0 +1,10 @@ +package com.google.firebase.ai.type + +public class ImagenEditMode private constructor(internal val value: String) { + + public companion object { + public val INPAINT_INSERTION: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_INSERTION") + public val INPAINT_REMOVAL: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_REMOVAL") + public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT") + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt new file mode 100644 index 00000000000..31cccbf9096 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt @@ -0,0 +1,68 @@ +package com.google.firebase.ai.type + +import kotlinx.serialization.Serializable + +@PublicPreviewAPI +public class ImagenEditingConfig( + public val image: ImagenInlineImage, + public val editMode: ImagenEditMode, + public val mask: ImagenInlineImage? = null, + public val maskDilation: Double? = null, + public val editSteps: Int? = null, +) { + public companion object { + public fun builder(): Builder = Builder() + } + + public class Builder { + @JvmField public var image: ImagenInlineImage? = null + @JvmField public var editMode: ImagenEditMode? = null + @JvmField public var mask: ImagenInlineImage? = null + @JvmField public var maskDilation: Double? = null + @JvmField public var editSteps: Int? = null + + public fun setImage(image: ImagenInlineImage): Builder = apply { this.image = image } + + public fun setEditMode(editMode: ImagenEditMode): Builder = apply { this.editMode = editMode } + + public fun setMask(mask: ImagenInlineImage): Builder = apply { this.mask = mask } + + public fun setMaskDilation(maskDilation: Double): Builder = apply { + this.maskDilation = maskDilation + } + + public fun setEditSteps(editSteps: Int): Builder = apply { this.editSteps = editSteps } + + public fun build(): ImagenEditingConfig { + if (image == null) { + throw IllegalStateException("ImagenEditingConfig must contain an image") + } + if (editMode == null) { + throw IllegalStateException("ImagenEditingConfig must contain an editMode") + } + return ImagenEditingConfig( + image = image!!, + editMode = editMode!!, + mask = mask, + maskDilation = maskDilation, + editSteps = editSteps, + ) + } + } + + internal fun toInternal(): Internal { + return Internal(baseSteps = editSteps) + } + + @Serializable + internal data class Internal( + val baseSteps: Int?, + ) +} + +@PublicPreviewAPI +public fun imagenEditingConfig(init: ImagenEditingConfig.Builder.() -> Unit): ImagenEditingConfig { + val builder = ImagenEditingConfig.builder() + builder.init() + return builder.build() +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt index 5fa1d0e183b..988e853d81c 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt @@ -18,6 +18,9 @@ package com.google.firebase.ai.type import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.util.Base64 +import java.io.ByteArrayOutputStream +import kotlinx.serialization.Serializable /** * Represents an Imagen-generated image that is returned as inline data. @@ -36,4 +39,19 @@ internal constructor(public val data: ByteArray, public val mimeType: String) { public fun asBitmap(): Bitmap { return BitmapFactory.decodeByteArray(data, 0, data.size) } + + @Serializable internal data class Internal(val bytesBase64Encoded: String) + + internal fun toInternal(): Internal { + val base64 = Base64.encodeToString(data, Base64.NO_WRAP) + return Internal(base64) + } +} + +@PublicPreviewAPI +public fun Bitmap.toImagenImage(): ImagenInlineImage { + val byteArrayOutputStream = ByteArrayOutputStream() + this.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream) + val byteArray = byteArrayOutputStream.toByteArray() + return ImagenInlineImage(data = byteArray, mimeType = "image/png") } From 9795fc39694914dbbd5579a18ca09e70759fe27a Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 25 Jun 2025 11:33:12 -0700 Subject: [PATCH 02/22] add java implementation and minor naming adjustments --- .../google/firebase/ai/java/ImagenModelFutures.kt | 13 +++++++++++++ .../google/firebase/ai/type/ImagenEditingConfig.kt | 10 +++++----- .../google/firebase/ai/type/ImagenInlineImage.kt | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt index 99d42d32732..11f3756eb0a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt @@ -19,6 +19,7 @@ package com.google.firebase.ai.java import androidx.concurrent.futures.SuspendToFutureAdapter import com.google.common.util.concurrent.ListenableFuture import com.google.firebase.ai.ImagenModel +import com.google.firebase.ai.type.ImagenEditingConfig import com.google.firebase.ai.type.ImagenGenerationResponse import com.google.firebase.ai.type.ImagenInlineImage import com.google.firebase.ai.type.PublicPreviewAPI @@ -39,6 +40,11 @@ public abstract class ImagenModelFutures internal constructor() { prompt: String, ): ListenableFuture> + public abstract fun editImage( + prompt: String, + config: ImagenEditingConfig + ): ListenableFuture> + /** Returns the [ImagenModel] object wrapped by this object. */ public abstract fun getImageModel(): ImagenModel @@ -48,6 +54,13 @@ public abstract class ImagenModelFutures internal constructor() { ): ListenableFuture> = SuspendToFutureAdapter.launchFuture { model.generateImages(prompt) } + override fun editImage( + prompt: String, + config: ImagenEditingConfig + ): ListenableFuture> = + SuspendToFutureAdapter.launchFuture { model.editImage(prompt, config) } + + override fun getImageModel(): ImagenModel = model } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt index 31cccbf9096..507d0d85704 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt @@ -4,11 +4,11 @@ import kotlinx.serialization.Serializable @PublicPreviewAPI public class ImagenEditingConfig( - public val image: ImagenInlineImage, - public val editMode: ImagenEditMode, - public val mask: ImagenInlineImage? = null, - public val maskDilation: Double? = null, - public val editSteps: Int? = null, + internal val image: ImagenInlineImage, + internal val editMode: ImagenEditMode, + internal val mask: ImagenInlineImage? = null, + internal val maskDilation: Double? = null, + internal val editSteps: Int? = null, ) { public companion object { public fun builder(): Builder = Builder() diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt index 988e853d81c..be09cb69c73 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt @@ -49,7 +49,7 @@ internal constructor(public val data: ByteArray, public val mimeType: String) { } @PublicPreviewAPI -public fun Bitmap.toImagenImage(): ImagenInlineImage { +public fun Bitmap.toImagenInlineImage(): ImagenInlineImage { val byteArrayOutputStream = ByteArrayOutputStream() this.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream) val byteArray = byteArrayOutputStream.toByteArray() From 60baab6270d38db81d79cdab874c86baaa743c3e Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 25 Jun 2025 11:34:18 -0700 Subject: [PATCH 03/22] update api.txt and format --- firebase-ai/api.txt | 47 +++++++++++++++++++ .../firebase/ai/java/ImagenModelFutures.kt | 1 - 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index 40aeb2b85d7..e45647ba69e 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -65,6 +65,7 @@ package com.google.firebase.ai { } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenModel { + method public suspend Object? editImage(String prompt, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation>); method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation>); } @@ -104,6 +105,7 @@ package com.google.firebase.ai.java { } @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenModelFutures { + method public abstract com.google.common.util.concurrent.ListenableFuture> editImage(String prompt, com.google.firebase.ai.type.ImagenEditingConfig config); method public static final com.google.firebase.ai.java.ImagenModelFutures from(com.google.firebase.ai.ImagenModel model); method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String prompt); method public abstract com.google.firebase.ai.ImagenModel getImageModel(); @@ -484,6 +486,47 @@ package com.google.firebase.ai.type { public static final class ImagenAspectRatio.Companion { } + public final class ImagenEditMode { + field public static final com.google.firebase.ai.type.ImagenEditMode.Companion Companion; + } + + public static final class ImagenEditMode.Companion { + method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_INSERTION(); + method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_REMOVAL(); + method public com.google.firebase.ai.type.ImagenEditMode getOUTPAINT(); + property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_INSERTION; + property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_REMOVAL; + property public final com.google.firebase.ai.type.ImagenEditMode OUTPAINT; + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenEditingConfig { + ctor public ImagenEditingConfig(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.ImagenEditMode editMode, com.google.firebase.ai.type.ImagenInlineImage? mask = null, Double? maskDilation = null, Integer? editSteps = null); + field public static final com.google.firebase.ai.type.ImagenEditingConfig.Companion Companion; + } + + public static final class ImagenEditingConfig.Builder { + ctor public ImagenEditingConfig.Builder(); + method public com.google.firebase.ai.type.ImagenEditingConfig build(); + method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setEditMode(com.google.firebase.ai.type.ImagenEditMode editMode); + method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setEditSteps(int editSteps); + method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setImage(com.google.firebase.ai.type.ImagenInlineImage image); + method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setMask(com.google.firebase.ai.type.ImagenInlineImage mask); + method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setMaskDilation(double maskDilation); + field public com.google.firebase.ai.type.ImagenEditMode? editMode; + field public Integer? editSteps; + field public com.google.firebase.ai.type.ImagenInlineImage? image; + field public com.google.firebase.ai.type.ImagenInlineImage? mask; + field public Double? maskDilation; + } + + public static final class ImagenEditingConfig.Companion { + method public com.google.firebase.ai.type.ImagenEditingConfig.Builder builder(); + } + + public final class ImagenEditingConfigKt { + method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenEditingConfig imagenEditingConfig(kotlin.jvm.functions.Function1 init); + } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenGenerationConfig { ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.ai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.ai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null); method public Boolean? getAddWatermark(); @@ -552,6 +595,10 @@ package com.google.firebase.ai.type { property public final String mimeType; } + public final class ImagenInlineImageKt { + method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenInlineImage toImagenInlineImage(android.graphics.Bitmap); + } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel { field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ADULT; field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ALL; diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt index 11f3756eb0a..e2703b71b01 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt @@ -60,7 +60,6 @@ public abstract class ImagenModelFutures internal constructor() { ): ListenableFuture> = SuspendToFutureAdapter.launchFuture { model.editImage(prompt, config) } - override fun getImageModel(): ImagenModel = model } From 806d4fbf67fa5e828d4e0e32d20183c541ca5513 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 25 Jun 2025 11:49:35 -0700 Subject: [PATCH 04/22] remove nullability from includeRaiReason --- .../src/main/kotlin/com/google/firebase/ai/common/Request.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index 0dabb8fef5e..e34eda4a31f 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -93,7 +93,7 @@ internal data class GenerateImageRequest( @Serializable internal data class ImagenParameters( val sampleCount: Int, - val includeRaiReason: Boolean?, + val includeRaiReason: Boolean, val storageUri: String?, val negativePrompt: String?, val aspectRatio: String?, From 66cff7ba99275a4fca9af6f610dd198e71958d72 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 25 Jun 2025 12:15:10 -0700 Subject: [PATCH 05/22] remove unneeded and gcsUri --- .../com/google/firebase/ai/ImagenModel.kt | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index a20354ba9f0..45ec18091ae 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -76,7 +76,7 @@ internal constructor( public suspend fun generateImages(prompt: String): ImagenGenerationResponse = try { controller - .generateImage(constructGenerationRequest(prompt, null, generationConfig)) + .generateImage(constructGenerationRequest(prompt, generationConfig)) .validate() .toPublicInline() } catch (e: Throwable) { @@ -88,17 +88,13 @@ internal constructor( config: ImagenEditingConfig ): ImagenGenerationResponse = try { - controller - .generateImage(constructEditRequest(prompt, null, config)) - .validate() - .toPublicInline() + controller.generateImage(constructEditRequest(prompt, config)).validate().toPublicInline() } catch (e: Throwable) { throw FirebaseAIException.from(e) } private fun constructGenerationRequest( prompt: String, - gcsUri: String? = null, generationConfig: ImagenGenerationConfig? = null, ): GenerateImageRequest { return GenerateImageRequest( @@ -106,13 +102,13 @@ internal constructor( GenerateImageRequest.ImagenParameters( sampleCount = generationConfig?.numberOfImages ?: 1, includeRaiReason = true, - addWatermark = this.generationConfig?.addWatermark, + addWatermark = generationConfig?.addWatermark, personGeneration = safetySettings?.personFilterLevel?.internalVal, negativePrompt = generationConfig?.negativePrompt, safetySetting = safetySettings?.safetyFilterLevel?.internalVal, - storageUri = gcsUri, + storageUri = null, aspectRatio = generationConfig?.aspectRatio?.internalVal, - imageOutputOptions = this.generationConfig?.imageFormat?.toInternal(), + imageOutputOptions = generationConfig?.imageFormat?.toInternal(), editMode = null, editConfig = null ), @@ -121,7 +117,6 @@ internal constructor( private fun constructEditRequest( prompt: String, - gcsUri: String? = null, editConfig: ImagenEditingConfig, ): GenerateImageRequest { return GenerateImageRequest( @@ -158,13 +153,13 @@ internal constructor( GenerateImageRequest.ImagenParameters( sampleCount = generationConfig?.numberOfImages ?: 1, includeRaiReason = true, - addWatermark = this.generationConfig?.addWatermark, + addWatermark = generationConfig?.addWatermark, personGeneration = safetySettings?.personFilterLevel?.internalVal, negativePrompt = generationConfig?.negativePrompt, safetySetting = safetySettings?.safetyFilterLevel?.internalVal, - storageUri = gcsUri, + storageUri = null, aspectRatio = generationConfig?.aspectRatio?.internalVal, - imageOutputOptions = this.generationConfig?.imageFormat?.toInternal(), + imageOutputOptions = generationConfig?.imageFormat?.toInternal(), editMode = editConfig.editMode.value, editConfig = editConfig.toInternal() ), From e7104987fd2cf3fc64c8fef4117213f9d14589a1 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 25 Jun 2025 12:32:25 -0700 Subject: [PATCH 06/22] rename constructGenerateImageRequest --- .../src/main/kotlin/com/google/firebase/ai/ImagenModel.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index 45ec18091ae..6e7d9cd6f35 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -76,7 +76,7 @@ internal constructor( public suspend fun generateImages(prompt: String): ImagenGenerationResponse = try { controller - .generateImage(constructGenerationRequest(prompt, generationConfig)) + .generateImage(constructGenerateImageRequest(prompt, generationConfig)) .validate() .toPublicInline() } catch (e: Throwable) { @@ -93,7 +93,7 @@ internal constructor( throw FirebaseAIException.from(e) } - private fun constructGenerationRequest( + private fun constructGenerateImageRequest( prompt: String, generationConfig: ImagenGenerationConfig? = null, ): GenerateImageRequest { From f632e94ede993476ed41fd1d6127b641ca072d5d Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 9 Jul 2025 15:27:15 -0700 Subject: [PATCH 07/22] rewrite to fit the referenceType style transfer, subject references, and more complex editing settings --- firebase-ai/gradle.properties | 2 +- .../com/google/firebase/ai/ImagenModel.kt | 73 +++++++++++-------- .../com/google/firebase/ai/common/Request.kt | 24 +----- .../firebase/ai/java/ImagenModelFutures.kt | 9 ++- .../com/google/firebase/ai/type/Dimensions.kt | 3 + .../firebase/ai/type/ImagenBackgroundMask.kt | 5 ++ .../firebase/ai/type/ImagenControlConfig.kt | 10 +++ .../ai/type/ImagenControlReference.kt | 19 +++++ .../firebase/ai/type/ImagenControlType.kt | 10 +++ .../firebase/ai/type/ImagenEditingConfig.kt | 52 +------------ .../firebase/ai/type/ImagenForegroundMask.kt | 5 ++ .../firebase/ai/type/ImagenImagePlacement.kt | 19 +++++ .../firebase/ai/type/ImagenMaskConfig.kt | 20 +++++ .../google/firebase/ai/type/ImagenMaskMode.kt | 10 +++ .../firebase/ai/type/ImagenMaskReference.kt | 20 +++++ .../google/firebase/ai/type/ImagenRawImage.kt | 4 + .../google/firebase/ai/type/ImagenRawMask.kt | 8 ++ .../firebase/ai/type/ImagenReferenceImage.kt | 51 +++++++++++++ .../firebase/ai/type/ImagenSemanticMask.kt | 5 ++ .../firebase/ai/type/ImagenStyleConfig.kt | 12 +++ .../firebase/ai/type/ImagenStyleReference.kt | 13 ++++ .../firebase/ai/type/ImagenSubjectConfig.kt | 16 ++++ .../ai/type/ImagenSubjectReference.kt | 14 ++++ .../ai/type/ImagenSubjectReferenceType.kt | 10 +++ 24 files changed, 306 insertions(+), 108 deletions(-) create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt create mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt diff --git a/firebase-ai/gradle.properties b/firebase-ai/gradle.properties index b686fdcb9db..d029f15815f 100644 --- a/firebase-ai/gradle.properties +++ b/firebase-ai/gradle.properties @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version=16.2.0 +version=16.3.0 latestReleasedVersion=16.1.0 diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index 6e7d9cd6f35..ee7ebbf8122 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -21,11 +21,17 @@ import com.google.firebase.ai.common.APIController import com.google.firebase.ai.common.AppCheckHeaderProvider import com.google.firebase.ai.common.ContentBlockedException import com.google.firebase.ai.common.GenerateImageRequest +import com.google.firebase.ai.type.Dimensions import com.google.firebase.ai.type.FirebaseAIException +import com.google.firebase.ai.type.ImagenEditMode import com.google.firebase.ai.type.ImagenEditingConfig import com.google.firebase.ai.type.ImagenGenerationConfig import com.google.firebase.ai.type.ImagenGenerationResponse +import com.google.firebase.ai.type.ImagenImagePlacement import com.google.firebase.ai.type.ImagenInlineImage +import com.google.firebase.ai.type.ImagenMaskReference +import com.google.firebase.ai.type.ImagenRawImage +import com.google.firebase.ai.type.ImagenReferenceImage import com.google.firebase.ai.type.ImagenSafetySettings import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.RequestOptions @@ -84,15 +90,42 @@ internal constructor( } public suspend fun editImage( + referenceImages: List, prompt: String, - config: ImagenEditingConfig + config: ImagenEditingConfig? = null, ): ImagenGenerationResponse = try { - controller.generateImage(constructEditRequest(prompt, config)).validate().toPublicInline() + controller + .generateImage(constructEditRequest(referenceImages, prompt, config)) + .validate() + .toPublicInline() } catch (e: Throwable) { throw FirebaseAIException.from(e) } + public suspend fun inpaintImage( + image: ImagenInlineImage, + prompt: String, + mask: ImagenMaskReference, + config: ImagenEditingConfig, + ): ImagenGenerationResponse { + return editImage(listOf(ImagenRawImage(image), mask), prompt, config) + } + + public suspend fun outpaintImage( + image: ImagenInlineImage, + newDimensions: Dimensions, + newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER, + prompt: String = "", + config: ImagenEditingConfig? = null, + ): ImagenGenerationResponse { + return editImage( + ImagenMaskReference.generateMaskAndPadForOutpainting(image, newDimensions, newPosition), + prompt, + ImagenEditingConfig(ImagenEditMode.OUTPAINT, config?.editSteps) + ) + } + private fun constructGenerateImageRequest( prompt: String, generationConfig: ImagenGenerationConfig? = null, @@ -110,44 +143,22 @@ internal constructor( aspectRatio = generationConfig?.aspectRatio?.internalVal, imageOutputOptions = generationConfig?.imageFormat?.toInternal(), editMode = null, - editConfig = null + editConfig = null, ), ) } private fun constructEditRequest( + referenceImages: List, prompt: String, - editConfig: ImagenEditingConfig, + editConfig: ImagenEditingConfig?, ): GenerateImageRequest { + var maxRefId = referenceImages.mapNotNull { it.referenceId }.maxOrNull() ?: 1 return GenerateImageRequest( listOf( GenerateImageRequest.ImagenPrompt( prompt = prompt, - referenceImages = - buildList { - add( - GenerateImageRequest.ReferenceImage( - referenceType = GenerateImageRequest.ReferenceType.RAW, - referenceId = 1, - referenceImage = editConfig.image.toInternal(), - maskImageConfig = null - ) - ) - if (editConfig.mask != null) { - add( - GenerateImageRequest.ReferenceImage( - referenceType = GenerateImageRequest.ReferenceType.MASK, - referenceId = 2, - referenceImage = editConfig.mask.toInternal(), - maskImageConfig = - GenerateImageRequest.MaskImageConfig( - maskMode = GenerateImageRequest.MaskMode.USER_PROVIDED, - dilation = editConfig.maskDilation - ) - ) - ) - } - } + referenceImages = referenceImages.map { it.toInternal(++maxRefId) }, ) ), GenerateImageRequest.ImagenParameters( @@ -160,8 +171,8 @@ internal constructor( storageUri = null, aspectRatio = generationConfig?.aspectRatio?.internalVal, imageOutputOptions = generationConfig?.imageFormat?.toInternal(), - editMode = editConfig.editMode.value, - editConfig = editConfig.toInternal() + editMode = editConfig?.editMode?.value, + editConfig = editConfig?.toInternal(), ), ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index e34eda4a31f..1375102df23 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -23,7 +23,7 @@ import com.google.firebase.ai.type.Content import com.google.firebase.ai.type.GenerationConfig import com.google.firebase.ai.type.ImagenEditingConfig import com.google.firebase.ai.type.ImagenImageFormat -import com.google.firebase.ai.type.ImagenInlineImage +import com.google.firebase.ai.type.ImagenReferenceImage import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.SafetySetting import com.google.firebase.ai.type.Tool @@ -85,8 +85,7 @@ internal data class GenerateImageRequest( @Serializable internal data class ImagenPrompt( val prompt: String? = null, - val image: ImagenInlineImage.Internal? = null, - val referenceImages: List? = null + val referenceImages: List? = null ) @OptIn(PublicPreviewAPI::class) @@ -116,23 +115,4 @@ internal data class GenerateImageRequest( @SerialName("REFERENCE_TYPE_MASKED_SUBJECT") MASKED_SUBJECT, @SerialName("REFERENCE_TYPE_PRODUCT") PRODUCT } - - @Serializable - internal enum class MaskMode { - @SerialName("MASK_MODE_DEFAULT") DEFAULT, - @SerialName("MASK_MODE_USER_PROVIDED") USER_PROVIDED, - @SerialName("MASK_MODE_BACKGROUND") BACKGROUND, - @SerialName("MASK_MODE_FOREGROUND") FOREGROUND, - @SerialName("MASK_MODE_SEMANTIC") SEMANTIC - } - - @Serializable internal data class MaskImageConfig(val maskMode: MaskMode, val dilation: Double?) - - @Serializable - internal data class ReferenceImage( - val referenceType: ReferenceType, - val referenceId: Int, - val referenceImage: ImagenInlineImage.Internal, - val maskImageConfig: MaskImageConfig? - ) } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt index e2703b71b01..a105c53a705 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt @@ -22,6 +22,7 @@ import com.google.firebase.ai.ImagenModel import com.google.firebase.ai.type.ImagenEditingConfig import com.google.firebase.ai.type.ImagenGenerationResponse import com.google.firebase.ai.type.ImagenInlineImage +import com.google.firebase.ai.type.ImagenReferenceImage import com.google.firebase.ai.type.PublicPreviewAPI /** @@ -41,8 +42,9 @@ public abstract class ImagenModelFutures internal constructor() { ): ListenableFuture> public abstract fun editImage( + referenceImages: List, prompt: String, - config: ImagenEditingConfig + config: ImagenEditingConfig? = null ): ListenableFuture> /** Returns the [ImagenModel] object wrapped by this object. */ @@ -55,10 +57,11 @@ public abstract class ImagenModelFutures internal constructor() { SuspendToFutureAdapter.launchFuture { model.generateImages(prompt) } override fun editImage( + referenceImages: List, prompt: String, - config: ImagenEditingConfig + config: ImagenEditingConfig? ): ListenableFuture> = - SuspendToFutureAdapter.launchFuture { model.editImage(prompt, config) } + SuspendToFutureAdapter.launchFuture { model.editImage(referenceImages, prompt, config) } override fun getImageModel(): ImagenModel = model } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt new file mode 100644 index 00000000000..350441a7c08 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt @@ -0,0 +1,3 @@ +package com.google.firebase.ai.type + +public class Dimensions(public val width: Int, public val height: Int) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt new file mode 100644 index 00000000000..79129a9d455 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt @@ -0,0 +1,5 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public class ImagenBackgroundMask(dilation: Double? = null) : + ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.BACKGROUND, dilation)) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt new file mode 100644 index 00000000000..bb670788a8e --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt @@ -0,0 +1,10 @@ +package com.google.firebase.ai.typeImage + +import com.google.firebase.ai.type.ImagenControlType + +internal class ImagenControlConfig( + internal val controlType: ImagenControlType, + internal val enableComputation: Boolean? = null, + internal val superpixelRegionSize: Int? = null, + internal val superpixelRuler: Int? = null +) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt new file mode 100644 index 00000000000..c0edf339298 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt @@ -0,0 +1,19 @@ +package com.google.firebase.ai.type + +import com.google.firebase.ai.typeImage.ImagenControlConfig + +@PublicPreviewAPI +public class ImagenControlReference( + image: ImagenInlineImage, + type: ImagenControlType, + referenceId: Int? = null, + enableComputation: Boolean? = null, + superpixelRegionSize: Int? = null, + superpixelRuler: Int? = null, +) : + ImagenReferenceImage( + controlConfig = + ImagenControlConfig(type, enableComputation, superpixelRegionSize, superpixelRuler), + image = image, + referenceId = referenceId, + ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt new file mode 100644 index 00000000000..e709ea4b93e --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt @@ -0,0 +1,10 @@ +package com.google.firebase.ai.type + +public class ImagenControlType internal constructor(internal val value: String) { + public companion object { + public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY") + public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE") + public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH") + public val COLOR_SUPERPIXEL: ImagenControlType = ImagenControlType("CONTROL_TYPE_COLOR_SUPERPIXEL") + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt index 507d0d85704..35b06879788 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt @@ -4,52 +4,9 @@ import kotlinx.serialization.Serializable @PublicPreviewAPI public class ImagenEditingConfig( - internal val image: ImagenInlineImage, - internal val editMode: ImagenEditMode, - internal val mask: ImagenInlineImage? = null, - internal val maskDilation: Double? = null, + internal val editMode: ImagenEditMode? = null, internal val editSteps: Int? = null, ) { - public companion object { - public fun builder(): Builder = Builder() - } - - public class Builder { - @JvmField public var image: ImagenInlineImage? = null - @JvmField public var editMode: ImagenEditMode? = null - @JvmField public var mask: ImagenInlineImage? = null - @JvmField public var maskDilation: Double? = null - @JvmField public var editSteps: Int? = null - - public fun setImage(image: ImagenInlineImage): Builder = apply { this.image = image } - - public fun setEditMode(editMode: ImagenEditMode): Builder = apply { this.editMode = editMode } - - public fun setMask(mask: ImagenInlineImage): Builder = apply { this.mask = mask } - - public fun setMaskDilation(maskDilation: Double): Builder = apply { - this.maskDilation = maskDilation - } - - public fun setEditSteps(editSteps: Int): Builder = apply { this.editSteps = editSteps } - - public fun build(): ImagenEditingConfig { - if (image == null) { - throw IllegalStateException("ImagenEditingConfig must contain an image") - } - if (editMode == null) { - throw IllegalStateException("ImagenEditingConfig must contain an editMode") - } - return ImagenEditingConfig( - image = image!!, - editMode = editMode!!, - mask = mask, - maskDilation = maskDilation, - editSteps = editSteps, - ) - } - } - internal fun toInternal(): Internal { return Internal(baseSteps = editSteps) } @@ -59,10 +16,3 @@ public class ImagenEditingConfig( val baseSteps: Int?, ) } - -@PublicPreviewAPI -public fun imagenEditingConfig(init: ImagenEditingConfig.Builder.() -> Unit): ImagenEditingConfig { - val builder = ImagenEditingConfig.builder() - builder.init() - return builder.build() -} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt new file mode 100644 index 00000000000..e2a9f244c3a --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt @@ -0,0 +1,5 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public class ImagenForegroundMask(dilation: Double? = null) : + ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.FOREGROUND, dilation)) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt new file mode 100644 index 00000000000..34ddb35af6f --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt @@ -0,0 +1,19 @@ +package com.google.firebase.ai.type + +public class ImagenImagePlacement private constructor(public val x: Int?, public val y: Int?) { + public companion object { + public fun fromCoordinate(x: Int, y: Int): ImagenImagePlacement { + return ImagenImagePlacement(x, y) + } + + public val CENTER: ImagenImagePlacement = ImagenImagePlacement(null, null) + public val TOP: ImagenImagePlacement = ImagenImagePlacement(null, null) + public val BOTTOM: ImagenImagePlacement = ImagenImagePlacement(null, null) + public val LEFT: ImagenImagePlacement = ImagenImagePlacement(null, null) + public val RIGHT: ImagenImagePlacement = ImagenImagePlacement(null, null) + public val TOP_LEFT: ImagenImagePlacement = ImagenImagePlacement(0, 0) + public val TOP_RIGHT: ImagenImagePlacement = ImagenImagePlacement(null, null) + public val BOTTOM_LEFT: ImagenImagePlacement = ImagenImagePlacement(null, null) + public val BOTTOM_RIGHT: ImagenImagePlacement = ImagenImagePlacement(null, null) + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt new file mode 100644 index 00000000000..51219677ff1 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt @@ -0,0 +1,20 @@ +package com.google.firebase.ai.type + +import kotlinx.serialization.Serializable + +internal class ImagenMaskConfig( + internal val maskType: ImagenMaskMode, + internal val dilation: Double? = null, + internal val classes: List? = null +) { + internal fun toInternal(): Internal { + return Internal(maskType.value, dilation, classes) + } + + @Serializable + internal data class Internal( + val maskMode: String, + val dilation: Double?, + val maskClasses: List? + ) +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt new file mode 100644 index 00000000000..3ed2323c3da --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt @@ -0,0 +1,10 @@ +package com.google.firebase.ai.type + +internal class ImagenMaskMode private constructor(internal val value: String) { + companion object { + val USER_PROVIDED: ImagenMaskMode = ImagenMaskMode("MASK_MODE_USER_PROVIDED") + val BACKGROUND: ImagenMaskMode = ImagenMaskMode("MASK_MODE_BACKGROUND") + val FOREGROUND: ImagenMaskMode = ImagenMaskMode("MASK_MODE_FOREGROUND") + val SEMANTIC: ImagenMaskMode = ImagenMaskMode("MASK_MODE_SEMANTIC") + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt new file mode 100644 index 00000000000..11dcb15db96 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt @@ -0,0 +1,20 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public abstract class ImagenMaskReference +internal constructor( + maskConfig: ImagenMaskConfig, + image: ImagenInlineImage? = null, +) : ImagenReferenceImage(maskConfig = maskConfig, image = image) { + + public companion object { + public fun generateMaskAndPadForOutpainting( + image: ImagenInlineImage, + newDimensions: Dimensions, + newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER, + ): List { + // TODO: Fill in + return emptyList() + } + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt new file mode 100644 index 00000000000..8776c0d04f2 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt @@ -0,0 +1,4 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public class ImagenRawImage(image: ImagenInlineImage) : ImagenReferenceImage(image = image) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt new file mode 100644 index 00000000000..d555227486a --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt @@ -0,0 +1,8 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public class ImagenRawMask(mask: ImagenInlineImage, dilation: Double? = null) : + ImagenMaskReference( + maskConfig = ImagenMaskConfig(ImagenMaskMode.USER_PROVIDED, dilation), + image = mask, + ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt new file mode 100644 index 00000000000..1fd676cb644 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -0,0 +1,51 @@ +package com.google.firebase.ai.type + +import com.google.firebase.ai.common.GenerateImageRequest +import com.google.firebase.ai.typeImage.ImagenControlConfig +import kotlinx.serialization.Serializable + +@PublicPreviewAPI +public abstract class ImagenReferenceImage +internal constructor( + internal val maskConfig: ImagenMaskConfig? = null, + internal val subjectConfig: ImagenSubjectConfig? = null, + internal val styleConfig: ImagenStyleConfig? = null, + internal val controlConfig: ImagenControlConfig? = null, + public val image: ImagenInlineImage? = null, + public val referenceId: Int? = null, +) { + + internal fun toInternal(optionalReferenceId: Int): Internal { + val referenceType = + when (this) { + is ImagenRawImage -> GenerateImageRequest.ReferenceType.RAW + is ImagenMaskReference -> GenerateImageRequest.ReferenceType.MASK + is ImagenSubjectReference -> GenerateImageRequest.ReferenceType.SUBJECT + is ImagenStyleReference -> GenerateImageRequest.ReferenceType.STYLE + is ImagenControlReference -> GenerateImageRequest.ReferenceType.CONTROL + else -> { + throw IllegalStateException( + "${this.javaClass.simpleName} is not a known subtype of ImagenReferenceImage" + ) + } + } + return Internal( + referenceType = referenceType, + referenceImage = image?.toInternal(), + referenceId = referenceId ?: optionalReferenceId, + subjectImageConfig = subjectConfig?.toInternal(), + maskImageConfig = maskConfig?.toInternal(), + styleImageConfig = styleConfig?.toInternal(), + ) + } + + @Serializable + internal data class Internal( + val referenceType: GenerateImageRequest.ReferenceType, + val referenceImage: ImagenInlineImage.Internal?, + val referenceId: Int, + val subjectImageConfig: ImagenSubjectConfig.Internal?, + val maskImageConfig: ImagenMaskConfig.Internal?, + val styleImageConfig: ImagenStyleConfig.Internal?, + ) +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt new file mode 100644 index 00000000000..e577a9fca54 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt @@ -0,0 +1,5 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public class ImagenSemanticMask(classes: List, dilation: Double? = null) : + ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.SEMANTIC, dilation, classes)) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt new file mode 100644 index 00000000000..917c660eeba --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt @@ -0,0 +1,12 @@ +package com.google.firebase.ai.type + +import kotlinx.serialization.Serializable + +internal class ImagenStyleConfig(val description: String?) { + + fun toInternal(): Internal { + return Internal(description) + } + + @Serializable internal data class Internal(val styleDescription: String?) +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt new file mode 100644 index 00000000000..c9fd278efe3 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt @@ -0,0 +1,13 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public class ImagenStyleReference( + image: ImagenInlineImage, + referenceId: Int? = null, + description: String? = null, +) : + ImagenReferenceImage( + image = image, + referenceId = referenceId, + styleConfig = ImagenStyleConfig(description) + ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt new file mode 100644 index 00000000000..10f333ccc91 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt @@ -0,0 +1,16 @@ +package com.google.firebase.ai.type + +import kotlinx.serialization.Serializable + +internal class ImagenSubjectConfig( + val description: String?, + val type: ImagenSubjectReferenceType?, +) { + + internal fun toInternal(): Internal { + return Internal(description, type?.value) + } + + @Serializable + internal data class Internal(val subjectDescription: String?, val subjectType: String?) +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt new file mode 100644 index 00000000000..c63abb5bf5d --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt @@ -0,0 +1,14 @@ +package com.google.firebase.ai.type + +@PublicPreviewAPI +public class ImagenSubjectReference( + image: ImagenInlineImage, + referenceId: Int? = null, + description: String? = null, + subjectType: ImagenSubjectReferenceType? = null, +) : + ImagenReferenceImage( + image = image, + referenceId = referenceId, + subjectConfig = ImagenSubjectConfig(description, subjectType), + ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt new file mode 100644 index 00000000000..a36b67dee06 --- /dev/null +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt @@ -0,0 +1,10 @@ +package com.google.firebase.ai.type + +public class ImagenSubjectReferenceType private constructor(internal val value: String) { + + public companion object { + public val PERSON: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PERSON") + public val ANIMAL: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_ANIMAL") + public val PRODUCT: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PRODUCT") + } +} From 0327b0a1c11b3bda7dd3bee08ee30ae5fd9d97bb Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Thu, 10 Jul 2025 10:15:58 -0700 Subject: [PATCH 08/22] added support for outpainting properly --- firebase-ai/api.txt | 141 ++++++++++++++---- .../firebase/ai/type/ImagenControlType.kt | 13 +- .../firebase/ai/type/ImagenImagePlacement.kt | 40 +++++ .../firebase/ai/type/ImagenMaskReference.kt | 52 ++++++- .../ai/type/ImagenSubjectReferenceType.kt | 9 +- 5 files changed, 215 insertions(+), 40 deletions(-) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index e45647ba69e..ce6071e89ca 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -65,8 +65,10 @@ package com.google.firebase.ai { } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenModel { - method public suspend Object? editImage(String prompt, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation>); + method public suspend Object? editImage(java.util.List referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation>); method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation>); + method public suspend Object? inpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, String prompt, com.google.firebase.ai.type.ImagenMaskReference mask, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation>); + method public suspend Object? outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation>); } @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveGenerativeModel { @@ -105,7 +107,7 @@ package com.google.firebase.ai.java { } @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenModelFutures { - method public abstract com.google.common.util.concurrent.ListenableFuture> editImage(String prompt, com.google.firebase.ai.type.ImagenEditingConfig config); + method public abstract com.google.common.util.concurrent.ListenableFuture> editImage(java.util.List referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null); method public static final com.google.firebase.ai.java.ImagenModelFutures from(com.google.firebase.ai.ImagenModel model); method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String prompt); method public abstract com.google.firebase.ai.ImagenModel getImageModel(); @@ -259,6 +261,14 @@ package com.google.firebase.ai.type { property public final int totalTokens; } + public final class Dimensions { + ctor public Dimensions(int width, int height); + method public int getHeight(); + method public int getWidth(); + property public final int height; + property public final int width; + } + public final class FileDataPart implements com.google.firebase.ai.type.Part { ctor public FileDataPart(String uri, String mimeType); method public String getMimeType(); @@ -486,6 +496,29 @@ package com.google.firebase.ai.type { public static final class ImagenAspectRatio.Companion { } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenBackgroundMask extends com.google.firebase.ai.type.ImagenMaskReference { + ctor public ImagenBackgroundMask(Double? dilation = null); + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenControlReference extends com.google.firebase.ai.type.ImagenReferenceImage { + ctor public ImagenControlReference(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.ImagenControlType type, Integer? referenceId = null, Boolean? enableComputation = null, Integer? superpixelRegionSize = null, Integer? superpixelRuler = null); + } + + public final class ImagenControlType { + field public static final com.google.firebase.ai.type.ImagenControlType.Companion Companion; + } + + public static final class ImagenControlType.Companion { + method public com.google.firebase.ai.type.ImagenControlType getCANNY(); + method public com.google.firebase.ai.type.ImagenControlType getCOLOR_SUPERPIXEL(); + method public com.google.firebase.ai.type.ImagenControlType getFACE_MESH(); + method public com.google.firebase.ai.type.ImagenControlType getSCRIBBLE(); + property public final com.google.firebase.ai.type.ImagenControlType CANNY; + property public final com.google.firebase.ai.type.ImagenControlType COLOR_SUPERPIXEL; + property public final com.google.firebase.ai.type.ImagenControlType FACE_MESH; + property public final com.google.firebase.ai.type.ImagenControlType SCRIBBLE; + } + public final class ImagenEditMode { field public static final com.google.firebase.ai.type.ImagenEditMode.Companion Companion; } @@ -500,31 +533,11 @@ package com.google.firebase.ai.type { } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenEditingConfig { - ctor public ImagenEditingConfig(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.ImagenEditMode editMode, com.google.firebase.ai.type.ImagenInlineImage? mask = null, Double? maskDilation = null, Integer? editSteps = null); - field public static final com.google.firebase.ai.type.ImagenEditingConfig.Companion Companion; - } - - public static final class ImagenEditingConfig.Builder { - ctor public ImagenEditingConfig.Builder(); - method public com.google.firebase.ai.type.ImagenEditingConfig build(); - method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setEditMode(com.google.firebase.ai.type.ImagenEditMode editMode); - method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setEditSteps(int editSteps); - method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setImage(com.google.firebase.ai.type.ImagenInlineImage image); - method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setMask(com.google.firebase.ai.type.ImagenInlineImage mask); - method public com.google.firebase.ai.type.ImagenEditingConfig.Builder setMaskDilation(double maskDilation); - field public com.google.firebase.ai.type.ImagenEditMode? editMode; - field public Integer? editSteps; - field public com.google.firebase.ai.type.ImagenInlineImage? image; - field public com.google.firebase.ai.type.ImagenInlineImage? mask; - field public Double? maskDilation; - } - - public static final class ImagenEditingConfig.Companion { - method public com.google.firebase.ai.type.ImagenEditingConfig.Builder builder(); + ctor public ImagenEditingConfig(com.google.firebase.ai.type.ImagenEditMode? editMode = null, Integer? editSteps = null); } - public final class ImagenEditingConfigKt { - method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenEditingConfig imagenEditingConfig(kotlin.jvm.functions.Function1 init); + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenForegroundMask extends com.google.firebase.ai.type.ImagenMaskReference { + ctor public ImagenForegroundMask(Double? dilation = null); } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenGenerationConfig { @@ -587,6 +600,36 @@ package com.google.firebase.ai.type { method public com.google.firebase.ai.type.ImagenImageFormat png(); } + public final class ImagenImagePlacement { + method public Integer? getX(); + method public Integer? getY(); + property public final Integer? x; + property public final Integer? y; + field public static final com.google.firebase.ai.type.ImagenImagePlacement.Companion Companion; + } + + public static final class ImagenImagePlacement.Companion { + method public com.google.firebase.ai.type.ImagenImagePlacement fromCoordinate(int x, int y); + method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM(); + method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_LEFT(); + method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_RIGHT(); + method public com.google.firebase.ai.type.ImagenImagePlacement getCENTER(); + method public com.google.firebase.ai.type.ImagenImagePlacement getLEFT(); + method public com.google.firebase.ai.type.ImagenImagePlacement getRIGHT(); + method public com.google.firebase.ai.type.ImagenImagePlacement getTOP(); + method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_LEFT(); + method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_RIGHT(); + property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM; + property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_LEFT; + property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_RIGHT; + property public final com.google.firebase.ai.type.ImagenImagePlacement CENTER; + property public final com.google.firebase.ai.type.ImagenImagePlacement LEFT; + property public final com.google.firebase.ai.type.ImagenImagePlacement RIGHT; + property public final com.google.firebase.ai.type.ImagenImagePlacement TOP; + property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_LEFT; + property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_RIGHT; + } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenInlineImage { method public android.graphics.Bitmap asBitmap(); method public byte[] getData(); @@ -599,6 +642,14 @@ package com.google.firebase.ai.type { method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenInlineImage toImagenInlineImage(android.graphics.Bitmap); } + @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenMaskReference extends com.google.firebase.ai.type.ImagenReferenceImage { + field public static final com.google.firebase.ai.type.ImagenMaskReference.Companion Companion; + } + + public static final class ImagenMaskReference.Companion { + method public java.util.List generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER); + } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel { field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ADULT; field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ALL; @@ -609,6 +660,21 @@ package com.google.firebase.ai.type { public static final class ImagenPersonFilterLevel.Companion { } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenRawImage extends com.google.firebase.ai.type.ImagenReferenceImage { + ctor public ImagenRawImage(com.google.firebase.ai.type.ImagenInlineImage image); + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenRawMask extends com.google.firebase.ai.type.ImagenMaskReference { + ctor public ImagenRawMask(com.google.firebase.ai.type.ImagenInlineImage mask, Double? dilation = null); + } + + @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenReferenceImage { + method public final com.google.firebase.ai.type.ImagenInlineImage? getImage(); + method public final Integer? getReferenceId(); + property public final com.google.firebase.ai.type.ImagenInlineImage? image; + property public final Integer? referenceId; + } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSafetyFilterLevel { field public static final com.google.firebase.ai.type.ImagenSafetyFilterLevel BLOCK_LOW_AND_ABOVE; field public static final com.google.firebase.ai.type.ImagenSafetyFilterLevel BLOCK_MEDIUM_AND_ABOVE; @@ -624,6 +690,31 @@ package com.google.firebase.ai.type { ctor public ImagenSafetySettings(com.google.firebase.ai.type.ImagenSafetyFilterLevel safetyFilterLevel, com.google.firebase.ai.type.ImagenPersonFilterLevel personFilterLevel); } + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSemanticMask extends com.google.firebase.ai.type.ImagenMaskReference { + ctor public ImagenSemanticMask(java.util.List classes, Double? dilation = null); + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenStyleReference extends com.google.firebase.ai.type.ImagenReferenceImage { + ctor public ImagenStyleReference(com.google.firebase.ai.type.ImagenInlineImage image, Integer? referenceId = null, String? description = null); + } + + @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSubjectReference extends com.google.firebase.ai.type.ImagenReferenceImage { + ctor public ImagenSubjectReference(com.google.firebase.ai.type.ImagenInlineImage image, Integer? referenceId = null, String? description = null, com.google.firebase.ai.type.ImagenSubjectReferenceType? subjectType = null); + } + + public final class ImagenSubjectReferenceType { + field public static final com.google.firebase.ai.type.ImagenSubjectReferenceType.Companion Companion; + } + + public static final class ImagenSubjectReferenceType.Companion { + method public com.google.firebase.ai.type.ImagenSubjectReferenceType getANIMAL(); + method public com.google.firebase.ai.type.ImagenSubjectReferenceType getPERSON(); + method public com.google.firebase.ai.type.ImagenSubjectReferenceType getPRODUCT(); + property public final com.google.firebase.ai.type.ImagenSubjectReferenceType ANIMAL; + property public final com.google.firebase.ai.type.ImagenSubjectReferenceType PERSON; + property public final com.google.firebase.ai.type.ImagenSubjectReferenceType PRODUCT; + } + public final class InlineDataPart implements com.google.firebase.ai.type.Part { ctor public InlineDataPart(byte[] inlineData, String mimeType); method public byte[] getInlineData(); diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt index e709ea4b93e..f41a5dea2e3 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt @@ -1,10 +1,11 @@ package com.google.firebase.ai.type public class ImagenControlType internal constructor(internal val value: String) { - public companion object { - public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY") - public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE") - public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH") - public val COLOR_SUPERPIXEL: ImagenControlType = ImagenControlType("CONTROL_TYPE_COLOR_SUPERPIXEL") - } + public companion object { + public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY") + public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE") + public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH") + public val COLOR_SUPERPIXEL: ImagenControlType = + ImagenControlType("CONTROL_TYPE_COLOR_SUPERPIXEL") + } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt index 34ddb35af6f..e521966b65f 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt @@ -1,6 +1,46 @@ package com.google.firebase.ai.type public class ImagenImagePlacement private constructor(public val x: Int?, public val y: Int?) { + + internal fun normalizeToDimensions( + imageDimensions: Dimensions, + canvasDimensions: Dimensions, + ): ImagenImagePlacement { + if (this.x != null && this.y != null) { + return this + } + val halfCanvasHeight = canvasDimensions.height / 2 + val halfCanvasWidth = canvasDimensions.width / 2 + val halfImageHeight = imageDimensions.height / 2 + val halfImageWidth = imageDimensions.width / 2 + return when (this) { + CENTER -> + ImagenImagePlacement(halfCanvasWidth - halfImageWidth, halfCanvasHeight - halfImageHeight) + TOP -> ImagenImagePlacement(halfCanvasWidth - halfImageWidth, 0) + BOTTOM -> + ImagenImagePlacement( + halfCanvasWidth - halfImageWidth, + canvasDimensions.height - imageDimensions.height, + ) + LEFT -> ImagenImagePlacement(0, halfCanvasHeight - halfImageHeight) + RIGHT -> + ImagenImagePlacement( + canvasDimensions.width - imageDimensions.width, + halfCanvasHeight - halfImageHeight, + ) + TOP_RIGHT -> ImagenImagePlacement(canvasDimensions.width - imageDimensions.width, 0) + BOTTOM_LEFT -> ImagenImagePlacement(0, canvasDimensions.height - imageDimensions.height) + BOTTOM_RIGHT -> + ImagenImagePlacement( + canvasDimensions.width - imageDimensions.width, + canvasDimensions.height - imageDimensions.height, + ) + else -> { + throw IllegalStateException("Unknown ImagenImagePlacement instance, cannot normalize") + } + } + } + public companion object { public fun fromCoordinate(x: Int, y: Int): ImagenImagePlacement { return ImagenImagePlacement(x, y) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt index 11dcb15db96..d008e47a2e4 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt @@ -1,11 +1,15 @@ package com.google.firebase.ai.type +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Rect + @PublicPreviewAPI public abstract class ImagenMaskReference -internal constructor( - maskConfig: ImagenMaskConfig, - image: ImagenInlineImage? = null, -) : ImagenReferenceImage(maskConfig = maskConfig, image = image) { +internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = null) : + ImagenReferenceImage(maskConfig = maskConfig, image = image) { public companion object { public fun generateMaskAndPadForOutpainting( @@ -13,8 +17,44 @@ internal constructor( newDimensions: Dimensions, newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER, ): List { - // TODO: Fill in - return emptyList() + val originalBitmap = image.asBitmap() + val normalizedPosition = + newPosition.normalizeToDimensions( + Dimensions(originalBitmap.width, originalBitmap.height), + newDimensions, + ) + val normalizedImageRectangle = + Rect( + normalizedPosition.x!!, + normalizedPosition.y!!, + normalizedPosition.x + originalBitmap.width, + normalizedPosition.y + originalBitmap.height, + ) + + val maskBitmap = + Bitmap.createBitmap(newDimensions.width, newDimensions.height, Bitmap.Config.RGB_565) + val newImageBitmap = + Bitmap.createBitmap(newDimensions.width, newDimensions.height, Bitmap.Config.RGB_565) + + val maskCanvas = Canvas(maskBitmap) + val newImageCanvas = Canvas(newImageBitmap) + + val blackPaint = Paint() + blackPaint.color = Color.BLACK + val whitePaint = Paint() + whitePaint.color = Color.WHITE + + // Fill the mask with white, then draw a black rectangle where the image is. + maskCanvas.drawPaint(whitePaint) + maskCanvas.drawRect(normalizedImageRectangle, blackPaint) + + // fill the image with black, and then draw the bitmap into the corresponding spot + newImageCanvas.drawPaint(blackPaint) + newImageCanvas.drawBitmap(originalBitmap, null, normalizedImageRectangle, null) + return listOf( + ImagenRawImage(newImageBitmap.toImagenInlineImage()), + ImagenRawMask(maskBitmap.toImagenInlineImage()), + ) } } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt index a36b67dee06..5a029c41e22 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt @@ -3,8 +3,11 @@ package com.google.firebase.ai.type public class ImagenSubjectReferenceType private constructor(internal val value: String) { public companion object { - public val PERSON: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PERSON") - public val ANIMAL: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_ANIMAL") - public val PRODUCT: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PRODUCT") + public val PERSON: ImagenSubjectReferenceType = + ImagenSubjectReferenceType("SUBJECT_TYPE_PERSON") + public val ANIMAL: ImagenSubjectReferenceType = + ImagenSubjectReferenceType("SUBJECT_TYPE_ANIMAL") + public val PRODUCT: ImagenSubjectReferenceType = + ImagenSubjectReferenceType("SUBJECT_TYPE_PRODUCT") } } From c0f616138624eb84b75af631123ed6fe79ca2a8e Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Thu, 10 Jul 2025 11:17:51 -0700 Subject: [PATCH 09/22] add copyright headers --- .../com/google/firebase/ai/type/Dimensions.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenBackgroundMask.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenControlConfig.kt | 17 ++++++++++++++++- .../firebase/ai/type/ImagenControlReference.kt | 17 +++++++++++++++-- .../firebase/ai/type/ImagenControlType.kt | 15 +++++++++++++++ .../google/firebase/ai/type/ImagenEditMode.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenEditingConfig.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenForegroundMask.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenImagePlacement.kt | 15 +++++++++++++++ .../google/firebase/ai/type/ImagenMaskConfig.kt | 15 +++++++++++++++ .../google/firebase/ai/type/ImagenMaskMode.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenMaskReference.kt | 15 +++++++++++++++ .../google/firebase/ai/type/ImagenRawImage.kt | 15 +++++++++++++++ .../google/firebase/ai/type/ImagenRawMask.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenReferenceImage.kt | 16 +++++++++++++++- .../firebase/ai/type/ImagenSemanticMask.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenStyleConfig.kt | 15 +++++++++++++++ .../firebase/ai/type/ImagenStyleReference.kt | 15 +++++++++++++++ .../ai/type/ImagenSubjectReferenceType.kt | 15 +++++++++++++++ 19 files changed, 286 insertions(+), 4 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt index 350441a7c08..548fa3b365e 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type public class Dimensions(public val width: Int, public val height: Int) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt index 79129a9d455..ad0d421ee6a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type @PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt index bb670788a8e..684fd69eb83 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt @@ -1,4 +1,19 @@ -package com.google.firebase.ai.typeImage +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.firebase.ai.type import com.google.firebase.ai.type.ImagenControlType diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt index c0edf339298..9fa7f592d08 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt @@ -1,7 +1,20 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type -import com.google.firebase.ai.typeImage.ImagenControlConfig - @PublicPreviewAPI public class ImagenControlReference( image: ImagenInlineImage, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt index f41a5dea2e3..8fdbbc47cbc 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type public class ImagenControlType internal constructor(internal val value: String) { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt index 339ac440ea1..0d613b55e1a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type public class ImagenEditMode private constructor(internal val value: String) { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt index 35b06879788..adf8dc86a4f 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type import kotlinx.serialization.Serializable diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt index e2a9f244c3a..1d019c54c69 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type @PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt index e521966b65f..c88a0fd1a29 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type public class ImagenImagePlacement private constructor(public val x: Int?, public val y: Int?) { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt index 51219677ff1..64a9a4376e3 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskConfig.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type import kotlinx.serialization.Serializable diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt index 3ed2323c3da..b83a20381e2 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskMode.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type internal class ImagenMaskMode private constructor(internal val value: String) { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt index d008e47a2e4..bde908dd8a9 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type import android.graphics.Bitmap diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt index 8776c0d04f2..11edeb3a4ec 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type @PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt index d555227486a..8169ded0482 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type @PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt index 1fd676cb644..24d6344d65e 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -1,7 +1,21 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type import com.google.firebase.ai.common.GenerateImageRequest -import com.google.firebase.ai.typeImage.ImagenControlConfig import kotlinx.serialization.Serializable @PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt index e577a9fca54..edf5e2764d7 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type @PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt index 917c660eeba..222f9b3f5a3 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleConfig.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type import kotlinx.serialization.Serializable diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt index c9fd278efe3..4c162784c72 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type @PublicPreviewAPI diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt index 5a029c41e22..d0afade387e 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type public class ImagenSubjectReferenceType private constructor(internal val value: String) { From 778f176f97a2e20f15e59c62b7f8d117201fb4ba Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Fri, 11 Jul 2025 11:23:11 -0700 Subject: [PATCH 10/22] javadocs1 --- firebase-ai/api.txt | 2 + .../com/google/firebase/ai/ImagenModel.kt | 29 ++++++++ .../firebase/ai/java/ImagenModelFutures.kt | 67 +++++++++++++++++++ .../firebase/ai/type/ImagenControlConfig.kt | 2 - 4 files changed, 98 insertions(+), 2 deletions(-) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index ce6071e89ca..b4812b80bd6 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -111,6 +111,8 @@ package com.google.firebase.ai.java { method public static final com.google.firebase.ai.java.ImagenModelFutures from(com.google.firebase.ai.ImagenModel model); method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String prompt); method public abstract com.google.firebase.ai.ImagenModel getImageModel(); + method public abstract com.google.common.util.concurrent.ListenableFuture> inpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, String prompt, com.google.firebase.ai.type.ImagenMaskReference mask, com.google.firebase.ai.type.ImagenEditingConfig config); + method public abstract com.google.common.util.concurrent.ListenableFuture> outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null); field public static final com.google.firebase.ai.java.ImagenModelFutures.Companion Companion; } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index ee7ebbf8122..acef248694c 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -89,6 +89,14 @@ internal constructor( throw FirebaseAIException.from(e) } + /** + * Generates an image from a single or set of base images, returning the result directly to the + * caller. + * + * @param prompt the text input given to the model as a prompt + * @param referenceImages the image inputs given to the model as a prompt + * @param config the editing configuration settings + */ public suspend fun editImage( referenceImages: List, prompt: String, @@ -103,6 +111,14 @@ internal constructor( throw FirebaseAIException.from(e) } + /** + * Generates an image by inpainting a masked off part of a base image. + * + * @param image the base image + * @param prompt the text input given to the model as a prompt + * @param mask the mask which defines where in the image can be painted by imagen. + * @param config the editing configuration settings, its important to include an [ImagenEditMode] + */ public suspend fun inpaintImage( image: ImagenInlineImage, prompt: String, @@ -112,6 +128,19 @@ internal constructor( return editImage(listOf(ImagenRawImage(image), mask), prompt, config) } + /** + * Generates an image by outpainting the image, extending its borders + * + * @param image the base image + * @param newDimensions the new dimensions for the image, *must* be larger than the original + * image. + * @param newPosition the placement of the base image within the new image. This can either be + * coordinates (0,0 is the top left corner) or an alignment (ex: [ImagenImagePlacement.BOTTOM]) + * @param prompt optional, but can be used to specify the background generated if context is + * insufficient + * @param config the editing configuration settings + * @see [ImagenMaskReference.generateMaskAndPadForOutpainting] + */ public suspend fun outpaintImage( image: ImagenInlineImage, newDimensions: Dimensions, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt index a105c53a705..03f1ae5d983 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt @@ -19,9 +19,13 @@ package com.google.firebase.ai.java import androidx.concurrent.futures.SuspendToFutureAdapter import com.google.common.util.concurrent.ListenableFuture import com.google.firebase.ai.ImagenModel +import com.google.firebase.ai.type.Dimensions +import com.google.firebase.ai.type.ImagenEditMode import com.google.firebase.ai.type.ImagenEditingConfig import com.google.firebase.ai.type.ImagenGenerationResponse +import com.google.firebase.ai.type.ImagenImagePlacement import com.google.firebase.ai.type.ImagenInlineImage +import com.google.firebase.ai.type.ImagenMaskReference import com.google.firebase.ai.type.ImagenReferenceImage import com.google.firebase.ai.type.PublicPreviewAPI @@ -41,12 +45,56 @@ public abstract class ImagenModelFutures internal constructor() { prompt: String, ): ListenableFuture> + /** + * Generates an image from a single or set of base images, returning the result directly to the + * caller. + * + * @param prompt the text input given to the model as a prompt + * @param referenceImages the image inputs given to the model as a prompt + * @param config the editing configuration settings + */ public abstract fun editImage( referenceImages: List, prompt: String, config: ImagenEditingConfig? = null ): ListenableFuture> + /** + * Generates an image by inpainting a masked off part of a base image. + * + * @param image the base image + * @param prompt the text input given to the model as a prompt + * @param mask the mask which defines where in the image can be painted by imagen. + * @param config the editing configuration settings, its important to include an [ImagenEditMode] + */ + public abstract fun inpaintImage( + image: ImagenInlineImage, + prompt: String, + mask: ImagenMaskReference, + config: ImagenEditingConfig, + ): ListenableFuture> + + /** + * Generates an image by outpainting the image, extending its borders + * + * @param image the base image + * @param newDimensions the new dimensions for the image, *must* be larger than the original + * image. + * @param newPosition the placement of the base image within the new image. This can either be + * coordinates (0,0 is the top left corner) or an alignment (ex: [ImagenImagePlacement.BOTTOM]) + * @param prompt optional, but can be used to specify the background generated if context is + * insufficient + * @param config the editing configuration settings + * @see [ImagenMaskReference.generateMaskAndPadForOutpainting] + */ + public abstract fun outpaintImage( + image: ImagenInlineImage, + newDimensions: Dimensions, + newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER, + prompt: String = "", + config: ImagenEditingConfig? = null, + ): ListenableFuture> + /** Returns the [ImagenModel] object wrapped by this object. */ public abstract fun getImageModel(): ImagenModel @@ -63,6 +111,25 @@ public abstract class ImagenModelFutures internal constructor() { ): ListenableFuture> = SuspendToFutureAdapter.launchFuture { model.editImage(referenceImages, prompt, config) } + override fun inpaintImage( + image: ImagenInlineImage, + prompt: String, + mask: ImagenMaskReference, + config: ImagenEditingConfig + ): ListenableFuture> = + SuspendToFutureAdapter.launchFuture { model.inpaintImage(image, prompt, mask, config) } + + override fun outpaintImage( + image: ImagenInlineImage, + newDimensions: Dimensions, + newPosition: ImagenImagePlacement, + prompt: String, + config: ImagenEditingConfig? + ): ListenableFuture> = + SuspendToFutureAdapter.launchFuture { + model.outpaintImage(image, newDimensions, newPosition, prompt, config) + } + override fun getImageModel(): ImagenModel = model } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt index 684fd69eb83..09d6122140b 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt @@ -15,8 +15,6 @@ */ package com.google.firebase.ai.type -import com.google.firebase.ai.type.ImagenControlType - internal class ImagenControlConfig( internal val controlType: ImagenControlType, internal val enableComputation: Boolean? = null, From 1131d228a3c247ef5aa6e4c600279fa6fd4874ae Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 15 Jul 2025 22:49:01 -0700 Subject: [PATCH 11/22] Add refdocs, and address a few review comments --- firebase-ai/api.txt | 18 +- .../com/google/firebase/ai/ImagenModel.kt | 3 +- .../firebase/ai/java/ImagenModelFutures.kt | 3 +- .../com/google/firebase/ai/type/Dimensions.kt | 5 + .../firebase/ai/type/ImagenBackgroundMask.kt | 20 -- .../firebase/ai/type/ImagenControlConfig.kt | 22 +- .../ai/type/ImagenControlReference.kt | 32 --- .../firebase/ai/type/ImagenControlType.kt | 10 + .../google/firebase/ai/type/ImagenEditMode.kt | 7 + .../firebase/ai/type/ImagenEditingConfig.kt | 5 + .../firebase/ai/type/ImagenForegroundMask.kt | 20 -- .../firebase/ai/type/ImagenImagePlacement.kt | 65 ++++-- .../firebase/ai/type/ImagenInlineImage.kt | 4 +- .../firebase/ai/type/ImagenMaskReference.kt | 75 ------- .../google/firebase/ai/type/ImagenRawImage.kt | 19 -- .../google/firebase/ai/type/ImagenRawMask.kt | 23 -- .../firebase/ai/type/ImagenReferenceImage.kt | 205 ++++++++++++++++++ .../firebase/ai/type/ImagenSemanticMask.kt | 20 -- .../firebase/ai/type/ImagenStyleReference.kt | 28 --- .../ai/type/ImagenSubjectReference.kt | 14 -- .../ai/type/ImagenSubjectReferenceType.kt | 4 + 21 files changed, 324 insertions(+), 278 deletions(-) delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt delete mode 100644 firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index b4812b80bd6..9f0f90a5aab 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -503,7 +503,7 @@ package com.google.firebase.ai.type { } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenControlReference extends com.google.firebase.ai.type.ImagenReferenceImage { - ctor public ImagenControlReference(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.ImagenControlType type, Integer? referenceId = null, Boolean? enableComputation = null, Integer? superpixelRegionSize = null, Integer? superpixelRuler = null); + ctor public ImagenControlReference(com.google.firebase.ai.type.ImagenControlType type, com.google.firebase.ai.type.ImagenInlineImage? image = null, Integer? referenceId = null, Boolean? enableComputation = null, Integer? superpixelRegionSize = null, Integer? superpixelRuler = null); } public final class ImagenControlType { @@ -612,22 +612,22 @@ package com.google.firebase.ai.type { public static final class ImagenImagePlacement.Companion { method public com.google.firebase.ai.type.ImagenImagePlacement fromCoordinate(int x, int y); - method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM(); + method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_CENTER(); method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_LEFT(); method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_RIGHT(); method public com.google.firebase.ai.type.ImagenImagePlacement getCENTER(); - method public com.google.firebase.ai.type.ImagenImagePlacement getLEFT(); - method public com.google.firebase.ai.type.ImagenImagePlacement getRIGHT(); - method public com.google.firebase.ai.type.ImagenImagePlacement getTOP(); + method public com.google.firebase.ai.type.ImagenImagePlacement getLEFT_CENTER(); + method public com.google.firebase.ai.type.ImagenImagePlacement getRIGHT_CENTER(); + method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_CENTER(); method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_LEFT(); method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_RIGHT(); - property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM; + property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_CENTER; property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_LEFT; property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_RIGHT; property public final com.google.firebase.ai.type.ImagenImagePlacement CENTER; - property public final com.google.firebase.ai.type.ImagenImagePlacement LEFT; - property public final com.google.firebase.ai.type.ImagenImagePlacement RIGHT; - property public final com.google.firebase.ai.type.ImagenImagePlacement TOP; + property public final com.google.firebase.ai.type.ImagenImagePlacement LEFT_CENTER; + property public final com.google.firebase.ai.type.ImagenImagePlacement RIGHT_CENTER; + property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_CENTER; property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_LEFT; property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_RIGHT; } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index acef248694c..86364996be2 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -135,7 +135,8 @@ internal constructor( * @param newDimensions the new dimensions for the image, *must* be larger than the original * image. * @param newPosition the placement of the base image within the new image. This can either be - * coordinates (0,0 is the top left corner) or an alignment (ex: [ImagenImagePlacement.BOTTOM]) + * coordinates (0,0 is the top left corner) or an alignment (ex: + * [ImagenImagePlacement.BOTTOM_CENTER]) * @param prompt optional, but can be used to specify the background generated if context is * insufficient * @param config the editing configuration settings diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt index 03f1ae5d983..91af7679115 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt @@ -81,7 +81,8 @@ public abstract class ImagenModelFutures internal constructor() { * @param newDimensions the new dimensions for the image, *must* be larger than the original * image. * @param newPosition the placement of the base image within the new image. This can either be - * coordinates (0,0 is the top left corner) or an alignment (ex: [ImagenImagePlacement.BOTTOM]) + * coordinates (0,0 is the top left corner) or an alignment (ex: + * [ImagenImagePlacement.BOTTOM_CENTER]) * @param prompt optional, but can be used to specify the background generated if context is * insufficient * @param config the editing configuration settings diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt index 548fa3b365e..98f256f39b2 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Dimensions.kt @@ -15,4 +15,9 @@ */ package com.google.firebase.ai.type +/** + * Represents the dimensions of an image in pixels + * @param width the width of the image in pixels + * @param height the height of the image in pixels + */ public class Dimensions(public val width: Int, public val height: Int) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt deleted file mode 100644 index ad0d421ee6a..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenBackgroundMask.kt +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenBackgroundMask(dilation: Double? = null) : - ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.BACKGROUND, dilation)) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt index 09d6122140b..c6f3b01af6c 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlConfig.kt @@ -15,9 +15,29 @@ */ package com.google.firebase.ai.type +import kotlinx.serialization.Serializable + internal class ImagenControlConfig( internal val controlType: ImagenControlType, internal val enableComputation: Boolean? = null, internal val superpixelRegionSize: Int? = null, internal val superpixelRuler: Int? = null -) {} +) { + + fun toInternal(): Internal { + return Internal( + controlType = controlType.value, + enableControlImageComputation = enableComputation, + superpixelRegionSize = superpixelRegionSize, + superpixelRuler = superpixelRuler + ) + } + + @Serializable + internal class Internal( + val controlType: String?, + val enableControlImageComputation: Boolean?, + val superpixelRegionSize: Int?, + val superpixelRuler: Int? + ) +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt deleted file mode 100644 index 9fa7f592d08..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlReference.kt +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenControlReference( - image: ImagenInlineImage, - type: ImagenControlType, - referenceId: Int? = null, - enableComputation: Boolean? = null, - superpixelRegionSize: Int? = null, - superpixelRuler: Int? = null, -) : - ImagenReferenceImage( - controlConfig = - ImagenControlConfig(type, enableComputation, superpixelRegionSize, superpixelRuler), - image = image, - referenceId = referenceId, - ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt index 8fdbbc47cbc..bfbdb3dae05 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt @@ -15,11 +15,21 @@ */ package com.google.firebase.ai.type +/** Represents a control type for controlled Imagen generation/editing */ public class ImagenControlType internal constructor(internal val value: String) { public companion object { + /** Canny Control uses edge detection to ensure the new image follow the same outlines */ public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY") + /** + * Scribble Control uses enhanced edge detection to ensure the new image follow similar outlines + */ public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE") + /** Face mesh control is used to ensure that the new image has the same facial expressions */ public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH") + /** + * Color superpixels are a control mechanism used to ensure that the new image looks similar in + * shape and color to the original + */ public val COLOR_SUPERPIXEL: ImagenControlType = ImagenControlType("CONTROL_TYPE_COLOR_SUPERPIXEL") } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt index 0d613b55e1a..5cc368c1abd 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt @@ -15,11 +15,18 @@ */ package com.google.firebase.ai.type +/** Represents the edit mode for Imagen */ public class ImagenEditMode private constructor(internal val value: String) { public companion object { + /** Inpainting insertion is used when inserting a new element into an image */ public val INPAINT_INSERTION: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_INSERTION") + /** + * Inpainting removal is used when removing an element from an image, and replacing it with + * background + */ public val INPAINT_REMOVAL: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_REMOVAL") + /** Outpainting is used to extend the borders of an image outwards */ public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT") } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt index adf8dc86a4f..60e54261c1e 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditingConfig.kt @@ -17,6 +17,11 @@ package com.google.firebase.ai.type import kotlinx.serialization.Serializable +/** + * Contains the editing settings which are not specific to a reference image + * @param editMode holds the editing mode if the request is for inpainting or outpainting + * @param editSteps the number of intermediate steps to include in the editing process + */ @PublicPreviewAPI public class ImagenEditingConfig( internal val editMode: ImagenEditMode? = null, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt deleted file mode 100644 index 1d019c54c69..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenForegroundMask.kt +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenForegroundMask(dilation: Double? = null) : - ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.FOREGROUND, dilation)) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt index c88a0fd1a29..5c907051d81 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt @@ -15,8 +15,17 @@ */ package com.google.firebase.ai.type -public class ImagenImagePlacement private constructor(public val x: Int?, public val y: Int?) { +/** + * Represents where the placement of an image within a new, larger image, usually in the context of + * an outpainting request. + */ +public class ImagenImagePlacement +private constructor(public val x: Int? = null, public val y: Int? = null) { + /** + * If this placement is represented by coordinates this is a no-op, if its one of the enumerated + * types below, then the position is calculated based on its description + */ internal fun normalizeToDimensions( imageDimensions: Dimensions, canvasDimensions: Dimensions, @@ -31,14 +40,14 @@ public class ImagenImagePlacement private constructor(public val x: Int?, public return when (this) { CENTER -> ImagenImagePlacement(halfCanvasWidth - halfImageWidth, halfCanvasHeight - halfImageHeight) - TOP -> ImagenImagePlacement(halfCanvasWidth - halfImageWidth, 0) - BOTTOM -> + TOP_CENTER -> ImagenImagePlacement(halfCanvasWidth - halfImageWidth, 0) + BOTTOM_CENTER -> ImagenImagePlacement( halfCanvasWidth - halfImageWidth, canvasDimensions.height - imageDimensions.height, ) - LEFT -> ImagenImagePlacement(0, halfCanvasHeight - halfImageHeight) - RIGHT -> + LEFT_CENTER -> ImagenImagePlacement(0, halfCanvasHeight - halfImageHeight) + RIGHT_CENTER -> ImagenImagePlacement( canvasDimensions.width - imageDimensions.width, halfCanvasHeight - halfImageHeight, @@ -57,18 +66,48 @@ public class ImagenImagePlacement private constructor(public val x: Int?, public } public companion object { + /** + * Creates an [ImagenImagePlacement] that represents a placement in the described by two + * coordinates. The coordinate system has 0,0 in the top left corner, and the x and y + * coordinates represent the location of the top left corner of the original image. + * @param x the x coordinate of the top left corner of the original image + * @param y the y coordinate of the top left corner of the original image + */ public fun fromCoordinate(x: Int, y: Int): ImagenImagePlacement { return ImagenImagePlacement(x, y) } - public val CENTER: ImagenImagePlacement = ImagenImagePlacement(null, null) - public val TOP: ImagenImagePlacement = ImagenImagePlacement(null, null) - public val BOTTOM: ImagenImagePlacement = ImagenImagePlacement(null, null) - public val LEFT: ImagenImagePlacement = ImagenImagePlacement(null, null) - public val RIGHT: ImagenImagePlacement = ImagenImagePlacement(null, null) + /** + * This [ImagenImagePlacement] is centered horizontally and vertically within the larger image + */ + public val CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** + * This [ImagenImagePlacement] is centered horizontally and aligned with the top edge of the + * larger image + */ + public val TOP_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** + * This [ImagenImagePlacement] is centered horizontally and aligned with the bottom edge of the + * larger image + */ + public val BOTTOM_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** + * This [ImagenImagePlacement] is centered vertically and aligned with the left edge of the + * larger image + */ + public val LEFT_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** + * This [ImagenImagePlacement] is centered vertically and aligned with the right edge of the + * larger image + */ + public val RIGHT_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** This [ImagenImagePlacement] is aligned with the top left corner of the larger image */ public val TOP_LEFT: ImagenImagePlacement = ImagenImagePlacement(0, 0) - public val TOP_RIGHT: ImagenImagePlacement = ImagenImagePlacement(null, null) - public val BOTTOM_LEFT: ImagenImagePlacement = ImagenImagePlacement(null, null) - public val BOTTOM_RIGHT: ImagenImagePlacement = ImagenImagePlacement(null, null) + /** This [ImagenImagePlacement] is aligned with the top right corner of the larger image */ + public val TOP_RIGHT: ImagenImagePlacement = ImagenImagePlacement() + /** This [ImagenImagePlacement] is aligned with the bottom left corner of the larger image */ + public val BOTTOM_LEFT: ImagenImagePlacement = ImagenImagePlacement() + /** This [ImagenImagePlacement] is caligned with the bottom right corner of the larger image */ + public val BOTTOM_RIGHT: ImagenImagePlacement = ImagenImagePlacement() } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt index be09cb69c73..fc6033d5390 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenInlineImage.kt @@ -51,7 +51,7 @@ internal constructor(public val data: ByteArray, public val mimeType: String) { @PublicPreviewAPI public fun Bitmap.toImagenInlineImage(): ImagenInlineImage { val byteArrayOutputStream = ByteArrayOutputStream() - this.compress(Bitmap.CompressFormat.PNG, 100, byteArrayOutputStream) + this.compress(Bitmap.CompressFormat.JPEG, 80, byteArrayOutputStream) val byteArray = byteArrayOutputStream.toByteArray() - return ImagenInlineImage(data = byteArray, mimeType = "image/png") + return ImagenInlineImage(data = byteArray, mimeType = "image/jpeg") } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt deleted file mode 100644 index bde908dd8a9..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenMaskReference.kt +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -import android.graphics.Bitmap -import android.graphics.Canvas -import android.graphics.Color -import android.graphics.Paint -import android.graphics.Rect - -@PublicPreviewAPI -public abstract class ImagenMaskReference -internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = null) : - ImagenReferenceImage(maskConfig = maskConfig, image = image) { - - public companion object { - public fun generateMaskAndPadForOutpainting( - image: ImagenInlineImage, - newDimensions: Dimensions, - newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER, - ): List { - val originalBitmap = image.asBitmap() - val normalizedPosition = - newPosition.normalizeToDimensions( - Dimensions(originalBitmap.width, originalBitmap.height), - newDimensions, - ) - val normalizedImageRectangle = - Rect( - normalizedPosition.x!!, - normalizedPosition.y!!, - normalizedPosition.x + originalBitmap.width, - normalizedPosition.y + originalBitmap.height, - ) - - val maskBitmap = - Bitmap.createBitmap(newDimensions.width, newDimensions.height, Bitmap.Config.RGB_565) - val newImageBitmap = - Bitmap.createBitmap(newDimensions.width, newDimensions.height, Bitmap.Config.RGB_565) - - val maskCanvas = Canvas(maskBitmap) - val newImageCanvas = Canvas(newImageBitmap) - - val blackPaint = Paint() - blackPaint.color = Color.BLACK - val whitePaint = Paint() - whitePaint.color = Color.WHITE - - // Fill the mask with white, then draw a black rectangle where the image is. - maskCanvas.drawPaint(whitePaint) - maskCanvas.drawRect(normalizedImageRectangle, blackPaint) - - // fill the image with black, and then draw the bitmap into the corresponding spot - newImageCanvas.drawPaint(blackPaint) - newImageCanvas.drawBitmap(originalBitmap, null, normalizedImageRectangle, null) - return listOf( - ImagenRawImage(newImageBitmap.toImagenInlineImage()), - ImagenRawMask(maskBitmap.toImagenInlineImage()), - ) - } - } -} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt deleted file mode 100644 index 11edeb3a4ec..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawImage.kt +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenRawImage(image: ImagenInlineImage) : ImagenReferenceImage(image = image) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt deleted file mode 100644 index 8169ded0482..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenRawMask.kt +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenRawMask(mask: ImagenInlineImage, dilation: Double? = null) : - ImagenMaskReference( - maskConfig = ImagenMaskConfig(ImagenMaskMode.USER_PROVIDED, dilation), - image = mask, - ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt index 24d6344d65e..8c852491fcc 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -15,9 +15,15 @@ */ package com.google.firebase.ai.type +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Rect import com.google.firebase.ai.common.GenerateImageRequest import kotlinx.serialization.Serializable +/** Represents an reference image for an Imagen editing request */ @PublicPreviewAPI public abstract class ImagenReferenceImage internal constructor( @@ -50,6 +56,7 @@ internal constructor( subjectImageConfig = subjectConfig?.toInternal(), maskImageConfig = maskConfig?.toInternal(), styleImageConfig = styleConfig?.toInternal(), + controlConfig = controlConfig?.toInternal(), ) } @@ -61,5 +68,203 @@ internal constructor( val subjectImageConfig: ImagenSubjectConfig.Internal?, val maskImageConfig: ImagenMaskConfig.Internal?, val styleImageConfig: ImagenStyleConfig.Internal?, + val controlConfig: ImagenControlConfig.Internal? ) } + +/** + * Represents a refrence image (provided or generated) to bound the created image via ControlNet + * @param image the image provided, required if enableComputation is false + * @param type the type of ControlNet reference image + * @param referenceId the reference ID for this image, to be referenced in the prompt + * @param enableComputation requests that the reference image be generated serverside instead of + * provided + * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and enableComputation is true, this will + * control the size of each superpixel region in pixels for the generated referenced image + * @param superpixelRuler if type is COLOR_SUPERPIXEL and enableComputation is true, this will + * control the superpixel smoothness factor for the generated referenced image + */ +@PublicPreviewAPI +public class ImagenControlReference( + type: ImagenControlType, + image: ImagenInlineImage? = null, + referenceId: Int? = null, + enableComputation: Boolean? = null, + superpixelRegionSize: Int? = null, + superpixelRuler: Int? = null, +) : + ImagenReferenceImage( + controlConfig = + ImagenControlConfig(type, enableComputation, superpixelRegionSize, superpixelRuler), + image = image, + referenceId = referenceId, + ) {} + +/** + * Represents a reference image for Imagen editing which will mask of a region to be edited. This + * image (generated or provided) should contain only black and white pixels, with black representing + * parts of the image which should not change. + */ +@PublicPreviewAPI +public abstract class ImagenMaskReference +internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = null) : + ImagenReferenceImage(maskConfig = maskConfig, image = image) { + + public companion object { + /** + * Generates two reference images: + * * One [ImagenRawImage] containing the original image, padded out to the new dimensions with + * black pixels, with the original image placed at the given placement + * * One [ImagenRawMask] of the same dimensions containing white everywhere except at the + * placement original image. + * + * This is the format expected by Imagen for outpainting requests. + * + * @param image the original image + * @param newDimensions the new dimensions for outpainting. This *must* be more than the + * original image. + * @param newPosition the placement of the original image within the new outpainted image. + */ + public fun generateMaskAndPadForOutpainting( + image: ImagenInlineImage, + newDimensions: Dimensions, + newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER, + ): List { + val originalBitmap = image.asBitmap() + val normalizedPosition = + newPosition.normalizeToDimensions( + Dimensions(originalBitmap.width, originalBitmap.height), + newDimensions, + ) + + if (normalizedPosition.x == null || normalizedPosition.y == null) { + throw IllegalStateException("Error normalizing position for mask and padding.") + } + + val normalizedImageRectangle = + Rect( + normalizedPosition.x, + normalizedPosition.y, + normalizedPosition.x + originalBitmap.width, + normalizedPosition.y + originalBitmap.height, + ) + + val maskBitmap = + Bitmap.createBitmap(newDimensions.width, newDimensions.height, Bitmap.Config.RGB_565) + val newImageBitmap = + Bitmap.createBitmap(newDimensions.width, newDimensions.height, Bitmap.Config.RGB_565) + + val maskCanvas = Canvas(maskBitmap) + val newImageCanvas = Canvas(newImageBitmap) + + val blackPaint = Paint().apply { color = Color.BLACK } + val whitePaint = Paint().apply { color = Color.WHITE } + + // Fill the mask with white, then draw a black rectangle where the image is. + maskCanvas.drawPaint(whitePaint) + maskCanvas.drawRect(normalizedImageRectangle, blackPaint) + + // fill the image with black, and then draw the bitmap into the corresponding spot + newImageCanvas.drawPaint(blackPaint) + newImageCanvas.drawBitmap(originalBitmap, null, normalizedImageRectangle, null) + return listOf( + ImagenRawImage(newImageBitmap.toImagenInlineImage()), + ImagenRawMask(maskBitmap.toImagenInlineImage()), + ) + } + } +} + +/** + * A generated mask image which will auto-detect and mask out the background. The background will be + * white, and the foreground black + * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and + * make it seem more convincing. For example, 0.05 would dilate the mask 5%. + */ +@PublicPreviewAPI +public class ImagenBackgroundMask(dilation: Double? = null) : + ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.BACKGROUND, dilation)) {} + +/** + * A generated mask image which will auto-detect and mask out the foreground. The background will be + * black, and the foreground white + * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and + * make it seem more convincing. For example, 0.05 would dilate the mask 5%. + */ +@PublicPreviewAPI +public class ImagenForegroundMask(dilation: Double? = null) : + ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.FOREGROUND, dilation)) {} + +/** + * Represents a reference image for Imagen editing which will mask of a region to be edited. This + * image should contain only black and white pixels, with black representing parts of the image + * which should not change. + * + * @param mask the mask image + * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and + * make it seem more convincing. For example, 0.05 would dilate the mask 5%. + */ +@PublicPreviewAPI +public class ImagenRawMask(mask: ImagenInlineImage, dilation: Double? = null) : + ImagenMaskReference( + maskConfig = ImagenMaskConfig(ImagenMaskMode.USER_PROVIDED, dilation), + image = mask, + ) {} + +/** + * Represents a generated mask for Imagen editing which masks out certain objects using object + * detection. + * @param classes the list of segmentation IDs for objects to detect and mask out. See + * [here](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api-edit#segment-ids) + * for a list of segmentation IDs + * @param dilation the amount to dilate the mask, this can help smooth the borders of an edit and + * make it seem more convincing. For example, 0.05 would dilate the mask 5%. + */ +@PublicPreviewAPI +public class ImagenSemanticMask(classes: List, dilation: Double? = null) : + ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.SEMANTIC, dilation, classes)) {} + +/** + * Represents a base image for Imagen editing + * @param image the image + */ +@PublicPreviewAPI +public class ImagenRawImage(image: ImagenInlineImage) : ImagenReferenceImage(image = image) {} + +/** + * A reference image for style transfer + * @param image the image representing the style you want to transfer to your original images + * @param referenceId the reference ID you can use to reference this style in your prompt + * @param description the description you can use to reference this style in your prompt + */ +@PublicPreviewAPI +public class ImagenStyleReference( + image: ImagenInlineImage, + referenceId: Int? = null, + description: String? = null, +) : + ImagenReferenceImage( + image = image, + referenceId = referenceId, + styleConfig = ImagenStyleConfig(description) + ) {} + +/** + * A reference image for generating an image with a specific subject + * @param image the image of the subject + * @param referenceId the reference ID you can use to reference this subject in your prompt + * @param description the description you can use to reference this subject in your prompt + * @param subjectType the type of the subject + */ +@PublicPreviewAPI +public class ImagenSubjectReference( + image: ImagenInlineImage, + referenceId: Int? = null, + description: String? = null, + subjectType: ImagenSubjectReferenceType? = null, +) : + ImagenReferenceImage( + image = image, + referenceId = referenceId, + subjectConfig = ImagenSubjectConfig(description, subjectType), + ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt deleted file mode 100644 index edf5e2764d7..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSemanticMask.kt +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenSemanticMask(classes: List, dilation: Double? = null) : - ImagenMaskReference(maskConfig = ImagenMaskConfig(ImagenMaskMode.SEMANTIC, dilation, classes)) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt deleted file mode 100644 index 4c162784c72..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenStyleReference.kt +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2025 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenStyleReference( - image: ImagenInlineImage, - referenceId: Int? = null, - description: String? = null, -) : - ImagenReferenceImage( - image = image, - referenceId = referenceId, - styleConfig = ImagenStyleConfig(description) - ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt deleted file mode 100644 index c63abb5bf5d..00000000000 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReference.kt +++ /dev/null @@ -1,14 +0,0 @@ -package com.google.firebase.ai.type - -@PublicPreviewAPI -public class ImagenSubjectReference( - image: ImagenInlineImage, - referenceId: Int? = null, - description: String? = null, - subjectType: ImagenSubjectReferenceType? = null, -) : - ImagenReferenceImage( - image = image, - referenceId = referenceId, - subjectConfig = ImagenSubjectConfig(description, subjectType), - ) {} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt index d0afade387e..9f07b5711b9 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt @@ -15,13 +15,17 @@ */ package com.google.firebase.ai.type +/** Represents a type for a subject reference, specifying how it should be interpreted. */ public class ImagenSubjectReferenceType private constructor(internal val value: String) { public companion object { + /** Marks the reference type as being of a person */ public val PERSON: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PERSON") + /** Marks the reference type as being of a animal */ public val ANIMAL: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_ANIMAL") + /** Marks the reference type as being of a product */ public val PRODUCT: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PRODUCT") } From eb3ac0ab0ffb6b4ea85878e0b0dd0d0ace606284 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 15 Jul 2025 23:14:45 -0700 Subject: [PATCH 12/22] bump version --- firebase-ai/gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firebase-ai/gradle.properties b/firebase-ai/gradle.properties index d029f15815f..40274ac5e9a 100644 --- a/firebase-ai/gradle.properties +++ b/firebase-ai/gradle.properties @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version=16.3.0 -latestReleasedVersion=16.1.0 +version=17.1.0 +latestReleasedVersion=17.0.0 From 7ed0bd745af0b91827f671b65140904455b1bf99 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 16 Jul 2025 13:18:40 -0700 Subject: [PATCH 13/22] fixes for comments --- .../src/main/kotlin/com/google/firebase/ai/ImagenModel.kt | 2 +- .../src/main/kotlin/com/google/firebase/ai/common/Request.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index 86364996be2..3ae3add166e 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -161,7 +161,7 @@ internal constructor( generationConfig: ImagenGenerationConfig? = null, ): GenerateImageRequest { return GenerateImageRequest( - listOf(GenerateImageRequest.ImagenPrompt(prompt)), + listOf(GenerateImageRequest.ImagenPrompt(prompt, null)), GenerateImageRequest.ImagenParameters( sampleCount = generationConfig?.numberOfImages ?: 1, includeRaiReason = true, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt index 1375102df23..3f8e0ae079d 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt @@ -84,8 +84,8 @@ internal data class GenerateImageRequest( ) : Request { @Serializable internal data class ImagenPrompt( - val prompt: String? = null, - val referenceImages: List? = null + val prompt: String?, + val referenceImages: List? ) @OptIn(PublicPreviewAPI::class) From a48f7ac78a1b065449f98844f4e80cb8300f3a7a Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Thu, 17 Jul 2025 15:55:27 -0700 Subject: [PATCH 14/22] fixing refdocs for comments, and adding error for incorrect call to outpainting --- .../com/google/firebase/ai/ImagenModel.kt | 15 +++++--- .../firebase/ai/java/ImagenModelFutures.kt | 2 +- .../firebase/ai/type/ImagenControlType.kt | 12 +++--- .../google/firebase/ai/type/ImagenEditMode.kt | 9 ++--- .../firebase/ai/type/ImagenImagePlacement.kt | 38 ++++++------------- .../firebase/ai/type/ImagenReferenceImage.kt | 13 ++++++- 6 files changed, 41 insertions(+), 48 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt index 3ae3add166e..399da4c77cf 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt @@ -93,8 +93,8 @@ internal constructor( * Generates an image from a single or set of base images, returning the result directly to the * caller. * - * @param prompt the text input given to the model as a prompt * @param referenceImages the image inputs given to the model as a prompt + * @param prompt the text input given to the model as a prompt * @param config the editing configuration settings */ public suspend fun editImage( @@ -112,12 +112,14 @@ internal constructor( } /** - * Generates an image by inpainting a masked off part of a base image. + * Generates an image by inpainting a masked off part of a base image. Inpainting is the process + * of filling in missing or masked off parts of the image using context from the original image + * and prompt. * * @param image the base image * @param prompt the text input given to the model as a prompt - * @param mask the mask which defines where in the image can be painted by imagen. - * @param config the editing configuration settings, its important to include an [ImagenEditMode] + * @param mask the mask which defines where in the image can be painted by Imagen. + * @param config the editing configuration settings, it should include an [ImagenEditMode] */ public suspend fun inpaintImage( image: ImagenInlineImage, @@ -129,7 +131,8 @@ internal constructor( } /** - * Generates an image by outpainting the image, extending its borders + * Generates an image by outpainting the given image, extending its content beyond the original + * borders using context from the original image, and optionally, the prompt. * * @param image the base image * @param newDimensions the new dimensions for the image, *must* be larger than the original @@ -137,7 +140,7 @@ internal constructor( * @param newPosition the placement of the base image within the new image. This can either be * coordinates (0,0 is the top left corner) or an alignment (ex: * [ImagenImagePlacement.BOTTOM_CENTER]) - * @param prompt optional, but can be used to specify the background generated if context is + * @param prompt optional, can be used to specify the background generated if context is * insufficient * @param config the editing configuration settings * @see [ImagenMaskReference.generateMaskAndPadForOutpainting] diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt index 91af7679115..8c60a9e856a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt @@ -65,7 +65,7 @@ public abstract class ImagenModelFutures internal constructor() { * @param image the base image * @param prompt the text input given to the model as a prompt * @param mask the mask which defines where in the image can be painted by imagen. - * @param config the editing configuration settings, its important to include an [ImagenEditMode] + * @param config the editing configuration settings, it should include an [ImagenEditMode] */ public abstract fun inpaintImage( image: ImagenInlineImage, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt index bfbdb3dae05..6a573bbcf13 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt @@ -18,17 +18,15 @@ package com.google.firebase.ai.type /** Represents a control type for controlled Imagen generation/editing */ public class ImagenControlType internal constructor(internal val value: String) { public companion object { - /** Canny Control uses edge detection to ensure the new image follow the same outlines */ + /** Use edge detection to ensure the new image follow the same outlines */ public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY") - /** - * Scribble Control uses enhanced edge detection to ensure the new image follow similar outlines - */ + /** Use enhanced edge detection to ensure the new image follow similar outlines */ public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE") - /** Face mesh control is used to ensure that the new image has the same facial expressions */ + /** Use face mesh control to ensure that the new image has the same facial expressions */ public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH") /** - * Color superpixels are a control mechanism used to ensure that the new image looks similar in - * shape and color to the original + * Use color superpixels to ensure that the new image is similar in shape and color to the + * original */ public val COLOR_SUPERPIXEL: ImagenControlType = ImagenControlType("CONTROL_TYPE_COLOR_SUPERPIXEL") diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt index 5cc368c1abd..75b3c493c00 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt @@ -19,14 +19,11 @@ package com.google.firebase.ai.type public class ImagenEditMode private constructor(internal val value: String) { public companion object { - /** Inpainting insertion is used when inserting a new element into an image */ + /** Inserts a new element into an image */ public val INPAINT_INSERTION: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_INSERTION") - /** - * Inpainting removal is used when removing an element from an image, and replacing it with - * background - */ + /** Removes an element from an image */ public val INPAINT_REMOVAL: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_REMOVAL") - /** Outpainting is used to extend the borders of an image outwards */ + /** Extend the borders of an image outwards */ public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT") } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt index 5c907051d81..0144a96a284 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt @@ -16,8 +16,8 @@ package com.google.firebase.ai.type /** - * Represents where the placement of an image within a new, larger image, usually in the context of - * an outpainting request. + * Represents where the placement of an image is within a new, larger image, usually in the context + * of an outpainting request. */ public class ImagenImagePlacement private constructor(public val x: Int? = null, public val y: Int? = null) { @@ -67,7 +67,7 @@ private constructor(public val x: Int? = null, public val y: Int? = null) { public companion object { /** - * Creates an [ImagenImagePlacement] that represents a placement in the described by two + * Creates an [ImagenImagePlacement] that represents a placement in an image described by two * coordinates. The coordinate system has 0,0 in the top left corner, and the x and y * coordinates represent the location of the top left corner of the original image. * @param x the x coordinate of the top left corner of the original image @@ -77,37 +77,23 @@ private constructor(public val x: Int? = null, public val y: Int? = null) { return ImagenImagePlacement(x, y) } - /** - * This [ImagenImagePlacement] is centered horizontally and vertically within the larger image - */ + /** Center the image horizontally and vertically within the larger image */ public val CENTER: ImagenImagePlacement = ImagenImagePlacement() - /** - * This [ImagenImagePlacement] is centered horizontally and aligned with the top edge of the - * larger image - */ + /** Center the image horizontally and aligned with the top edge of the larger image */ public val TOP_CENTER: ImagenImagePlacement = ImagenImagePlacement() - /** - * This [ImagenImagePlacement] is centered horizontally and aligned with the bottom edge of the - * larger image - */ + /** Center the image horizontally and aligned with the bottom edge of the larger image */ public val BOTTOM_CENTER: ImagenImagePlacement = ImagenImagePlacement() - /** - * This [ImagenImagePlacement] is centered vertically and aligned with the left edge of the - * larger image - */ + /** Center the image vertically and aligned with the left edge of the larger image */ public val LEFT_CENTER: ImagenImagePlacement = ImagenImagePlacement() - /** - * This [ImagenImagePlacement] is centered vertically and aligned with the right edge of the - * larger image - */ + /** Center the image vertically and aligned with the right edge of the larger image */ public val RIGHT_CENTER: ImagenImagePlacement = ImagenImagePlacement() - /** This [ImagenImagePlacement] is aligned with the top left corner of the larger image */ + /** Align the image with the top left corner of the larger image */ public val TOP_LEFT: ImagenImagePlacement = ImagenImagePlacement(0, 0) - /** This [ImagenImagePlacement] is aligned with the top right corner of the larger image */ + /** Align the image with the top right corner of the larger image */ public val TOP_RIGHT: ImagenImagePlacement = ImagenImagePlacement() - /** This [ImagenImagePlacement] is aligned with the bottom left corner of the larger image */ + /** Align the image with the bottom left corner of the larger image */ public val BOTTOM_LEFT: ImagenImagePlacement = ImagenImagePlacement() - /** This [ImagenImagePlacement] is caligned with the bottom right corner of the larger image */ + /** Align the image with the bottom right corner of the larger image */ public val BOTTOM_RIGHT: ImagenImagePlacement = ImagenImagePlacement() } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt index 8c852491fcc..90351a7eb30 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -73,7 +73,7 @@ internal constructor( } /** - * Represents a refrence image (provided or generated) to bound the created image via ControlNet + * Represents a reference image (provided or generated) to bound the created image via ControlNet * @param image the image provided, required if enableComputation is false * @param type the type of ControlNet reference image * @param referenceId the reference ID for this image, to be referenced in the prompt @@ -112,7 +112,7 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n public companion object { /** - * Generates two reference images: + * Generates these two reference images in order: * * One [ImagenRawImage] containing the original image, padded out to the new dimensions with * black pixels, with the original image placed at the given placement * * One [ImagenRawMask] of the same dimensions containing white everywhere except at the @@ -131,6 +131,15 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER, ): List { val originalBitmap = image.asBitmap() + if ( + originalBitmap.width > newDimensions.width || originalBitmap.height > newDimensions.height + ) { + throw IllegalArgumentException( + "New Dimensions must be strictly larger than original image dimensions. Original image " + + "is:${originalBitmap.width}x${originalBitmap.height}, new dimensions are " + + "${newDimensions.width}x${newDimensions.height}" + ) + } val normalizedPosition = newPosition.normalizeToDimensions( Dimensions(originalBitmap.width, originalBitmap.height), From 21262c3aa64ff41861bde63cee5f36a7be71835d Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 22 Jul 2025 10:45:22 -0700 Subject: [PATCH 15/22] add tests, and JvmStatic and JvmField annotations --- .../firebase/ai/type/ImagenControlType.kt | 11 +++-- .../google/firebase/ai/type/ImagenEditMode.kt | 4 +- .../firebase/ai/type/ImagenImagePlacement.kt | 27 ++++++++----- .../firebase/ai/type/ImagenReferenceImage.kt | 4 +- .../ai/type/ImagenSubjectReferenceType.kt | 6 +++ .../google/firebase/ai/SerializationTests.kt | 40 +++++++++++++++++++ .../google/firebase/ai/JavaCompileTests.java | 20 ++++++++++ 7 files changed, 98 insertions(+), 14 deletions(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt index 6a573bbcf13..0732b9a0553 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenControlType.kt @@ -18,16 +18,21 @@ package com.google.firebase.ai.type /** Represents a control type for controlled Imagen generation/editing */ public class ImagenControlType internal constructor(internal val value: String) { public companion object { + /** Use edge detection to ensure the new image follow the same outlines */ - public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY") + @JvmField public val CANNY: ImagenControlType = ImagenControlType("CONTROL_TYPE_CANNY") + /** Use enhanced edge detection to ensure the new image follow similar outlines */ - public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE") + @JvmField public val SCRIBBLE: ImagenControlType = ImagenControlType("CONTROL_TYPE_SCRIBBLE") + /** Use face mesh control to ensure that the new image has the same facial expressions */ - public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH") + @JvmField public val FACE_MESH: ImagenControlType = ImagenControlType("CONTROL_TYPE_FACE_MESH") + /** * Use color superpixels to ensure that the new image is similar in shape and color to the * original */ + @JvmField public val COLOR_SUPERPIXEL: ImagenControlType = ImagenControlType("CONTROL_TYPE_COLOR_SUPERPIXEL") } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt index 75b3c493c00..a852ea88a2a 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenEditMode.kt @@ -20,10 +20,12 @@ public class ImagenEditMode private constructor(internal val value: String) { public companion object { /** Inserts a new element into an image */ + @JvmField public val INPAINT_INSERTION: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_INSERTION") /** Removes an element from an image */ + @JvmField public val INPAINT_REMOVAL: ImagenEditMode = ImagenEditMode("EDIT_MODE_INPAINT_REMOVAL") /** Extend the borders of an image outwards */ - public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT") + @JvmField public val OUTPAINT: ImagenEditMode = ImagenEditMode("EDIT_MODE_OUTPAINT") } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt index 0144a96a284..db003a15344 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenImagePlacement.kt @@ -73,27 +73,36 @@ private constructor(public val x: Int? = null, public val y: Int? = null) { * @param x the x coordinate of the top left corner of the original image * @param y the y coordinate of the top left corner of the original image */ + @JvmStatic public fun fromCoordinate(x: Int, y: Int): ImagenImagePlacement { return ImagenImagePlacement(x, y) } /** Center the image horizontally and vertically within the larger image */ - public val CENTER: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** Center the image horizontally and aligned with the top edge of the larger image */ - public val TOP_CENTER: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val TOP_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** Center the image horizontally and aligned with the bottom edge of the larger image */ - public val BOTTOM_CENTER: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val BOTTOM_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** Center the image vertically and aligned with the left edge of the larger image */ - public val LEFT_CENTER: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val LEFT_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** Center the image vertically and aligned with the right edge of the larger image */ - public val RIGHT_CENTER: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val RIGHT_CENTER: ImagenImagePlacement = ImagenImagePlacement() + /** Align the image with the top left corner of the larger image */ - public val TOP_LEFT: ImagenImagePlacement = ImagenImagePlacement(0, 0) + @JvmField public val TOP_LEFT: ImagenImagePlacement = ImagenImagePlacement(0, 0) + /** Align the image with the top right corner of the larger image */ - public val TOP_RIGHT: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val TOP_RIGHT: ImagenImagePlacement = ImagenImagePlacement() + /** Align the image with the bottom left corner of the larger image */ - public val BOTTOM_LEFT: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val BOTTOM_LEFT: ImagenImagePlacement = ImagenImagePlacement() + /** Align the image with the bottom right corner of the larger image */ - public val BOTTOM_RIGHT: ImagenImagePlacement = ImagenImagePlacement() + @JvmField public val BOTTOM_RIGHT: ImagenImagePlacement = ImagenImagePlacement() } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt index 90351a7eb30..a56e0318294 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -74,7 +74,7 @@ internal constructor( /** * Represents a reference image (provided or generated) to bound the created image via ControlNet - * @param image the image provided, required if enableComputation is false + * @param image the image provided, required if [enableComputation] is false * @param type the type of ControlNet reference image * @param referenceId the reference ID for this image, to be referenced in the prompt * @param enableComputation requests that the reference image be generated serverside instead of @@ -125,6 +125,8 @@ internal constructor(maskConfig: ImagenMaskConfig, image: ImagenInlineImage? = n * original image. * @param newPosition the placement of the original image within the new outpainted image. */ + @JvmOverloads + @JvmStatic public fun generateMaskAndPadForOutpainting( image: ImagenInlineImage, newDimensions: Dimensions, diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt index 9f07b5711b9..dfe77f9adf5 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectReferenceType.kt @@ -19,13 +19,19 @@ package com.google.firebase.ai.type public class ImagenSubjectReferenceType private constructor(internal val value: String) { public companion object { + /** Marks the reference type as being of a person */ + @JvmField public val PERSON: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PERSON") + /** Marks the reference type as being of a animal */ + @JvmField public val ANIMAL: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_ANIMAL") + /** Marks the reference type as being of a product */ + @JvmField public val PRODUCT: ImagenSubjectReferenceType = ImagenSubjectReferenceType("SUBJECT_TYPE_PRODUCT") } diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt index 9a16f6b5b46..2fbaf011460 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt @@ -20,11 +20,14 @@ import com.google.firebase.ai.common.util.descriptorToJson import com.google.firebase.ai.type.Candidate import com.google.firebase.ai.type.CountTokensResponse import com.google.firebase.ai.type.GenerateContentResponse +import com.google.firebase.ai.type.ImagenReferenceImage import com.google.firebase.ai.type.ModalityTokenCount +import com.google.firebase.ai.type.PublicPreviewAPI import com.google.firebase.ai.type.Schema import io.kotest.assertions.json.shouldEqualJson import org.junit.Test +@OptIn(PublicPreviewAPI::class) internal class SerializationTests { @Test fun `test countTokensResponse serialization as Json`() { @@ -233,4 +236,41 @@ internal class SerializationTests { val actualJson = descriptorToJson(Schema.Internal.serializer().descriptor) expectedJsonAsString shouldEqualJson actualJson.toString() } + + @Test + fun `test ReferenceImage serialization as Json`() { + val expectedJsonAsString = + """ + { + "id": "ImagenReferenceImage", + "type": "object", + "properties": { + "referenceType": { + "type": "string" + }, + "referenceImage": { + "${'$'}ref": "ImagenInlineImage" + }, + "referenceId": { + "type": "integer" + }, + "subjectImageConfig": { + "${'$'}ref": "ImagenSubjectConfig" + }, + "maskImageConfig": { + "${'$'}ref": "ImagenMaskConfig" + }, + "styleImageConfig": { + "${'$'}ref": "ImagenStyleConfig" + }, + "controlConfig": { + "${'$'}ref": "ImagenControlConfig" + } + } +} + """ + .trimIndent() + val actualJson = descriptorToJson(ImagenReferenceImage.Internal.serializer().descriptor) + expectedJsonAsString shouldEqualJson actualJson.toString() + } } diff --git a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java index 559c4ac8a04..3422c00e195 100644 --- a/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java +++ b/firebase-ai/src/testUtil/java/com/google/firebase/ai/JavaCompileTests.java @@ -21,9 +21,11 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.firebase.ai.FirebaseAI; import com.google.firebase.ai.GenerativeModel; +import com.google.firebase.ai.ImagenModel; import com.google.firebase.ai.LiveGenerativeModel; import com.google.firebase.ai.java.ChatFutures; import com.google.firebase.ai.java.GenerativeModelFutures; +import com.google.firebase.ai.java.ImagenModelFutures; import com.google.firebase.ai.java.LiveModelFutures; import com.google.firebase.ai.java.LiveSessionFutures; import com.google.firebase.ai.type.BlockReason; @@ -33,6 +35,7 @@ import com.google.firebase.ai.type.Content; import com.google.firebase.ai.type.ContentModality; import com.google.firebase.ai.type.CountTokensResponse; +import com.google.firebase.ai.type.Dimensions; import com.google.firebase.ai.type.FileDataPart; import com.google.firebase.ai.type.FinishReason; import com.google.firebase.ai.type.FunctionCallPart; @@ -43,6 +46,11 @@ import com.google.firebase.ai.type.HarmProbability; import com.google.firebase.ai.type.HarmSeverity; import com.google.firebase.ai.type.ImagePart; +import com.google.firebase.ai.type.ImagenBackgroundMask; +import com.google.firebase.ai.type.ImagenEditMode; +import com.google.firebase.ai.type.ImagenEditingConfig; +import com.google.firebase.ai.type.ImagenInlineImage; +import com.google.firebase.ai.type.ImagenMaskReference; import com.google.firebase.ai.type.InlineDataPart; import com.google.firebase.ai.type.LiveGenerationConfig; import com.google.firebase.ai.type.LiveServerContent; @@ -65,6 +73,7 @@ import com.google.firebase.concurrent.FirebaseExecutors; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; @@ -141,6 +150,17 @@ private LiveGenerationConfig getLiveConfig() { .build(); } + private void testImagen() { + ImagenModel modelSuspend = FirebaseAI.getInstance().imagenModel(""); + ImagenModelFutures model = ImagenModelFutures.from(modelSuspend); + model.editImage( + Collections.singletonList(new ImagenBackgroundMask()), + "", + new ImagenEditingConfig(ImagenEditMode.OUTPAINT, 25)); + ImagenMaskReference.generateMaskAndPadForOutpainting( + new ImagenInlineImage(new byte[0], ""), new Dimensions(0, 0)); + } + private void testFutures(GenerativeModelFutures futures) throws Exception { Content content = new Content.Builder() From adf495797497620a8bd247341d9696b985e6f25c Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 22 Jul 2025 11:13:37 -0700 Subject: [PATCH 16/22] add copyright and update api.txt --- firebase-ai/api.txt | 67 +++++++------------ .../firebase/ai/type/ImagenSubjectConfig.kt | 15 +++++ .../google/firebase/ai/SerializationTests.kt | 4 +- 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index ffb4b6fb61b..ebecf342156 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -71,7 +71,7 @@ package com.google.firebase.ai { method public suspend Object? editImage(java.util.List referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation>); method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation>); method public suspend Object? inpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, String prompt, com.google.firebase.ai.type.ImagenMaskReference mask, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation>); - method public suspend Object? outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation>); + method public suspend Object? outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation>); } @com.google.firebase.ai.type.PublicPreviewAPI public final class LiveGenerativeModel { @@ -115,7 +115,7 @@ package com.google.firebase.ai.java { method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String prompt); method public abstract com.google.firebase.ai.ImagenModel getImageModel(); method public abstract com.google.common.util.concurrent.ListenableFuture> inpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, String prompt, com.google.firebase.ai.type.ImagenMaskReference mask, com.google.firebase.ai.type.ImagenEditingConfig config); - method public abstract com.google.common.util.concurrent.ListenableFuture> outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null); + method public abstract com.google.common.util.concurrent.ListenableFuture> outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null); field public static final com.google.firebase.ai.java.ImagenModelFutures.Companion Companion; } @@ -556,31 +556,24 @@ package com.google.firebase.ai.type { } public final class ImagenControlType { + field public static final com.google.firebase.ai.type.ImagenControlType CANNY; + field public static final com.google.firebase.ai.type.ImagenControlType COLOR_SUPERPIXEL; field public static final com.google.firebase.ai.type.ImagenControlType.Companion Companion; + field public static final com.google.firebase.ai.type.ImagenControlType FACE_MESH; + field public static final com.google.firebase.ai.type.ImagenControlType SCRIBBLE; } public static final class ImagenControlType.Companion { - method public com.google.firebase.ai.type.ImagenControlType getCANNY(); - method public com.google.firebase.ai.type.ImagenControlType getCOLOR_SUPERPIXEL(); - method public com.google.firebase.ai.type.ImagenControlType getFACE_MESH(); - method public com.google.firebase.ai.type.ImagenControlType getSCRIBBLE(); - property public final com.google.firebase.ai.type.ImagenControlType CANNY; - property public final com.google.firebase.ai.type.ImagenControlType COLOR_SUPERPIXEL; - property public final com.google.firebase.ai.type.ImagenControlType FACE_MESH; - property public final com.google.firebase.ai.type.ImagenControlType SCRIBBLE; } public final class ImagenEditMode { field public static final com.google.firebase.ai.type.ImagenEditMode.Companion Companion; + field public static final com.google.firebase.ai.type.ImagenEditMode INPAINT_INSERTION; + field public static final com.google.firebase.ai.type.ImagenEditMode INPAINT_REMOVAL; + field public static final com.google.firebase.ai.type.ImagenEditMode OUTPAINT; } public static final class ImagenEditMode.Companion { - method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_INSERTION(); - method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_REMOVAL(); - method public com.google.firebase.ai.type.ImagenEditMode getOUTPAINT(); - property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_INSERTION; - property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_REMOVAL; - property public final com.google.firebase.ai.type.ImagenEditMode OUTPAINT; } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenEditingConfig { @@ -652,33 +645,25 @@ package com.google.firebase.ai.type { } public final class ImagenImagePlacement { + method public static com.google.firebase.ai.type.ImagenImagePlacement fromCoordinate(int x, int y); method public Integer? getX(); method public Integer? getY(); property public final Integer? x; property public final Integer? y; + field public static final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_CENTER; + field public static final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_LEFT; + field public static final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_RIGHT; + field public static final com.google.firebase.ai.type.ImagenImagePlacement CENTER; field public static final com.google.firebase.ai.type.ImagenImagePlacement.Companion Companion; + field public static final com.google.firebase.ai.type.ImagenImagePlacement LEFT_CENTER; + field public static final com.google.firebase.ai.type.ImagenImagePlacement RIGHT_CENTER; + field public static final com.google.firebase.ai.type.ImagenImagePlacement TOP_CENTER; + field public static final com.google.firebase.ai.type.ImagenImagePlacement TOP_LEFT; + field public static final com.google.firebase.ai.type.ImagenImagePlacement TOP_RIGHT; } public static final class ImagenImagePlacement.Companion { method public com.google.firebase.ai.type.ImagenImagePlacement fromCoordinate(int x, int y); - method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_CENTER(); - method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_LEFT(); - method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_RIGHT(); - method public com.google.firebase.ai.type.ImagenImagePlacement getCENTER(); - method public com.google.firebase.ai.type.ImagenImagePlacement getLEFT_CENTER(); - method public com.google.firebase.ai.type.ImagenImagePlacement getRIGHT_CENTER(); - method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_CENTER(); - method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_LEFT(); - method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_RIGHT(); - property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_CENTER; - property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_LEFT; - property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_RIGHT; - property public final com.google.firebase.ai.type.ImagenImagePlacement CENTER; - property public final com.google.firebase.ai.type.ImagenImagePlacement LEFT_CENTER; - property public final com.google.firebase.ai.type.ImagenImagePlacement RIGHT_CENTER; - property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_CENTER; - property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_LEFT; - property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_RIGHT; } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenInlineImage { @@ -694,11 +679,14 @@ package com.google.firebase.ai.type { } @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenMaskReference extends com.google.firebase.ai.type.ImagenReferenceImage { + method public static final java.util.List generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions); + method public static final java.util.List generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER); field public static final com.google.firebase.ai.type.ImagenMaskReference.Companion Companion; } public static final class ImagenMaskReference.Companion { - method public java.util.List generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER); + method public java.util.List generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions); + method public java.util.List generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = com.google.firebase.ai.type.ImagenImagePlacement.CENTER); } @com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel { @@ -754,16 +742,13 @@ package com.google.firebase.ai.type { } public final class ImagenSubjectReferenceType { + field public static final com.google.firebase.ai.type.ImagenSubjectReferenceType ANIMAL; field public static final com.google.firebase.ai.type.ImagenSubjectReferenceType.Companion Companion; + field public static final com.google.firebase.ai.type.ImagenSubjectReferenceType PERSON; + field public static final com.google.firebase.ai.type.ImagenSubjectReferenceType PRODUCT; } public static final class ImagenSubjectReferenceType.Companion { - method public com.google.firebase.ai.type.ImagenSubjectReferenceType getANIMAL(); - method public com.google.firebase.ai.type.ImagenSubjectReferenceType getPERSON(); - method public com.google.firebase.ai.type.ImagenSubjectReferenceType getPRODUCT(); - property public final com.google.firebase.ai.type.ImagenSubjectReferenceType ANIMAL; - property public final com.google.firebase.ai.type.ImagenSubjectReferenceType PERSON; - property public final com.google.firebase.ai.type.ImagenSubjectReferenceType PRODUCT; } public final class InlineDataPart implements com.google.firebase.ai.type.Part { diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt index 10f333ccc91..603a580a25e 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenSubjectConfig.kt @@ -1,3 +1,18 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.google.firebase.ai.type import kotlinx.serialization.Serializable diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt index 177fde19a8b..befb3a2d180 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt @@ -390,8 +390,8 @@ internal class SerializationTests { @Test fun `test ReferenceImage serialization as Json`() { - val expectedJsonAsString = - """ + val expectedJsonAsString = + """ { "id": "ImagenReferenceImage", "type": "object", From c4a626dae5b93b827cc36bb746db99e3cde2de8f Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 22 Jul 2025 11:18:28 -0700 Subject: [PATCH 17/22] format --- .../test/java/com/google/firebase/ai/SerializationTests.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt index befb3a2d180..177fde19a8b 100644 --- a/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt +++ b/firebase-ai/src/test/java/com/google/firebase/ai/SerializationTests.kt @@ -390,8 +390,8 @@ internal class SerializationTests { @Test fun `test ReferenceImage serialization as Json`() { - val expectedJsonAsString = - """ + val expectedJsonAsString = + """ { "id": "ImagenReferenceImage", "type": "object", From 41427fa2b7992b23c7dece47edd55c358743e24a Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 22 Jul 2025 12:51:15 -0700 Subject: [PATCH 18/22] removed references to ControlNet and added an integration test --- .../com/google/firebase/ai/ImagenTests.kt | 46 +++++++++++++++++++ .../firebase/ai/type/ImagenReferenceImage.kt | 11 +++-- 2 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt diff --git a/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt b/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt new file mode 100644 index 00000000000..3926c6283cb --- /dev/null +++ b/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.firebase.ai + +import com.google.firebase.ai.AIModels.Companion.app +import com.google.firebase.ai.type.ImagenBackgroundMask +import com.google.firebase.ai.type.ImagenEditMode +import com.google.firebase.ai.type.ImagenEditingConfig +import com.google.firebase.ai.type.ImagenRawImage +import com.google.firebase.ai.type.PublicPreviewAPI +import kotlinx.coroutines.runBlocking +import org.junit.Test + +@OptIn(PublicPreviewAPI::class) +class ImagenTests { + @Test + fun testGenerateAndEditImage() { + val imageGenerationModel = FirebaseAI.getInstance(app()).imagenModel("imagen-3.0-generate-002") + val imageEditingModel = FirebaseAI.getInstance(app()).imagenModel("imagen-3.0-capability-001") + + runBlocking { + val catImage = imageGenerationModel.generateImages("A cat").images.first() + val editedCatImage = imageEditingModel.editImage( + listOf(ImagenRawImage(catImage), ImagenBackgroundMask()), + "A cat flying through space", + ImagenEditingConfig( + ImagenEditMode.INPAINT_INSERTION + ) + ) + assert(editedCatImage.images.size == 1) + } + } +} diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt index a56e0318294..202fb35e2a5 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -73,15 +73,16 @@ internal constructor( } /** - * Represents a reference image (provided or generated) to bound the created image via ControlNet + * Represents a reference image (provided or generated) to bound the created image via controlled + * generation. * @param image the image provided, required if [enableComputation] is false - * @param type the type of ControlNet reference image + * @param type the type of control reference image * @param referenceId the reference ID for this image, to be referenced in the prompt - * @param enableComputation requests that the reference image be generated serverside instead of + * @param [enableComputation] requests that the reference image be generated serverside instead of * provided - * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and enableComputation is true, this will + * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and [enableComputation] is true, this will * control the size of each superpixel region in pixels for the generated referenced image - * @param superpixelRuler if type is COLOR_SUPERPIXEL and enableComputation is true, this will + * @param superpixelRuler if type is COLOR_SUPERPIXEL and [enableComputation] is true, this will * control the superpixel smoothness factor for the generated referenced image */ @PublicPreviewAPI From b172b2289ee8ec18796ca0ea6d22c14e203b86b9 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Tue, 22 Jul 2025 12:53:38 -0700 Subject: [PATCH 19/22] format --- .../com/google/firebase/ai/ImagenTests.kt | 29 +++++++++---------- .../firebase/ai/type/ImagenReferenceImage.kt | 4 +-- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt b/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt index 3926c6283cb..dbc35a71dad 100644 --- a/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt +++ b/firebase-ai/src/androidTest/kotlin/com/google/firebase/ai/ImagenTests.kt @@ -26,21 +26,20 @@ import org.junit.Test @OptIn(PublicPreviewAPI::class) class ImagenTests { - @Test - fun testGenerateAndEditImage() { - val imageGenerationModel = FirebaseAI.getInstance(app()).imagenModel("imagen-3.0-generate-002") - val imageEditingModel = FirebaseAI.getInstance(app()).imagenModel("imagen-3.0-capability-001") + @Test + fun testGenerateAndEditImage() { + val imageGenerationModel = FirebaseAI.getInstance(app()).imagenModel("imagen-3.0-generate-002") + val imageEditingModel = FirebaseAI.getInstance(app()).imagenModel("imagen-3.0-capability-001") - runBlocking { - val catImage = imageGenerationModel.generateImages("A cat").images.first() - val editedCatImage = imageEditingModel.editImage( - listOf(ImagenRawImage(catImage), ImagenBackgroundMask()), - "A cat flying through space", - ImagenEditingConfig( - ImagenEditMode.INPAINT_INSERTION - ) - ) - assert(editedCatImage.images.size == 1) - } + runBlocking { + val catImage = imageGenerationModel.generateImages("A cat").images.first() + val editedCatImage = + imageEditingModel.editImage( + listOf(ImagenRawImage(catImage), ImagenBackgroundMask()), + "A cat flying through space", + ImagenEditingConfig(ImagenEditMode.INPAINT_INSERTION) + ) + assert(editedCatImage.images.size == 1) } + } } diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt index 202fb35e2a5..a39c2721faf 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -80,8 +80,8 @@ internal constructor( * @param referenceId the reference ID for this image, to be referenced in the prompt * @param [enableComputation] requests that the reference image be generated serverside instead of * provided - * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and [enableComputation] is true, this will - * control the size of each superpixel region in pixels for the generated referenced image + * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and [enableComputation] is true, this + * will control the size of each superpixel region in pixels for the generated referenced image * @param superpixelRuler if type is COLOR_SUPERPIXEL and [enableComputation] is true, this will * control the superpixel smoothness factor for the generated referenced image */ From c4d4034dfbf8d33936ade0a3cd9cd125a4aa7ac3 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 23 Jul 2025 10:03:18 -0700 Subject: [PATCH 20/22] remove extra square brackets --- .../kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt index a39c2721faf..b9ba2ff48b2 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/type/ImagenReferenceImage.kt @@ -78,7 +78,7 @@ internal constructor( * @param image the image provided, required if [enableComputation] is false * @param type the type of control reference image * @param referenceId the reference ID for this image, to be referenced in the prompt - * @param [enableComputation] requests that the reference image be generated serverside instead of + * @param enableComputation requests that the reference image be generated serverside instead of * provided * @param superpixelRegionSize if type is COLOR_SUPERPIXEL and [enableComputation] is true, this * will control the size of each superpixel region in pixels for the generated referenced image From 9647fd78107f446ba07771da125f71be3b1f4c19 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 23 Jul 2025 10:05:07 -0700 Subject: [PATCH 21/22] add changelog --- firebase-ai/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/firebase-ai/CHANGELOG.md b/firebase-ai/CHANGELOG.md index 36e2d4f3309..7f86ca48d2b 100644 --- a/firebase-ai/CHANGELOG.md +++ b/firebase-ai/CHANGELOG.md @@ -1,5 +1,6 @@ # Unreleased - +* [feature] added support for Imagen Editing, including inpainting, outpainting, control, style + transfer, and subject references (#7075) # 17.0.0 * [feature] Added support for configuring the "thinking" budget when using Gemini From 09805054438feaac3b5ad3639e1dded4bbc30c94 Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Wed, 23 Jul 2025 10:10:25 -0700 Subject: [PATCH 22/22] make config optional for editing images --- firebase-ai/api.txt | 1 + .../firebase/ai/java/ImagenModelFutures.kt | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/firebase-ai/api.txt b/firebase-ai/api.txt index ebecf342156..b979c361b52 100644 --- a/firebase-ai/api.txt +++ b/firebase-ai/api.txt @@ -110,6 +110,7 @@ package com.google.firebase.ai.java { } @com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenModelFutures { + method public abstract com.google.common.util.concurrent.ListenableFuture> editImage(java.util.List referenceImages, String prompt); method public abstract com.google.common.util.concurrent.ListenableFuture> editImage(java.util.List referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null); method public static final com.google.firebase.ai.java.ImagenModelFutures from(com.google.firebase.ai.ImagenModel model); method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String prompt); diff --git a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt index 8c60a9e856a..2f0299da406 100644 --- a/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt +++ b/firebase-ai/src/main/kotlin/com/google/firebase/ai/java/ImagenModelFutures.kt @@ -59,6 +59,18 @@ public abstract class ImagenModelFutures internal constructor() { config: ImagenEditingConfig? = null ): ListenableFuture> + /** + * Generates an image from a single or set of base images, returning the result directly to the + * caller. + * + * @param prompt the text input given to the model as a prompt + * @param referenceImages the image inputs given to the model as a prompt + */ + public abstract fun editImage( + referenceImages: List, + prompt: String, + ): ListenableFuture> + /** * Generates an image by inpainting a masked off part of a base image. * @@ -112,6 +124,12 @@ public abstract class ImagenModelFutures internal constructor() { ): ListenableFuture> = SuspendToFutureAdapter.launchFuture { model.editImage(referenceImages, prompt, config) } + override fun editImage( + referenceImages: List, + prompt: String, + ): ListenableFuture> = + editImage(referenceImages, prompt, null) + override fun inpaintImage( image: ImagenInlineImage, prompt: String,