Skip to content

Imagen Editing #7075

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 24 commits into from
Jul 23, 2025
Merged

Imagen Editing #7075

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
dec0990
Add imagen editing options like inpainting and outpainting
Jun 24, 2025
9795fc3
add java implementation and minor naming adjustments
Jun 25, 2025
60baab6
update api.txt and format
Jun 25, 2025
806d4fb
remove nullability from includeRaiReason
Jun 25, 2025
66cff7b
remove unneeded and gcsUri
Jun 25, 2025
e710498
rename constructGenerateImageRequest
Jun 25, 2025
f632e94
rewrite to fit the referenceType style transfer, subject references, …
Jul 9, 2025
0327b0a
added support for outpainting properly
Jul 10, 2025
c0f6161
add copyright headers
Jul 10, 2025
778f176
javadocs1
Jul 11, 2025
1131d22
Add refdocs, and address a few review comments
Jul 16, 2025
eb3ac0a
bump version
Jul 16, 2025
7ed0bd7
fixes for comments
Jul 16, 2025
a48f7ac
fixing refdocs for comments, and adding error for incorrect call to o…
Jul 17, 2025
21262c3
add tests, and JvmStatic and JvmField annotations
Jul 22, 2025
2f85544
Merge branch 'main' into davidmotson.imagen_editing
Jul 22, 2025
adf4957
add copyright and update api.txt
Jul 22, 2025
c4a626d
format
Jul 22, 2025
41427fa
removed references to ControlNet and added an integration test
Jul 22, 2025
b172b22
format
Jul 22, 2025
c4d4034
remove extra square brackets
Jul 23, 2025
9647fd7
add changelog
Jul 23, 2025
0980505
make config optional for editing images
Jul 23, 2025
83df056
Merge branch 'main' into davidmotson.imagen_editing
davidmotson Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 138 additions & 0 deletions firebase-ai/api.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,10 @@ package com.google.firebase.ai {
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenModel {
method public suspend Object? editImage(java.util.List<? extends com.google.firebase.ai.type.ImagenReferenceImage> referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
method public suspend Object? inpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, String prompt, com.google.firebase.ai.type.ImagenMaskReference mask, com.google.firebase.ai.type.ImagenEditingConfig config, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
method public suspend Object? outpaintImage(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER, String prompt = "", com.google.firebase.ai.type.ImagenEditingConfig? config = null, kotlin.coroutines.Continuation<? super com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>>);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class LiveGenerativeModel {
Expand Down Expand Up @@ -104,6 +107,7 @@ package com.google.firebase.ai.java {
}

@com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenModelFutures {
method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>> editImage(java.util.List<? extends com.google.firebase.ai.type.ImagenReferenceImage> referenceImages, String prompt, com.google.firebase.ai.type.ImagenEditingConfig? config = null);
method public static final com.google.firebase.ai.java.ImagenModelFutures from(com.google.firebase.ai.ImagenModel model);
method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.ai.type.ImagenGenerationResponse<com.google.firebase.ai.type.ImagenInlineImage>> generateImages(String prompt);
method public abstract com.google.firebase.ai.ImagenModel getImageModel();
Expand Down Expand Up @@ -257,6 +261,14 @@ package com.google.firebase.ai.type {
property public final int totalTokens;
}

public final class Dimensions {
ctor public Dimensions(int width, int height);
method public int getHeight();
method public int getWidth();
property public final int height;
property public final int width;
}

public final class FileDataPart implements com.google.firebase.ai.type.Part {
ctor public FileDataPart(String uri, String mimeType);
method public String getMimeType();
Expand Down Expand Up @@ -484,6 +496,50 @@ package com.google.firebase.ai.type {
public static final class ImagenAspectRatio.Companion {
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenBackgroundMask extends com.google.firebase.ai.type.ImagenMaskReference {
ctor public ImagenBackgroundMask(Double? dilation = null);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenControlReference extends com.google.firebase.ai.type.ImagenReferenceImage {
ctor public ImagenControlReference(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.ImagenControlType type, Integer? referenceId = null, Boolean? enableComputation = null, Integer? superpixelRegionSize = null, Integer? superpixelRuler = null);
}

public final class ImagenControlType {
field public static final com.google.firebase.ai.type.ImagenControlType.Companion Companion;
}

public static final class ImagenControlType.Companion {
method public com.google.firebase.ai.type.ImagenControlType getCANNY();
method public com.google.firebase.ai.type.ImagenControlType getCOLOR_SUPERPIXEL();
method public com.google.firebase.ai.type.ImagenControlType getFACE_MESH();
method public com.google.firebase.ai.type.ImagenControlType getSCRIBBLE();
property public final com.google.firebase.ai.type.ImagenControlType CANNY;
property public final com.google.firebase.ai.type.ImagenControlType COLOR_SUPERPIXEL;
property public final com.google.firebase.ai.type.ImagenControlType FACE_MESH;
property public final com.google.firebase.ai.type.ImagenControlType SCRIBBLE;
}

public final class ImagenEditMode {
field public static final com.google.firebase.ai.type.ImagenEditMode.Companion Companion;
}

public static final class ImagenEditMode.Companion {
method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_INSERTION();
method public com.google.firebase.ai.type.ImagenEditMode getINPAINT_REMOVAL();
method public com.google.firebase.ai.type.ImagenEditMode getOUTPAINT();
property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_INSERTION;
property public final com.google.firebase.ai.type.ImagenEditMode INPAINT_REMOVAL;
property public final com.google.firebase.ai.type.ImagenEditMode OUTPAINT;
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenEditingConfig {
ctor public ImagenEditingConfig(com.google.firebase.ai.type.ImagenEditMode? editMode = null, Integer? editSteps = null);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenForegroundMask extends com.google.firebase.ai.type.ImagenMaskReference {
ctor public ImagenForegroundMask(Double? dilation = null);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenGenerationConfig {
ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.ai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.ai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null);
method public Boolean? getAddWatermark();
Expand Down Expand Up @@ -544,6 +600,36 @@ package com.google.firebase.ai.type {
method public com.google.firebase.ai.type.ImagenImageFormat png();
}

public final class ImagenImagePlacement {
method public Integer? getX();
method public Integer? getY();
property public final Integer? x;
property public final Integer? y;
field public static final com.google.firebase.ai.type.ImagenImagePlacement.Companion Companion;
}

public static final class ImagenImagePlacement.Companion {
method public com.google.firebase.ai.type.ImagenImagePlacement fromCoordinate(int x, int y);
method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM();
method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_LEFT();
method public com.google.firebase.ai.type.ImagenImagePlacement getBOTTOM_RIGHT();
method public com.google.firebase.ai.type.ImagenImagePlacement getCENTER();
method public com.google.firebase.ai.type.ImagenImagePlacement getLEFT();
method public com.google.firebase.ai.type.ImagenImagePlacement getRIGHT();
method public com.google.firebase.ai.type.ImagenImagePlacement getTOP();
method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_LEFT();
method public com.google.firebase.ai.type.ImagenImagePlacement getTOP_RIGHT();
property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM;
property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_LEFT;
property public final com.google.firebase.ai.type.ImagenImagePlacement BOTTOM_RIGHT;
property public final com.google.firebase.ai.type.ImagenImagePlacement CENTER;
property public final com.google.firebase.ai.type.ImagenImagePlacement LEFT;
property public final com.google.firebase.ai.type.ImagenImagePlacement RIGHT;
property public final com.google.firebase.ai.type.ImagenImagePlacement TOP;
property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_LEFT;
property public final com.google.firebase.ai.type.ImagenImagePlacement TOP_RIGHT;
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenInlineImage {
method public android.graphics.Bitmap asBitmap();
method public byte[] getData();
Expand All @@ -552,6 +638,18 @@ package com.google.firebase.ai.type {
property public final String mimeType;
}

public final class ImagenInlineImageKt {
method @com.google.firebase.ai.type.PublicPreviewAPI public static com.google.firebase.ai.type.ImagenInlineImage toImagenInlineImage(android.graphics.Bitmap);
}

@com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenMaskReference extends com.google.firebase.ai.type.ImagenReferenceImage {
field public static final com.google.firebase.ai.type.ImagenMaskReference.Companion Companion;
}

public static final class ImagenMaskReference.Companion {
method public java.util.List<com.google.firebase.ai.type.ImagenReferenceImage> generateMaskAndPadForOutpainting(com.google.firebase.ai.type.ImagenInlineImage image, com.google.firebase.ai.type.Dimensions newDimensions, com.google.firebase.ai.type.ImagenImagePlacement newPosition = ImagenImagePlacement.CENTER);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {
field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ADULT;
field public static final com.google.firebase.ai.type.ImagenPersonFilterLevel ALLOW_ALL;
Expand All @@ -562,6 +660,21 @@ package com.google.firebase.ai.type {
public static final class ImagenPersonFilterLevel.Companion {
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenRawImage extends com.google.firebase.ai.type.ImagenReferenceImage {
ctor public ImagenRawImage(com.google.firebase.ai.type.ImagenInlineImage image);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenRawMask extends com.google.firebase.ai.type.ImagenMaskReference {
ctor public ImagenRawMask(com.google.firebase.ai.type.ImagenInlineImage mask, Double? dilation = null);
}

@com.google.firebase.ai.type.PublicPreviewAPI public abstract class ImagenReferenceImage {
method public final com.google.firebase.ai.type.ImagenInlineImage? getImage();
method public final Integer? getReferenceId();
property public final com.google.firebase.ai.type.ImagenInlineImage? image;
property public final Integer? referenceId;
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSafetyFilterLevel {
field public static final com.google.firebase.ai.type.ImagenSafetyFilterLevel BLOCK_LOW_AND_ABOVE;
field public static final com.google.firebase.ai.type.ImagenSafetyFilterLevel BLOCK_MEDIUM_AND_ABOVE;
Expand All @@ -577,6 +690,31 @@ package com.google.firebase.ai.type {
ctor public ImagenSafetySettings(com.google.firebase.ai.type.ImagenSafetyFilterLevel safetyFilterLevel, com.google.firebase.ai.type.ImagenPersonFilterLevel personFilterLevel);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSemanticMask extends com.google.firebase.ai.type.ImagenMaskReference {
ctor public ImagenSemanticMask(java.util.List<java.lang.Integer> classes, Double? dilation = null);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenStyleReference extends com.google.firebase.ai.type.ImagenReferenceImage {
ctor public ImagenStyleReference(com.google.firebase.ai.type.ImagenInlineImage image, Integer? referenceId = null, String? description = null);
}

@com.google.firebase.ai.type.PublicPreviewAPI public final class ImagenSubjectReference extends com.google.firebase.ai.type.ImagenReferenceImage {
ctor public ImagenSubjectReference(com.google.firebase.ai.type.ImagenInlineImage image, Integer? referenceId = null, String? description = null, com.google.firebase.ai.type.ImagenSubjectReferenceType? subjectType = null);
}

public final class ImagenSubjectReferenceType {
field public static final com.google.firebase.ai.type.ImagenSubjectReferenceType.Companion Companion;
}

public static final class ImagenSubjectReferenceType.Companion {
method public com.google.firebase.ai.type.ImagenSubjectReferenceType getANIMAL();
method public com.google.firebase.ai.type.ImagenSubjectReferenceType getPERSON();
method public com.google.firebase.ai.type.ImagenSubjectReferenceType getPRODUCT();
property public final com.google.firebase.ai.type.ImagenSubjectReferenceType ANIMAL;
property public final com.google.firebase.ai.type.ImagenSubjectReferenceType PERSON;
property public final com.google.firebase.ai.type.ImagenSubjectReferenceType PRODUCT;
}

public final class InlineDataPart implements com.google.firebase.ai.type.Part {
ctor public InlineDataPart(byte[] inlineData, String mimeType);
method public byte[] getInlineData();
Expand Down
2 changes: 1 addition & 1 deletion firebase-ai/gradle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.

version=16.2.0
version=16.3.0
latestReleasedVersion=16.1.0
90 changes: 82 additions & 8 deletions firebase-ai/src/main/kotlin/com/google/firebase/ai/ImagenModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,17 @@ import com.google.firebase.ai.common.APIController
import com.google.firebase.ai.common.AppCheckHeaderProvider
import com.google.firebase.ai.common.ContentBlockedException
import com.google.firebase.ai.common.GenerateImageRequest
import com.google.firebase.ai.type.Dimensions
import com.google.firebase.ai.type.FirebaseAIException
import com.google.firebase.ai.type.ImagenEditMode
import com.google.firebase.ai.type.ImagenEditingConfig
import com.google.firebase.ai.type.ImagenGenerationConfig
import com.google.firebase.ai.type.ImagenGenerationResponse
import com.google.firebase.ai.type.ImagenImagePlacement
import com.google.firebase.ai.type.ImagenInlineImage
import com.google.firebase.ai.type.ImagenMaskReference
import com.google.firebase.ai.type.ImagenRawImage
import com.google.firebase.ai.type.ImagenReferenceImage
import com.google.firebase.ai.type.ImagenSafetySettings
import com.google.firebase.ai.type.PublicPreviewAPI
import com.google.firebase.ai.type.RequestOptions
Expand Down Expand Up @@ -75,30 +82,97 @@ internal constructor(
public suspend fun generateImages(prompt: String): ImagenGenerationResponse<ImagenInlineImage> =
try {
controller
.generateImage(constructRequest(prompt, null, generationConfig))
.generateImage(constructGenerateImageRequest(prompt, generationConfig))
.validate()
.toPublicInline()
} catch (e: Throwable) {
throw FirebaseAIException.from(e)
}

private fun constructRequest(
public suspend fun editImage(
referenceImages: List<ImagenReferenceImage>,
prompt: String,
gcsUri: String?,
config: ImagenGenerationConfig?,
config: ImagenEditingConfig? = null,
): ImagenGenerationResponse<ImagenInlineImage> =
try {
controller
.generateImage(constructEditRequest(referenceImages, prompt, config))
.validate()
.toPublicInline()
} catch (e: Throwable) {
throw FirebaseAIException.from(e)
}

public suspend fun inpaintImage(
image: ImagenInlineImage,
prompt: String,
mask: ImagenMaskReference,
config: ImagenEditingConfig,
): ImagenGenerationResponse<ImagenInlineImage> {
return editImage(listOf(ImagenRawImage(image), mask), prompt, config)
}

public suspend fun outpaintImage(
image: ImagenInlineImage,
newDimensions: Dimensions,
newPosition: ImagenImagePlacement = ImagenImagePlacement.CENTER,
prompt: String = "",
config: ImagenEditingConfig? = null,
): ImagenGenerationResponse<ImagenInlineImage> {
return editImage(
ImagenMaskReference.generateMaskAndPadForOutpainting(image, newDimensions, newPosition),
prompt,
ImagenEditingConfig(ImagenEditMode.OUTPAINT, config?.editSteps)
)
}

private fun constructGenerateImageRequest(
prompt: String,
generationConfig: ImagenGenerationConfig? = null,
): GenerateImageRequest {
return GenerateImageRequest(
listOf(GenerateImageRequest.ImagenPrompt(prompt)),
GenerateImageRequest.ImagenParameters(
sampleCount = config?.numberOfImages ?: 1,
sampleCount = generationConfig?.numberOfImages ?: 1,
includeRaiReason = true,
addWatermark = generationConfig?.addWatermark,
personGeneration = safetySettings?.personFilterLevel?.internalVal,
negativePrompt = generationConfig?.negativePrompt,
safetySetting = safetySettings?.safetyFilterLevel?.internalVal,
storageUri = null,
aspectRatio = generationConfig?.aspectRatio?.internalVal,
imageOutputOptions = generationConfig?.imageFormat?.toInternal(),
editMode = null,
editConfig = null,
),
)
}

private fun constructEditRequest(
referenceImages: List<ImagenReferenceImage>,
prompt: String,
editConfig: ImagenEditingConfig?,
): GenerateImageRequest {
var maxRefId = referenceImages.mapNotNull { it.referenceId }.maxOrNull() ?: 1
return GenerateImageRequest(
listOf(
GenerateImageRequest.ImagenPrompt(
prompt = prompt,
referenceImages = referenceImages.map { it.toInternal(++maxRefId) },
)
),
GenerateImageRequest.ImagenParameters(
sampleCount = generationConfig?.numberOfImages ?: 1,
includeRaiReason = true,
addWatermark = generationConfig?.addWatermark,
personGeneration = safetySettings?.personFilterLevel?.internalVal,
negativePrompt = config?.negativePrompt,
negativePrompt = generationConfig?.negativePrompt,
safetySetting = safetySettings?.safetyFilterLevel?.internalVal,
storageUri = gcsUri,
aspectRatio = config?.aspectRatio?.internalVal,
storageUri = null,
aspectRatio = generationConfig?.aspectRatio?.internalVal,
imageOutputOptions = generationConfig?.imageFormat?.toInternal(),
editMode = editConfig?.editMode?.value,
editConfig = editConfig?.toInternal(),
),
)
}
Expand Down
23 changes: 22 additions & 1 deletion firebase-ai/src/main/kotlin/com/google/firebase/ai/common/Request.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import com.google.firebase.ai.common.util.fullModelName
import com.google.firebase.ai.common.util.trimmedModelName
import com.google.firebase.ai.type.Content
import com.google.firebase.ai.type.GenerationConfig
import com.google.firebase.ai.type.ImagenEditingConfig
import com.google.firebase.ai.type.ImagenImageFormat
import com.google.firebase.ai.type.ImagenReferenceImage
import com.google.firebase.ai.type.PublicPreviewAPI
import com.google.firebase.ai.type.SafetySetting
import com.google.firebase.ai.type.Tool
Expand Down Expand Up @@ -75,11 +77,16 @@ internal data class CountTokensRequest(
}

@Serializable
@PublicPreviewAPI
internal data class GenerateImageRequest(
val instances: List<ImagenPrompt>,
val parameters: ImagenParameters,
) : Request {
@Serializable internal data class ImagenPrompt(val prompt: String)
@Serializable
internal data class ImagenPrompt(
val prompt: String? = null,
val referenceImages: List<ImagenReferenceImage.Internal>? = null
)

@OptIn(PublicPreviewAPI::class)
@Serializable
Expand All @@ -93,5 +100,19 @@ internal data class GenerateImageRequest(
val personGeneration: String?,
val addWatermark: Boolean?,
val imageOutputOptions: ImagenImageFormat.Internal?,
val editMode: String?,
val editConfig: ImagenEditingConfig.Internal?,
)

@Serializable
internal enum class ReferenceType {
@SerialName("REFERENCE_TYPE_UNSPECIFIED") UNSPECIFIED,
@SerialName("REFERENCE_TYPE_RAW") RAW,
@SerialName("REFERENCE_TYPE_MASK") MASK,
@SerialName("REFERENCE_TYPE_CONTROL") CONTROL,
@SerialName("REFERENCE_TYPE_STYLE") STYLE,
@SerialName("REFERENCE_TYPE_SUBJECT") SUBJECT,
@SerialName("REFERENCE_TYPE_MASKED_SUBJECT") MASKED_SUBJECT,
@SerialName("REFERENCE_TYPE_PRODUCT") PRODUCT
}
}
Loading