feat(ui): make ref images croppable

Author: psychedelicious

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/modelsLoaded.ts

@@ -12,7 +12,13 @@ import {
 } from 'features/controlLayers/store/paramsSlice';
 import { refImageModelChanged, selectRefImagesSlice } from 'features/controlLayers/store/refImagesSlice';
 import { selectCanvasSlice } from 'features/controlLayers/store/selectors';
-import { getEntityIdentifier, isFLUXReduxConfig, isIPAdapterConfig } from 'features/controlLayers/store/types';
+import {
+  getEntityIdentifier,
+  isFLUXReduxConfig,
+  isIPAdapterConfig,
+  isRegionalGuidanceFLUXReduxConfig,
+  isRegionalGuidanceIPAdapterConfig,
+} from 'features/controlLayers/store/types';
 import { zModelIdentifierField } from 'features/nodes/types/common';
 import { modelSelected } from 'features/parameters/store/actions';
 import {
@@ -252,7 +258,7 @@ const handleIPAdapterModels: ModelHandler = (models, state, dispatch, log) => {
 
   selectCanvasSlice(state).regionalGuidance.entities.forEach((entity) => {
     entity.referenceImages.forEach(({ id: referenceImageId, config }) => {
-      if (!isIPAdapterConfig(config)) {
+      if (!isRegionalGuidanceIPAdapterConfig(config)) {
         return;
       }
 
@@ -295,7 +301,7 @@ const handleFLUXReduxModels: ModelHandler = (models, state, dispatch, log) => {
 
   selectCanvasSlice(state).regionalGuidance.entities.forEach((entity) => {
     entity.referenceImages.forEach(({ id: referenceImageId, config }) => {
-      if (!isFLUXReduxConfig(config)) {
+      if (!isRegionalGuidanceFLUXReduxConfig(config)) {
         return;
       }
 

invokeai/frontend/web/src/features/controlLayers/components/RefImage/RefImageImage.tsx

@@ -1,27 +1,31 @@
 import { Flex } from '@invoke-ai/ui-library';
 import { useStore } from '@nanostores/react';
+import { objectEquals } from '@observ33r/object-equals';
 import { skipToken } from '@reduxjs/toolkit/query';
 import { useAppSelector, useAppStore } from 'app/store/storeHooks';
 import { UploadImageIconButton } from 'common/hooks/useImageUploadButton';
 import { bboxSizeOptimized, bboxSizeRecalled } from 'features/controlLayers/store/canvasSlice';
 import { useCanvasIsStaging } from 'features/controlLayers/store/canvasStagingAreaSlice';
 import { sizeOptimized, sizeRecalled } from 'features/controlLayers/store/paramsSlice';
-import type { ImageWithDims } from 'features/controlLayers/store/types';
+import type { CroppableImageWithDims } from 'features/controlLayers/store/types';
+import { imageDTOToCroppableImage, imageDTOToImageWithDims } from 'features/controlLayers/store/util';
+import { Editor } from 'features/cropper/lib/editor';
+import { cropImageModalApi } from 'features/cropper/store';
 import type { setGlobalReferenceImageDndTarget, setRegionalGuidanceReferenceImageDndTarget } from 'features/dnd/dnd';
 import { DndDropTarget } from 'features/dnd/DndDropTarget';
 import { DndImage } from 'features/dnd/DndImage';
 import { DndImageIcon } from 'features/dnd/DndImageIcon';
 import { selectActiveTab } from 'features/ui/store/uiSelectors';
 import { memo, useCallback, useEffect } from 'react';
 import { useTranslation } from 'react-i18next';
-import { PiArrowCounterClockwiseBold, PiRulerBold } from 'react-icons/pi';
-import { useGetImageDTOQuery } from 'services/api/endpoints/images';
+import { PiArrowCounterClockwiseBold, PiCropBold, PiRulerBold } from 'react-icons/pi';
+import { useGetImageDTOQuery, useUploadImageMutation } from 'services/api/endpoints/images';
 import type { ImageDTO } from 'services/api/types';
 import { $isConnected } from 'services/events/stores';
 
 type Props<T extends typeof setGlobalReferenceImageDndTarget | typeof setRegionalGuidanceReferenceImageDndTarget> = {
-  image: ImageWithDims | null;
-  onChangeImage: (imageDTO: ImageDTO | null) => void;
+  image: CroppableImageWithDims | null;
+  onChangeImage: (croppableImage: CroppableImageWithDims | null) => void;
   dndTarget: T;
   dndTargetData: ReturnType<T['getData']>;
 };
@@ -38,20 +42,28 @@ export const RefImageImage = memo(
     const isConnected = useStore($isConnected);
     const tab = useAppSelector(selectActiveTab);
     const isStaging = useCanvasIsStaging();
-    const { currentData: imageDTO, isError } = useGetImageDTOQuery(image?.image_name ?? skipToken);
+    const imageWithDims = image?.crop?.image ?? image?.original.image ?? null;
+    const croppedImageDTOReq = useGetImageDTOQuery(image?.crop?.image?.image_name ?? skipToken);
+    const originalImageDTOReq = useGetImageDTOQuery(image?.original.image.image_name ?? skipToken);
+    const [uploadImage] = useUploadImageMutation();
+
+    const originalImageDTO = originalImageDTOReq.currentData;
+    const croppedImageDTO = croppedImageDTOReq.currentData;
+    const imageDTO = croppedImageDTO ?? originalImageDTO;
+
     const handleResetControlImage = useCallback(() => {
       onChangeImage(null);
     }, [onChangeImage]);
 
     useEffect(() => {
-      if (isConnected && isError) {
+      if ((isConnected && croppedImageDTOReq.isError) || originalImageDTOReq.isError) {
         handleResetControlImage();
       }
-    }, [handleResetControlImage, isError, isConnected]);
+    }, [handleResetControlImage, isConnected, croppedImageDTOReq.isError, originalImageDTOReq.isError]);
 
     const onUpload = useCallback(
       (imageDTO: ImageDTO) => {
-        onChangeImage(imageDTO);
+        onChangeImage(imageDTOToCroppableImage(imageDTO));
       },
       [onChangeImage]
     );
@@ -70,13 +82,67 @@ export const RefImageImage = memo(
       }
     }, [imageDTO, isStaging, store, tab]);
 
+    const edit = useCallback(() => {
+      if (!originalImageDTO) {
+        return;
+      }
+
+      // We will create a new editor instance each time the user wants to edit
+      const editor = new Editor();
+
+      // When the user applies the crop, we will upload the cropped image and store the applied crop box so if the user
+      // re-opens the editor they see the same crop
+      const onApplyCrop = async () => {
+        const box = editor.getCropBox();
+        if (objectEquals(box, image?.crop?.box)) {
+          // If the box hasn't changed, don't do anything
+          return;
+        }
+        if (!box || objectEquals(box, { x: 0, y: 0, width: originalImageDTO.width, height: originalImageDTO.height })) {
+          // There is a crop applied but it is the whole iamge - revert to original image
+          onChangeImage(imageDTOToCroppableImage(originalImageDTO));
+          return;
+        }
+        const blob = await editor.exportImage('blob');
+        const file = new File([blob], 'image.png', { type: 'image/png' });
+
+        const newCroppedImageDTO = await uploadImage({
+          file,
+          is_intermediate: true,
+          image_category: 'user',
+        }).unwrap();
+
+        onChangeImage(
+          imageDTOToCroppableImage(originalImageDTO, {
+            image: imageDTOToImageWithDims(newCroppedImageDTO),
+            box,
+            ratio: editor.getCropAspectRatio(),
+          })
+        );
+      };
+
+      const onReady = async () => {
+        const initial = image?.crop ? { cropBox: image.crop.box, aspectRatio: image.crop.ratio } : undefined;
+        // Load the image into the editor and open the modal once it's ready
+        await editor.loadImage(originalImageDTO.image_url, initial);
+      };
+
+      cropImageModalApi.open({ editor, onApplyCrop, onReady });
+    }, [image?.crop, onChangeImage, originalImageDTO, uploadImage]);
+
     return (
-      <Flex position="relative" w="full" h="full" alignItems="center" data-error={!imageDTO && !image?.image_name}>
+      <Flex
+        position="relative"
+        w="full"
+        h="full"
+        alignItems="center"
+        data-error={!imageDTO && !imageWithDims?.image_name}
+      >
         {!imageDTO && (
           <UploadImageIconButton
             w="full"
             h="full"
-            isError={!imageDTO && !image?.image_name}
+            isError={!imageDTO && !imageWithDims?.image_name}
             onUpload={onUpload}
             fontSize={36}
           />
@@ -99,6 +165,15 @@ export const RefImageImage = memo(
                 isDisabled={!imageDTO || (tab === 'canvas' && isStaging)}
               />
             </Flex>
+
+            <Flex position="absolute" flexDir="column" top={2} insetInlineStart={2} gap={1}>
+              <DndImageIcon
+                onClick={edit}
+                icon={<PiCropBold size={16} />}
+                tooltip={t('common.crop')}
+                isDisabled={!imageDTO}
+              />
+            </Flex>
           </>
         )}
         <DndDropTarget dndTarget={dndTarget} dndTargetData={dndTargetData} label={t('gallery.drop')} />

invokeai/frontend/web/src/features/controlLayers/components/RefImage/RefImageList.tsx

@@ -13,7 +13,7 @@ import {
   selectRefImageEntityIds,
   selectSelectedRefEntityId,
 } from 'features/controlLayers/store/refImagesSlice';
-import { imageDTOToImageWithDims } from 'features/controlLayers/store/util';
+import { imageDTOToCroppableImage } from 'features/controlLayers/store/util';
 import { addGlobalReferenceImageDndTarget } from 'features/dnd/dnd';
 import { DndDropTarget } from 'features/dnd/DndDropTarget';
 import { selectActiveTab } from 'features/ui/store/uiSelectors';
@@ -92,7 +92,7 @@ const AddRefImageDropTargetAndButton = memo(() => {
       ({
         onUpload: (imageDTO: ImageDTO) => {
           const config = getDefaultRefImageConfig(getState);
-          config.image = imageDTOToImageWithDims(imageDTO);
+          config.image = imageDTOToCroppableImage(imageDTO);
           dispatch(refImageAdded({ overrides: { config } }));
         },
         allowMultiple: false,

invokeai/frontend/web/src/features/controlLayers/components/RefImage/RefImagePreview.tsx

@@ -1,6 +1,5 @@
 import type { SystemStyleObject } from '@invoke-ai/ui-library';
 import { Flex, Icon, IconButton, Image, Skeleton, Text, Tooltip } from '@invoke-ai/ui-library';
-import { skipToken } from '@reduxjs/toolkit/query';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { round } from 'es-toolkit/compat';
 import { useRefImageEntity } from 'features/controlLayers/components/RefImage/useRefImageEntity';
@@ -15,7 +14,7 @@ import { isIPAdapterConfig } from 'features/controlLayers/store/types';
 import { getGlobalReferenceImageWarnings } from 'features/controlLayers/store/validators';
 import { memo, useCallback, useEffect, useMemo, useState } from 'react';
 import { PiExclamationMarkBold, PiEyeSlashBold, PiImageBold } from 'react-icons/pi';
-import { useGetImageDTOQuery } from 'services/api/endpoints/images';
+import { useImageDTOFromCroppableImage } from 'services/api/endpoints/images';
 
 import { RefImageWarningTooltipContent } from './RefImageWarningTooltipContent';
 
@@ -72,7 +71,8 @@ export const RefImagePreview = memo(() => {
   const selectedEntityId = useAppSelector(selectSelectedRefEntityId);
   const isPanelOpen = useAppSelector(selectIsRefImagePanelOpen);
   const [showWeightDisplay, setShowWeightDisplay] = useState(false);
-  const { data: imageDTO } = useGetImageDTOQuery(entity.config.image?.image_name ?? skipToken);
+
+  const imageDTO = useImageDTOFromCroppableImage(entity.config.image);
 
   const sx = useMemo(() => {
     if (!isIPAdapterConfig(entity.config)) {
@@ -145,7 +145,7 @@ export const RefImagePreview = memo(() => {
         overflow="hidden"
       >
         <Image
-          src={imageDTO?.thumbnail_url}
+          src={imageDTO?.image_url}
           objectFit="contain"
           aspectRatio="1/1"
           height={imageDTO?.height}

invokeai/frontend/web/src/features/controlLayers/components/RefImage/RefImageSettings.tsx

@@ -30,6 +30,7 @@ import {
 } from 'features/controlLayers/store/refImagesSlice';
 import type {
   CLIPVisionModelV2,
+  CroppableImageWithDims,
   FLUXReduxImageInfluence as FLUXReduxImageInfluenceType,
   IPMethodV2,
 } from 'features/controlLayers/store/types';
@@ -42,7 +43,6 @@ import type {
   ChatGPT4oModelConfig,
   FLUXKontextModelConfig,
   FLUXReduxModelConfig,
-  ImageDTO,
   IPAdapterModelConfig,
 } from 'services/api/types';
 
@@ -104,15 +104,19 @@ const RefImageSettingsContent = memo(() => {
   );
 
   const onChangeImage = useCallback(
-    (imageDTO: ImageDTO | null) => {
-      dispatch(refImageImageChanged({ id, imageDTO }));
+    (croppableImage: CroppableImageWithDims | null) => {
+      dispatch(refImageImageChanged({ id, croppableImage }));
     },
     [dispatch, id]
   );
 
   const dndTargetData = useMemo<SetGlobalReferenceImageDndTargetData>(
-    () => setGlobalReferenceImageDndTarget.getData({ id }, config.image?.image_name),
-    [id, config.image?.image_name]
+    () =>
+      setGlobalReferenceImageDndTarget.getData(
+        { id },
+        config.image?.crop?.image.image_name ?? config.image?.original.image.image_name
+      ),
+    [id, config.image?.crop?.image.image_name, config.image?.original.image.image_name]
   );
 
   const isFLUX = useAppSelector(selectIsFLUX);

invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettings.tsx

@@ -6,7 +6,6 @@ import { FLUXReduxImageInfluence } from 'features/controlLayers/components/commo
 import { IPAdapterCLIPVisionModel } from 'features/controlLayers/components/common/IPAdapterCLIPVisionModel';
 import { Weight } from 'features/controlLayers/components/common/Weight';
 import { IPAdapterMethod } from 'features/controlLayers/components/RefImage/IPAdapterMethod';
-import { RefImageImage } from 'features/controlLayers/components/RefImage/RefImageImage';
 import { RegionalGuidanceIPAdapterSettingsEmptyState } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceIPAdapterSettingsEmptyState';
 import { RegionalReferenceImageModel } from 'features/controlLayers/components/RegionalGuidance/RegionalReferenceImageModel';
 import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
@@ -37,6 +36,8 @@ import { PiBoundingBoxBold, PiXBold } from 'react-icons/pi';
 import type { FLUXReduxModelConfig, ImageDTO, IPAdapterModelConfig } from 'services/api/types';
 import { assert } from 'tsafe';
 
+import { RegionalGuidanceRefImageImage } from './RegionalGuidanceRefImageImage';
+
 type Props = {
   referenceImageId: string;
 };
@@ -114,7 +115,7 @@ const RegionalGuidanceIPAdapterSettingsContent = memo(({ referenceImageId }: Pro
         { entityIdentifier, referenceImageId },
         config.image?.image_name
       ),
-    [entityIdentifier, config.image?.image_name, referenceImageId]
+    [entityIdentifier, config.image, referenceImageId]
   );
 
   const pullBboxIntoIPAdapter = usePullBboxIntoRegionalGuidanceReferenceImage(entityIdentifier, referenceImageId);
@@ -170,7 +171,7 @@ const RegionalGuidanceIPAdapterSettingsContent = memo(({ referenceImageId }: Pro
             </Flex>
           )}
           <Flex alignItems="center" justifyContent="center" h={32} w={32} aspectRatio="1/1" flexGrow={1}>
-            <RefImageImage
+            <RegionalGuidanceRefImageImage
               image={config.image}
               onChangeImage={onChangeImage}
               dndTarget={setRegionalGuidanceReferenceImageDndTarget}