feat(storage-resize-image): add param for backfill batch size (#2478)

* feat(storage-resize-image): add param for backfill batch size

* chore(storage-resize-images): format

* fix(storage-resize-image): tests

* feat: add validation of batch size param

Author: CorieW

storage-resize-images/CHANGELOG.md

@@ -1,3 +1,7 @@
+## Version 0.2.10
+
+feat - added param for adjusting backfill max batch size
+
 ## Version 0.2.9
 
 fixed - run an npm audit and bump vulnerable dependencies

storage-resize-images/README.md

@@ -185,6 +185,10 @@ Leave this field empty if you do not want to store failed images in a separate d
 * Backfill existing images: Should existing, unresized images in the Storage bucket be resized as well?
 
 
+* Backfill batch size: The maximum number of images to resize in a single batch. By default, the function is configured to resize 3 images at a time. This is a conservative default to work with smallest memory option (512 MB).
+WARNING: Ensure your function has enough memory to handle the batch size.
+
+
 * Assign new access token: Should resized images have a new access token assigned to them,  different from the original image?
 
 

storage-resize-images/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: storage-resize-images
-version: 0.2.9
+version: 0.2.10
 specVersion: v1beta
 
 displayName: Resize Images
@@ -329,6 +329,20 @@ params:
       - label: No
         value: false
 
+  - param: BACKFILL_BATCH_SIZE
+    label: Backfill batch size
+    description: >
+      The maximum number of images to resize in a single batch. By default, the
+      function is configured to resize 3 images at a time. This is a
+      conservative default to work with smallest memory option (512 MB).
+
+      WARNING: Ensure your function has enough memory to handle the batch size.
+    type: number
+    default: 3
+    validationRegex: ^[1-9][0-9]*$
+    validationErrorMessage: Please provide a valid number.
+    required: false
+
   - param: REGENERATE_TOKEN
     label: Assign new access token
     description: >

storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap

@@ -3,6 +3,7 @@
 exports[`extension configuration detected from environment variables 1`] = `
 Object {
   "animated": false,
+  "backfillBatchSize": 3,
   "bucket": "extensions-testing.appspot.com",
   "cacheControlHeader": undefined,
   "contentFilterLevel": null,

storage-resize-images/functions/__tests__/filters.test.ts

@@ -11,6 +11,7 @@ jest.mock("../src/config", () => {
       imgSizes: ["200x200"],
       resizedImagesPath: undefined,
       deleteOriginalFile: "true",
+      backfillBatchSize: 3,
     },
   };
 });

storage-resize-images/functions/__tests__/function.test.ts

@@ -27,6 +27,7 @@ jest.mock("../src/config", () => {
       imgSizes: ["200x200"],
       resizedImagesPath: undefined,
       deleteOriginalFile: "true",
+      backfillBatchSize: 3,
     },
   };
 });

storage-resize-images/functions/src/config.ts

@@ -88,6 +88,7 @@ export const config = {
   ),
   customFilterPrompt: process.env.CUSTOM_FILTER_PROMPT || null,
   placeholderImagePath: process.env.PLACEHOLDER_IMAGE_PATH || null,
+  backfillBatchSize: Number(process.env.BACKFILL_BATCH_SIZE) || 3,
 };
 
 export type Config = typeof config;

storage-resize-images/functions/src/index.ts

@@ -175,7 +175,7 @@ export const backfillResizedImages = functions.tasks
     const bucket = admin.storage().bucket(process.env.IMG_BUCKET);
     const query = data.nextPageQuery || {
       autoPaginate: false,
-      maxResults: 3, // We only grab 3 images at a time to minimize the chance of OOM errors.
+      maxResults: config.backfillBatchSize,
     };
     const [files, nextPageQuery] = await bucket.getFiles(query);
     const filesToResize = files.filter((f: File) => {