Public Folders

Author: TimCsaky

app/src/components/constants.js

@@ -57,7 +57,7 @@ module.exports = Object.freeze({
   /** Maximum Content Length supported by S3 CopyObjectCommand */
   MAXCOPYOBJECTLENGTH: 5 * 1024 * 1024 * 1024, // 5 GB
 
-  /** Maximum Content Length supported by S3 CopyObjectCommand */
+  /** Maximum Content Length (file size) supported by S3 */
   MAXFILEOBJECTLENGTH: 5 * 1024 * 1024 * 1024 * 1024, // 5 TB
 
   /** Allowable values for the Metadata Directive parameter */

app/src/components/utils.js

@@ -93,6 +93,7 @@ const utils = {
         data.bucket = bucketData.bucket;
         data.endpoint = bucketData.endpoint;
         data.key = bucketData.key;
+        // data.public = bucketData.public;
         data.secretAccessKey = bucketData.secretAccessKey;
         if (bucketData.region) data.region = bucketData.region;
       } else if (utils.getConfigBoolean('objectStorage.enabled')) {
@@ -357,6 +358,21 @@ const utils = {
       && pathParts.filter(part => !prefixParts.includes(part)).length === 1;
   },
 
+  /**
+   * @function isBelowPrefix
+   * Predicate function determining if a path is 'below' a prefix
+   * @param {string} prefix The base "folder"
+   * @param {string} path The "sub-folder" to check
+   * @returns {boolean} True if path is below of prefix. False in all other cases.
+   */
+  isBelowPrefix(prefix, path) {
+    if (typeof prefix !== 'string' || typeof path !== 'string') return false;
+    else if (path === prefix) return false;
+    else if (prefix === DELIMITER) return true;
+    else if (path.startsWith(prefix)) return true;
+    else return false;
+  },
+
   /**
    * @function isTruthy
    * Returns true if the element name in the object contains a truthy value

app/src/controllers/bucket.js

@@ -9,6 +9,7 @@ const log = require('../components/log')(module.filename);
 const {
   addDashesToUuid,
   getCurrentIdentity,
+  getBucket,
   isTruthy,
   joinPath,
   mixedQueryToArray,
@@ -323,6 +324,48 @@ const controller = {
     }
   },
 
+
+  /**
+   * @function togglePublic
+   * Sets the public flag of a bucket (or folder)
+   * @param {object} req Express request object
+   * @param {object} res Express response object
+   * @param {function} next The next callback function
+   * @returns {function} Express middleware function
+   */
+  async togglePublic(req, res, next) {
+    try {
+      const bucketId = addDashesToUuid(req.params.bucketId);
+      const publicFlag = isTruthy(req.query.public) ?? false;
+      const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER);
+
+      const bucket = await getBucket(bucketId);
+      const data = {
+        bucketId: bucketId,
+        path: bucket.key + '/',
+        public: publicFlag,
+        userId: userId
+      };
+      // Update S3 Policy
+      await storageService.updatePublic(data).catch((e) => {
+        log.warn('Failed to apply permission changes to S3' + e, { function: 'togglePublic', ...data });
+      });
+
+      // update public flag for this bucket and all child buckets and objects!
+      const response = await bucketService.updatePublic({
+        ...bucket,
+        bucketId: bucketId,
+        public: publicFlag,
+        userId: userId
+      });
+
+      res.status(200).json(response);
+    } catch (e) {
+      next(errorToProblem(SERVICE, e));
+    }
+  },
+
+
   /**
    * @function updateBucket
    * Updates a bucket

app/src/controllers/object.js

@@ -274,7 +274,7 @@ const controller = {
       }
 
       // Preflight existence check for bucketId
-      const { key: bucketKey } = await getBucket(bucketId);
+      const { key: bucketKey, public: bucketPublic } = await getBucket(bucketId);
 
       const objId = uuidv4();
       const data = {
@@ -308,7 +308,9 @@ const controller = {
           existingObjectId: objectId,
         });
 
-      } catch (err) {
+      }
+      // headObject threw an error because object was not found
+      catch (err) {
         if (err instanceof Problem) throw err; // Rethrow Problem type errors
 
         // Object is soft deleted from the bucket
@@ -366,7 +368,8 @@ const controller = {
         const object = await objectService.create({
           ...data,
           userId: userId,
-          path: joinPath(bucketKey, data.name)
+          path: joinPath(bucketKey, data.name),
+          public: bucketPublic // set public status to match that of parent 'folder'
         }, trx);
 
         // Create Version
@@ -1074,19 +1077,17 @@ const controller = {
       const data = {
         id: objId,
         bucketId: req.currentObject?.bucketId,
-        filePath: req.currentObject?.path,
+        path: req.currentObject?.path,
         public: publicFlag,
         userId: userId,
-        // TODO: Implement if/when we proceed with version-scoped public permission management
-        // s3VersionId: await getS3VersionId(
-        //   req.query.s3VersionId, addDashesToUuid(req.query.versionId), objId
-        // )
       };
 
-      storageService.putObjectPublic(data).catch(() => {
-        // Gracefully continue even when S3 ACL management operation fails
-        log.warn('Failed to apply ACL permission changes to S3', { function: 'togglePublic', ...data });
+      // Update S3 Policy
+      await storageService.updatePublic(data).catch((error) => {
+        log.warn('Failed to apply permission changes to S3', { function: 'togglePublic', ...data });
+        throw new Error(error);
       });
+      // Update object record in COMS database
       const response = await objectService.update(data);
 
       res.status(200).json(response);

app/src/controllers/sync.js

@@ -56,10 +56,10 @@ const controller = {
       /**
        * sync (ie create or delete) bucket records in COMS db to match 'folders' (S3 key prefixes) that exist in S3
       */
-      // parent + child bucket records already in COMS db
+      // get parent + child bucket records already in COMS db
       const dbChildBuckets = await bucketService.searchChildBuckets(parentBucket, false, userId);
       let dbBuckets = [parentBucket].concat(dbChildBuckets);
-      // 'folders' that exist below (and including) the parent 'folder' in S3
+
       const s3Response = await storageService.listAllObjectVersions({ bucketId: bucketId, precisePath: false });
       const s3Keys = [...new Set([
         ...s3Response.DeleteMarkers.map(object => formatS3KeyForCompare(object.Key)),
@@ -68,7 +68,7 @@ const controller = {
 
       // Wrap sync sql operations in a single transaction
       const response = await utils.trxWrapper(async (trx) => {
-
+        // sync bucket records
         const syncedBuckets = await this.syncBucketRecords(
           dbBuckets,
           s3Keys,
@@ -106,6 +106,9 @@ const controller = {
       const bucket = await bucketService.read(bucketId);
       const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER);
 
+      // sync bucket.public flag
+      await this.syncBucketPublic(bucket.key, bucket.bucketId, userId);
+
       const s3Objects = await storageService.listAllObjectVersions({ bucketId: bucketId, filterLatest: true });
 
       const response = await utils.trxWrapper(async (trx) => {
@@ -141,17 +144,8 @@ const controller = {
               dbBuckets = dbBuckets.filter(b => b.bucketId !== dbBucket.bucketId);
             })
         )
-      );
-      // add current user's permissions to all buckets
-      await Promise.all(
-        dbBuckets.map(bucket => {
-          return bucketPermissionService.addPermissions(
-            bucket.bucketId,
-            currentUserParentBucketPerms.map(permCode => ({ userId, permCode })),
-            undefined,
-            trx
-          );
-        })
+        // TODO: delete COMS S3 Policies for deleted COMS buckets and child objects.
+        // Also consider when using DEL /Bucket endpoint, should we delete policies?
       );
 
       // Create buckets only found in S3 in COMS db
@@ -176,6 +170,22 @@ const controller = {
             });
         })
       );
+
+      // Update permissions and Sync Public status
+      await Promise.all(
+        // for each bucket
+        dbBuckets.map(async bucket => {
+          // --- Add current user's permissions that exist on parent bucket if they dont already exist
+          await bucketPermissionService.addPermissions(
+            bucket.bucketId,
+            currentUserParentBucketPerms.map(permCode => ({ userId, permCode })),
+            undefined,
+            trx
+          );
+          // --- Sync S3 Bucket Policies applied by COMS
+          await this.syncBucketPublic(bucket.key, bucket.bucketId, userId);
+        })
+      );
       return dbBuckets;
     }
     catch (err) {
@@ -184,6 +194,18 @@ const controller = {
     }
   },
 
+  async syncBucketPublic(key, bucketId, userId) {
+    let public = false;
+    public = await storageService.getPublic({ path: key, bucketId: bucketId });
+    bucketService.update({
+      bucketId: bucketId,
+      updatedBy: userId,
+      public: public
+      // TODO: consider changing this to actual lastSyncDate
+      // lastSyncRequestedDate: now(),
+    });
+  },
+
   /**
    * @function queueObjectRecords
    * Synchronizes (creates / prunes) COMS db object records with state in S3

app/src/db/migrations/20250422000000_016-bucket-public.js

@@ -0,0 +1,35 @@
+exports.up = function (knex) {
+  return Promise.resolve()
+    // // allow null for object.public
+    // .then(() => knex.schema.alterTable('object', table => {
+    //   table.boolean('public').nullable().alter();
+    // }))
+    // // where object.public is false, set to null
+    // .then(() => knex('object')
+    //   .where({ 'public': false })
+    //   .update({ 'public': null }))
+    // .then(() => knex.schema.alterTable('object', table => {
+    //   table.boolean('public').nullable().alter();
+    // }))
+    // add public column to bucket table
+    .then(() => knex.schema.alterTable('bucket', table => {
+      table.boolean('public').notNullable().defaultTo(false);
+    }));
+};
+
+exports.down = function (knex) {
+  return Promise.resolve()
+    // drop public column in bucket table
+    .then(() => knex.schema.alterTable('bucket', table => {
+      table.dropColumn('public');
+    }));
+  // // where object.public is null, set to false
+  // .then(() => knex('object')
+  //   .where({ 'public': null })
+  //   .update({ 'public': false }))
+
+  // disallow null for object.public
+  // .then(() => knex.schema.alterTable('object', table => {
+  //   table.boolean('public').notNullable().defaultTo(false).alter();
+  // }));
+};

app/src/db/models/tables/bucket.js

@@ -99,6 +99,7 @@ class Bucket extends mixin(Model, [
         region: { type: 'string', maxLength: 255 },
         active: { type: 'boolean' },
         lastSyncRequestedDate: { type: ['string', 'null'], format: 'date-time' },
+        public: { type: 'boolean' },
         ...stamps
       },
       additionalProperties: false