From 7a8cb310c4ca85e58de3a34fd24b34ac14de045f Mon Sep 17 00:00:00 2001 From: samartnik Date: Thu, 27 Dec 2018 17:49:46 -0500 Subject: [PATCH 1/2] Fixed receiving objects in chunk --- client/requestUtil.js | 5 +- client/sync.js | 13 ++++- lib/s3Helper.js | 6 +- test/client/requestUtil.js | 117 ++++++++++++++++++++++++++++++++++++- 4 files changed, 136 insertions(+), 5 deletions(-) diff --git a/client/requestUtil.js b/client/requestUtil.js index 1c55822..f6b0742 100644 --- a/client/requestUtil.js +++ b/client/requestUtil.js @@ -173,13 +173,16 @@ RequestUtil.prototype.parseAWSResponse = function (bytes) { * @param {number=} maxRecords Limit response to a given number of recods. By default the Sync lib will fetch all matching records, which might take a long time. If falsey, fetch all records. * @returns {Promise(Array.)} */ -RequestUtil.prototype.list = function (category, startAt, maxRecords) { +RequestUtil.prototype.list = function (category, startAt, maxRecords, nextContinuationToken) { const prefix = `${this.apiVersion}/${this.userId}/${category}` let options = { MaxKeys: maxRecords || 1000, Bucket: this.bucket, Prefix: prefix } + if (nextContinuationToken != '') { + options.ContinuationToken = nextContinuationToken + } if (startAt) { options.StartAfter = `${prefix}/${startAt}` } return this.withRetry(() => { if (this.shouldListObject(startAt, category)) { diff --git a/client/sync.js b/client/sync.js index fe8c1c9..8e43d55 100644 --- a/client/sync.js +++ b/client/sync.js @@ -22,6 +22,7 @@ var clientUserId = null var clientKeys = {} var config = {} var seed +var nextContinuationTokens = {} /** * Logs stuff on the visible HTML page. @@ -105,7 +106,11 @@ const startSync = (requester) => { if (!proto.categories[category]) { throw new Error(`Unsupported sync category: ${category}`) } - requester.list(proto.categories[category], startAt, limitResponse).then((s3Objects) => { + let continuationToken = '' + if (nextContinuationTokens[category]) { + continuationToken = nextContinuationTokens[category] + } + requester.list(proto.categories[category], startAt, limitResponse, continuationToken).then((s3Objects) => { const jsRecords = getJSRecords(s3Objects.contents) logSync(`got ${jsRecords.length} decrypted records in ${category} after ${startAt}`) let lastRecordTimestamp @@ -117,6 +122,12 @@ const startSync = (requester) => { } else if (!s3Objects.isTruncated) { requester.setListInProgress(false) } + if (s3Objects.isTruncated) { + // When is it truncated we need to provide continuation token, so system could understand where to continue from next time + nextContinuationTokens[category] = s3Objects.nextContinuationToken + } else { + nextContinuationTokens[category] = '' + } ipc.send(messages.GET_EXISTING_OBJECTS, category, jsRecords, lastRecordTimestamp, s3Objects.isTruncated) }) }) diff --git a/lib/s3Helper.js b/lib/s3Helper.js index 6f284bb..a789425 100644 --- a/lib/s3Helper.js +++ b/lib/s3Helper.js @@ -221,7 +221,8 @@ module.exports.listObjects = function (s3, options, limitResponse) { } else { resolve({ contents: data.Contents, - isTruncated: data.IsTruncated + isTruncated: data.IsTruncated, + nextContinuationToken : data.NextContinuationToken }) } }) @@ -230,7 +231,8 @@ module.exports.listObjects = function (s3, options, limitResponse) { if (error) { reject(error) } resolve({ contents: data, - isTruncated: false + isTruncated: false, + nextContinuationToken : '' }) }) } diff --git a/test/client/requestUtil.js b/test/client/requestUtil.js index 47aa424..d221a5f 100644 --- a/test/client/requestUtil.js +++ b/test/client/requestUtil.js @@ -392,16 +392,131 @@ test('client RequestUtil', (t) => { const testCanLimitResponseToOne = (t) => { t.test('limitResponse to 1', (t) => { - t.plan(2) + t.plan(3) requestUtil.list(proto.categories.PREFERENCES, 0, 1) .then((s3Objects) => { t.assert(s3Objects.isTruncated === true, `${t.name} has true isTruncated value`) t.assert(s3Objects.contents.length === 1, `${t.name} has one record`) + testCanGetBookmarksInChunks(t) }) .catch((error) => t.fail(error)) }) } + const testCanGetBookmarksInChunks = (t) => { + const records = [ + { + action: 'CREATE', + deviceId: new Uint8Array([0]), + objectId: testHelper.newUuid(), + bookmark: { + site: { + location: `https://brave.com?q=1`, + title: 'lulz', + lastAccessedTime: 1480000000 * 1000, + creationTime: 1480000000 * 1000 + }, + isFolder: false, + hideInToolbar: false, + order: '1.0.0.1' + }}, + { + action: 'CREATE', + deviceId: new Uint8Array([0]), + objectId: testHelper.newUuid(), + bookmark: { + site: { + location: `https://brave.com?q=2`, + title: 'lulz', + lastAccessedTime: 1480000000 * 1000, + creationTime: 1480000000 * 1000 + }, + isFolder: false, + hideInToolbar: false, + order: '1.0.0.2' + }}, + { + action: 'CREATE', + deviceId: new Uint8Array([0]), + objectId: testHelper.newUuid(), + bookmark: { + site: { + location: `https://brave.com?q=3`, + title: 'lulz', + lastAccessedTime: 1480000000 * 1000, + creationTime: 1480000000 * 1000 + }, + isFolder: false, + hideInToolbar: false, + order: '1.0.0.3' + }}, + { + action: 'CREATE', + deviceId: new Uint8Array([0]), + objectId: testHelper.newUuid(), + bookmark: { + site: { + location: `https://brave.com?q=4`, + title: 'lulz', + lastAccessedTime: 1480000000 * 1000, + creationTime: 1480000000 * 1000 + }, + isFolder: false, + hideInToolbar: false, + order: '1.0.0.4' + }}, + { + action: 'CREATE', + deviceId: new Uint8Array([0]), + objectId: testHelper.newUuid(), + bookmark: { + site: { + location: `https://brave.com?q=5`, + title: 'lulz', + lastAccessedTime: 1480000000 * 1000, + creationTime: 1480000000 * 1000 + }, + isFolder: false, + hideInToolbar: false, + order: '1.0.0.5' + }} + ] + + records.forEach((record) => { + requestUtil.put(proto.categories.BOOKMARKS, record) + }) + + t.test('#getBookmarksInChunks', (t) => { + t.plan(1) + getBookmarksInChunks(t, '', 1) + }) + + const getBookmarksInChunks = (t, continuationToken, iterationNumber) => { + if ((continuationToken === undefined || continuationToken === '') && iterationNumber > 1) { + t.assert(true, 'getBookmarksInChunks exit recurtion') + return + } + t.test('#getBookmarksInChunks attempt #' + iterationNumber, (t) => { + t.plan(3) + requestUtil.list(proto.categories.BOOKMARKS, 0, 3, continuationToken) + .then((s3Objects) => { + t.assert(s3Objects.contents.length <= 3, `${t.name} has less or exactly 3 records`) + if (s3Objects.isTruncated === true && s3Objects.nextContinuationToken !== '' && s3Objects.nextContinuationToken !== undefined) { + t.assert(true, `${t.name} isTruncated is true and nextContinuationToken is not empty`) + } else if (s3Objects.isTruncated === false && (s3Objects.nextContinuationToken === '' || s3Objects.nextContinuationToken === undefined) && iterationNumber > 1) { + t.assert(true, `${t.name} isTruncated is false and nextContinuationToken is empty`) + } else { + t.assert(false, `${t.name} isTruncated and nextContinuationToken values doesn't match, iterationNumber: ${iterationNumber}`) + } + continuationToken = s3Objects.nextContinuationToken + iterationNumber = iterationNumber + 1 + getBookmarksInChunks(t, continuationToken, iterationNumber) + }) + .catch((error) => t.fail(error)) + }) + } + } + const expiredCredentials = { aws: clientTestHelper.EXPIRED_CREDENTIALS.aws, s3Post: clientTestHelper.EXPIRED_CREDENTIALS.s3Post, From 5428838791f709b24f321e6e5d8d43119b8ce93c Mon Sep 17 00:00:00 2001 From: samartnik Date: Wed, 2 Jan 2019 15:54:52 -0500 Subject: [PATCH 2/2] Fixed receiving objects in chunk (lint issues fix) --- client/requestUtil.js | 2 +- lib/s3Helper.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/requestUtil.js b/client/requestUtil.js index f6b0742..6211bff 100644 --- a/client/requestUtil.js +++ b/client/requestUtil.js @@ -180,7 +180,7 @@ RequestUtil.prototype.list = function (category, startAt, maxRecords, nextContin Bucket: this.bucket, Prefix: prefix } - if (nextContinuationToken != '') { + if (nextContinuationToken !== '') { options.ContinuationToken = nextContinuationToken } if (startAt) { options.StartAfter = `${prefix}/${startAt}` } diff --git a/lib/s3Helper.js b/lib/s3Helper.js index a789425..456ef8b 100644 --- a/lib/s3Helper.js +++ b/lib/s3Helper.js @@ -222,7 +222,7 @@ module.exports.listObjects = function (s3, options, limitResponse) { resolve({ contents: data.Contents, isTruncated: data.IsTruncated, - nextContinuationToken : data.NextContinuationToken + nextContinuationToken: data.NextContinuationToken }) } }) @@ -232,7 +232,7 @@ module.exports.listObjects = function (s3, options, limitResponse) { resolve({ contents: data, isTruncated: false, - nextContinuationToken : '' + nextContinuationToken: '' }) }) }