Skip to content
This repository was archived by the owner on Jul 31, 2020. It is now read-only.

Fixed receiving objects in chunk #269

Open
samartnik wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fixed receiving objects in chunk #269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7a8cb31
Fixed receiving objects in chunk
samartnik Dec 27, 2018
5428838
Fixed receiving objects in chunk (lint issues fix)
samartnik Jan 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion client/requestUtil.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,13 +173,16 @@ RequestUtil.prototype.parseAWSResponse = function (bytes) {
* @param {number=} maxRecords Limit response to a given number of recods. By default the Sync lib will fetch all matching records, which might take a long time. If falsey, fetch all records.
* @returns {Promise(Array.<Object>)}
*/
RequestUtil.prototype.list = function (category, startAt, maxRecords) {
RequestUtil.prototype.list = function (category, startAt, maxRecords, nextContinuationToken) {
const prefix = `${this.apiVersion}/${this.userId}/${category}`
let options = {
MaxKeys: maxRecords || 1000,
Bucket: this.bucket,
Prefix: prefix
}
if (nextContinuationToken !== '') {
options.ContinuationToken = nextContinuationToken
}
if (startAt) { options.StartAfter = `${prefix}/${startAt}` }
return this.withRetry(() => {
if (this.shouldListObject(startAt, category)) {
Expand Down
13 changes: 12 additions & 1 deletion client/sync.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ var clientUserId = null
var clientKeys = {}
var config = {}
var seed
var nextContinuationTokens = {}

/**
* Logs stuff on the visible HTML page.
Expand Down Expand Up @@ -105,7 +106,11 @@ const startSync = (requester) => {
if (!proto.categories[category]) {
throw new Error(`Unsupported sync category: ${category}`)
}
requester.list(proto.categories[category], startAt, limitResponse).then((s3Objects) => {
let continuationToken = ''
if (nextContinuationTokens[category]) {
continuationToken = nextContinuationTokens[category]
}
requester.list(proto.categories[category], startAt, limitResponse, continuationToken).then((s3Objects) => {
const jsRecords = getJSRecords(s3Objects.contents)
logSync(`got ${jsRecords.length} decrypted records in ${category} after ${startAt}`)
let lastRecordTimestamp
Expand All @@ -117,6 +122,12 @@ const startSync = (requester) => {
} else if (!s3Objects.isTruncated) {
requester.setListInProgress(false)
}
if (s3Objects.isTruncated) {
// When is it truncated we need to provide continuation token, so system could understand where to continue from next time
nextContinuationTokens[category] = s3Objects.nextContinuationToken
} else {
nextContinuationTokens[category] = ''
}
ipc.send(messages.GET_EXISTING_OBJECTS, category, jsRecords, lastRecordTimestamp, s3Objects.isTruncated)
})
})
Expand Down
6 changes: 4 additions & 2 deletions lib/s3Helper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,8 @@ module.exports.listObjects = function (s3, options, limitResponse) {
} else {
resolve({
contents: data.Contents,
isTruncated: data.IsTruncated
isTruncated: data.IsTruncated,
nextContinuationToken: data.NextContinuationToken
})
}
})
Expand All @@ -230,7 +231,8 @@ module.exports.listObjects = function (s3, options, limitResponse) {
if (error) { reject(error) }
resolve({
contents: data,
isTruncated: false
isTruncated: false,
nextContinuationToken: ''
})
})
}
Expand Down
117 changes: 116 additions & 1 deletion test/client/requestUtil.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,16 +392,131 @@ test('client RequestUtil', (t) => {

const testCanLimitResponseToOne = (t) => {
t.test('limitResponse to 1', (t) => {
t.plan(2)
t.plan(3)
requestUtil.list(proto.categories.PREFERENCES, 0, 1)
.then((s3Objects) => {
t.assert(s3Objects.isTruncated === true, `${t.name} has true isTruncated value`)
t.assert(s3Objects.contents.length === 1, `${t.name} has one record`)
testCanGetBookmarksInChunks(t)
})
.catch((error) => t.fail(error))
})
}

const testCanGetBookmarksInChunks = (t) => {
const records = [
{
action: 'CREATE',
deviceId: new Uint8Array([0]),
objectId: testHelper.newUuid(),
bookmark: {
site: {
location: `https://brave.com?q=1`,
title: 'lulz',
lastAccessedTime: 1480000000 * 1000,
creationTime: 1480000000 * 1000
},
isFolder: false,
hideInToolbar: false,
order: '1.0.0.1'
}},
{
action: 'CREATE',
deviceId: new Uint8Array([0]),
objectId: testHelper.newUuid(),
bookmark: {
site: {
location: `https://brave.com?q=2`,
title: 'lulz',
lastAccessedTime: 1480000000 * 1000,
creationTime: 1480000000 * 1000
},
isFolder: false,
hideInToolbar: false,
order: '1.0.0.2'
}},
{
action: 'CREATE',
deviceId: new Uint8Array([0]),
objectId: testHelper.newUuid(),
bookmark: {
site: {
location: `https://brave.com?q=3`,
title: 'lulz',
lastAccessedTime: 1480000000 * 1000,
creationTime: 1480000000 * 1000
},
isFolder: false,
hideInToolbar: false,
order: '1.0.0.3'
}},
{
action: 'CREATE',
deviceId: new Uint8Array([0]),
objectId: testHelper.newUuid(),
bookmark: {
site: {
location: `https://brave.com?q=4`,
title: 'lulz',
lastAccessedTime: 1480000000 * 1000,
creationTime: 1480000000 * 1000
},
isFolder: false,
hideInToolbar: false,
order: '1.0.0.4'
}},
{
action: 'CREATE',
deviceId: new Uint8Array([0]),
objectId: testHelper.newUuid(),
bookmark: {
site: {
location: `https://brave.com?q=5`,
title: 'lulz',
lastAccessedTime: 1480000000 * 1000,
creationTime: 1480000000 * 1000
},
isFolder: false,
hideInToolbar: false,
order: '1.0.0.5'
}}
]

records.forEach((record) => {
requestUtil.put(proto.categories.BOOKMARKS, record)
})

t.test('#getBookmarksInChunks', (t) => {
t.plan(1)
getBookmarksInChunks(t, '', 1)
})

const getBookmarksInChunks = (t, continuationToken, iterationNumber) => {
if ((continuationToken === undefined || continuationToken === '') && iterationNumber > 1) {
t.assert(true, 'getBookmarksInChunks exit recurtion')
return
}
t.test('#getBookmarksInChunks attempt #' + iterationNumber, (t) => {
t.plan(3)
requestUtil.list(proto.categories.BOOKMARKS, 0, 3, continuationToken)
.then((s3Objects) => {
t.assert(s3Objects.contents.length <= 3, `${t.name} has less or exactly 3 records`)
if (s3Objects.isTruncated === true && s3Objects.nextContinuationToken !== '' && s3Objects.nextContinuationToken !== undefined) {
t.assert(true, `${t.name} isTruncated is true and nextContinuationToken is not empty`)
} else if (s3Objects.isTruncated === false && (s3Objects.nextContinuationToken === '' || s3Objects.nextContinuationToken === undefined) && iterationNumber > 1) {
t.assert(true, `${t.name} isTruncated is false and nextContinuationToken is empty`)
} else {
t.assert(false, `${t.name} isTruncated and nextContinuationToken values doesn't match, iterationNumber: ${iterationNumber}`)
}
continuationToken = s3Objects.nextContinuationToken
iterationNumber = iterationNumber + 1
getBookmarksInChunks(t, continuationToken, iterationNumber)
})
.catch((error) => t.fail(error))
})
}
}

const expiredCredentials = {
aws: clientTestHelper.EXPIRED_CREDENTIALS.aws,
s3Post: clientTestHelper.EXPIRED_CREDENTIALS.s3Post,
Expand Down