Skip to content

Commit 6ed464b

Browse files
jloukjlouk
committed
Updating aws sdk to v3
1 parent 9e1d4b5 commit 6ed464b

13 files changed

+2432
-382
lines changed

‎backup.js‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
var AWS = require('aws-sdk');
1+
var { S3Client } = require('@aws-sdk/client-s3');
2+
var { Upload } = require('@aws-sdk/lib-storage');
23
var Dyno = require('@mapbox/dyno');
34
var stream = require('stream');
45
var zlib = require('zlib');
56

67
module.exports = function(config, done) {
78
var primary = Dyno(config);
8-
var s3 = new AWS.S3();
9+
var s3Client = new S3Client({ region: config.region });
910

1011
var log = config.log || console.log;
1112

@@ -40,20 +41,30 @@ module.exports = function(config, done) {
4041

4142
log('[segment %s] Starting backup job %s of %s', index, config.backup.jobid, config.region + '/' + config.table);
4243

43-
s3.upload({
44-
Bucket: config.backup.bucket,
45-
Key: key,
46-
Body: data
47-
}, function(err) {
48-
if (err) return next(err);
49-
log('[segment %s] Uploaded dynamo backup to s3://%s/%s', index, config.backup.bucket, key);
50-
log('[segment %s] Wrote %s items to backup', index, count);
51-
next();
52-
}).on('httpUploadProgress', function(progress) {
44+
const upload = new Upload({
45+
client: s3Client,
46+
params: {
47+
Bucket: config.backup.bucket,
48+
Key: key,
49+
Body: data
50+
}
51+
});
52+
53+
upload.on('httpUploadProgress', function(progress) {
5354
log('[segment %s] Uploaded %s bytes', index, progress.loaded);
54-
size = progress.total;
55+
size = progress.loaded;
5556
});
5657

58+
upload.done()
59+
.then(() => {
60+
log('[segment %s] Uploaded dynamo backup to s3://%s/%s', index, config.backup.bucket, key);
61+
log('[segment %s] Wrote %s items to backup', index, count);
62+
next();
63+
})
64+
.catch(err => {
65+
next(err);
66+
});
67+
5768
function next(err) {
5869
if (err) return done(err);
5970
done(null, { size: size, count: count });

‎bin/backup-table.js‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ var fastlog = require('../fastlog');
55
var args = require('minimist')(process.argv.slice(2));
66
var crypto = require('crypto');
77
var s3urls = require('s3urls');
8-
var AWS = require('aws-sdk');
8+
var { CloudWatchClient, PutMetricDataCommand } = require('@aws-sdk/client-cloudwatch');
99

1010
function usage() {
1111
console.error('');
@@ -64,7 +64,7 @@ backup(config, function(err, details) {
6464
if (err) log.error(err);
6565

6666
if (args.metric) {
67-
var cw = new AWS.CloudWatch({ region: config.region });
67+
var cwClient = new CloudWatchClient({ region: config.region });
6868
var params = {
6969
Namespace: args.metric,
7070
MetricData: []
@@ -107,8 +107,12 @@ backup(config, function(err, details) {
107107
});
108108
}
109109

110-
cw.putMetricData(params, function(err) {
111-
if (err) log.error(err);
112-
});
110+
cwClient.send(new PutMetricDataCommand(params))
111+
.then(() => {
112+
// Success
113+
})
114+
.catch(err => {
115+
log.error(err);
116+
});
113117
}
114118
});

‎bin/incremental-diff-record.js‎

Lines changed: 50 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ var minimist = require('minimist');
44
var s3urls = require('s3urls');
55
var Dyno = require('@mapbox/dyno');
66
var crypto = require('crypto');
7-
var AWS = require('aws-sdk');
8-
var s3 = new AWS.S3();
7+
var { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3');
98
var assert = require('assert');
109

1110
var args = minimist(process.argv.slice(2));
@@ -34,6 +33,8 @@ if (!table) {
3433
var region = table.split('/')[0];
3534
table = table.split('/')[1];
3635

36+
var s3Client = new S3Client({ region: region });
37+
3738
var s3url = args._[1];
3839

3940
if (!s3url) {
@@ -86,30 +87,51 @@ dyno.getItem({ Key: key }, function(err, data) {
8687
if (err) throw err;
8788
var dynamoRecord = data.Item;
8889

89-
s3.getObject(s3url, function(err, data) {
90-
if (err && err.statusCode !== 404) throw err;
91-
var s3data = err ? undefined : Dyno.deserialize(data.Body.toString());
92-
93-
console.log('DynamoDB record');
94-
console.log('--------------');
95-
console.log(dynamoRecord);
96-
console.log('');
97-
98-
console.log('Incremental backup record (%s)', s3url.Key);
99-
console.log('--------------');
100-
console.log(s3data);
101-
console.log('');
102-
103-
try {
104-
assert.deepEqual(s3data, dynamoRecord);
105-
console.log('----------------------------');
106-
console.log('✔ The records are equivalent');
107-
console.log('----------------------------');
108-
}
109-
catch (err) {
110-
console.log('--------------------------------');
111-
console.log('✘ The records are not equivalent');
112-
console.log('--------------------------------');
113-
}
114-
});
90+
s3Client.send(new GetObjectCommand(s3url))
91+
.then(data => {
92+
var s3data = Dyno.deserialize(data.Body.toString());
93+
94+
console.log('DynamoDB record');
95+
console.log('--------------');
96+
console.log(dynamoRecord);
97+
console.log('');
98+
99+
console.log('Incremental backup record (%s)', s3url.Key);
100+
console.log('--------------');
101+
console.log(s3data);
102+
console.log('');
103+
104+
try {
105+
assert.deepEqual(s3data, dynamoRecord);
106+
console.log('----------------------------');
107+
console.log('✔ The records are equivalent');
108+
console.log('----------------------------');
109+
}
110+
catch (err) {
111+
console.log('--------------------------------');
112+
console.log('✘ The records are not equivalent');
113+
console.log('--------------------------------');
114+
}
115+
})
116+
.catch(err => {
117+
if (err.$metadata && err.$metadata.httpStatusCode === 404) {
118+
var s3data = undefined;
119+
120+
console.log('DynamoDB record');
121+
console.log('--------------');
122+
console.log(dynamoRecord);
123+
console.log('');
124+
125+
console.log('Incremental backup record (%s)', s3url.Key);
126+
console.log('--------------');
127+
console.log(s3data);
128+
console.log('');
129+
130+
console.log('--------------------------------');
131+
console.log('✘ The records are not equivalent');
132+
console.log('--------------------------------');
133+
} else {
134+
throw err;
135+
}
136+
});
115137
});

‎bin/incremental-snapshot.js‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env node
22

3-
var AWS = require('aws-sdk');
3+
var { CloudWatchClient, PutMetricDataCommand } = require('@aws-sdk/client-cloudwatch');
44
var args = require('minimist')(process.argv.slice(2));
55
var s3urls = require('s3urls');
66
var fastlog = require('../fastlog');
@@ -59,7 +59,7 @@ snapshot(config, function(err, details) {
5959
var namespace = args.metric.split('/')[1];
6060
var table = args.metric.split('/')[2];
6161

62-
var cw = new AWS.CloudWatch({ region: region });
62+
var cwClient = new CloudWatchClient({ region: region });
6363

6464
var params = {
6565
Namespace: namespace,
@@ -103,10 +103,13 @@ snapshot(config, function(err, details) {
103103
});
104104
}
105105

106-
cw.putMetricData(params, function(err) {
107-
if (err) return log.error(err);
108-
if (!details) return log.info('Snapshot failed, wrote error metric to %s', args.metric);
109-
log.info('Wrote %s size / %s count metrics to %s', details.size, details.count, args.metric);
110-
});
106+
cwClient.send(new PutMetricDataCommand(params))
107+
.then(() => {
108+
if (!details) return log.info('Snapshot failed, wrote error metric to %s', args.metric);
109+
log.info('Wrote %s size / %s count metrics to %s', details.size, details.count, args.metric);
110+
})
111+
.catch(err => {
112+
log.error(err);
113+
});
111114
}
112115
});

‎index.js‎

Lines changed: 40 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
var AWS = require('aws-sdk');
1+
var { S3Client, PutObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3');
22
var Dyno = require('@mapbox/dyno');
33
var queue = require('queue-async');
44
var crypto = require('crypto');
@@ -118,16 +118,16 @@ function replicate(event, context, callback) {
118118

119119
function incrementalBackup(event, context, callback) {
120120
var params = {
121-
maxRetries: 1000,
122-
httpOptions: {
123-
timeout: 1000,
124-
agent: module.exports.agent
121+
maxAttempts: 1000,
122+
requestHandler: {
123+
connectionTimeout: 1000,
124+
httpAgent: module.exports.agent
125125
}
126126
};
127127

128128
if (process.env.BackupRegion) params.region = process.env.BackupRegion;
129129

130-
var s3 = new AWS.S3(params);
130+
var s3Client = new S3Client(params);
131131

132132
var filterer;
133133
if (process.env.TurnoverRole && process.env.TurnoverAt) {
@@ -185,28 +185,40 @@ function incrementalBackup(event, context, callback) {
185185
Key: [process.env.BackupPrefix, table, id].join('/')
186186
};
187187

188-
var req = change.eventName === 'REMOVE' ? 'deleteObject' : 'putObject';
189-
if (req === 'putObject') params.Body = JSON.stringify(change.dynamodb.NewImage);
190-
191-
s3[req](params, function(err) {
192-
if (err) console.log(
193-
'[error] %s | %s s3://%s/%s | %s',
194-
JSON.stringify(change.dynamodb.Keys),
195-
req, params.Bucket, params.Key,
196-
err.message
197-
);
198-
next(err);
199-
}).on('retry', function(res) {
200-
if (!res.error || !res.httpResponse || !res.httpResponse.headers) return;
201-
if (res.error.name === 'TimeoutError') res.error.retryable = true;
202-
console.log(
203-
'[failed-request] request-id: %s | id-2: %s | %s s3://%s/%s | %s',
204-
res.httpResponse.headers['x-amz-request-id'],
205-
res.httpResponse.headers['x-amz-id-2'],
206-
req, params.Bucket, params.Key,
207-
res.error
208-
);
209-
});
188+
var command;
189+
if (change.eventName === 'REMOVE') {
190+
command = new DeleteObjectCommand(params);
191+
} else {
192+
params.Body = JSON.stringify(change.dynamodb.NewImage);
193+
command = new PutObjectCommand(params);
194+
}
195+
196+
s3Client.send(command)
197+
.then(() => {
198+
next();
199+
})
200+
.catch(err => {
201+
console.log(
202+
'[error] %s | %s s3://%s/%s | %s',
203+
JSON.stringify(change.dynamodb.Keys),
204+
change.eventName === 'REMOVE' ? 'deleteObject' : 'putObject',
205+
params.Bucket, params.Key,
206+
err.message
207+
);
208+
209+
// Log retry information if available
210+
if (err.$metadata && err.$metadata.requestId) {
211+
console.log(
212+
'[failed-request] request-id: %s | %s s3://%s/%s | %s',
213+
err.$metadata.requestId,
214+
change.eventName === 'REMOVE' ? 'deleteObject' : 'putObject',
215+
params.Bucket, params.Key,
216+
err
217+
);
218+
}
219+
220+
next(err);
221+
});
210222
});
211223
});
212224

0 commit comments

Comments
 (0)