From eb456ef2195f8a006271997c12096942752132f6 Mon Sep 17 00:00:00 2001 From: Chris Silivestru Date: Tue, 13 Jun 2017 13:58:58 -0400 Subject: [PATCH 1/7] Going to 1 for read is never a good idea. We can afford the extra 20c per month. --- src/Provisioner.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Provisioner.js b/src/Provisioner.js index e214b0d..47ed866 100644 --- a/src/Provisioner.js +++ b/src/Provisioner.js @@ -41,6 +41,15 @@ export default class Provisioner extends ProvisionerConfigurableBase { // Option 3 - DynamoDB / S3 sourced table specific settings // return await ...; + + // Option 4 - Modify certain defaults with env vars + DefaultProvisioner.ReadCapacity.Min = process.env.AWS_MIN_READ || 5; + DefaultProvisioner.ReadCapacity.Max = process.env.AWS_MIN_WRITE || 100; + DefaultProvisioner.WriteCapacity.Min = process.env.AWS_MIN_WRITE || 1; + DefaultProvisioner.WriteCapacity.Max = process.env.AWS_MIN_WRITE || 100; + + console.log(DefaultProvisioner); + return DefaultProvisioner; } isReadCapacityIncrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { From 00ed01f5e97a38374f28cff914bfab0a8d009555 Mon Sep 17 00:00:00 2001 From: Chris Silivestru Date: Wed, 14 Jun 2017 10:16:17 -0400 Subject: [PATCH 2/7] Need a way to bring in custom confgis. Make the keys in this file the same as the table you want the config to apply to. --- src/Provisioner.js | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Provisioner.js b/src/Provisioner.js index 47ed866..9a87b20 100644 --- a/src/Provisioner.js +++ b/src/Provisioner.js @@ -6,6 +6,7 @@ import Throughput from './utils/Throughput'; import ProvisionerLogging from './provisioning/ProvisionerLogging'; import { Region } from './configuration/Region'; import DefaultProvisioner from './configuration/DefaultProvisioner'; +import CustomProvisioners from './configuration/CustomProvisioners'; import { invariant } from './Global'; import type { TableProvisionedAndConsumedThroughput, ProvisionerConfig, AdjustmentContext } from './flow/FlowTypes'; @@ -34,22 +35,16 @@ export default class Provisioner extends ProvisionerConfigurableBase { getTableConfig(data: TableProvisionedAndConsumedThroughput): ProvisionerConfig { // Option 1 - Default settings for all tables - return DefaultProvisioner; + // return DefaultProvisioner; // Option 2 - Bespoke table specific settings // return data.TableName === 'Table1' ? Climbing : Default; // Option 3 - DynamoDB / S3 sourced table specific settings // return await ...; - - // Option 4 - Modify certain defaults with env vars - DefaultProvisioner.ReadCapacity.Min = process.env.AWS_MIN_READ || 5; - DefaultProvisioner.ReadCapacity.Max = process.env.AWS_MIN_WRITE || 100; - DefaultProvisioner.WriteCapacity.Min = process.env.AWS_MIN_WRITE || 1; - DefaultProvisioner.WriteCapacity.Max = process.env.AWS_MIN_WRITE || 100; - - console.log(DefaultProvisioner); - return DefaultProvisioner; + + // Option 4 - Get the table specific config based on table name + return CustomProvisioners[data.TableName] || DefaultProvisioner; } isReadCapacityIncrementRequired(data: TableProvisionedAndConsumedThroughput): boolean { From bd4d4c5f90a74e5836c4c6acc5030a8d5144fb0a Mon Sep 17 00:00:00 2001 From: Chris Silivestru Date: Tue, 13 Jun 2017 10:28:07 -0400 Subject: [PATCH 3/7] Should be able to autoscale tables based on a special tag put on them. This would allow us to turn autoscaling on and off of tables without requiring changing the lambda function code and redeploying it. --- src/Provisioner.js | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/Provisioner.js b/src/Provisioner.js index 9a87b20..d33abf9 100644 --- a/src/Provisioner.js +++ b/src/Provisioner.js @@ -21,13 +21,37 @@ export default class Provisioner extends ProvisionerConfigurableBase { async getTableNamesAsync(): Promise { // Option 1 - All tables (Default) - return await this.db.listAllTableNamesAsync(); + //return await this.db.listAllTableNamesAsync(); // Option 2 - Hardcoded list of tables // return ['Table1', 'Table2', 'Table3']; // Option 3 - DynamoDB / S3 configured list of tables // return await ...; + + // Option 4 - Select all tables with a specific tag + return await this.db.listAllTableNamesAsync() + .then(list => { + return Promise.all(list.map(name => { + const params = { ResourceArn: `arn:aws:dynamodb:${process.env.AWS_REGION}:${process.env.AWS_ACCOUNT_NUMBER}:table/${name}` }; + + return new Promise((resolve, reject) => { + // Required to throttle the requests (AWS only accepts 10 of these calls per second per account) + setTimeout(() => { + db.listTagsOfResource(params, (err, {Tags}) => { + if (err) return reject(err); + + return resolve({ tableName: name, tags: Tags }); + }); + }, 100) + }); + })) + .then(list => { + return list + .filter(pkg => { return pkg.tags.some(tag => tag.Key === process.env.AWS_AUTOSCALE_TAG_NAME || 'autoscaled' && tag.Value.match(/true/g)); }) + .map(pkg => pkg.tableName); + }) + }) } // Gets the json settings which control how the specifed table will be autoscaled From b15305aaaf5101005dcbf6e025999bcdaa1025f8 Mon Sep 17 00:00:00 2001 From: Chris Silivestru Date: Tue, 13 Jun 2017 13:33:19 -0400 Subject: [PATCH 4/7] Added all the flow support needed. --- flow/aws-sdk.js | 10 ++++++ src/Provisioner.js | 50 ++++++++++++++------------ src/aws/DynamoDB.js | 28 +++++++++++++++ src/provisioning/ProvisionerLogging.js | 4 +++ 4 files changed, 70 insertions(+), 22 deletions(-) diff --git a/flow/aws-sdk.js b/flow/aws-sdk.js index 4ebae82..c3e391f 100644 --- a/flow/aws-sdk.js +++ b/flow/aws-sdk.js @@ -62,6 +62,16 @@ declare module 'aws-sdk' { TableNames: string[] }; + declare type ListTagsRequest = { + NextToken?: string, + ResourceArn?: string + }; + + declare type ListTagsResponse = { + NextToken?: string, + Tags: string[] + }; + declare type DeleteTableRequest = { TableName: string, }; diff --git a/src/Provisioner.js b/src/Provisioner.js index d33abf9..a12e6e4 100644 --- a/src/Provisioner.js +++ b/src/Provisioner.js @@ -21,7 +21,7 @@ export default class Provisioner extends ProvisionerConfigurableBase { async getTableNamesAsync(): Promise { // Option 1 - All tables (Default) - //return await this.db.listAllTableNamesAsync(); + // return await this.db.listAllTableNamesAsync(); // Option 2 - Hardcoded list of tables // return ['Table1', 'Table2', 'Table3']; @@ -30,28 +30,34 @@ export default class Provisioner extends ProvisionerConfigurableBase { // return await ...; // Option 4 - Select all tables with a specific tag + if (!process.env.AWS_REGION || !process.env.AWS_ACCOUNT_NUMBER || !process.env.AWS_AUTOSCALE_TAG_NAME) { + throw new Error('Missing environemnt variables to build the AWS ARN'); + } + return await this.db.listAllTableNamesAsync() - .then(list => { - return Promise.all(list.map(name => { - const params = { ResourceArn: `arn:aws:dynamodb:${process.env.AWS_REGION}:${process.env.AWS_ACCOUNT_NUMBER}:table/${name}` }; - - return new Promise((resolve, reject) => { - // Required to throttle the requests (AWS only accepts 10 of these calls per second per account) - setTimeout(() => { - db.listTagsOfResource(params, (err, {Tags}) => { - if (err) return reject(err); - - return resolve({ tableName: name, tags: Tags }); - }); - }, 100) - }); - })) - .then(list => { - return list - .filter(pkg => { return pkg.tags.some(tag => tag.Key === process.env.AWS_AUTOSCALE_TAG_NAME || 'autoscaled' && tag.Value.match(/true/g)); }) - .map(pkg => pkg.tableName); - }) - }) + .then(list => { + return Promise.all(list.map(name => { + const params = { ResourceArn: `arn:aws:dynamodb:${process.env.AWS_REGION}:${process.env.AWS_ACCOUNT_NUMBER}:table/${name}` }; + + return new Promise((resolve, reject) => { + // Required to throttle the requests (AWS only accepts 10 of these calls per second per account) + setTimeout(() => { + this.db.listTagsOfResourceAsync(params) + .then(tags => resolve({ tableName: name, tags })) + .catch(reject); + }, 100); + }); + })); + }) + .then(tableNamesWithTags => { + return tableNamesWithTags + .filter(pkg => { return pkg.tags.some(tag => tag.Key === process.env.AWS_AUTOSCALE_TAG_NAME || 'autoscaled' && tag.Value.match(/true/g)); }) + .map(pkg => pkg.tableName); + }) + .then(tableNames => { + ProvisionerLogging.logIdentifiedTables(tableNames); + return tableNames; + }); } // Gets the json settings which control how the specifed table will be autoscaled diff --git a/src/aws/DynamoDB.js b/src/aws/DynamoDB.js index e46174f..ce20549 100644 --- a/src/aws/DynamoDB.js +++ b/src/aws/DynamoDB.js @@ -11,6 +11,8 @@ import type { UpdateTableResponse, ListTablesRequest, ListTablesResponse, + ListTagsRequest, + ListTagsResponse, } from 'aws-sdk'; export default class DynamoDB { @@ -52,6 +54,21 @@ export default class DynamoDB { } } + async listTagsAsync(params: ?ListTagsRequest): Promise { + let sw = stats.timer('DynamoDB.listTagsAsync').start(); + try { + return await this._db.listTagsOfResource(params).promise(); + } catch (ex) { + warning(JSON.stringify({ + class: 'DynamoDB', + function: 'listTagsAsync' + }, null, json.padding)); + throw ex; + } finally { + sw.end(); + } + } + async listAllTableNamesAsync(): Promise { let tableNames = []; let lastTable; @@ -63,6 +80,17 @@ export default class DynamoDB { return tableNames; } + async listTagsOfResourceAsync(params): Promise { + let tagValues = []; + let nextToken; + do { + let listTagsResponse = await this.listTagsAsync(params); + tagValues = tagValues.concat(listTagsResponse.Tags); + nextToken = listTagsResponse.NextToken; + } while (nextToken); + return tagValues; + } + async describeTableAsync(params: DescribeTableRequest): Promise { let sw = stats.timer('DynamoDB.describeTableAsync').start(); try { diff --git a/src/provisioning/ProvisionerLogging.js b/src/provisioning/ProvisionerLogging.js index 2bb4bc8..872abec 100644 --- a/src/provisioning/ProvisionerLogging.js +++ b/src/provisioning/ProvisionerLogging.js @@ -6,6 +6,10 @@ import type { } from '../flow/FlowTypes'; export default class ConfigLogging { + static logIdentifiedTables(tableNames) { + log('The following tables were identified for autoscaling:', tableNames); + } + static isAdjustmentRequiredLog( adjustmentContext: AdjustmentContext, adjustmentData: AdjustmentData, From 53877961e23b1d73b6c833b10df513dbc7033e8e Mon Sep 17 00:00:00 2001 From: Chris Silivestru Date: Wed, 14 Jun 2017 10:30:39 -0400 Subject: [PATCH 5/7] Make the use of tags optional so that the default is still all tables. --- src/Provisioner.js | 70 ++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/src/Provisioner.js b/src/Provisioner.js index a12e6e4..21b8d18 100644 --- a/src/Provisioner.js +++ b/src/Provisioner.js @@ -20,44 +20,46 @@ export default class Provisioner extends ProvisionerConfigurableBase { // Gets the list of tables which we want to autoscale async getTableNamesAsync(): Promise { - // Option 1 - All tables (Default) - // return await this.db.listAllTableNamesAsync(); + // Option 1: Identify tables by custom tag + if (process.env.DDB_AUTOSCALE_USE_TAGS) { + if (!process.env.AWS_REGION || !process.env.AWS_ACCOUNT_NUMBER || !process.env.AWS_AUTOSCALE_TAG_NAME) { + throw new Error('Missing environemnt variables to build the AWS ARN'); + } + + return await this.db.listAllTableNamesAsync() + .then(list => { + return Promise.all(list.map(name => { + const params = { ResourceArn: `arn:aws:dynamodb:${process.env.AWS_REGION}:${process.env.AWS_ACCOUNT_NUMBER}:table/${name}` }; + + return new Promise((resolve, reject) => { + // Required to throttle the requests (AWS only accepts 10 of these calls per second per account) + setTimeout(() => { + this.db.listTagsOfResourceAsync(params) + .then(tags => resolve({ tableName: name, tags })) + .catch(reject); + }, 100); + }); + })); + }) + .then(tableNamesWithTags => { + return tableNamesWithTags + .filter(pkg => { return pkg.tags.some(tag => tag.Key === process.env.AWS_AUTOSCALE_TAG_NAME || 'autoscaled' && tag.Value.match(/true/g)); }) + .map(pkg => pkg.tableName); + }) + .then(tableNames => { + ProvisionerLogging.logIdentifiedTables(tableNames); + return tableNames; + }); + } + + // Option 2 - All tables (Default) + return await this.db.listAllTableNamesAsync(); - // Option 2 - Hardcoded list of tables + // Option 3 - Hardcoded list of tables // return ['Table1', 'Table2', 'Table3']; - // Option 3 - DynamoDB / S3 configured list of tables + // Option 4 - DynamoDB / S3 configured list of tables // return await ...; - - // Option 4 - Select all tables with a specific tag - if (!process.env.AWS_REGION || !process.env.AWS_ACCOUNT_NUMBER || !process.env.AWS_AUTOSCALE_TAG_NAME) { - throw new Error('Missing environemnt variables to build the AWS ARN'); - } - - return await this.db.listAllTableNamesAsync() - .then(list => { - return Promise.all(list.map(name => { - const params = { ResourceArn: `arn:aws:dynamodb:${process.env.AWS_REGION}:${process.env.AWS_ACCOUNT_NUMBER}:table/${name}` }; - - return new Promise((resolve, reject) => { - // Required to throttle the requests (AWS only accepts 10 of these calls per second per account) - setTimeout(() => { - this.db.listTagsOfResourceAsync(params) - .then(tags => resolve({ tableName: name, tags })) - .catch(reject); - }, 100); - }); - })); - }) - .then(tableNamesWithTags => { - return tableNamesWithTags - .filter(pkg => { return pkg.tags.some(tag => tag.Key === process.env.AWS_AUTOSCALE_TAG_NAME || 'autoscaled' && tag.Value.match(/true/g)); }) - .map(pkg => pkg.tableName); - }) - .then(tableNames => { - ProvisionerLogging.logIdentifiedTables(tableNames); - return tableNames; - }); } // Gets the json settings which control how the specifed table will be autoscaled From 245673ef7c82e2025142651537a7dec896924b55 Mon Sep 17 00:00:00 2001 From: Chris Silivestru Date: Wed, 14 Jun 2017 10:33:21 -0400 Subject: [PATCH 6/7] More consistent environment names. --- src/Provisioner.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Provisioner.js b/src/Provisioner.js index 21b8d18..2846636 100644 --- a/src/Provisioner.js +++ b/src/Provisioner.js @@ -22,7 +22,7 @@ export default class Provisioner extends ProvisionerConfigurableBase { // Option 1: Identify tables by custom tag if (process.env.DDB_AUTOSCALE_USE_TAGS) { - if (!process.env.AWS_REGION || !process.env.AWS_ACCOUNT_NUMBER || !process.env.AWS_AUTOSCALE_TAG_NAME) { + if (!process.env.AWS_REGION || !process.env.AWS_ACCOUNT_NUMBER || !process.env.DDB_AUTOSCALE_TAG_NAME) { throw new Error('Missing environemnt variables to build the AWS ARN'); } @@ -43,7 +43,7 @@ export default class Provisioner extends ProvisionerConfigurableBase { }) .then(tableNamesWithTags => { return tableNamesWithTags - .filter(pkg => { return pkg.tags.some(tag => tag.Key === process.env.AWS_AUTOSCALE_TAG_NAME || 'autoscaled' && tag.Value.match(/true/g)); }) + .filter(pkg => { return pkg.tags.some(tag => tag.Key === process.env.DDB_AUTOSCALE_TAG_NAME || 'autoscaled' && tag.Value.match(/true/g)); }) .map(pkg => pkg.tableName); }) .then(tableNames => { From d0b47d7f3d6dcc7199c4a03ec9241ae8c0fd6251 Mon Sep 17 00:00:00 2001 From: Chris Silivestru Date: Wed, 14 Jun 2017 10:44:22 -0400 Subject: [PATCH 7/7] Make CustomProvisioners part of the default case. If it's there, great! Use it. Otherwise, just stick with the default. --- src/Provisioner.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Provisioner.js b/src/Provisioner.js index 2846636..610b3bf 100644 --- a/src/Provisioner.js +++ b/src/Provisioner.js @@ -66,17 +66,14 @@ export default class Provisioner extends ProvisionerConfigurableBase { // eslint-disable-next-line no-unused-vars getTableConfig(data: TableProvisionedAndConsumedThroughput): ProvisionerConfig { - // Option 1 - Default settings for all tables - // return DefaultProvisioner; + // Option 1 - Default settings for all tables unless included in CustomProvisioners.json + return (CustomProvisioners || {})[data.TableName] || DefaultProvisioner; // Option 2 - Bespoke table specific settings // return data.TableName === 'Table1' ? Climbing : Default; // Option 3 - DynamoDB / S3 sourced table specific settings // return await ...; - - // Option 4 - Get the table specific config based on table name - return CustomProvisioners[data.TableName] || DefaultProvisioner; } isReadCapacityIncrementRequired(data: TableProvisionedAndConsumedThroughput): boolean {