From c42089663b6360a73d68ca8e983096897ec42003 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Wed, 7 Aug 2019 11:34:11 -0400 Subject: [PATCH 1/2] add support for PAY_PER_REQUEST BillingMode --- dynamorm/exceptions.py | 4 +++ dynamorm/table.py | 75 +++++++++++++++++++++++++++++------------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/dynamorm/exceptions.py b/dynamorm/exceptions.py index b4917e0..160f0e1 100644 --- a/dynamorm/exceptions.py +++ b/dynamorm/exceptions.py @@ -41,6 +41,10 @@ class MissingTableAttribute(DynamoTableException): """A required attribute is missing""" +class InvalidTableAttribute(DynamoTableException): + """An attribute has an invalid value""" + + class InvalidSchemaField(DynamoTableException): """A field provided does not exist in the schema""" diff --git a/dynamorm/table.py b/dynamorm/table.py index 7f6cc4a..3f48009 100644 --- a/dynamorm/table.py +++ b/dynamorm/table.py @@ -4,25 +4,27 @@ The attributes you define on your inner ``Table`` class map to underlying boto data structures. This mapping is expressed through the following data model: -========= ======== ==== =========== -Attribute Required Type Description -========= ======== ==== =========== -name True str The name of the table, as stored in Dynamo. +========= ======== ==== =========== +Attribute Required Type Description +========= ======== ==== =========== +name True str The name of the table, as stored in Dynamo. -hash_key True str The name of the field to use as the hash key. +hash_key True str The name of the field to use as the hash key. It must exist in the schema. -range_key False str The name of the field to use as the range_key, if one is used. +range_key False str The name of the field to use as the range_key, if one is used. It must exist in the schema. -read True int The provisioned read throughput. +read Cond int The provisioned read throughput. Required for 'PROVISIONED' billing_mode (default). -write True int The provisioned write throughput. +write Cond int The provisioned write throughput. Required for 'PROVISIONED' billing_mode (default). -stream False str The stream view type, either None or one of: - 'NEW_IMAGE'|'OLD_IMAGE'|'NEW_AND_OLD_IMAGES'|'KEYS_ONLY' +billing_mode True str The billing mode. One of: 'PROVISIONED'|'PAY_PER_REQUEST' -========= ======== ==== =========== +stream False str The stream view type, either None or one of: + 'NEW_IMAGE'|'OLD_IMAGE'|'NEW_AND_OLD_IMAGES'|'KEYS_ONLY' + +========= ======== ==== =========== Indexes @@ -68,6 +70,7 @@ class DynamoCommon3(object): range_key = None read = None write = None + billing_mode = 'PROVISIONED' def __init__(self): for attr in self.REQUIRED_ATTRS: @@ -96,6 +99,9 @@ def as_schema(name, key_type): @property def provisioned_throughput(self): """Return an appropriate ProvisionedThroughput, based on our attributes""" + if self.billing_mode != 'PROVISIONED': + return None + return { 'ReadCapacityUnits': self.read, 'WriteCapacityUnits': self.write @@ -119,6 +125,7 @@ def lookup_by_type(cls, index_type): def __init__(self, table, schema): self.table = table self.schema = schema + self.billing_mode = table.billing_mode super(DynamoIndex3, self).__init__() @@ -163,7 +170,8 @@ class DynamoGlobalIndex3(DynamoIndex3): @property def index_args(self): args = super(DynamoGlobalIndex3, self).index_args - args['ProvisionedThroughput'] = self.provisioned_throughput + if self.billing_mode == 'PROVISIONED': + args['ProvisionedThroughput'] = self.provisioned_throughput return args @@ -340,21 +348,28 @@ def create_table(self, wait=True): :param bool wait: If set to True, the default, this call will block until the table is created """ - if not self.read or not self.write: - raise MissingTableAttribute("The read/write attributes are required to create a table") + if self.billing_mode not in ('PROVISIONED', 'PAY_PER_REQUEST'): + raise InvalidTableAttribute("valid values for billing_mode are: PROVISIONED|PAY_PER_REQUEST") + + if self.billing_mode == 'PROVISIONED' and (not self.read or not self.write): + raise MissingTableAttribute("The read/write attributes are required to create " + "a table when billing_mode is 'PROVISIONED'") - index_args = collections.defaultdict(list) + extra_args = collections.defaultdict(list) for index in six.itervalues(self.indexes): - index_args[index.ARG_KEY].append(index.index_args) + extra_args[index.ARG_KEY].append(index.index_args) + + if self.billing_mode == 'PROVISIONED': + extra_args['ProvisionedThroughput'] = self.provisioned_throughput log.info("Creating table %s", self.name) table = self.resource.create_table( TableName=self.name, KeySchema=self.key_schema, AttributeDefinitions=self.attribute_definitions, - ProvisionedThroughput=self.provisioned_throughput, StreamSpecification=self.stream_specification, - **index_args + BillingMode=self.billing_mode, + **extra_args ) if wait: log.info("Waiting for table creation...") @@ -430,8 +445,19 @@ def do_update(**kwargs): wait_for_active() + billing_args = {} + + # check if we're going to change our billing mode + current_billing_mode = table.billing_mode_summary['BillingMode'] + if self.billing_mode != current_billing_mode: + log.info("Updating billing mode on table %s (%s -> %s)", + self.name, + current_billing_mode, + self.billing_mode) + billing_args['BillingMode'] = self.billing_mode + # check if we're going to change our capacity - if (self.read and self.write) and \ + if (self.billing_mode == 'PROVISIONED' and self.read and self.write) and \ (self.read != table.provisioned_throughput['ReadCapacityUnits'] or self.write != table.provisioned_throughput['WriteCapacityUnits']): @@ -443,7 +469,10 @@ def do_update(**kwargs): if k.endswith('Units') ), self.provisioned_throughput) - do_update(ProvisionedThroughput=self.provisioned_throughput) + billing_args['ProvisionedThroughput'] = self.provisioned_throughput + + if billing_args: + do_update(**billing_args) return self.update_table() # check if we're going to modify the stream @@ -475,7 +504,9 @@ def do_update(**kwargs): for index in six.itervalues(self.indexes): if index.name in existing_indexes: current_capacity = existing_indexes[index.name]['ProvisionedThroughput'] - if (index.read and index.write) and \ + update_args = {} + + if (index.billing_mode == 'PROVISIONED' and index.read and index.write) and \ (index.read != current_capacity['ReadCapacityUnits'] or index.write != current_capacity['WriteCapacityUnits']): @@ -484,7 +515,7 @@ def do_update(**kwargs): do_update(GlobalSecondaryIndexUpdates=[{ 'Update': { - 'IndexName': index['IndexName'], + 'IndexName': index.name, 'ProvisionedThroughput': index.provisioned_throughput } }]) From 723f0462720499185879d47084ab3db8e78a65a6 Mon Sep 17 00:00:00 2001 From: Matthew Williams Date: Wed, 14 Aug 2019 02:36:08 -0400 Subject: [PATCH 2/2] bump version; update CHANGELOG; bump boto3 min version to 1.9.54 --- CHANGELOG.rst | 5 +++++ setup.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf46ad3..5629a8a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,8 @@ +0.9.5 - 2019-08-14 +################## + +* Add support for PAY_PER_REQUEST billing mode. + 0.9.3 - 2019.04.30 ################## diff --git a/setup.py b/setup.py index 90e5cd8..e01b72b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='dynamorm', - version='0.9.3', + version='0.9.5', description='DynamORM is a Python object & relation mapping library for Amazon\'s DynamoDB service.', long_description=long_description, author='Evan Borgstrom', @@ -14,7 +14,7 @@ license='Apache License Version 2.0', install_requires=[ 'blinker>=1.4,<2.0', - 'boto3>=1.3,<2.0', + 'boto3>=1.9.54,<2.0', 'six', ], extras_require={