From f01740b1cafda7d7c9953598f9591a9b5a6f4b1a Mon Sep 17 00:00:00 2001 From: Matt Bornski Date: Mon, 9 Dec 2013 16:49:28 -0800 Subject: [PATCH] Implement and test BatchPutItem --- lib/dynamoid/adapter.rb | 28 +++++++++++++++----- lib/dynamoid/adapter/aws_sdk.rb | 37 +++++++++++++++++++++++++++ spec/dynamoid/adapter/aws_sdk_spec.rb | 24 +++++++++++++++++ spec/dynamoid/adapter_spec.rb | 7 ++++- 4 files changed, 88 insertions(+), 8 deletions(-) diff --git a/lib/dynamoid/adapter.rb b/lib/dynamoid/adapter.rb index 6f74cd7..efdc66e 100644 --- a/lib/dynamoid/adapter.rb +++ b/lib/dynamoid/adapter.rb @@ -44,18 +44,32 @@ def benchmark(method, *args) # Write an object to the adapter. Partition it to a randomly selected key first if necessary. # # @param [String] table the name of the table to write the object to - # @param [Object] object the object itself + # @param [Array] objects array of objects to insert, can also be a singular object # @param [Hash] options Options that are passed to the put_item call # # @return [Object] the persisted object # # @since 0.2.0 - def write(table, object, options = nil) - if Dynamoid::Config.partitioning? && object[:id] - object[:id] = "#{object[:id]}.#{Random.rand(Dynamoid::Config.partition_size)}" - object[:updated_at] = Time.now.to_f + def write(table, objects, options = nil) + if objects.respond_to?(:each) && !objects.respond_to?(:keys) + if Dynamoid::Config.partitioning? + objects.each do |object| + if object[:id] + object[:id] = "#{object[:id]}.#{Random.rand(Dynamoid::Config.partition_size)}" + object[:updated_at] = Time.now.to_f + end + end + batch_put_item({table => objects}, options) + else + batch_put_item({table => objects}, options) + end + else + if Dynamoid::Config.partitioning? && objects[:id] + objects[:id] = "#{objects[:id]}.#{Random.rand(Dynamoid::Config.partition_size)}" + objects[:updated_at] = Time.now.to_f + end + put_item(table, objects, options) end - put_item(table, object, options) end # Read one or many keys from the selected table. This method intelligently calls batch_get or get on the underlying adapter depending on @@ -138,7 +152,7 @@ def scan(table, query, opts = {}) end end - [:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item].each do |m| + [:batch_get_item, :create_table, :delete_item, :delete_table, :get_item, :list_tables, :put_item, :batch_put_item].each do |m| # Method delegation with benchmark to the underlying adapter. Faster than relying on method_missing. # # @since 0.2.0 diff --git a/lib/dynamoid/adapter/aws_sdk.rb b/lib/dynamoid/adapter/aws_sdk.rb index 0804e4e..5ec22c7 100644 --- a/lib/dynamoid/adapter/aws_sdk.rb +++ b/lib/dynamoid/adapter/aws_sdk.rb @@ -75,6 +75,43 @@ def batch_get_item(table_ids, options = {}) end hash end + + # Persist many items at once from DynamoDB. More efficient than persisting each item individually. + # + # @example Persist {"foo": "bar"} and {"foo": "bear"} to table1, assuming "foo" is primary key (must include key). + # Dynamoid::Adapter::AwsSdk.batch_put_item({'table1' => [{"foo" => "bar"}, {"foo" => "bear"}]}) + # + # @param [Hash] table_objects the hash of tables and objects to persist + # @param [Hash] options to be passed to underlying BatchPut call + # + # @return nil + # + def batch_put_item(table_objects, options = {}) + return nil if table_objects.all?{|k, v| v.empty?} + table_objects.each do |t, objects| + Array(objects).in_groups_of(25, false) do |group| + batch = AWS::DynamoDB::BatchWrite.new(:config => @@connection.config) + batch.put(t, group) + batch.process! + end + end + nil + end + # Persists an item on DynamoDB. + # + # @param [String] table_name the name of the table + # @param [Object] object a hash or Dynamoid object to persist + # + # @since 0.2.0 + def put_item(table_name, object, options = nil) + table = get_table(table_name) + table.items.create( + object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)}, + options || {} + ) + rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e + raise Dynamoid::Errors::ConditionalCheckFailedException + end # Delete many items at once from DynamoDB. More efficient than delete each item individually. # diff --git a/spec/dynamoid/adapter/aws_sdk_spec.rb b/spec/dynamoid/adapter/aws_sdk_spec.rb index c30f59f..986a3ed 100644 --- a/spec/dynamoid/adapter/aws_sdk_spec.rb +++ b/spec/dynamoid/adapter/aws_sdk_spec.rb @@ -204,6 +204,30 @@ def key_partition results[test_table3].should include({:name => 'Josh', :id => '1', :range => 1.0}) results[test_table3].should include({:name => 'Justin', :id => '2', :range => 2.0}) end + + # BatchPutItem + it "performs BatchPutItem with singular keys" do + Dynamoid::Adapter.batch_put_item(test_table1 => [{:id => '1', :name => 'Josh'}], test_table2 => [{:id => '1', :name => 'Justin'}]) + + results = Dynamoid::Adapter.batch_get_item(test_table1 => '1', test_table2 => '1') + results.size.should == 2 + results[test_table1].size.should == 1 + results[test_table2].size.should == 1 + + results[test_table1].should include({:name => 'Josh', :id => '1'}) + results[test_table2].should include({:name => 'Justin', :id => '1'}) + end + + it "performs BatchPutItem with multiple keys" do + Dynamoid::Adapter.batch_put_item(test_table1 => [{:id => '1', :name => 'Josh'}, {:id => '2', :name => 'Justin'}]) + + results = Dynamoid::Adapter.batch_get_item(test_table1 => ['1', '2']) + results.size.should == 1 + results[test_table1].size.should == 2 + + results[test_table1].should include({:name => 'Josh', :id => '1'}) + results[test_table1].should include({:name => 'Justin', :id => '2'}) + end # BatchDeleteItem it "performs BatchDeleteItem with singular keys" do diff --git a/spec/dynamoid/adapter_spec.rb b/spec/dynamoid/adapter_spec.rb index 337a567..2027e62 100644 --- a/spec/dynamoid/adapter_spec.rb +++ b/spec/dynamoid/adapter_spec.rb @@ -28,11 +28,16 @@ def test_table; 'dynamoid_tests_TestTable'; end Dynamoid::Config.partitioning = @previous_value end - it 'writes through the adapter' do + it 'writes through the adapter for one object' do described_class.expects(:put_item).with(test_table, {:id => single_id}, nil).returns(true) described_class.write(test_table, {:id => single_id}) end + it 'writes through the adapter for many objects' do + described_class.expects(:batch_put_item).with({test_table => many_ids.collect { |id| {:id => id} }}, nil).returns(true) + described_class.write(test_table, many_ids.collect { |id| {:id => id} }) + end + it 'reads through the adapter for one ID' do described_class.expects(:get_item).with(test_table, single_id, {}).returns(true) described_class.read(test_table, single_id)