From d8f3774ca7a13a71d7ffb7bd6ce22c0b31296a62 Mon Sep 17 00:00:00 2001 From: Logan Bowers Date: Tue, 18 Jun 2013 21:01:02 -0700 Subject: [PATCH 1/4] V2 client implementation with ~80% tests passing --- Gemfile | 2 +- Gemfile.lock | 4 +- lib/dynamoid/adapter.rb | 4 +- lib/dynamoid/adapter/client_v2.rb | 548 ++++++++++++++++++++++++++++++ spec/spec_helper.rb | 10 +- 5 files changed, 559 insertions(+), 9 deletions(-) create mode 100644 lib/dynamoid/adapter/client_v2.rb diff --git a/Gemfile b/Gemfile index 3804871..e321df6 100644 --- a/Gemfile +++ b/Gemfile @@ -19,6 +19,6 @@ group :development do gem "redcarpet", '1.17.2' gem 'github-markup' gem 'pry' - gem 'fake_dynamo', '~>0.1.3' + gem 'fake_dynamo', '>=0.2' gem "mocha", '0.10.0' end diff --git a/Gemfile.lock b/Gemfile.lock index 9bd6e1c..a6da3c2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,7 +14,7 @@ GEM builder (3.0.4) coderay (1.0.9) diff-lcs (1.2.4) - fake_dynamo (0.1.3) + fake_dynamo (0.2.1) activesupport json sinatra @@ -69,7 +69,7 @@ DEPENDENCIES activemodel aws-sdk bundler - fake_dynamo (~> 0.1.3) + fake_dynamo (>= 0.2) github-markup jeweler mocha (= 0.10.0) diff --git a/lib/dynamoid/adapter.rb b/lib/dynamoid/adapter.rb index 30e3d8f..40c2e1b 100644 --- a/lib/dynamoid/adapter.rb +++ b/lib/dynamoid/adapter.rb @@ -182,7 +182,7 @@ def get_original_id_and_partition id # # @since 0.2.0 def result_for_partition(results, table_name) - table = Dynamoid::Adapter::AwsSdk.get_table(table_name) + table = @adapter.get_table(table_name) if table.range_key range_key_name = table.range_key.name.to_sym @@ -243,7 +243,7 @@ def query(table_name, opts = {}) unless Dynamoid::Config.partitioning? #no paritioning? just pass to the standard query method - Dynamoid::Adapter::AwsSdk.query(table_name, opts) + @adapter.query(table_name, opts) else #get all the hash_values that could be possible ids = id_with_partitions(opts[:hash_value]) diff --git a/lib/dynamoid/adapter/client_v2.rb b/lib/dynamoid/adapter/client_v2.rb new file mode 100644 index 0000000..371157e --- /dev/null +++ b/lib/dynamoid/adapter/client_v2.rb @@ -0,0 +1,548 @@ +# encoding: utf-8 +require 'aws' + +require 'pp' + +module Dynamoid + module Adapter + + + # + # Uses the low-level V2 client API + # + module ClientV2 + extend self + + # Establish the connection to DynamoDB. + # + # @return [AWS::DynamoDB::ClientV2] the raw DynamoDB connection + + def connect! + @client = AWS::DynamoDB::ClientV2.new + end + + # Return the client object. + # + # + # @since 0.2.0 + def client + @client + end + + # Get many items at once from DynamoDB. More efficient than getting each item individually. + # + # @example Retrieve IDs 1 and 2 from the table testtable + # Dynamoid::Adapter::AwsSdk.batch_get_item({'table1' => ['1', '2']}, :consistent_read => true) + # + # @param [Hash] table_ids the hash of tables and IDs to retrieve + # @param [Hash] options to be passed to underlying BatchGet call + # + # @return [Hash] a hash where keys are the table names and the values are the retrieved items + # + # @since 0.2.0 + def batch_get_item(table_ids, options = {}) + request_items = {} + table_ids.each do |t, ids| + next if ids.empty? + hk = describe_table(t).hash_key.to_s + request_items[t] = { + keys: ids.map { |id| { hk => attribute_value(id) } } + } + end + + raise "Unhandled options remaining" unless options.empty? + results = client.batch_get_item( + request_items: request_items + ) + + results.data + ret = {} + results.data[:responses].each do |table, rows| + ret[table] = rows.collect { |r| result_item_to_hash(r) } + end + ret + rescue + STDERR.puts("batch_get_item FAILED") + PP.pp(request_items) + raise + end + + # Delete many items at once from DynamoDB. More efficient than delete each item individually. + # + # @example Delete IDs 1 and 2 from the table testtable + # Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => ['1', '2']) + #or + # Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => [['hk1', 'rk2'], ['hk1', 'rk2']]])) + # + # @param [Hash] options the hash of tables and IDs to delete + # + # @return nil + # + def batch_delete_item(options) + raise "TODO" + end + + # Create a table on DynamoDB. This usually takes a long time to complete. + # + # @param [String] table_name the name of the table to create + # @param [Symbol] key the table's primary key (defaults to :id) + # @param [Hash] options provide a range_key here if you want one for the table + # + # @since 0.2.0 + def create_table(table_name, key = :id, options = {}) + Dynamoid.logger.info "Creating #{table_name} table. This could take a while." + read_capacity = options.delete(:read_capacity) || Dynamoid::Config.read_capacity + write_capacity = options.delete(:write_capacity) || Dynamoid::Config.write_capacity + range_key = options.delete(:range_key) + + key_schema = [ + { attribute_name: key.to_s, key_type: HASH_KEY } + ] + key_schema << { + attribute_name: range_key.keys.first.to_s, key_type: RANGE_KEY + } if(range_key) + + attribute_definitions = [ + { attribute_name: key.to_s, attribute_type: 'S' } + ] + attribute_definitions << { + attribute_name: range_key.keys.first.to_s, attribute_type: api_type(range_key.values.first) + } if(range_key) + + client.create_table(table_name: table_name, + provisioned_throughput: { + read_capacity_units: read_capacity, + write_capacity_units: write_capacity + }, + key_schema: key_schema, + attribute_definitions: attribute_definitions + ) + + [:id, :table_name].each { |k| options.delete(k) } + raise "Not empty options: #{options.keys.join(',')}" unless options.empty? + + rescue AWS::DynamoDB::Errors::ResourceInUseException => e + #STDERR.puts("SWALLOWED AN EXCEPTION creating table #{table_name}") + rescue + STDERR.puts("create_table FAILED") + PP.pp(key_schema) + raise + end + + # Removes an item from DynamoDB. + # + # @param [String] table_name the name of the table + # @param [String] key the hash key of the item to delete + # @param [Number] range_key the range key of the item to delete, required if the table has a composite key + # + # @since 0.2.0 + def delete_item(table_name, key, options = nil) + table = describe_table(table_name) + client.delete_item(table_name: table_name, key: key_stanza(table, key, options && options[:range_key])) + + rescue + STDERR.puts("delete_item FAILED on #{table_name}, #{key}, #{options}") + PP.pp(table.schema) + raise + end + + # Deletes an entire table from DynamoDB. Only 10 tables can be in the deleting state at once, + # so if you have more this method may raise an exception. + # + # @param [String] table_name the name of the table to destroy + # + # @since 0.2.0 + def delete_table(table_name) + client.delete_table(table_name: table_name) + end + + # @todo Add a DescribeTable method. + + # Fetches an item from DynamoDB. + # + # @param [String] table_name the name of the table + # @param [String] key the hash key of the item to find + # @param [Number] range_key the range key of the item to find, required if the table has a composite key + # + # @return [Hash] a hash representing the raw item in DynamoDB + # + # @since 0.2.0 + def get_item(table_name, key, options = {}) + table = describe_table(table_name) + range_key = options.delete(:range_key) + + result = {} + + item = client.get_item(table_name: table_name, + key: key_stanza(table, key, range_key) + )[:item] + item ? result_item_to_hash(item) : nil + rescue + STDERR.puts("get_item FAILED ON #{key}, #{options}") + STDERR.puts("----") + PP.pp(item) + raise + end + + # + # @return new attributes for the record + # + def update_item(table_name, key, options = {}) + range_key = options.delete(:range_key) + conditions = options.delete(:conditions) + table = describe_table(table_name) + + yield(iu = ItemUpdater.new(table, key, range_key)) + + raise "non-empty options: #{options}" unless options.empty? + + result = client.update_item(table_name: table_name, + key: key_stanza(table, key, range_key), + attribute_updates: iu.to_h, + expected: expected_stanza(conditions), + return_values: "ALL_NEW" + ) + + result_item_to_hash(result[:attributes]) + rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException + raise Dynamoid::Errors::ConditionalCheckFailedException + end + + # List all tables on DynamoDB. + # + # @since 0.2.0 + def list_tables + client.list_tables[:table_names] + end + + # Persists an item on DynamoDB. + # + # @param [String] table_name the name of the table + # @param [Object] object a hash or Dynamoid object to persist + # + # @since 0.2.0 + def put_item(table_name, object, options = nil) + item = {} + + object.each do |k, v| + next if v.nil? || (v.respond_to?(:empty?) && v.empty?) + item[k.to_s] = attribute_value(v) + end + + result = client.put_item(table_name: table_name, + item: item, + expected: expected_stanza(options) + ) + #STDERR.puts("DATA: #{result.data}") + rescue + STDERR.puts("put_item FAILED ON") + PP.pp(object) + STDERR.puts('--- options:') + PP.pp(options) + STDERR.puts('---- item:') + PP.pp(item) + STDERR.puts('--- expected:') + PP.pp(expected_stanza(options)) + STDERR.puts("---") + STDERR.puts(table_name) + STDERR.puts("---") + PP.pp describe_table(table_name) + raise + end + + # Query the DynamoDB table. This employs DynamoDB's indexes so is generally faster than scanning, but is + # only really useful for range queries, since it can only find by one hash key at once. Only provide + # one range key to the hash. + # + # @param [String] table_name the name of the table + # @param [Hash] opts the options to query the table with + # @option opts [String] :hash_value the value of the hash key to find + # @option opts [Range] :range_value find the range key within this range + # @option opts [Number] :range_greater_than find range keys greater than this + # @option opts [Number] :range_less_than find range keys less than this + # @option opts [Number] :range_gte find range keys greater than or equal to this + # @option opts [Number] :range_lte find range keys less than or equal to this + # + # @return [Enumerator] an iterator of all matching items + # + # @since 0.2.0 + def query(table_name, opts = {}) + raise "TODO" + end + + # Scan the DynamoDB table. This is usually a very slow operation as it naively filters all data on + # the DynamoDB servers. + # + # @param [String] table_name the name of the table + # @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan + # + # @return [Enumerator] an iterator of all matching items + # + # @since 0.2.0 + def scan(table_name, scan_hash, select_opts) + limit = select_opts.delete(:limit) + + request = { table_name: table_name } + request[:limit] = limit if limit + request[:scan_filter] = scan_hash.reduce({}) do |memo, kvp| + memo[kvp[0].to_s] = { + attribute_value_list: [attribute_value(kvp[1])], + comparison_operator: "EQ" + } + memo + end if(scan_hash && !scan_hash.empty?) + + results = client.scan(request) + + raise "non-empty select_opts" if(select_opts && !select_opts.empty?) + + Enumerator.new do |y| + results.data[:member].each { |row| y << result_item_to_hash(row) } + end + rescue + STDERR.puts("FAILED scan") + PP.pp(scan_hash) + STDERR.puts("---") + PP.pp(select_opts) + STDERR.puts("---") + PP.pp(request) + raise + end + + + # + # Truncates all records in the given table + # + def truncate(table_name) + table = describe_table(table_name) + hk = table.hash_key + rk = table.range_key + + scan(table_name, {}, {}).each do |attributes| + opts = {range_key: attributes[rk.to_sym] } if rk + delete_item(table_name, attributes[hk], opts) + end + end + + # + # Legacy method that exposes a DynamoDB v1-list table object + # + def get_table(table_name) + LegacyTable.new(describe_table(table_name)) + end + + protected + + STRING_TYPE = "S".freeze + STRING_SET = "SS".freeze + NUM_TYPE = "N".freeze + + # + # Given a value and an options typedef, returns an AttributeValue hash + # + # @param value The value to convert to an AttributeValue hash + # @param [String] type The target api_type (e.g. "N", "SS") for value. If not supplied, + # the type will be inferred from the Ruby type + # + def attribute_value(value, type = nil) + if(type) + value = value.to_s + else + case(value) + when String then + type = STRING_TYPE + when Enumerable then + type = STRING_SET + value = value.to_a + when Numeric then + type = NUM_TYPE + value = value.to_s + else raise "Not sure how to infer type for #{value}" + end + end + { type => value } + end + + #Converts from symbol to the API string for the given data type + # E.g. :number -> 'N' + def api_type(type) + case(type) + when :string then STRING_TYPE + when :number then NUM_TYPE + when :datetime then NUM_TYPE + else raise "Unknown type: #{type}" + end + end + + def load_value(value, type) + case(type) + when :s then value + when :n then value.to_f + when :ss then Set.new(value) + else raise "Not sure how to load type #{type} for #{value}" + end + end + + # + # The key hash passed on get_item, put_item, delete_item, update_item, etc + # + def key_stanza(table, hash_key, range_key = nil) + key = { table.hash_key.to_s => attribute_value(hash_key.to_s, STRING_TYPE) } + key[table.range_key.to_s] = { table.range_type => range_key.to_s } if range_key + key + end + + # + # @param [Hash] conditions Condidtions to enforce on operation (e.g. { :if => { :count => 5 }, :unless_exists => ['id']}) + # @return an Expected stanza for the given conditions hash + # + def expected_stanza(conditions = nil) + expected = Hash.new { |h,k| h[k] = {} } + return expected unless conditions + + conditions[:unless_exists].try(:each) do |col| + expected[col.to_s][:exists] = false + end + conditions[:if].try(:each) do |col,val| + expected[col.to_s][:value] = attribute_value(val) + end + + expected + end + + HASH_KEY = "HASH".freeze + RANGE_KEY = "RANGE".freeze + def describe_table(table_name, reload = false) + table_def = client.describe_table(table_name: table_name).data + Table.new(table_def) + end + + # + # Converts a hash returned by get_item, scan, etc. into a key-value hash + # + def result_item_to_hash(item) + {}.tap do |r| + item.each { |k,v| r[k.to_sym] = load_value(v.values.first, v.keys.first) } + end + end + + # + # Represents a table. Exposes data from the "DescribeTable" API call, and also + # provides methods for coercing values to the proper types based on the table's schema data + # + class Table + attr_reader :schema + + # + # @param [Hash] schema Data returns from a "DescribeTable" call + # + def initialize(schema) + @schema = schema[:table] + end + + def range_key + @range_key ||= schema[:key_schema].find { |d| d[:key_type] == RANGE_KEY }.try(:fetch,:attribute_name) + end + + def range_type + range_type ||= schema[:attribute_definitions].find { |d| + d[:attribute_name] == range_key + }.try(:fetch,:attribute_type, nil) + end + + def hash_key + schema[:key_schema].find { |d| d[:key_type] == HASH_KEY }.try(:fetch,:attribute_name).to_sym + end + + # + # Returns the API type (e.g. "N", "SS") for the given column, if the schema defines it, + # nil otherwise + # + def col_type(col) + col = col.to_s + col_def = schema[:attribute_definitions].find { |d| d[:attribute_name] == col.to_s } + col_def && col_def[:attribute_type] + end + end + + class LegacyTable + def initialize(table) + @table = table + end + + def range_key + rk = @table.range_key + rk && Column.new(@table.range_key) + end + + class Column + attr_reader :name + + def initialize(name) + @name = name + end + end + end + + # + # Mimics behavior of the yielded object on DynamoDB's update_item API (high level). + # + class ItemUpdater + attr_reader :table, :key, :range_key + + def initialize(table, key, range_key = nil) + @table = table; @key = key, @range_key = range_key + @additions = {} + @deletions = {} + end + + # + # Adds the given values to the values already stored in the corresponding columns. + # The column must contain a Set or a number. + # + # @param [Hash] vals keys of the hash are the columns to update, vals are the values to + # add. values must be a Set, Array, or Numeric + # + def add(values) + @additions.merge!(values) + end + + # + # Removes values from the sets of the given columns + # + # @param [Hash] values keys of the hash are the columns, values are Arrays/Sets of items + # to remove + # + def delete(values) + @deletions.merge!(values) + end + + # + # Returns an AttributeUpdates hash suitable for passing to the V2 Client API + # + def to_h + ret = {} + + @additions.each do |k,v| + ret[k.to_s] = { + action: ADD, + value: ClientV2.send(:attribute_value, v) + } + end + @deletions.each do |k,v| + ret[k.to_s] = { + action: DELETE, + value: ClientV2.send(:attribute_value, v) + } + end + + ret + end + + ADD = "ADD".freeze + DELETE = "DELETE".freeze + PUT = "PUT".freeze + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2979684..0c2c770 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -15,13 +15,13 @@ AWS.config({ :access_key_id => ENV['ACCESS_KEY'], :secret_access_key => ENV['SECRET_KEY'], - :dynamo_db_endpoint => 'localhost', + :dynamo_db_endpoint => '127.0.0.1', :dynamo_db_port => '4567', :use_ssl => false }) Dynamoid.configure do |config| - config.adapter = 'aws_sdk' + config.adapter = 'client_v2' config.namespace = 'dynamoid_tests' config.warn_on_scan = false end @@ -41,8 +41,10 @@ config.before(:each) do Dynamoid::Adapter.list_tables.each do |table| if table =~ /^#{Dynamoid::Config.namespace}/ - table = Dynamoid::Adapter.get_table(table) - table.items.each {|i| i.delete} + + Dynamoid::Adapter.truncate(table) + # table = Dynamoid::Adapter.get_table(table) + # table.items.each {|i| i.delete} end end end From 33763d2b094900d4a8192d248eba1de2ac5276bb Mon Sep 17 00:00:00 2001 From: Logan Bowers Date: Sat, 31 Aug 2013 19:54:25 -0700 Subject: [PATCH 2/4] More, query-related specs passing --- Gemfile.lock | 91 +++++++++++++++++-------- lib/dynamoid/adapter/client_v2.rb | 106 ++++++++++++++++++++++++++++-- 2 files changed, 162 insertions(+), 35 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a6da3c2..d42cbeb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,38 +1,71 @@ GEM remote: https://rubygems.org/ specs: - activemodel (3.2.13) - activesupport (= 3.2.13) - builder (~> 3.0.0) - activesupport (3.2.13) - i18n (= 0.6.1) - multi_json (~> 1.0) - aws-sdk (1.10.0) + activemodel (4.0.0) + activesupport (= 4.0.0) + builder (~> 3.1.0) + activesupport (4.0.0) + i18n (~> 0.6, >= 0.6.4) + minitest (~> 4.2) + multi_json (~> 1.3) + thread_safe (~> 0.1) + tzinfo (~> 0.3.37) + addressable (2.3.5) + atomic (1.1.13) + aws-sdk (1.15.0) json (~> 1.4) - nokogiri (>= 1.4.4) + nokogiri (< 1.6.0) uuidtools (~> 2.1) - builder (3.0.4) + builder (3.1.4) coderay (1.0.9) diff-lcs (1.2.4) - fake_dynamo (0.2.1) + fake_dynamo (0.2.4) activesupport json sinatra - git (1.2.5) + faraday (0.8.8) + multipart-post (~> 1.2.0) + git (1.2.6) github-markup (0.7.5) - i18n (0.6.1) - jeweler (1.8.4) + github_api (0.10.1) + addressable + faraday (~> 0.8.1) + hashie (>= 1.2) + multi_json (~> 1.4) + nokogiri (~> 1.5.2) + oauth2 + hashie (2.0.5) + highline (1.6.19) + httpauth (0.2.0) + i18n (0.6.5) + jeweler (1.8.7) + builder bundler (~> 1.0) git (>= 1.2.5) + github_api (= 0.10.1) + highline (>= 1.6.15) + nokogiri (= 1.5.10) rake rdoc json (1.8.0) + jwt (0.1.8) + multi_json (>= 1.5) metaclass (0.0.1) - method_source (0.8.1) + method_source (0.8.2) + minitest (4.7.5) mocha (0.10.0) metaclass (~> 0.0.1) - multi_json (1.7.3) - nokogiri (1.5.9) + multi_json (1.7.9) + multi_xml (0.5.5) + multipart-post (1.2.0) + nokogiri (1.5.10) + oauth2 (0.9.2) + faraday (~> 0.8) + httpauth (~> 0.2) + jwt (~> 0.1.4) + multi_json (~> 1.0) + multi_xml (~> 0.5) + rack (~> 1.2) pry (0.9.12.2) coderay (~> 1.0.5) method_source (~> 0.8) @@ -40,27 +73,29 @@ GEM rack (1.5.2) rack-protection (1.5.0) rack - rake (10.0.4) + rake (10.1.0) rdoc (4.0.1) json (~> 1.4) redcarpet (1.17.2) - rspec (2.13.0) - rspec-core (~> 2.13.0) - rspec-expectations (~> 2.13.0) - rspec-mocks (~> 2.13.0) - rspec-core (2.13.1) - rspec-expectations (2.13.0) + rspec (2.14.1) + rspec-core (~> 2.14.0) + rspec-expectations (~> 2.14.0) + rspec-mocks (~> 2.14.0) + rspec-core (2.14.5) + rspec-expectations (2.14.2) diff-lcs (>= 1.1.3, < 2.0) - rspec-mocks (2.13.1) - sinatra (1.4.2) - rack (~> 1.5, >= 1.5.2) + rspec-mocks (2.14.3) + sinatra (1.4.3) + rack (~> 1.4) rack-protection (~> 1.4) tilt (~> 1.3, >= 1.3.4) - slop (3.4.5) + slop (3.4.6) + thread_safe (0.1.2) + atomic tilt (1.4.1) tzinfo (0.3.37) uuidtools (2.1.4) - yard (0.8.6.1) + yard (0.8.7) PLATFORMS ruby diff --git a/lib/dynamoid/adapter/client_v2.rb b/lib/dynamoid/adapter/client_v2.rb index 371157e..cac11dd 100644 --- a/lib/dynamoid/adapter/client_v2.rb +++ b/lib/dynamoid/adapter/client_v2.rb @@ -18,7 +18,7 @@ module ClientV2 # @return [AWS::DynamoDB::ClientV2] the raw DynamoDB connection def connect! - @client = AWS::DynamoDB::ClientV2.new + @client = AWS::DynamoDB::Client.new(:api_version => '2012-08-10') end # Return the client object. @@ -44,9 +44,22 @@ def batch_get_item(table_ids, options = {}) request_items = {} table_ids.each do |t, ids| next if ids.empty? - hk = describe_table(t).hash_key.to_s + tbl = describe_table(t) + hk = tbl.hash_key.to_s + rng = tbl.range_key.try :to_s + + keys = if(rng) + ids.map do |h,r| + { hk => attribute_value(h), rng => attribute_value(r) } + end + else + ids.map do |id| + { hk => attribute_value(id) } + end + end + request_items[t] = { - keys: ids.map { |id| { hk => attribute_value(id) } } + keys: keys } end @@ -56,7 +69,7 @@ def batch_get_item(table_ids, options = {}) ) results.data - ret = {} + ret = Hash.new([].freeze) #Default for tables where no rows are returned results.data[:responses].each do |table, rows| ret[table] = rows.collect { |r| result_item_to_hash(r) } end @@ -234,6 +247,8 @@ def put_item(table_name, object, options = nil) expected: expected_stanza(options) ) #STDERR.puts("DATA: #{result.data}") + rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e + raise Errors::ConditionalCheckFailedException rescue STDERR.puts("put_item FAILED ON") PP.pp(object) @@ -267,9 +282,72 @@ def put_item(table_name, object, options = nil) # # @since 0.2.0 def query(table_name, opts = {}) - raise "TODO" + STDERR.puts("GOT:") + PP.pp opts + table = describe_table(table_name) + hk = table.hash_key.to_s + rng = table.range_key.to_s + q = opts.slice(:consistent_read, :scan_index_forward, :limit) + + opts.delete(:consistent_read) + opts.delete(:scan_index_forward) + opts.delete(:limit) + opts.delete(:next_token).tap do |token| + break unless token + q[:exclusive_start_key] = { + hk => token[:hash_key_element], + rng => token[:range_key_element] + } + end + + key_conditions = { + hk => { + comparison_operator: EQ, + attribute_value_list: [ + { STRING_TYPE => opts.delete(:hash_value).to_s.freeze } + ] + } + } + opts.each_pair do |k, v| + next unless(op = RANGE_MAP[k]) + key_conditions[rng] = { + comparison_operator: op, + attribute_value_list: [ + { NUM_TYPE => opts.delete(k).to_s.freeze } + ] + } + end + + q[:table_name] = table_name + q[:key_conditions] = key_conditions + + STDERR.puts("Q: ") + PP.pp(q) + + STDERR.puts("OPTS: ") + PP.pp(opts) + raise "MOAR STUFF" unless opts.empty? + Enumerator.new { |y| + result = client.query(q) + STDERR.puts("RESULT: ") + PP.pp(result) + result.member.each { |r| + y << result_item_to_hash(r) + } + } end + EQ = "EQ".freeze + ID = "id".freeze + + RANGE_MAP = { + range_greater_than: 'GT', + range_less_than: 'LT', + range_gte: 'GE', + range_lte: 'LE', + range_begins_with: 'BEGINS_WITH' + } + # Scan the DynamoDB table. This is usually a very slow operation as it naively filters all data on # the DynamoDB servers. # @@ -378,7 +456,7 @@ def load_value(value, type) case(type) when :s then value when :n then value.to_f - when :ss then Set.new(value) + when :ss then Set.new(value.to_a) else raise "Not sure how to load type #{type} for #{value}" end end @@ -494,6 +572,7 @@ def initialize(table, key, range_key = nil) @table = table; @key = key, @range_key = range_key @additions = {} @deletions = {} + @updates = {} end # @@ -516,6 +595,13 @@ def add(values) def delete(values) @deletions.merge!(values) end + + # + # Replaces the values of one or more attributes + # + def set(values) + @updates.merge!(values) + end # # Returns an AttributeUpdates hash suitable for passing to the V2 Client API @@ -535,7 +621,13 @@ def to_h value: ClientV2.send(:attribute_value, v) } end - + @updates.each do |k,v| + ret[k.to_s] = { + action: PUT, + value: ClientV2.send(:attribute_value, v) + } + end + ret end From debeab3713132371ac10b5145a89c9c1a0c54f6d Mon Sep 17 00:00:00 2001 From: Logan Bowers Date: Sat, 31 Aug 2013 21:35:51 -0700 Subject: [PATCH 3/4] All tests pass using the ClientV2 API --- lib/dynamoid/adapter/client_v2.rb | 60 +++++++++++++++++++------------ lib/dynamoid/document.rb | 2 +- spec/dynamoid/document_spec.rb | 2 +- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/lib/dynamoid/adapter/client_v2.rb b/lib/dynamoid/adapter/client_v2.rb index cac11dd..17faaf7 100644 --- a/lib/dynamoid/adapter/client_v2.rb +++ b/lib/dynamoid/adapter/client_v2.rb @@ -5,14 +5,12 @@ module Dynamoid module Adapter - + module ClientV2; end # - # Uses the low-level V2 client API + # Uses the low-level V2 client API. # - module ClientV2 - extend self - + class < Date: Sat, 31 Aug 2013 21:54:48 -0700 Subject: [PATCH 4/4] Cache table schema data --- lib/dynamoid/adapter/client_v2.rb | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/dynamoid/adapter/client_v2.rb b/lib/dynamoid/adapter/client_v2.rb index 17faaf7..9a6df9f 100644 --- a/lib/dynamoid/adapter/client_v2.rb +++ b/lib/dynamoid/adapter/client_v2.rb @@ -11,12 +11,14 @@ module ClientV2; end # Uses the low-level V2 client API. # class < '2012-08-10') + @client = AWS::DynamoDB::Client.new(:api_version => '2012-08-10') + @table_cache = {} end # Return the client object. @@ -174,6 +176,7 @@ def delete_item(table_name, key, options = nil) # @since 0.2.0 def delete_table(table_name) client.delete_table(table_name: table_name) + table_cache.clear end # @todo Add a DescribeTable method. @@ -418,7 +421,7 @@ def get_table(table_name) end def count(table_name) - describe_table(table_name).item_count + describe_table(table_name, true).item_count end protected @@ -502,9 +505,14 @@ def expected_stanza(conditions = nil) HASH_KEY = "HASH".freeze RANGE_KEY = "RANGE".freeze + + # + # New, semi-arbitrary API to get data on the table + # def describe_table(table_name, reload = false) - table_def = client.describe_table(table_name: table_name).data - Table.new(table_def) + (!reload && table_cache[table_name]) || begin + table_cache[table_name] = Table.new(client.describe_table(table_name: table_name).data) + end end #