Retry on mongos operation failures

Author: durran

lib/mongo/client.rb

@@ -151,6 +151,10 @@ def hash
     # @option options [ Logger ] :logger A custom logger if desired.
     # @option options [ true, false ] :truncate_logs Whether to truncate the
     #   logs at the default 250 characters.
+    # @option options [ Integer ] :max_read_retries The maximum number of read
+    #   retries on mongos query failures.
+    # @option options [ Float ] :read_retry_interval The interval, in seconds,
+    #   in which reads on a mongos are retried.
     #
     # @since 2.0.0
     def initialize(addresses_or_uri, options = Options::Redacted.new)

lib/mongo/cluster.rb

@@ -25,6 +25,16 @@ class Cluster
     include Event::Subscriber
     include Loggable
 
+    # The default number of mongos read retries.
+    #
+    # @since 2.1.1
+    MAX_READ_RETRIES = 1
+
+    # The default mongos read retry interval, in seconds.
+    #
+    # @since 2.1.1
+    READ_RETRY_INTERVAL = 5
+
     # @return [ Hash ] The options hash.
     attr_reader :options
 
@@ -143,6 +153,32 @@ def elect_primary!(description)
       @topology = topology.elect_primary(description, servers_list)
     end
 
+    # Get the maximum number of times the cluster can retry a read operation on
+    # a mongos.
+    #
+    # @example Get the max read retries.
+    #   cluster.max_read_retries
+    #
+    # @return [ Integer ] The maximum retries.
+    #
+    # @since 2.1.1
+    def max_read_retries
+      options[:max_read_retries] || MAX_READ_RETRIES
+    end
+
+    # Get the interval, in seconds, in which a mongos read operation is
+    # retried.
+    #
+    # @example Get the read retry interval.
+    #   cluster.read_retry_interval
+    #
+    # @return [ Float ] The interval.
+    #
+    # @since 2.1.1
+    def read_retry_interval
+      options[:read_retry_interval] || READ_RETRY_INTERVAL
+    end
+
     # Notify the cluster that a standalone server was discovered so that the
     # topology can be updated accordingly.
     #

lib/mongo/error/operation_failure.rb

@@ -18,6 +18,35 @@ class Error
     # Raised when an operation fails for some reason.
     #
     # @since 2.0.0
-    class OperationFailure < Error; end
+    class OperationFailure < Error
+
+      # These are magic error messages that could indicate a cluster
+      # reconfiguration behind a mongos. We cannot check error codes as they
+      # change between versions, for example 15988 which has 2 completely
+      # different meanings between 2.4 and 3.0.
+      #
+      # @since 2.1.1
+      RETRY_MESSAGES = [
+        'transport error',
+        'socket exception',
+        "can't connect",
+        'no master',
+        'not master',
+        'connect failed',
+        'error querying'
+      ].freeze
+
+      # Can the operation that caused the error be retried?
+      #
+      # @example Is the error retryable?
+      #   error.retryable?
+      #
+      # @return [ true, false ] If the error is retryable.
+      #
+      # @since 2.1.1
+      def retryable?
+        RETRY_MESSAGES.any?{ |m| message.include?(m) }
+      end
+    end
   end
 end

lib/mongo/retryable.rb

@@ -26,28 +26,44 @@ module Retryable
 
     # Execute a read operation with a retry.
     #
+    # @api private
+    #
     # @example Execute the read.
     #   read_with_retry do
     #     ...
     #   end
     #
     # @note This only retries read operations on socket errors.
     #
+    # @param [ Integer ] attempt The retry attempt count - for internal use.
     # @param [ Proc ] block The block to execute.
     #
     # @return [ Result ] The result of the operation.
     #
     # @since 2.1.0
-    def read_with_retry(&block)
+    def read_with_retry(attempt = 0, &block)
       begin
         block.call
       rescue Error::SocketError, Error::SocketTimeoutError
         retry_operation(&block)
+      rescue Error::OperationFailure => e
+        if cluster.sharded? && e.retryable?
+          if attempt < max_read_retries
+            # We don't scan the cluster in this case as Mongos always returns
+            # ready after a ping whether no matter what the state behind it is.
+            sleep(read_retry_interval)
+            read_with_retry(attempt - 1, &block)
+          end
+        else
+          raise e
+        end
       end
     end
 
     # Execute a write operation with a retry.
     #
+    # @api private
+    #
     # @example Execute the write.
     #   write_with_retry do
     #     ...

spec/mongo/retryable_spec.rb

@@ -14,6 +14,14 @@ def initialize(operation, cluster)
         @cluster = cluster
       end
 
+      def max_read_retries
+        cluster.max_read_retries
+      end
+
+      def read_retry_interval
+        cluster.read_retry_interval
+      end
+
       def read
         read_with_retry do
           operation.execute
@@ -81,14 +89,80 @@ def write
 
     context 'when an operation failure occurs' do
 
-      before do
-        expect(operation).to receive(:execute).and_raise(Mongo::Error::OperationFailure).ordered
+      context 'when the cluster is not a mongos' do
+
+        before do
+          expect(operation).to receive(:execute).and_raise(Mongo::Error::OperationFailure).ordered
+          expect(cluster).to receive(:sharded?).and_return(false)
+        end
+
+        it 'raises an exception' do
+          expect {
+            retryable.read
+          }.to raise_error(Mongo::Error::OperationFailure)
+        end
       end
 
-      it 'raises an exception' do
-        expect {
-          retryable.read
-        }.to raise_error(Mongo::Error::OperationFailure)
+      context 'when the cluster is a mongos' do
+
+        context 'when the operation failure is not retryable' do
+
+          let(:error) do
+            Mongo::Error::OperationFailure.new('not authorized')
+          end
+
+          before do
+            expect(operation).to receive(:execute).and_raise(error).ordered
+            expect(cluster).to receive(:sharded?).and_return(true)
+          end
+
+          it 'raises the exception' do
+            expect {
+              retryable.read
+            }.to raise_error(Mongo::Error::OperationFailure)
+          end
+        end
+
+        context 'when the operation failure is retryable' do
+
+          let(:error) do
+            Mongo::Error::OperationFailure.new('no master')
+          end
+
+          context 'when the retry succeeds' do
+
+            before do
+              expect(operation).to receive(:execute).and_raise(error).ordered
+              expect(cluster).to receive(:sharded?).and_return(true)
+              expect(cluster).to receive(:max_read_retries).and_return(1).ordered
+              expect(cluster).to receive(:read_retry_interval).and_return(0.1).ordered
+              expect(operation).to receive(:execute).and_return(true).ordered
+            end
+
+            it 'returns the result' do
+              expect(retryable.read).to be true
+            end
+          end
+
+          context 'when the retry fails once and then succeeds' do
+
+            before do
+              expect(operation).to receive(:execute).and_raise(error).ordered
+              expect(cluster).to receive(:sharded?).and_return(true)
+              expect(cluster).to receive(:max_read_retries).and_return(1).ordered
+              expect(cluster).to receive(:read_retry_interval).and_return(0.1).ordered
+              expect(operation).to receive(:execute).and_raise(error).ordered
+              expect(cluster).to receive(:sharded?).and_return(true)
+              expect(cluster).to receive(:max_read_retries).and_return(1).ordered
+              expect(cluster).to receive(:read_retry_interval).and_return(0.1).ordered
+              expect(operation).to receive(:execute).and_return(true).ordered
+            end
+
+            it 'returns the result' do
+              expect(retryable.read).to be true
+            end
+          end
+        end
       end
     end
   end