[4.4] Backport new timeout config options [ADR 006] (#746)

* Backport new connection timeout config options
  Backport of #745

Author: robsdedude

docs/source/api.rst

@@ -160,6 +160,8 @@ Driver Configuration
 Additional configuration can be provided via the :class:`neo4j.Driver` constructor.
 
 
++ :ref:`session-connection-timeout-ref`
++ :ref:`update-routing-table-timeout-ref`
 + :ref:`connection-acquisition-timeout-ref`
 + :ref:`connection-timeout-ref`
 + :ref:`encrypted-ref`
@@ -172,12 +174,59 @@ Additional configuration can be provided via the :class:`neo4j.Driver` construct
 + :ref:`user-agent-ref`
 
 
+.. _session-connection-timeout-ref:
+
+``session_connection_timeout``
+------------------------------
+The maximum amount of time in seconds the session will wait when trying to
+establish a usable read/write connection to the remote host.
+This encompasses *everything* that needs to happen for this, including,
+if necessary, updating the routing table, fetching a connection from the pool,
+and, if necessary fully establishing a new connection with the reader/writer.
+
+Since this process may involve updating the routing table, acquiring a
+connection from the pool, or establishing a new connection, it should be chosen
+larger than :ref:`update-routing-table-timeout-ref`,
+:ref:`connection-acquisition-timeout-ref`, and :ref:`connection-timeout-ref`.
+
+:Type: ``float``
+:Default: ``float("inf")``
+
+.. versionadded:: 4.4.5
+
+
+.. _update-routing-table-timeout-ref:
+
+``update_routing_table_timeout``
+--------------------------------
+The maximum amount of time in seconds the driver will attempt to fetch a new
+routing table. This encompasses *everything* that needs to happen for this,
+including fetching connections from the pool, performing handshakes, and
+requesting and receiving a fresh routing table.
+
+Since this process may involve acquiring a connection from the pool, or
+establishing a new connection, it should be chosen larger than
+:ref:`connection-acquisition-timeout-ref` and :ref:`connection-timeout-ref`.
+
+This setting only has an effect for :ref:`neo4j-driver-ref`, but not for
+:ref:`bolt-driver-ref` as it does no routing at all.
+
+:Type: ``float``
+:Default: ``90.0``
+
+.. versionadded:: 4.4.5
+
+
 .. _connection-acquisition-timeout-ref:
 
 ``connection_acquisition_timeout``
 ----------------------------------
-The maximum amount of time in seconds a session will wait when requesting a connection from the connection pool.
-Since the process of acquiring a connection may involve creating a new connection, ensure that the value of this configuration is higher than the configured :ref:`connection-timeout-ref`.
+The maximum amount of time in seconds the driver will wait to either acquire an
+idle connection from the pool (including potential liveness checks) or create a
+new connection when the pool is not full and all existing connection are in use.
+
+Since this process may involve opening a new connection including handshakes,
+it should be chosen larger than :ref:`connection-timeout-ref`.
 
 :Type: ``float``
 :Default: ``60.0``
@@ -187,7 +236,11 @@ Since the process of acquiring a connection may involve creating a new connectio
 
 ``connection_timeout``
 ----------------------
-The maximum amount of time in seconds to wait for a TCP connection to be established.
+The maximum amount of time in seconds to wait for a TCP connection to be
+established.
+
+This *does not* include any handshake(s), or authentication required before the
+connection can be used to perform database related work.
 
 :Type: ``float``
 :Default: ``30.0``

neo4j/__init__.py

@@ -60,6 +60,7 @@
 from logging import getLogger
 
 
+from neo4j._deadline import Deadline
 from neo4j.addressing import (
     Address,
     IPv4Address,
@@ -451,6 +452,7 @@ def _verify_routing_connectivity(self):
         )
 
         table = self._pool.get_routing_table_for_default_database()
+        timeout = self._default_workspace_config.connection_acquisition_timeout
         routing_info = {}
         for ix in list(table.routers):
             try:
@@ -459,8 +461,7 @@ def _verify_routing_connectivity(self):
                     database=self._default_workspace_config.database,
                     imp_user=self._default_workspace_config.impersonated_user,
                     bookmarks=None,
-                    timeout=self._default_workspace_config
-                                .connection_acquisition_timeout
+                    deadline=Deadline(timeout)
                 )
             except (ServiceUnavailable, SessionExpired, Neo4jError):
                 routing_info[ix] = None

neo4j/_deadline.py

@@ -0,0 +1,98 @@
+# Copyright (c) "Neo4j"
+# Neo4j Sweden AB [https://neo4j.com]
+#
+# This file is part of Neo4j.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from contextlib import contextmanager
+from time import perf_counter
+
+
+class Deadline:
+    def __init__(self, timeout):
+        if timeout is None or timeout == float("inf"):
+            self._deadline = float("inf")
+        else:
+            self._deadline = perf_counter() + timeout
+        self._original_timeout = timeout
+
+    @property
+    def original_timeout(self):
+        return self._original_timeout
+
+    def expired(self):
+        return self.to_timeout() == 0
+
+    def to_timeout(self):
+        if self._deadline == float("inf"):
+            return None
+        timeout = self._deadline - perf_counter()
+        return timeout if timeout > 0 else 0
+
+    def __eq__(self, other):
+        if isinstance(other, Deadline):
+            return self._deadline == other._deadline
+        return NotImplemented
+
+    def __gt__(self, other):
+        if isinstance(other, Deadline):
+            return self._deadline > other._deadline
+        return NotImplemented
+
+    def __ge__(self, other):
+        if isinstance(other, Deadline):
+            return self._deadline >= other._deadline
+        return NotImplemented
+
+    def __lt__(self, other):
+        if isinstance(other, Deadline):
+            return self._deadline < other._deadline
+        return NotImplemented
+
+    def __le__(self, other):
+        if isinstance(other, Deadline):
+            return self._deadline <= other._deadline
+        return NotImplemented
+
+    @classmethod
+    def from_timeout_or_deadline(cls, timeout):
+        if isinstance(timeout, cls):
+            return timeout
+        return cls(timeout)
+
+
+merge_deadlines = min
+
+
+def merge_deadlines_and_timeouts(*deadline):
+    deadlines = map(Deadline.from_timeout_or_deadline, deadline)
+    return merge_deadlines(deadlines)
+
+
+@contextmanager
+def connection_deadline(connection, deadline):
+    original_deadline = connection.socket.get_deadline()
+    if deadline is None and original_deadline is not None:
+        # nothing to do here
+        yield
+        return
+    deadline = merge_deadlines(
+        (d for d in (deadline, original_deadline) if d is not None)
+    )
+    connection.socket.set_deadline(deadline)
+    try:
+        yield
+    finally:
+        connection.socket.set_deadline(original_deadline)

neo4j/_exceptions.py

@@ -172,3 +172,7 @@ def transaction(self):
 class BoltProtocolError(BoltError):
     """ Raised when an unexpected or unsupported protocol event occurs.
     """
+
+
+class SocketDeadlineExceeded(RuntimeError):
+    """Raised from sockets with deadlines when a timeout occurs."""

neo4j/conf.py

@@ -185,6 +185,13 @@ class PoolConfig(Config):
     connection_timeout = 30.0  # seconds
     # The maximum amount of time to wait for a TCP connection to be established.
 
+    #: Update Routing Table Timout
+    update_routing_table_timeout = 90.0  # seconds
+    # The maximum amount of time to wait for updating the routing table.
+    # This includes everything necessary for this to happen.
+    # Including opening sockets, requesting and receiving the routing table,
+    # etc.
+
     #: Trust
     trust = TRUST_SYSTEM_CA_SIGNED_CERTIFICATES
     # Specify how to determine the authenticity of encryption certificates provided by the Neo4j instance on connection.
@@ -256,6 +263,12 @@ class WorkspaceConfig(Config):
     """ WorkSpace configuration.
     """
 
+    #: Session Connection Timeout
+    session_connection_timeout = float("inf")  # seconds
+    # The maximum amount of time to wait for a session to obtain a usable
+    # read/write connection. This includes everything necessary for this to
+    # happen. Including fetching routing tables, opening sockets, etc.
+
     #: Connection Acquisition Timeout
     connection_acquisition_timeout = 60.0  # seconds
     # The maximum amount of time a session will wait when requesting a connection from the connection pool.

neo4j/exceptions.py

@@ -152,6 +152,8 @@ def __str__(self):
 class ClientError(Neo4jError):
     """ The Client sent a bad request - changing the request might yield a successful outcome.
     """
+    def __str__(self):
+        return super(Neo4jError, self).__str__()
 
 
 class DatabaseError(Neo4jError):