From ea38a5408fcce135fd8e9611c61d98721ad22e21 Mon Sep 17 00:00:00 2001 From: Hans Glad Date: Wed, 5 Mar 2025 12:05:18 +0100 Subject: [PATCH 1/2] IS-18152: documented new retry_strategy option and batch_retries --- hub/changelog.rst | 9 ++++ .../pipes/configuration-pump.rst | 7 +++ .../configuration-systems-microservice.rst | 9 +++- .../systems/configuration-systems-url.rst | 53 ++++++++++++++++--- 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/hub/changelog.rst b/hub/changelog.rst index 392cedbb76..defd32b067 100644 --- a/hub/changelog.rst +++ b/hub/changelog.rst @@ -1,6 +1,15 @@ Changelog ========= +.. _changelog_2025-03-10: + +2025-03-10 +---------- +* Added the option to specify retry strategies for different HTTP status codes in the URL, REST and microservice systems. + This can be configured in the new ``retry_strategy`` :ref:`property `. +* Documented ``batch_retries`` in the :ref:`pump properties `. + + .. _changelog_2025-03-09: 2025-03-09 diff --git a/hub/documentation/service-configuration/pipes/configuration-pump.rst b/hub/documentation/service-configuration/pipes/configuration-pump.rst index 9337ae10ba..9a9a87c7ec 100644 --- a/hub/documentation/service-configuration/pipes/configuration-pump.rst +++ b/hub/documentation/service-configuration/pipes/configuration-pump.rst @@ -268,6 +268,13 @@ they are formatted in the :doc:`Cron Expressions <../../../cron-expressions>` do - 0 - + * - ``batch_retries`` + - Integer + - The maximum number of retries allowed when using batched writes. This will only have an effect if the sink + supports batching. + - 3 + - + * - ``fallback_to_single_entities_on_batch_fail`` - Boolean - A flag that controls if the pipes should attempt to process a single entity at a time if a batch diff --git a/hub/documentation/service-configuration/systems/configuration-systems-microservice.rst b/hub/documentation/service-configuration/systems/configuration-systems-microservice.rst index 9d0dbd8ebf..89a477f948 100644 --- a/hub/documentation/service-configuration/systems/configuration-systems-microservice.rst +++ b/hub/documentation/service-configuration/systems/configuration-systems-microservice.rst @@ -48,7 +48,7 @@ The table below shows the total resource quotas available for the microservices Prototype ^^^^^^^^^ -:: +.. code-block:: json { "_id": "id-of-microservice", @@ -237,6 +237,13 @@ Properties - ``true`` - + * - ``retry_strategy`` + - Object + - See the documentation on the :ref:`URL system `. + - + - + + Documentation for deprecated properties can be found :ref:`here `. diff --git a/hub/documentation/service-configuration/systems/configuration-systems-url.rst b/hub/documentation/service-configuration/systems/configuration-systems-url.rst index 4906947d87..bce8ec7e5a 100644 --- a/hub/documentation/service-configuration/systems/configuration-systems-url.rst +++ b/hub/documentation/service-configuration/systems/configuration-systems-url.rst @@ -8,10 +8,12 @@ serving `HTTP requests - - A optional set of properties that specifies support for automatic fetching of JWT access tokens from a oauth2 + - An optional set of properties that specifies support for automatic fetching of JWT access tokens from a oauth2 enabled provider. The grant types supported are "client credentials" and "refresh token". For the "client credentials" grant type you need to supply a ``client_id`` and ``client_secret`` from your oauth2 provider. You must also specify a ``token_url`` URL to a service which generates JWT access tokens. For the "refresh token" @@ -153,7 +176,7 @@ Properties * - ``tripletex`` - Dict - - A optional set of properties that specifies support for automatic generation and refreshing of Tripletex access + - An optional set of properties that specifies support for automatic generation and refreshing of Tripletex access tokens. See the `Tripletex API documentation `_ for details. You need to supply a ``consumer_token`` and ``employee_token`` from your Tripletex account. You must also specify a ``token_url`` URL to the Tripletex API service which generates access tokens. Optionally you can define a @@ -164,7 +187,7 @@ Properties * - ``proxies`` - Dict - - A optional set of properties that specifies a set of SOCKS5 proxies for the URL system. The keys represents url- + - An optional set of properties that specifies a set of SOCKS5 proxies for the URL system. The keys represents url- prefixes (for example 'http' and 'https') and the values of the HTTP(S) or SOCKS5 servers that the requests matching the prefixes should be passed through. The values should be on the form ``socks5://username:password@domain_or_ip:port`` or .``http(s)://username:password@domain_or_ip:port`` @@ -193,12 +216,30 @@ Properties - ``false`` - + .. _url_system_retry_strategy: + * - ``retry_strategy`` + - Object + - Specifies the retry strategies for different HTTP status codes. This essentially allows setting defaults for + retry-related properties on the :ref:`pump ` depending on the received status code. The status + codes are set as keys, and each value must be an object that contains the desired properties. + The special ``default`` key can also be used to set a default for all status codes that are not specified in the + strategy. If a retry property is set directly on the pump, that value will take priority over the ``retry_strategy``. + The following properties can be set as part of the retry strategy: ``max_read_retries``, ``read_retry_delay``, + ``write_retry_delay``, ``max_retries_per_entity``, ``max_consecutive_write_errors``, + ``max_write_errors_in_retry_dataset``, ``batch_retries``. + + See the :ref:`prototype ` for an example configuration. + + - + - + + [1] Exactly one of ``base_url`` and ``url_pattern`` must be specified. Example configuration ^^^^^^^^^^^^^^^^^^^^^ -:: +.. code-block:: json { "_id": "our-http-server", @@ -209,7 +250,7 @@ Example configuration Example with ntlm configuration: -:: +.. code-block:: json { "_id": "our-http-server", From d7e133df1842f39120c1e66e3184ac7c56ce5306 Mon Sep 17 00:00:00 2001 From: Hans Glad Date: Wed, 5 Mar 2025 13:38:22 +0100 Subject: [PATCH 2/2] IS-18152: moved retry_strategy example from prototype to example configurations --- .../systems/configuration-systems-url.rst | 63 ++++++++++++------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/hub/documentation/service-configuration/systems/configuration-systems-url.rst b/hub/documentation/service-configuration/systems/configuration-systems-url.rst index bce8ec7e5a..f9d0383f41 100644 --- a/hub/documentation/service-configuration/systems/configuration-systems-url.rst +++ b/hub/documentation/service-configuration/systems/configuration-systems-url.rst @@ -53,27 +53,6 @@ Prototype "http": "socks5://mysocksproxy:1234", "https": "socks5://user:password@mysslsocksproxy:1234", }, - "retry_strategy": { - "400-428": { - "write_retry_delay": 0, - "max_retries_per_entity": 0 - }, - "429": { - "write_retry_delay": 30, - "max_retries_per_entity": 100, - "read_retry_delay": 30 - }, - "500,502,504": { - "write_retry_delay": 60, - "max_retries_per_entity": 100, - "read_retry_delay": 60 - }, - "default": { - "write_retry_delay": 30, - "max_retries_per_entity": 3, - "read_retry_delay": 0 - } - }, "authentication": "basic", "connect_timeout": 60, "read_timeout": 1800 @@ -228,7 +207,7 @@ Properties ``write_retry_delay``, ``max_retries_per_entity``, ``max_consecutive_write_errors``, ``max_write_errors_in_retry_dataset``, ``batch_retries``. - See the :ref:`prototype ` for an example configuration. + An example of a URL system with a ``retry_strategy`` can be found :ref:`here `. - - @@ -236,8 +215,12 @@ Properties [1] Exactly one of ``base_url`` and ``url_pattern`` must be specified. -Example configuration -^^^^^^^^^^^^^^^^^^^^^ +.. _url_system_example_configurations: + +Example configurations +^^^^^^^^^^^^^^^^^^^^^^ + +Basic example configuration: .. code-block:: json @@ -261,3 +244,35 @@ Example with ntlm configuration: "password": "$SECRET(password-variable)", "base_url": "http://our.domain.com/files" } + +Example with retry strategies for different HTTP status codes: + +.. code-block:: json + + { + "_id": "our-http-server", + "name": "Our HTTP Server", + "type": "system:url", + "base_url": "http://our.domain.com/files", + "retry_strategy": { + "400-428": { + "write_retry_delay": 0, + "max_retries_per_entity": 0 + }, + "429": { + "write_retry_delay": 30, + "max_retries_per_entity": 100, + "read_retry_delay": 30 + }, + "500,502,504": { + "write_retry_delay": 60, + "max_retries_per_entity": 100, + "read_retry_delay": 60 + }, + "default": { + "write_retry_delay": 30, + "max_retries_per_entity": 3, + "read_retry_delay": 0 + } + } + } \ No newline at end of file