feat(http_exporter): allow to run retry loop on connection errors

Author: pafi-code

exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py

@@ -186,30 +186,34 @@ def export(
         serialized_data = encode_logs(batch).SerializeToString()
         deadline_sec = time() + self._timeout
         for retry_num in range(_MAX_RETRYS):
+            # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff.
+            backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2)
             try:
                 resp = self._export(serialized_data, deadline_sec - time())
+                if resp.ok:
+                    return LogExportResult.SUCCESS
+                if not _is_retryable(resp):
+                    _logger.error(
+                        "Failed to export logs batch code: %s, reason: %s",
+                        resp.status_code,
+                        resp.text,
+                    )
+                    return LogExportResult.FAILURE
             except Exception as error:
                 _logger.error("Failed to export logs batch reason: %s", error)
-                return LogExportResult.FAILURE
-            if resp.ok:
-                return LogRecordExportResult.SUCCESS
-            # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff.
-            backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2)
+
             if (
-                not _is_retryable(resp)
-                or retry_num + 1 == _MAX_RETRYS
+                retry_num + 1 == _MAX_RETRYS
                 or backoff_seconds > (deadline_sec - time())
                 or self._shutdown
             ):
                 _logger.error(
-                    "Failed to export logs batch code: %s, reason: %s",
-                    resp.status_code,
-                    resp.text,
+                    "Failed to export logs batch due to timeout,"
+                    "max retries or shutdown."
                 )
                 return LogRecordExportResult.FAILURE
             _logger.warning(
-                "Transient error %s encountered while exporting logs batch, retrying in %.2fs.",
-                resp.reason,
+                "Transient error encountered while exporting logs batch, retrying in %.2fs.",
                 backoff_seconds,
             )
             shutdown = self._shutdown_is_occuring.wait(backoff_seconds)

exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py

@@ -231,32 +231,36 @@ def export(
         serialized_data = encode_metrics(metrics_data).SerializeToString()
         deadline_sec = time() + self._timeout
         for retry_num in range(_MAX_RETRYS):
+            # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff.
+            backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2)
             try:
                 resp = self._export(serialized_data, deadline_sec - time())
+                if resp.ok:
+                    return MetricExportResult.SUCCESS
+                if not _is_retryable(resp):
+                    _logger.error(
+                        "Failed to export metrics batch code: %s, reason: %s",
+                        resp.status_code,
+                        resp.text,
+                    )
+                    return MetricExportResult.FAILURE
             except Exception as error:
                 _logger.error(
                     "Failed to export metrics batch reason: %s", error
                 )
-                return MetricExportResult.FAILURE
-            if resp.ok:
-                return MetricExportResult.SUCCESS
-            # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff.
-            backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2)
+
             if (
-                not _is_retryable(resp)
-                or retry_num + 1 == _MAX_RETRYS
+                retry_num + 1 == _MAX_RETRYS
                 or backoff_seconds > (deadline_sec - time())
                 or self._shutdown
             ):
                 _logger.error(
-                    "Failed to export metrics batch code: %s, reason: %s",
-                    resp.status_code,
-                    resp.text,
+                    "Failed to export metrics batch due to timeout,"
+                    "max retries or shutdown."
                 )
                 return MetricExportResult.FAILURE
             _logger.warning(
-                "Transient error %s encountered while exporting metrics batch, retrying in %.2fs.",
-                resp.reason,
+                "Transient error encountered while exporting metrics batch, retrying in %.2fs.",
                 backoff_seconds,
             )
             shutdown = self._shutdown_in_progress.wait(backoff_seconds)

exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/trace_exporter/__init__.py

@@ -179,30 +179,34 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
         serialized_data = encode_spans(spans).SerializePartialToString()
         deadline_sec = time() + self._timeout
         for retry_num in range(_MAX_RETRYS):
+            # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff.
+            backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2)
             try:
                 resp = self._export(serialized_data, deadline_sec - time())
+                if resp.ok:
+                    return SpanExportResult.SUCCESS
+                if not _is_retryable(resp):
+                    _logger.error(
+                        "Failed to export span batch code: %s, reason: %s",
+                        resp.status_code,
+                        resp.text,
+                    )
+                    return SpanExportResult.FAILURE
             except Exception as error:
                 _logger.error("Failed to export span batch reason: %s", error)
-                return SpanExportResult.FAILURE
-            if resp.ok:
-                return SpanExportResult.SUCCESS
-            # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff.
-            backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2)
+
             if (
-                not _is_retryable(resp)
-                or retry_num + 1 == _MAX_RETRYS
+                retry_num + 1 == _MAX_RETRYS
                 or backoff_seconds > (deadline_sec - time())
                 or self._shutdown
             ):
                 _logger.error(
-                    "Failed to export span batch code: %s, reason: %s",
-                    resp.status_code,
-                    resp.text,
+                    "Failed to export span batch due to timeout,"
+                    "max retries or shutdown."
                 )
                 return SpanExportResult.FAILURE
             _logger.warning(
-                "Transient error %s encountered while exporting span batch, retrying in %.2fs.",
-                resp.reason,
+                "Transient error encountered while exporting span batch, retrying in %.2fs.",
                 backoff_seconds,
             )
             shutdown = self._shutdown_in_progress.wait(backoff_seconds)