From 75819de7e8b2f450b4b9479dd9393352dd0bc719 Mon Sep 17 00:00:00 2001
From: Rob Rudin <rudin@progress.com>
Date: Fri, 29 Aug 2025 14:16:37 -0400
Subject: [PATCH] MLE-23230 Trying out retry interceptor

This is a prototype; we don't want to apply it automatically. Intent for now is to see if this helps avoid connection errors during the regression piplines in Jenkins.
---
 .../client/impl/okhttp/OkHttpUtil.java        |  9 +-
 .../client/impl/okhttp/RetryInterceptor.java  | 87 +++++++++++++++++++
 2 files changed, 91 insertions(+), 5 deletions(-)
 create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryInterceptor.java

diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java
index b14da8ada..93a273db8 100644
--- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java
+++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/OkHttpUtil.java
@@ -7,11 +7,7 @@
 import com.marklogic.client.impl.HTTPKerberosAuthInterceptor;
 import com.marklogic.client.impl.HTTPSamlAuthInterceptor;
 import com.marklogic.client.impl.SSLUtil;
-import okhttp3.ConnectionPool;
-import okhttp3.CookieJar;
-import okhttp3.Dns;
-import okhttp3.Interceptor;
-import okhttp3.OkHttpClient;
+import okhttp3.*;
 
 import javax.net.SocketFactory;
 import javax.net.ssl.HostnameVerifier;
@@ -82,6 +78,9 @@ public static OkHttpClient.Builder newOkHttpClientBuilder(String host, DatabaseC
 		OkHttpUtil.configureSocketFactory(clientBuilder, sslContext, trustManager);
 		OkHttpUtil.configureHostnameVerifier(clientBuilder, sslVerifier);
 
+		// Trying this out for all calls initially to see how the regression test piplines do.
+		clientBuilder.addInterceptor(new RetryInterceptor(3, 1000, 2, 8000));
+
 		return clientBuilder;
 	}
 
diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryInterceptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryInterceptor.java
new file mode 100644
index 000000000..2f54430a8
--- /dev/null
+++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryInterceptor.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
+ */
+package com.marklogic.client.impl.okhttp;
+
+import okhttp3.Interceptor;
+import okhttp3.Request;
+import okhttp3.Response;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+
+/**
+ * OkHttp interceptor that retries requests on certain connection failures,
+ * which can be helpful when MarkLogic is temporarily unavailable during restarts.
+ */
+class RetryInterceptor implements Interceptor {
+
+	private final static Logger logger = org.slf4j.LoggerFactory.getLogger(RetryInterceptor.class);
+
+	private final int maxRetries;
+	private final long initialDelayMs;
+	private final double backoffMultiplier;
+	private final long maxDelayMs;
+
+	RetryInterceptor(int maxRetries, long initialDelayMs, double backoffMultiplier, long maxDelayMs) {
+		this.maxRetries = maxRetries;
+		this.initialDelayMs = initialDelayMs;
+		this.backoffMultiplier = backoffMultiplier;
+		this.maxDelayMs = maxDelayMs;
+	}
+
+	@Override
+	public Response intercept(Chain chain) throws IOException {
+		Request request = chain.request();
+		IOException lastException = null;
+
+		for (int attempt = 0; attempt <= maxRetries; attempt++) {
+			try {
+				return chain.proceed(request);
+			} catch (IOException e) {
+				lastException = e;
+
+				if (attempt == maxRetries || !isRetryableException(e)) {
+					logger.warn("Not retryable: {}; {}", e.getClass(), e.getMessage());
+					throw e;
+				}
+
+				long delay = calculateDelay(attempt);
+				logger.warn("Request to {} failed (attempt {}/{}): {}. Retrying in {}ms",
+					request.url(), attempt + 1, maxRetries, e.getMessage(), delay);
+
+				sleep(delay);
+			}
+		}
+
+		throw lastException;
+	}
+
+	private boolean isRetryableException(IOException e) {
+		return e instanceof ConnectException ||
+			e instanceof SocketTimeoutException ||
+			e instanceof UnknownHostException ||
+			(e.getMessage() != null && (
+				e.getMessage().contains("Failed to connect") ||
+					e.getMessage().contains("unexpected end of stream") ||
+					e.getMessage().contains("Connection reset") ||
+					e.getMessage().contains("Read timed out")
+			));
+	}
+
+	private long calculateDelay(int attempt) {
+		long delay = (long) (initialDelayMs * Math.pow(backoffMultiplier, attempt));
+		return Math.min(delay, maxDelayMs);
+	}
+
+	private void sleep(long delay) {
+		try {
+			Thread.sleep(delay);
+		} catch (InterruptedException ie) {
+			logger.warn("Ignoring InterruptedException while sleeping for retry delay: {}", ie.getMessage());
+		}
+	}
+}