Fix issue where low traffic prevents timely detection and refresh of RS when the peer goes offline (#297)

Author: maochongxin

src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java

@@ -1309,6 +1309,10 @@ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean r
                                 tableEntryRefreshContinuousFailureCeiling);
                             syncRefreshMetadata();
                             tableEntryRefreshContinuousFailureCount.set(0);
+                        } else if (e.isConnectInactive()) {
+                            // getMetaRefreshConnection failed, maybe the server is down, so we need to refresh metadata directly
+                            syncRefreshMetadata();
+                            tableEntryRefreshContinuousFailureCount.set(0);
                         }
                     } catch (Throwable t) {
                         RUNTIME.error("getOrRefreshTableEntry meet exception", t);
@@ -1414,6 +1418,10 @@ public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String t
                                     tableEntryRefreshContinuousFailureCeiling);
                             syncRefreshMetadata();
                             tableEntryRefreshContinuousFailureCount.set(0);
+                        } else if (e.isConnectInactive()) {
+                            // getMetaRefreshConnection failed, maybe the server is down, so we need to refresh metadata directly
+                            syncRefreshMetadata();
+                            tableEntryRefreshContinuousFailureCount.set(0);
                         }
                     } catch (Throwable t) {
                         RUNTIME.error("getOrRefreshTableEntry meet exception", t);
@@ -1504,9 +1512,15 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName, bo
             throw e;
         } catch (Exception e) {
             RUNTIME.error(LCD.convert("01-00020"), tableEntryKey, tableEntry, e);
-            throw new ObTableEntryRefreshException(String.format(
-                "failed to get table entry key=%s original tableEntry=%s ", tableEntryKey,
-                tableEntry), e);
+            if (e instanceof ObTableEntryRefreshException) {
+                throw new ObTableEntryRefreshException(String.format(
+                        "failed to get table entry key=%s original tableEntry=%s ", tableEntryKey,
+                        tableEntry), e, ((ObTableEntryRefreshException) e).isConnectInactive());
+            } else {
+                throw new ObTableEntryRefreshException(String.format(
+                        "failed to get table entry key=%s original tableEntry=%s ", tableEntryKey,
+                        tableEntry), e);
+            }
         }
         tableLocations.put(tableName, tableEntry);
         if (fetchAll) {

src/main/java/com/alipay/oceanbase/rpc/exception/ObTableEntryRefreshException.java

@@ -18,6 +18,8 @@
 package com.alipay.oceanbase.rpc.exception;
 
 public class ObTableEntryRefreshException extends ObTableException {
+    
+    private boolean connectInactive = false;
 
     /*
      * Ob table entry refresh exception.
@@ -52,7 +54,15 @@ public ObTableEntryRefreshException(String message) {
     public ObTableEntryRefreshException(String message, Throwable cause) {
         super(message, cause);
     }
+    
+    public ObTableEntryRefreshException(String message, Throwable cause, boolean connectInactive) {
+        super(message, cause);
+        this.connectInactive = connectInactive;
+    }
 
+    public boolean isConnectInactive() {
+        return connectInactive;
+    }
     /*
      * Ob table entry refresh exception.
      */

src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java

@@ -304,7 +304,10 @@ private static Connection getMetaRefreshConnection(String url, ObUserAuth sysUA)
             return DriverManager.getConnection(url, sysUA.getUserName(), sysUA.getPassword());
         } catch (Exception e) {
             RUNTIME.error(LCD.convert("01-00005"), e.getMessage(), e);
-            throw new ObTableEntryRefreshException("fail to connect meta server", e);
+            // Since the JDBC connection fails here, it is likely that the server has crashed or scaling down.   
+            // Therefore, we need to set the Inactive flag of the ObTableEntryRefreshException to true.   
+            // This allows the upper-layer retry mechanism to catch this exception and immediately refresh the metadata.
+            throw new ObTableEntryRefreshException("fail to connect meta server", e, true /* connect inactive */);
         }
     }
 
@@ -403,9 +406,15 @@ private static TableEntry callTableEntryRefresh(ObServerAddr obServerAddr, Table
             } else {
                 RUNTIME.error(LCD.convert("01-00007"), url, key, e);
             }
-            throw new ObTableEntryRefreshException(format(
-                "fail to refresh table entry from remote url=%s, key=%s, message=%s", url, key,
-                e.getMessage()), e);
+            if (e instanceof ObTableEntryRefreshException) {
+                throw new ObTableEntryRefreshException(format(
+                        "fail to refresh table entry from remote url=%s, key=%s, message=%s", url, key,
+                        e.getMessage()), e, ((ObTableEntryRefreshException) e).isConnectInactive());
+            } else {
+                throw new ObTableEntryRefreshException(format(
+                        "fail to refresh table entry from remote url=%s, key=%s, message=%s", url, key,
+                        e.getMessage()), e.getCause());
+            }
         } finally {
             try {
                 if (null != connection) {
@@ -543,6 +552,10 @@ public static String loadTableNameWithGroupName(final ServerRoster serverRoster,
             RUNTIME.error("callTableEntryNameWithPriority meet exception", e);
             serverRoster.downgradePriority(addr);
             throw e;
+        } catch (ObTableEntryRefreshException e) {
+            RUNTIME.error("callTableEntryNameWithPriority meet exception", e);
+            throw new ObTableEntryRefreshException(format(
+                "fail to get table name from remote url=%s, key=%s", url, key), e, e.isConnectInactive());
         } catch (Exception e) {
             throw new ObTableNotExistException(format(
                 "fail to get table name from remote url=%s, key=%s", url, key), e);
@@ -979,8 +992,13 @@ public static Long getTableIdFromRemote(ObServerAddr obServerAddr, ObUserAuth sy
                                                        + " table_id from remote");
             }
         } catch (Exception e) {
-            throw new ObTableEntryRefreshException("fail to get " + tableName
-                                                   + " table_id from remote", e);
+            if (e instanceof ObTableEntryRefreshException) {
+                throw new ObTableEntryRefreshException(format(
+                    "fail to get " + tableName + " table_id from remote", e), e, ((ObTableEntryRefreshException) e).isConnectInactive());
+            } else {
+                throw new ObTableEntryRefreshException(format(
+                    "fail to get " + tableName + " table_id from remote", e), e);
+            }
         } finally {
             try {
                 if (null != rs) {
@@ -1019,8 +1037,13 @@ public static ObIndexInfo getIndexInfoFromRemote(ObServerAddr obServerAddr, ObUs
                 throw new ObTableEntryRefreshException("index is not exist");
             }
         } catch (Exception e) {
-            throw new ObTableEntryRefreshException(format(
-                "fail to get index info from remote, indexTableName: %s, error message: %s", indexTableName, e.getMessage()), e);
+            if (e instanceof ObTableEntryRefreshException) {
+                throw new ObTableEntryRefreshException(format(
+                        "fail to get index info from remote, indexTableName: %s, error message: %s", indexTableName, e.getMessage()), e, ((ObTableEntryRefreshException) e).isConnectInactive());
+            } else {
+                throw new ObTableEntryRefreshException(format(
+                        "fail to get index info from remote, indexTableName: %s, error message: %s", indexTableName, e.getMessage()), e);
+            }
         } finally {
             try {
                 if (null != rs) {