Online Indexer: replace the synchronized runner with a heartbeat

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/logging/LogMessageKeys.java

@@ -58,6 +58,7 @@ public enum LogMessageKeys {
     TRANSACTION_ID,
     TRANSACTION_NAME,
     AGE_SECONDS,
+    AGE_MILLISECONDS,
     CONSTITUENT,
     TOTAL_MICROS,
     // record splitting/unsplitting
@@ -162,7 +163,7 @@ public enum LogMessageKeys {
     RECORDS_PER_SECOND,
     DOCUMENT,
     SESSION_ID,
-    INDEXER_SESSION_ID,
+    EXISTING_INDEXER_ID,
     INDEXER_ID,
     INDEX_STATE_PRECONDITION,
     INITIAL_INDEX_STATE,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStore.java

@@ -83,6 +83,7 @@
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
 import com.apple.foundationdb.record.provider.common.DynamicMessageRecordSerializer;
 import com.apple.foundationdb.record.provider.common.RecordSerializer;
+import com.apple.foundationdb.record.provider.foundationdb.indexing.IndexingHeartbeat;
 import com.apple.foundationdb.record.provider.foundationdb.indexing.IndexingRangeSet;
 import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath;
 import com.apple.foundationdb.record.provider.foundationdb.storestate.FDBRecordStoreStateCache;
@@ -4965,7 +4966,9 @@ public void removeFormerIndex(FormerIndex formerIndex) {
     }
 
     private void clearReadableIndexBuildData(Index index) {
+        // Clear index maintenance data that is unneeded once the index becomes readable
         IndexingRangeSet.forIndexBuild(this, index).clear();
+        IndexingHeartbeat.clearAllHeartbeats(this, index);
     }
 
     @SuppressWarnings("PMD.CloseResource")

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java

@@ -413,6 +413,10 @@ public enum Waits implements Wait {
         WAIT_INDEX_OPERATION("wait for index operation"),
         /** Wait for indexing type stamp operation. */
         WAIT_INDEX_TYPESTAMP_OPERATION("wait for indexing type stamp operation"),
+        /** Wait for clearing indexing heartbeats. */
+        WAIT_INDEX_CLEAR_HEARTBEATS("Wait for clearing indexing heartbeats"),
+        /** Wait for reading indexing heartbeats. */
+        WAIT_INDEX_READ_HEARTBEATS("Wait for reading indexing heartbeats"),
         /** Wait for adding an index. */
         WAIT_ADD_INDEX("wait for adding an index"),
         /** Wait for dropping an index. */

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingByIndex.java

@@ -103,7 +103,7 @@ private Index getSourceIndex(RecordMetaData metaData) {
         throw new ValidationException("no source index",
                 LogMessageKeys.INDEX_NAME, common.getIndex().getName(),
                 LogMessageKeys.SOURCE_INDEX, policy.getSourceIndex(),
-                LogMessageKeys.INDEXER_ID, common.getUuid());
+                LogMessageKeys.INDEXER_ID, common.getIndexerId());
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingCommon.java

@@ -27,7 +27,6 @@
 import com.apple.foundationdb.record.metadata.Index;
 import com.apple.foundationdb.record.metadata.MetaDataException;
 import com.apple.foundationdb.record.metadata.RecordType;
-import com.apple.foundationdb.record.provider.foundationdb.synchronizedsession.SynchronizedSessionRunner;
 import com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner;
 import com.apple.foundationdb.tuple.Tuple;
 
@@ -50,10 +49,9 @@
 
 @API(API.Status.INTERNAL)
 public class IndexingCommon {
-    private final UUID uuid = UUID.randomUUID();
+    private final UUID indexerId = UUID.randomUUID();
 
     @Nonnull private final FDBDatabaseRunner runner;
-    @Nullable private SynchronizedSessionRunner synchronizedSessionRunner = null;
 
     @Nonnull private final FDBRecordStore.Builder recordStoreBuilder;
     @Nonnull private final AtomicLong totalRecordsScanned;
@@ -137,8 +135,8 @@ private void fillTargetIndexers(@Nonnull List<Index> targetIndexes, @Nullable Co
         }
     }
 
-    public UUID getUuid() {
-        return uuid;
+    public UUID getIndexerId() {
+        return indexerId;
     }
 
     public List<Object> indexLogMessageKeyValues() {
@@ -158,7 +156,7 @@ public List<Object> indexLogMessageKeyValues(@Nullable String transactionName, @
         logIf(true, keyValues,
                 LogMessageKeys.TARGET_INDEX_NAME, getTargetIndexesNames(),
                 LogMessageKeys.RECORDS_SCANNED, totalRecordsScanned.get(),
-                LogMessageKeys.INDEXER_ID, uuid);
+                LogMessageKeys.INDEXER_ID, indexerId);
 
         if (moreKeyValues != null && !moreKeyValues.isEmpty()) {
             keyValues.addAll(moreKeyValues);
@@ -176,11 +174,6 @@ private void logIf(boolean condition, List<Object> list, @Nonnull Object... a) {
 
     @Nonnull
     public FDBDatabaseRunner getRunner() {
-        return synchronizedSessionRunner == null ? runner : synchronizedSessionRunner;
-    }
-
-    @Nonnull
-    public FDBDatabaseRunner getNonSynchronizedRunner() {
         return runner;
     }
 
@@ -258,15 +251,6 @@ public FDBRecordStore.Builder getRecordStoreBuilder() {
         return recordStoreBuilder;
     }
 
-    @Nullable
-    public SynchronizedSessionRunner getSynchronizedSessionRunner() {
-        return synchronizedSessionRunner;
-    }
-
-    public void setSynchronizedSessionRunner(@Nullable final SynchronizedSessionRunner synchronizedSessionRunner) {
-        this.synchronizedSessionRunner = synchronizedSessionRunner;
-    }
-
     @Nonnull
     public AtomicLong getTotalRecordsScanned() {
         return totalRecordsScanned;
@@ -287,8 +271,5 @@ public boolean loadConfig() {
 
     public void close() {
         runner.close();
-        if (synchronizedSessionRunner != null) {
-            synchronizedSessionRunner.close();
-        }
     }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingMerger.java

@@ -77,7 +77,7 @@ CompletableFuture<Void> mergeIndex(@Nullable SubspaceProvider subspaceProvider)
                 // Merge operation may take a long time, hence the runner's context must be a read-only. Ensure that it
                 // isn't a synchronized one, which may attempt a heartbeat write
                 // Note: this runAsync will retry according to the runner's "maxAttempts" setting
-                common.getNonSynchronizedRunner().runAsync(context -> openRecordStore(context)
+                common.getRunner().runAsync(context -> openRecordStore(context)
                                 .thenCompose(store -> {
                                     mergeStartTime.set(System.nanoTime());
                                     final IndexDeferredMaintenanceControl mergeControl = store.getIndexDeferredMaintenanceControl();

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexingSubspaces.java

@@ -27,6 +27,7 @@
 import com.apple.foundationdb.tuple.Tuple;
 
 import javax.annotation.Nonnull;
+import java.util.UUID;
 
 /**
  * List of subspaces related to the indexing/index-scrubbing processes.
@@ -40,6 +41,7 @@ public final class IndexingSubspaces {
     private static final Object INDEX_SCRUBBED_RECORDS_RANGES_ZERO = 4L;
     private static final Object INDEX_SCRUBBED_RECORDS_RANGES = 5L;
     private static final Object INDEX_SCRUBBED_INDEX_RANGES = 6L;
+    private static final Object INDEX_BUILD_HEARTBEAT_PREFIX = 7L;
 
     private IndexingSubspaces() {
         throw new IllegalStateException("Utility class");
@@ -83,6 +85,29 @@ public static Subspace indexBuildTypeSubspace(@Nonnull FDBRecordStoreBase<?> sto
         return indexBuildSubspace(store, index, INDEX_BUILD_TYPE_VERSION);
     }
 
+    /**
+     * Subspace that stores the indexing heartbeat.
+     * @param store store
+     * @param index index
+     * @return subspace
+     */
+    @Nonnull
+    public static Subspace indexHeartbeatSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index) {
+        return indexBuildSubspace(store, index, INDEX_BUILD_HEARTBEAT_PREFIX);
+    }
+
+    /**
+     * Subspace that stores the indexing heartbeat.
+     * @param store store
+     * @param index index
+     * @param indexerId session id
+     * @return subspace
+     */
+    @Nonnull
+    public static Subspace indexHeartbeatSubspace(@Nonnull FDBRecordStoreBase<?> store, @Nonnull Index index, @Nonnull UUID indexerId) {
+        return indexHeartbeatSubspace(store, index).subspace(Tuple.from(indexerId));
+    }
+
     /**
      * Subspace that stores scrubbed records ranges of the zero range-id. This subspace is backward compatible
      * to record ranges scrubbed before range-id was introduced.
@@ -184,5 +209,7 @@ public static void eraseAllIndexingDataButTheLock(@Nonnull FDBRecordContext cont
         eraseAllIndexingScrubbingData(context, store, index);
         context.clear(Range.startsWith(indexBuildScannedRecordsSubspace(store, index).pack()));
         context.clear(Range.startsWith(indexBuildTypeSubspace(store, index).pack()));
+        // The heartbeats, unlike the sync lock, may be erased here. If needed, an appropriate heartbeat will be set after this clear & within the same transaction.
+        context.clear(Range.startsWith(indexHeartbeatSubspace(store, index).pack()));
     }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexOperationBaseBuilder.java

@@ -27,7 +27,6 @@
 import com.apple.foundationdb.record.metadata.Index;
 import com.apple.foundationdb.record.metadata.MetaDataException;
 import com.apple.foundationdb.record.provider.common.RecordSerializer;
-import com.apple.foundationdb.record.provider.foundationdb.synchronizedsession.SynchronizedSessionRunner;
 import com.apple.foundationdb.subspace.Subspace;
 import com.google.protobuf.Message;
 
@@ -709,26 +708,32 @@ public B setTransactionTimeLimitMilliseconds(long timeLimitMilliseconds) {
     }
 
     /**
-     * <em>Deprecated</em>. This will soon be determined by the indexing session type
+     * <em>Deprecated</em>. Synchronization is handled by shared heartbeats and exclusive access is being determined by
+     * the indexing session type.
+     *
+     * <em> When upgrading from Synchronized sessions to a version that uses Shared Heartbeats</em>:
+     * During graduate code upgrade on multiple servers, there may be a state of one server indexing
+     * with a synchronized session lock, while another server builds the same index with an exclusive heartbeat
+     * "lock". If that happens:
+     * a) There will be no more than two concurrent active sessions (one per each lock type).
+     * b) The indexing sessions will conflict each other until one of the indexers will give up. While this is
+     * not optimal, the generated index will be valid.
+     *
      * Set the use of a synchronized session during the index operation. Synchronized sessions help performing
      * the multiple transactions operations in a resource efficient way.
      * Normally this should be {@code true}.
      *
-     * @see SynchronizedSessionRunner
      * @param useSynchronizedSession use synchronize session if true, otherwise false
      * @return this builder
      */
     @API(API.Status.DEPRECATED)
     public B setUseSynchronizedSession(boolean useSynchronizedSession) {
-        configBuilder.setUseSynchronizedSession(useSynchronizedSession);
         return self();
     }
 
     /**
-     * Set the lease length in milliseconds if the synchronized session is used. The default value is {@link OnlineIndexOperationConfig#DEFAULT_LEASE_LENGTH_MILLIS}.
-     * @see #setUseSynchronizedSession(boolean)
-     * @see com.apple.foundationdb.synchronizedsession.SynchronizedSession
-     * @param leaseLengthMillis length between last access and lease's end time in milliseconds
+     * Set the max age of an indexing heartbeat to define an "active" indexing session.
+     * @param leaseLengthMillis max heartbeat age to be considered "active". In milliseconds
      * @return this builder
      */
     public B setLeaseLengthMillis(long leaseLengthMillis) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexOperationConfig.java

@@ -21,7 +21,6 @@
 package com.apple.foundationdb.record.provider.foundationdb;
 
 import com.apple.foundationdb.annotation.API;
-import com.apple.foundationdb.record.provider.foundationdb.synchronizedsession.SynchronizedSessionRunner;
 
 import javax.annotation.Nonnull;
 
@@ -53,7 +52,7 @@
      */
     public static final int DEFAULT_PROGRESS_LOG_INTERVAL = -1;
     /**
-     * Default synchronized session lease time in milliseconds. This allows a lock expiration, if the online operation stops unexpectedly.
+     * Default indexing heartbeat age, in milliseconds, to define an "active" session.
      */
     public static final long DEFAULT_LEASE_LENGTH_MILLIS = 10_000;
 
@@ -77,14 +76,13 @@
     private final int increaseLimitAfter;
     private final long timeLimitMilliseconds;
     private final long transactionTimeLimitMilliseconds;
-    private final boolean useSynchronizedSession;
     private final long leaseLengthMillis;
 
     public static final long UNLIMITED_TIME = 0;
 
     OnlineIndexOperationConfig(int maxLimit, int initialLimit, int maxRetries, int recordsPerSecond, long progressLogIntervalMillis, int increaseLimitAfter,
                                int maxWriteLimitBytes, long timeLimitMilliseconds, long transactionTimeLimitMilliseconds,
-                               boolean useSynchronizedSession, long leaseLengthMillis) {
+                               long leaseLengthMillis) {
         this.maxLimit = maxLimit;
         this.initialLimit = initialLimit;
         this.maxRetries = maxRetries;
@@ -94,7 +92,6 @@
         this.maxWriteLimitBytes = maxWriteLimitBytes;
         this.timeLimitMilliseconds = timeLimitMilliseconds;
         this.transactionTimeLimitMilliseconds = transactionTimeLimitMilliseconds;
-        this.useSynchronizedSession = useSynchronizedSession;
         this.leaseLengthMillis = leaseLengthMillis;
     }
 
@@ -188,9 +185,17 @@
         return new Builder();
     }
 
+
+    /**
+     * <em>Deprecated</em>. Synchronized sessions are now automatically determined by the indexing method.
+     * Mutual indexing and index scrubbing (if applicable) do not expect to run exclusively, other indexing methods will
+     * throw an exception if they another active indexing session is detected.
+     * @return always true;
+     */
+    @API(API.Status.DEPRECATED)
     public boolean shouldUseSynchronizedSession() {
-        return useSynchronizedSession;
+        return true;
     }
 
     public long getLeaseLengthMillis() {
         return leaseLengthMillis;
@@ -213,7 +218,6 @@
                 .setMaxRetries(this.maxRetries)
                 .setTimeLimitMilliseconds(timeLimitMilliseconds)
                 .setTransactionTimeLimitMilliseconds(this.transactionTimeLimitMilliseconds)
-                .setUseSynchronizedSession(useSynchronizedSession)
                 .setLeaseLengthMillis(leaseLengthMillis);
     }
 
@@ -234,7 +238,6 @@
         private long timeLimitMilliseconds = UNLIMITED_TIME;
         private long transactionTimeLimitMilliseconds = DEFAULT_TRANSACTION_TIME_LIMIT;
         private long leaseLengthMillis = DEFAULT_LEASE_LENGTH_MILLIS;
-        private boolean useSynchronizedSession = true;
 
         protected Builder() {
 
@@ -485,23 +488,23 @@
         }
 
         /**
-         * Set the use of a synchronized session during the index operation. Synchronized sessions help performing
-         * the multiple transactions operation in a resource efficient way.
-         * Normally this should be {@code true}.
-         *
-         * @see SynchronizedSessionRunner
-         * @param useSynchronizedSession use synchronize session if true, otherwise false
+         * <em>Deprecated</em>. Synchronized sessions are now automatically determined by the indexing method.
+         * Mutual indexing and index scrubbing (if applicable) do not expect to run exclusively, other indexing methods will
+         * throw an exception if they another active indexing session is detected.
+         * @param useSynchronizedSession ignored.
          * @return this builder
          */
+        @API(API.Status.DEPRECATED)
         public Builder setUseSynchronizedSession(boolean useSynchronizedSession) {
-            this.useSynchronizedSession = useSynchronizedSession;
+            // no-op
             return this;
         }
 
         /**
-         * Set the lease length in milliseconds if the synchronized session is used. By default this is {@link #DEFAULT_LEASE_LENGTH_MILLIS}.
-         * @see #setUseSynchronizedSession(boolean)
-         * @see com.apple.foundationdb.synchronizedsession.SynchronizedSession
+         * Defines the maximum age of another session's heartbeat to be considered an active session.
+         * The default value is {@link #DEFAULT_LEASE_LENGTH_MILLIS}.
+         * Mutual indexing and index scrubbing (if applicable) do not expect to run exclusively, other indexing methods will
+         * throw an exception if they another active indexing session is detected.
          * @param leaseLengthMillis length between last access and lease's end time in milliseconds
          * @return this builder
          */
@@ -519,7 +522,7 @@
         public OnlineIndexOperationConfig build() {
             return new OnlineIndexOperationConfig(maxLimit, initialLimit, maxRetries, recordsPerSecond, progressLogIntervalMillis, increaseLimitAfter,
                     maxWriteLimitBytes, timeLimitMilliseconds, transactionTimeLimitMilliseconds,
-                    useSynchronizedSession, leaseLengthMillis);
+                    leaseLengthMillis);
         }
     }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/OnlineIndexScrubber.java

@@ -75,7 +75,7 @@ private IndexingBase getScrubber(IndexScrubbingTools.ScrubbingType type, AtomicL
     @Nonnull
     private CompletableFuture<Void> scrubIndexAsync(IndexScrubbingTools.ScrubbingType type, AtomicLong count) {
         return AsyncUtil.composeHandle(
-                getScrubber(type, count).buildIndexAsync(false, common.config.shouldUseSynchronizedSession()),
+                getScrubber(type, count).buildIndexAsync(false),
                 (ignore, ex) -> {
                     if (ex != null) {
                         throw FDBExceptions.wrapException(ex);