diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java index b39468318311..2f29c6985322 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java @@ -113,6 +113,15 @@ public abstract class StorageVolume implements Checkable ioTestSlidingWindow; private int healthCheckFileSize; + /* + Counter for consecutive latch or per-check timeouts. Incremented by + recordTimeoutAsIOFailure() which must NOT be synchronized (check() may be + holding the lock — that is exactly why the timeout fired). AtomicInteger + provides the necessary thread safety without locking. Reset to 0 by + resetTimeoutCount() whenever a check completes successfully. + */ + private final AtomicInteger consecutiveTimeoutCount; + /** * Type for StorageVolume. */ @@ -164,6 +173,7 @@ protected StorageVolume(Builder b) throws IOException { this.ioTestSlidingWindow = new LinkedList<>(); this.currentIOFailureCount = new AtomicInteger(0); this.healthCheckFileSize = dnConf.getVolumeHealthCheckFileSize(); + this.consecutiveTimeoutCount = new AtomicInteger(0); } else { storageDir = new File(b.volumeRootStr); volumeUsage = null; @@ -174,6 +184,7 @@ protected StorageVolume(Builder b) throws IOException { this.ioFailureTolerance = 0; this.conf = null; this.dnConf = null; + this.consecutiveTimeoutCount = new AtomicInteger(0); } this.storageDirStr = storageDir.getAbsolutePath(); } @@ -708,23 +719,11 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused) return VolumeCheckResult.HEALTHY; } - // Move the sliding window of IO test results forward 1 by adding the - // latest entry and removing the oldest entry from the window. - // Update the failure counter for the new window. - ioTestSlidingWindow.add(diskChecksPassed); - if (!diskChecksPassed) { - currentIOFailureCount.incrementAndGet(); - } - if (ioTestSlidingWindow.size() > ioTestCount && - Objects.equals(ioTestSlidingWindow.poll(), Boolean.FALSE)) { - currentIOFailureCount.decrementAndGet(); - } - - // If the failure threshold has been crossed, fail the volume without - // further scans. - // Once the volume is failed, it will not be checked anymore. - // The failure counts can be left as is. - if (currentIOFailureCount.get() > ioFailureTolerance) { + // Move the sliding window of IO test results forward 1 and check threshold. + if (advanceIOWindow(diskChecksPassed)) { + // If the failure threshold has been crossed, fail the volume without + // further scans. Once the volume is failed, it will not be checked + // anymore. The failure counts can be left as is. LOG.error("Failed IO test for volume {}: the last {} runs " + "encountered {} out of {} tolerated failures.", this, ioTestSlidingWindow.size(), currentIOFailureCount, @@ -740,6 +739,92 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused) return VolumeCheckResult.HEALTHY; } + /** + * Called by {@link StorageVolumeChecker} when a volume check times out — + * either because the global {@code checkAllVolumes()} latch expired before + * this volume's async check completed, or because the per-check timeout + * inside {@link ThrottledAsyncChecker} fired. + * + *

Must not be {@code synchronized}. When a timeout fires, + * {@link #check} may still be executing and holding the object lock — that + * is precisely why the timeout occurred. Acquiring the same lock here would + * deadlock or stall {@link StorageVolumeChecker#checkAllVolumes} until the + * hung check finally returns. + * + *

Instead, a dedicated {@link AtomicInteger} ({@code + * consecutiveTimeoutCount}) tracks consecutive timeouts without any locking. + * The threshold reuses the existing {@code ioFailureTolerance} so no + * additional configuration key is required. + * + *

Recovery: call {@link #resetTimeoutCount()} when a check completes + * successfully to break the timeout streak. + * + * @return {@code true} if {@code consecutiveTimeoutCount > ioFailureTolerance}, + * meaning the volume should now be marked FAILED; {@code false} if + * the timeout is still within tolerance this round. + */ + public boolean recordTimeoutAsIOFailure() { + int count = consecutiveTimeoutCount.incrementAndGet(); + if (count > ioFailureTolerance) { + LOG.error("Volume {} check timed out {} consecutive time(s)," + + " exceeding tolerance of {}. Marking FAILED.", + this, count, ioFailureTolerance); + return true; + } + LOG.warn("Volume {} check timed out ({}/{} consecutive timeouts tolerated)." + + " Common transient causes: kernel I/O scheduler saturation" + + " or JVM GC pressure. Volume will be failed if the next check" + + " also times out.", + this, count, ioFailureTolerance); + return false; + } + + /** + * Resets the consecutive-timeout counter to 0. + * + *

Called by {@link StorageVolumeChecker} when this volume's check + * completes successfully, indicating that the transient stall has resolved + * and any accumulated timeout count should not carry over to the next cycle. + * + *

No synchronization needed — operates on an {@link AtomicInteger}. + */ + public void resetTimeoutCount() { + int prev = consecutiveTimeoutCount.getAndSet(0); + if (prev > 0 && LOG.isDebugEnabled()) { + LOG.debug("Volume {} completed a healthy check. Consecutive timeout" + + " count reset from {} to 0.", this, prev); + } + } + + @VisibleForTesting + public int getConsecutiveTimeoutCount() { + return consecutiveTimeoutCount.get(); + } + + /** + * Advances the IO-test sliding window by one entry and updates the rolling + * failure counter. + * + *

Called by both {@link #check} (genuine IO test result) and + * {@link #recordTimeoutAsIOFailure} (synthetic failure for a check timeout), + * keeping the window-update logic in a single place. + * + * @param passed {@code true} if the IO test passed; {@code false} otherwise. + * @return {@code true} if {@code currentIOFailureCount} now exceeds + * {@code ioFailureTolerance}; {@code false} if still within bounds. + */ + private boolean advanceIOWindow(boolean passed) { + ioTestSlidingWindow.add(passed); + if (!passed) { + currentIOFailureCount.incrementAndGet(); + } + if (ioTestSlidingWindow.size() > ioTestCount && + Objects.equals(ioTestSlidingWindow.poll(), Boolean.FALSE)) { + currentIOFailureCount.decrementAndGet(); + } + return currentIOFailureCount.get() > ioFailureTolerance; + } + @Override public int hashCode() { return Objects.hash(storageDir); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java index b48b0dac1180..83ca783fc40c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolumeChecker.java @@ -42,6 +42,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.conf.ConfigurationSource; @@ -224,6 +225,13 @@ public Set checkAllVolumes( final AtomicLong numVolumes = new AtomicLong(volumes.size()); final CountDownLatch latch = new CountDownLatch(1); + // Shared set used to guarantee exactly-one call to + // recordTimeoutAsIOFailure() per volume, regardless of whether the + // per-check timeout (ResultHandler.onFailure) or the global latch timeout + // (pending-volumes loop below) fires first. The first path to CAS-add the + // volume owns the tolerance decision; the other path skips it. + final Set timeoutHandledSet = ConcurrentHashMap.newKeySet(); + for (StorageVolume v : volumes) { Optional> olf = delegateChecker.schedule(v, null); @@ -232,7 +240,8 @@ public Set checkAllVolumes( allVolumes.add(v); Futures.addCallback(olf.get(), new ResultHandler(v, healthyVolumes, failedVolumes, - numVolumes, (ignored1, ignored2) -> latch.countDown()), + numVolumes, (ignored1, ignored2) -> latch.countDown(), + timeoutHandledSet), MoreExecutors.directExecutor()); } else { if (v instanceof HddsVolume) { @@ -246,18 +255,51 @@ public Set checkAllVolumes( // Wait until our timeout elapses, after which we give up on // the remaining volumes. - if (!latch.await(maxAllowedTimeForCheckMs, TimeUnit.MILLISECONDS)) { - LOG.warn("checkAllVolumes timed out after {} ms", - maxAllowedTimeForCheckMs); - } + boolean completedOnTime = + latch.await(maxAllowedTimeForCheckMs, TimeUnit.MILLISECONDS); synchronized (this) { - // All volumes that have not been detected as healthy should be - // considered failed. This is a superset of 'failedVolumes'. - // - // Make a copy under the mutex as Sets.difference() returns a view - // of a potentially changing set. - return new HashSet<>(Sets.difference(allVolumes, healthyVolumes)); + if (!completedOnTime) { + LOG.warn("checkAllVolumes timed out after {} ms." + + " Evaluating per-volume latch-timeout tolerance.", + maxAllowedTimeForCheckMs); + } + + // Volumes that explicitly reported FAILED via check() are always + // returned — the IO-failure sliding window in StorageVolume.check() + // already applied its own tolerance. + final Set result = new HashSet<>(failedVolumes); + + // Volumes that completed healthy: reset their consecutive-timeout + // counter so a single transient timeout is not combined with an + // unrelated future one after a healthy gap. + for (StorageVolume v : healthyVolumes) { + v.resetTimeoutCount(); + } + + // Volumes still pending (neither healthy nor explicitly failed) at + // latch-timeout time. onFailure() may have already handled some of + // these via timeoutHandledSet; skip those to avoid double-counting. + final Set pendingVolumes = + new HashSet<>(Sets.difference(allVolumes, + Sets.union(healthyVolumes, failedVolumes))); + + for (StorageVolume v : pendingVolumes) { + if (!timeoutHandledSet.add(v)) { + // onFailure() already handled this volume's timeout (per-check + // timeout fired before the latch). The tolerance decision was + // already made there; nothing left to do. + continue; + } + // Latch fired first — this is the first (and only) handler. + if (v.recordTimeoutAsIOFailure()) { + // Tolerance exceeded — mark as failed. + result.add(v); + } + // else: within tolerance this round — omit from failed set. + } + + return result; } } @@ -298,7 +340,7 @@ public boolean checkVolume(final StorageVolume volume, Callback callback) { Futures.addCallback(olf.get(), new ResultHandler(volume, ConcurrentHashMap.newKeySet(), ConcurrentHashMap.newKeySet(), - new AtomicLong(1), callback), + new AtomicLong(1), callback, null), checkVolumeResultHandlerExecutorService ); return true; @@ -320,23 +362,39 @@ private static class ResultHandler private final Callback callback; /** - * @param healthyVolumes set of healthy volumes. If the disk check is - * successful, add the volume here. - * @param failedVolumes set of failed volumes. If the disk check fails, - * add the volume here. - * @param volumeCounter volumeCounter used to trigger callback invocation. - * @param callback invoked when the volumeCounter reaches 0. + * Shared set used to guarantee exactly-one call to + * {@link StorageVolume#recordTimeoutAsIOFailure()} per volume when both + * the per-check timeout ({@link #onFailure}) and the global latch timeout + * (pending-volumes loop in {@code checkAllVolumes}) can race for the same + * volume. + *

+ * {@code null} for the {@code checkVolume()} path, where no latch exists + * and {@link #onFailure} is the sole timeout handler. + */ + @Nullable + private final Set timeoutHandledSet; + + /** + * @param healthyVolumes set of healthy volumes. + * @param failedVolumes set of failed volumes. + * @param volumeCounter triggers callback when it reaches 0. + * @param callback invoked when volumeCounter reaches 0. + * @param timeoutHandledSet shared CAS set for exactly-once timeout + * handling; {@code null} for + * {@code checkVolume()}. */ ResultHandler(StorageVolume volume, Set healthyVolumes, Set failedVolumes, AtomicLong volumeCounter, - @Nullable Callback callback) { + @Nullable Callback callback, + @Nullable Set timeoutHandledSet) { this.volume = volume; this.healthyVolumes = healthyVolumes; this.failedVolumes = failedVolumes; this.volumeCounter = volumeCounter; this.callback = callback; + this.timeoutHandledSet = timeoutHandledSet; } @Override @@ -376,14 +434,41 @@ public void onFailure(@Nonnull Throwable t) { volume, exception); // If the scan was interrupted, do not count it as a volume failure. // This should only happen if the volume checker is being shut down. - if (!(t instanceof InterruptedException)) { - markFailed(); - cleanup(); + if (t instanceof InterruptedException) { + return; } + // Detect a per-check timeout from ThrottledAsyncChecker. + // Guava 28+ (including 33.5.0-jre used here) fails the TimeoutFuture + // with TimeoutException on timeout. + boolean isTimeout = exception instanceof TimeoutException; + if (isTimeout) { + // timeoutHandledSet is null for checkVolume() (sole timeout handler). + // For checkAllVolumes(), the set is shared with the pending-volumes + // loop; CAS-add determines which path owns the tolerance decision. + boolean firstToHandle = + (timeoutHandledSet == null) || timeoutHandledSet.add(volume); + if (firstToHandle) { + if (!volume.recordTimeoutAsIOFailure()) { + // Within tolerance: do NOT trigger the failure callback. + // The volume is not marked failed; the next check cycle will + // re-evaluate its health. cleanup() is intentionally not called + // to avoid firing handleVolumeFailures() with an empty failed set. + return; + } + // Tolerance exceeded — fall through to markFailed()/cleanup(). + } + // else: the pending-volumes loop already handled this timeout. + // Fall through to markFailed()/cleanup() for counter bookkeeping only. + } + markFailed(); + cleanup(); } private void markHealthy() { healthyVolumes.add(volume); + // A successful completion resets any accumulated timeout count so that + // an earlier transient timeout does not carry over to future cycles. + volume.resetTimeoutCount(); } private void markFailed() { diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java index 71cb7af04b71..2554147b952b 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeChecker.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hdfs.server.datanode.checker.VolumeCheckResult.FAILED; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.isNull; @@ -28,8 +29,11 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; import java.io.File; import java.nio.file.Path; import java.time.Duration; @@ -39,8 +43,13 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -301,6 +310,165 @@ public void testNumScansSkipped() throws Exception { checker.shutdownAndWait(0, TimeUnit.SECONDS); } + /** + * Explicitly captures the {@link Throwable} type that + * {@link Futures#withTimeout} delivers to {@code onFailure()} when the + * deadline fires. + * + *

{@link ThrottledAsyncChecker} uses {@code Futures.withTimeout()} + * internally; this test replicates that exact call to confirm that Guava + * 28+ (including the 33.5.0-jre version used by Ozone) delivers a + * {@link TimeoutException} — NOT a {@link java.util.concurrent.CancellationException}. + * {@code CancellationException} in new Guava means external cancellation + * (e.g. executor shutdown), not a disk-check timeout, so + * {@link StorageVolumeChecker} {@code ResultHandler.onFailure()} only + * treats {@link TimeoutException} as a timeout. + */ + @Test + public void testFuturesWithTimeoutExceptionType() throws Exception { + ScheduledExecutorService scheduler = + Executors.newSingleThreadScheduledExecutor(); + AtomicReference captured = new AtomicReference<>(); + CountDownLatch done = new CountDownLatch(1); + + try { + // A future that never completes on its own — same situation as a + // check() thread blocked inside fsync(). + SettableFuture neverCompletes = SettableFuture.create(); + + // Wrap with a real Futures.withTimeout(), identical to what + // ThrottledAsyncChecker does. + ListenableFuture timedFuture = + Futures.withTimeout(neverCompletes, 100, TimeUnit.MILLISECONDS, + scheduler); + + Futures.addCallback(timedFuture, new FutureCallback() { + @Override + public void onSuccess(VolumeCheckResult result) { + done.countDown(); + } + + @Override + public void onFailure(Throwable t) { + captured.set(t); + done.countDown(); + } + }, MoreExecutors.directExecutor()); + + assertTrue(done.await(2, TimeUnit.SECONDS), + "Timeout should have fired within 2 seconds"); + + Throwable thrown = captured.get(); + LOG.info("Futures.withTimeout() delivered to onFailure(): {}", + thrown.getClass().getName()); + + // Guava 28+ delivers TimeoutException for a timeout. + // CancellationException would mean external cancellation, not a timeout. + assertTrue(thrown instanceof TimeoutException, + "Expected TimeoutException from Futures.withTimeout() but got: " + + thrown.getClass().getName()); + } finally { + scheduler.shutdownNow(); + } + } + + /** + * Verifies that when the per-check timeout inside {@link ThrottledAsyncChecker} + * fires on {@link StorageVolumeChecker#checkVolume}, the first timeout is + * tolerated (callback not invoked, volume not failed) and the second + * consecutive timeout causes the volume to be failed. + * + *

This test uses the real {@link ThrottledAsyncChecker} (not + * {@link DummyChecker}) so that the actual {@code TimeoutException} + * path in {@code ResultHandler.onFailure()} fires. + */ + @Test + public void testCheckVolumeTimeoutTolerance() throws Exception { + setup(); + // Use a very short check timeout so the test completes quickly. + OzoneConfiguration timeoutConf = new OzoneConfiguration(); + DatanodeConfiguration dnConf = timeoutConf.getObject(DatanodeConfiguration.class); + dnConf.setDiskCheckTimeout(Duration.ofMillis(200)); + dnConf.setDiskCheckMinGap(Duration.ZERO); + timeoutConf.setFromObject(dnConf); + + // A latch-controlled mock volume: check() blocks until released. + HddsVolume volume = mock(HddsVolume.class); + CountDownLatch blockLatch = new CountDownLatch(1); + when(volume.check(any())).thenAnswer(inv -> { + blockLatch.await(); + return VolumeCheckResult.HEALTHY; + }); + // First timeout returns false (within tolerance), second returns true. + when(volume.recordTimeoutAsIOFailure()).thenReturn(false).thenReturn(true); + + AtomicLong callbackCount = new AtomicLong(0); + StorageVolumeChecker checker = + new StorageVolumeChecker(timeoutConf, new FakeTimer(), "test-"); + + // First checkVolume — should time out and be tolerated (callback NOT fired). + checker.checkVolume(volume, (healthy, failed) -> callbackCount.incrementAndGet()); + + // Wait long enough for the timeout to fire. + Thread.sleep(600); + assertEquals(0, callbackCount.get(), + "Callback must NOT fire for a tolerated timeout"); + verify(volume, times(1)).recordTimeoutAsIOFailure(); + + // Second checkVolume — should time out and exceed tolerance (callback fired). + checker.checkVolume(volume, (healthy, failed) -> callbackCount.incrementAndGet()); + Thread.sleep(600); + + assertEquals(1, callbackCount.get(), + "Callback must fire when tolerance is exceeded"); + + blockLatch.countDown(); + checker.shutdownAndWait(5, TimeUnit.SECONDS); + } + + /** + * Verifies that when the {@code checkAllVolumes()} latch times out and + * pending volumes have not yet reported a result, their consecutive-timeout + * counter is incremented and the first timeout is tolerated. + * + *

This test confirms that {@code recordTimeoutAsIOFailure()} is called on + * pending volumes, and that the volume is NOT in the returned failed set on + * the first timeout. + */ + @Test + public void testCheckAllVolumesLatchTimeoutTolerance() throws Exception { + setup(); + OzoneConfiguration timeoutConf = new OzoneConfiguration(); + DatanodeConfiguration dnConf = timeoutConf.getObject(DatanodeConfiguration.class); + dnConf.setDiskCheckTimeout(Duration.ofMillis(200)); + dnConf.setDiskCheckMinGap(Duration.ZERO); + timeoutConf.setFromObject(dnConf); + + HddsVolume volume = mock(HddsVolume.class); + CountDownLatch blockLatch = new CountDownLatch(1); + when(volume.check(any())).thenAnswer(inv -> { + blockLatch.await(); + return VolumeCheckResult.HEALTHY; + }); + // Simulate: first timeout is within tolerance. + when(volume.recordTimeoutAsIOFailure()).thenReturn(false); + when(volume.getVolumeInfoStats()).thenReturn( + new VolumeInfoMetrics("test-vol", volume)); + + StorageVolumeChecker checker = + new StorageVolumeChecker(timeoutConf, new FakeTimer(), "test-"); + + Set failed = + checker.checkAllVolumes(Arrays.asList(volume)); + + assertFalse(failed.contains(volume), + "Volume must NOT be in failed set on first tolerated timeout"); + verify(volume, times(1)).recordTimeoutAsIOFailure(); + + blockLatch.countDown(); + checker.shutdownAndWait(5, TimeUnit.SECONDS); + } + /** * A checker to wraps the result of {@link HddsVolume#check} in * an ImmediateFuture. diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java index 1c9b8bec8c8f..fa9d8907b58f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestStorageVolumeHealthChecks.java @@ -19,6 +19,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.nio.file.Path; @@ -341,6 +343,81 @@ public void testCorrectDirectoryChecked(StorageVolume.Builder builder) volume.check(false); } + /** + * With the default settings (ioFailureTolerance=1), the first simulated + * check timeout must be tolerated: {@code consecutiveTimeoutCount} becomes 1 + * which is NOT {@code > 1}, so {@code recordTimeoutAsIOFailure()} returns + * {@code false}. + */ + @ParameterizedTest + @MethodSource("volumeBuilders") + public void testFirstTimeoutIsTolerated(StorageVolume.Builder builder) + throws Exception { + StorageVolume volume = builder.build(); + volume.format(CLUSTER_ID); + volume.createTmpDirs(CLUSTER_ID); + + assertEquals(0, volume.getConsecutiveTimeoutCount()); + assertFalse(volume.recordTimeoutAsIOFailure(), + "First timeout should be tolerated (count 1 is not > tolerance 1)"); + assertEquals(1, volume.getConsecutiveTimeoutCount()); + } + + /** + * With the default settings (ioFailureTolerance=1), the second consecutive + * timeout must cause {@code recordTimeoutAsIOFailure()} to return + * {@code true}: count becomes 2 which IS {@code > 1}. + */ + @ParameterizedTest + @MethodSource("volumeBuilders") + public void testSecondConsecutiveTimeoutFails(StorageVolume.Builder builder) + throws Exception { + StorageVolume volume = builder.build(); + volume.format(CLUSTER_ID); + volume.createTmpDirs(CLUSTER_ID); + + assertFalse(volume.recordTimeoutAsIOFailure(), "First timeout should be tolerated"); + assertEquals(1, volume.getConsecutiveTimeoutCount()); + + assertTrue(volume.recordTimeoutAsIOFailure(), + "Second consecutive timeout should exceed tolerance and return true"); + assertEquals(2, volume.getConsecutiveTimeoutCount()); + } + + /** + * {@code resetTimeoutCount()} resets the consecutive-timeout counter to 0, + * so a subsequent single timeout is tolerated again — the streak does not + * carry over past a successful check cycle. + * + *

{@code resetTimeoutCount()} is called by {@link StorageVolumeChecker} + * whenever a volume completes a healthy check (either via + * {@code checkAllVolumes()} or via {@code checkVolume()}). + */ + @ParameterizedTest + @MethodSource("volumeBuilders") + public void testResetTimeoutCountResetsConsecutiveCounter( + StorageVolume.Builder builder) throws Exception { + StorageVolume volume = builder.build(); + volume.format(CLUSTER_ID); + volume.createTmpDirs(CLUSTER_ID); + + // Simulate one tolerated timeout. + assertFalse(volume.recordTimeoutAsIOFailure(), + "First timeout should be tolerated"); + assertEquals(1, volume.getConsecutiveTimeoutCount()); + + // StorageVolumeChecker calls resetTimeoutCount() when the volume's check + // eventually completes successfully. Simulate that here. + volume.resetTimeoutCount(); + assertEquals(0, volume.getConsecutiveTimeoutCount(), + "Counter must be reset to 0 after a successful check"); + + // A new single timeout after reset is tolerated again. + assertFalse(volume.recordTimeoutAsIOFailure(), + "Timeout after reset should be tolerated again"); + assertEquals(1, volume.getConsecutiveTimeoutCount()); + } + /** * Asserts that the disk checks are being done on the correct directory for * each volume type.