diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java index 552592f624f..ca8f2bcda1a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java @@ -25,6 +25,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.common.util.concurrent.RateLimiter; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.IOException; @@ -123,6 +124,7 @@ public class GarbageCollectorThread implements Runnable { private static final AtomicLong threadNum = new AtomicLong(0); final AbstractLogCompactor.Throttler throttler; + private final RateLimiter compactionReadByteRateLimiter; /** * Create a garbage collector thread. @@ -266,6 +268,7 @@ public void removeEntryLog(long logToRemove) { + majorCompactionThreshold + ", interval=" + majorCompactionInterval); lastMinorCompactionTime = lastMajorCompactionTime = System.currentTimeMillis(); + compactionReadByteRateLimiter = RateLimiter.create(conf.getCompactionReadRateByBytes()); } private EntryLogMetadataMap createEntryLogMetadataMap() throws IOException { @@ -619,9 +622,10 @@ void doCompactEntryLogs(double threshold, long maxTimeMillis) throws EntryLogMet meta.getEntryLogId(), meta.getUsage(), threshold); } - long priorRemainingSize = meta.getRemainingSize(); + long compactSize = meta.getTotalSize() - meta.getRemainingSize(); + compactionReadByteRateLimiter.acquire((int) (compactSize)); compactEntryLog(meta); - gcStats.getReclaimedSpaceViaCompaction().addCount(meta.getTotalSize() - priorRemainingSize); + gcStats.getReclaimedSpaceViaCompaction().addCount(compactSize); compactedBuckets[bucketIndex]++; }); } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java index 3b586108481..628bab5ed67 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java @@ -105,6 +105,7 @@ public class ServerConfiguration extends AbstractConfiguration { c.setIsThrottleByBytes(true); c.setCompactionRateByBytes(ENTRY_SIZE / 1000); + c.setCompactionReadRateByBytes(ENTRY_SIZE / 1000); c.setMinorCompactionThreshold(0.2f); c.setMajorCompactionThreshold(0.5f); return c; diff --git a/conf/bk_server.conf b/conf/bk_server.conf index 18c553fba02..2ff9a1d0342 100755 --- a/conf/bk_server.conf +++ b/conf/bk_server.conf @@ -581,6 +581,9 @@ ledgerDirectories=/tmp/bk-data # Set the rate at which compaction will readd entries. The unit is bytes added per second. # compactionRateByBytes=1000000 +# Set the rate at which compaction will read entries. The unit is bytes read per second. +# compactionReadRateByBytes=1000000 + # Flag to enable/disable transactional compaction. If it is set to true, it will use transactional compaction, # which it will use new entry log files to store compacted entries during compaction; if it is set to false, # it will use normal compaction, which it shares same entry log file with normal add operations. diff --git a/site3/website/docs/reference/config.md b/site3/website/docs/reference/config.md index 11953b80b3f..07cb2ac8e17 100644 --- a/site3/website/docs/reference/config.md +++ b/site3/website/docs/reference/config.md @@ -181,6 +181,7 @@ The table below lists parameters that you can set to configure bookies. All conf | isThrottleByBytes | Throttle compaction by bytes or by entries. | false | | compactionRateByEntries | Set the rate at which compaction will read entries. The unit is adds per second. | 1000 | | compactionRateByBytes | Set the rate at which compaction will read entries. The unit is bytes added per second. | 1000000 | +| compactionReadRateByBytes | Set the rate at which compaction will read entries. The unit is bytes read per second. | 1000000 | | useTransactionalCompaction | Flag to enable/disable transactional compaction. If it is set to true, it will use transactional compaction, which uses
new entry log files to store entries after compaction; otherwise, it will use normal compaction, which shares same entry
log file with normal add operations.
| false |