From dcad02417adf13ed1bc5df15604138eaa5fee9f7 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 23 May 2025 16:44:36 +0800 Subject: [PATCH 1/4] [improve][ml]Offload ledgers without check ledger length --- .../filesystem/impl/FileSystemManagedLedgerOffloader.java | 6 +++++- .../jcloud/impl/BlobStoreManagedLedgerOffloader.java | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java index 2431e61c91cab..7f22bed360820 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java @@ -194,11 +194,15 @@ private LedgerReader(ReadHandle readHandle, @Override public void run() { - if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) { + if (!readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) { promise.completeExceptionally( new IllegalArgumentException("An empty or open ledger should never be offloaded")); return; } + if (readHandle.getLength() <= 0) { + log.warn("Ledger [{}] has a zero length, but it contains {} entries. since it contains entries," + + " attempts to offload", readHandle.getLastAddConfirmed() + 1, readHandle.getId()); + } long ledgerId = readHandle.getId(); final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME); String storagePath = getStoragePath(storageBasePath, managedLedgerName); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java index 5ee62591b42fe..19802076105d9 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java @@ -204,11 +204,16 @@ public CompletableFuture offload(ReadHandle readHandle, final BlobStore writeBlobStore = getBlobStore(config.getBlobStoreLocation()); log.info("offload {} uuid {} extraMetadata {} to {} {}", readHandle.getId(), uuid, extraMetadata, config.getBlobStoreLocation(), writeBlobStore); - if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) { + if (!readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) { promise.completeExceptionally( new IllegalArgumentException("An empty or open ledger should never be offloaded")); return; } + if (readHandle.getLength() <= 0) { + log.warn("[{}] has a ledger [{}] with a zero length, but it contains {} entries. since it contains" + + " entries, attempts to offload", topicName, readHandle.getLastAddConfirmed() + 1, + readHandle.getId()); + } OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create() .withLedgerMetadata(readHandle.getLedgerMetadata()) .withDataBlockHeaderLength(BlockAwareSegmentInputStreamImpl.getHeaderSize()); From 082e8c1af826f25e541a7c53b890ea2c72d2b71e Mon Sep 17 00:00:00 2001 From: fengyubiao <9947090@qq.com> Date: Mon, 26 May 2025 15:54:04 +0800 Subject: [PATCH 2/4] Update tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java Co-authored-by: Lari Hotari --- .../filesystem/impl/FileSystemManagedLedgerOffloader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java index 7f22bed360820..2c312b92f62f3 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java @@ -200,8 +200,8 @@ public void run() { return; } if (readHandle.getLength() <= 0) { - log.warn("Ledger [{}] has a zero length, but it contains {} entries. since it contains entries," - + " attempts to offload", readHandle.getLastAddConfirmed() + 1, readHandle.getId()); + log.warn("Ledger [{}] has zero length, but it contains {} entries. " + + " Attemping to offload ledger since it contains entries.", readHandle.getId(), readHandle.getLastAddConfirmed() + 1); } long ledgerId = readHandle.getId(); final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME); From 0d82d93e3bccfcde84430b5f0324eace0e76f9f2 Mon Sep 17 00:00:00 2001 From: fengyubiao <9947090@qq.com> Date: Mon, 26 May 2025 15:54:13 +0800 Subject: [PATCH 3/4] Update tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java Co-authored-by: Lari Hotari --- .../offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java index 19802076105d9..55e8a1195da65 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java @@ -210,9 +210,8 @@ public CompletableFuture offload(ReadHandle readHandle, return; } if (readHandle.getLength() <= 0) { - log.warn("[{}] has a ledger [{}] with a zero length, but it contains {} entries. since it contains" - + " entries, attempts to offload", topicName, readHandle.getLastAddConfirmed() + 1, - readHandle.getId()); + log.warn("[{}] Ledger [{}] has zero length, but it contains {} entries." + + " Attemping to offload ledger since it contains entries.", topicName, readHandle.getId(), readHandle.getLastAddConfirmed() + 1); } OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create() .withLedgerMetadata(readHandle.getLedgerMetadata()) From 4cf5b10ca070af5031c134e78054b07c8abd37a7 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Mon, 26 May 2025 15:56:48 +0800 Subject: [PATCH 4/4] fix typo --- .../filesystem/impl/FileSystemManagedLedgerOffloader.java | 3 ++- .../offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java index 2c312b92f62f3..40ae52d7b7910 100644 --- a/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java +++ b/tiered-storage/file-system/src/main/java/org/apache/bookkeeper/mledger/offload/filesystem/impl/FileSystemManagedLedgerOffloader.java @@ -201,7 +201,8 @@ public void run() { } if (readHandle.getLength() <= 0) { log.warn("Ledger [{}] has zero length, but it contains {} entries. " - + " Attemping to offload ledger since it contains entries.", readHandle.getId(), readHandle.getLastAddConfirmed() + 1); + + " Attempting to offload ledger since it contains entries.", readHandle.getId(), + readHandle.getLastAddConfirmed() + 1); } long ledgerId = readHandle.getId(); final String managedLedgerName = extraMetadata.get(MANAGED_LEDGER_NAME); diff --git a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java index 55e8a1195da65..33bbc49ee2223 100644 --- a/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java +++ b/tiered-storage/jcloud/src/main/java/org/apache/bookkeeper/mledger/offload/jcloud/impl/BlobStoreManagedLedgerOffloader.java @@ -211,7 +211,8 @@ public CompletableFuture offload(ReadHandle readHandle, } if (readHandle.getLength() <= 0) { log.warn("[{}] Ledger [{}] has zero length, but it contains {} entries." - + " Attemping to offload ledger since it contains entries.", topicName, readHandle.getId(), readHandle.getLastAddConfirmed() + 1); + + " Attempting to offload ledger since it contains entries.", topicName, readHandle.getId(), + readHandle.getLastAddConfirmed() + 1); } OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create() .withLedgerMetadata(readHandle.getLedgerMetadata())