From fb8f519be64487b60881d8d325190914d21e575d Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Thu, 5 Jun 2025 15:20:48 -0700 Subject: [PATCH] [llvm][cas] Prevent corruption on ENOSPC on sparse filesystems If a platform and filesystem do not detect disk out of space errors during page fault but instead defer it to page flush, we can corrupt the CAS data without receiving an error until it is too late. Fix that on platforms that support preallocating disk space by ensuring all CAS allocations are allocated to disk before writing. For standalone files, we ensure the entire file is allocated up front. For bump pointer allocated files (index, datapool, cache), we allocate in chunks of 1 MB to amortize the cost, and store the current disk-allocated size into the database file next to the bump pointer so that every thread and process can check their allocations. The overhead from these additional checks is <1% in a store-heavy stress test on Darwin/APFS, and effectively zero overhead on a non-sparse filesystem Darwin/tmpfs. rdar://152273395 (cherry picked from commit cca6e01fd75ee02af9980ed5079774050c2c32da) --- clang/test/CAS/depscan-cas-log.c | 6 +- clang/test/CAS/validate-once.c | 2 +- llvm/cmake/config-ix.cmake | 1 + .../llvm/CAS/MappedFileRegionBumpPtr.h | 10 +- llvm/include/llvm/Config/config.h.cmake | 3 + llvm/lib/CAS/MappedFileRegionBumpPtr.cpp | 102 +++++++++++++++--- llvm/lib/CAS/OnDiskCommon.cpp | 35 +++++- llvm/lib/CAS/OnDiskCommon.h | 8 ++ llvm/lib/CAS/OnDiskGraphDB.cpp | 5 +- llvm/lib/CAS/OnDiskKeyValueDB.cpp | 2 +- llvm/test/CAS/logging.test | 24 ++--- llvm/test/CAS/validate-if-needed.test | 8 +- llvm/test/tools/llvm-cas/validation.test | 4 +- llvm/tools/llvm-cas/llvm-cas.cpp | 2 +- .../CAS/OnDiskHashMappedTrieTest.cpp | 2 +- 15 files changed, 167 insertions(+), 47 deletions(-) diff --git a/clang/test/CAS/depscan-cas-log.c b/clang/test/CAS/depscan-cas-log.c index 714b03aa1a297..f00634343f44a 100644 --- a/clang/test/CAS/depscan-cas-log.c +++ b/clang/test/CAS/depscan-cas-log.c @@ -10,11 +10,11 @@ // RUN: -cc1-args -cc1 -triple x86_64-apple-macosx11.0.0 -emit-obj %s -o %t/t.o -fcas-path %t/cas // RUN: FileCheck %s --input-file %t/cas/v1.log -// CHECK: [[PID1:[0-9]*]] {{[0-9]*}}: mmap '{{.*}}v8.index' +// CHECK: [[PID1:[0-9]*]] {{[0-9]*}}: mmap '{{.*}}v9.index' // CHECK: [[PID1]] {{[0-9]*}}: create subtrie -// CHECK: [[PID2:[0-9]*]] {{[0-9]*}}: mmap '{{.*}}v8.index' +// CHECK: [[PID2:[0-9]*]] {{[0-9]*}}: mmap '{{.*}}v9.index' // Even a minimal compilation involves at least 9 records for the cache key. // CHECK-COUNT-9: [[PID2]] {{[0-9]*}}: create record -// CHECK: [[PID1]] {{[0-9]*}}: close mmap '{{.*}}v8.index' +// CHECK: [[PID1]] {{[0-9]*}}: close mmap '{{.*}}v9.index' diff --git a/clang/test/CAS/validate-once.c b/clang/test/CAS/validate-once.c index 774727fc64581..152c812cb976b 100644 --- a/clang/test/CAS/validate-once.c +++ b/clang/test/CAS/validate-once.c @@ -1,7 +1,7 @@ // RUN: rm -rf %t // RUN: llvm-cas --cas %t/cas --ingest %s -// RUN: mv %t/cas/v1.1/v8.data %t/cas/v1.1/v8.data.bak +// RUN: mv %t/cas/v1.1/v9.data %t/cas/v1.1/v9.data.bak // RUN: %clang -cc1depscand -execute %{clang-daemon-dir}/%basename_t -cas-args -fcas-path %t/cas -- \ // RUN: %clang -target x86_64-apple-macos11 -I %S/Inputs \ diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake index 0aae13e30f2ab..acc806f41afa8 100644 --- a/llvm/cmake/config-ix.cmake +++ b/llvm/cmake/config-ix.cmake @@ -298,6 +298,7 @@ check_symbol_exists(malloc_zone_statistics malloc/malloc.h HAVE_MALLOC_ZONE_STATISTICS) check_symbol_exists(getrlimit "sys/types.h;sys/time.h;sys/resource.h" HAVE_GETRLIMIT) check_symbol_exists(posix_spawn spawn.h HAVE_POSIX_SPAWN) +check_symbol_exists(posix_fallocate fcntl.h HAVE_POSIX_FALLOCATE) check_symbol_exists(pread unistd.h HAVE_PREAD) check_symbol_exists(sbrk unistd.h HAVE_SBRK) check_symbol_exists(strerror_r string.h HAVE_STRERROR_R) diff --git a/llvm/include/llvm/CAS/MappedFileRegionBumpPtr.h b/llvm/include/llvm/CAS/MappedFileRegionBumpPtr.h index 04e4cf2faca69..17f78225e30a2 100644 --- a/llvm/include/llvm/CAS/MappedFileRegionBumpPtr.h +++ b/llvm/include/llvm/CAS/MappedFileRegionBumpPtr.h @@ -78,7 +78,7 @@ class MappedFileRegionBumpPtr { Expected allocateOffset(uint64_t AllocSize); char *data() const { return Region.data(); } - uint64_t size() const { return *BumpPtr; } + uint64_t size() const { return H->BumpPtr; } uint64_t capacity() const { return Region.size(); } RegionT &getRegion() { return Region; } @@ -100,7 +100,7 @@ class MappedFileRegionBumpPtr { void destroyImpl(); void moveImpl(MappedFileRegionBumpPtr &RHS) { std::swap(Region, RHS.Region); - std::swap(BumpPtr, RHS.BumpPtr); + std::swap(H, RHS.H); std::swap(Path, RHS.Path); std::swap(FD, RHS.FD); std::swap(SharedLockFD, RHS.SharedLockFD); @@ -108,8 +108,12 @@ class MappedFileRegionBumpPtr { } private: + struct Header { + std::atomic BumpPtr; + std::atomic AllocatedSize; + }; RegionT Region; - std::atomic *BumpPtr = nullptr; + Header *H = nullptr; std::string Path; std::optional FD; std::optional SharedLockFD; diff --git a/llvm/include/llvm/Config/config.h.cmake b/llvm/include/llvm/Config/config.h.cmake index ff30741c8f360..0bce6d8f8f6ee 100644 --- a/llvm/include/llvm/Config/config.h.cmake +++ b/llvm/include/llvm/Config/config.h.cmake @@ -149,6 +149,9 @@ /* Define to 1 if you have the `posix_spawn' function. */ #cmakedefine HAVE_POSIX_SPAWN ${HAVE_POSIX_SPAWN} +/* Define to 1 if you have the `posix_fallocate' function. */ +#cmakedefine HAVE_POSIX_FALLOCATE ${HAVE_POSIX_FALLOCATE} + /* Define to 1 if you have the `pread' function. */ #cmakedefine HAVE_PREAD ${HAVE_PREAD} diff --git a/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp b/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp index e1f73dd9fc048..ed94387143431 100644 --- a/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp +++ b/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp @@ -54,6 +54,19 @@ #include "llvm/CAS/MappedFileRegionBumpPtr.h" #include "OnDiskCommon.h" #include "llvm/CAS/OnDiskCASLogger.h" +#include "llvm/Support/Compiler.h" + +#if LLVM_ON_UNIX +#include +#if __has_include() +#include +#endif +#ifdef DEV_BSIZE +#define MAPPED_FILE_BSIZE DEV_BSIZE +#elif __linux__ +#define MAPPED_FILE_BSIZE 512 +#endif +#endif using namespace llvm; using namespace llvm::cas; @@ -85,6 +98,13 @@ struct FileLockRAII { return Error::success(); } }; + +struct FileSizeInfo { + uint64_t Size; + uint64_t AllocatedSize; + + static ErrorOr get(sys::fs::file_t File); +}; } // end anonymous namespace Expected MappedFileRegionBumpPtr::create( @@ -123,39 +143,41 @@ Expected MappedFileRegionBumpPtr::create( return std::move(E); sys::fs::file_t File = sys::fs::convertFDToNativeFile(FD); - sys::fs::file_status Status; - if (std::error_code EC = sys::fs::status(File, Status)) - return createFileError(Result.Path, EC); + auto FileSize = FileSizeInfo::get(File); + if (!FileSize) + return createFileError(Result.Path, FileSize.getError()); - if (Status.getSize() < Capacity) { + if (FileSize->Size < Capacity) { // Lock the file exclusively so only one process will do the initialization. if (Error E = InitLock.unlock()) return std::move(E); if (Error E = InitLock.lock(FileLockRAII::Exclusive)) return std::move(E); // Retrieve the current size now that we have exclusive access. - if (std::error_code EC = sys::fs::status(File, Status)) - return createFileError(Result.Path, EC); + FileSize = FileSizeInfo::get(File); + if (!FileSize) + return createFileError(Result.Path, FileSize.getError()); } // At this point either the file is still under-sized, or we have the size for // the completely initialized file. - if (Status.getSize() < Capacity) { + if (FileSize->Size < Capacity) { // We are initializing the file; it may be empty, or may have been shrunk // during a previous close. // FIXME: Detect a case where someone opened it with a smaller capacity. // FIXME: On Windows we should use FSCTL_SET_SPARSE and FSCTL_SET_ZERO_DATA // to make this a sparse region, if supported. + assert(InitLock.Locked == FileLockRAII::Exclusive); if (std::error_code EC = sys::fs::resize_file(FD, Capacity)) return createFileError(Result.Path, EC); if (Result.Logger) Result.Logger->log_MappedFileRegionBumpPtr_resizeFile( - Result.Path, Status.getSize(), Capacity); + Result.Path, FileSize->Size, Capacity); } else { // Someone else initialized it. - Capacity = Status.getSize(); + Capacity = FileSize->Size; } // Create the mapped region. @@ -168,7 +190,8 @@ Expected MappedFileRegionBumpPtr::create( Result.Region = std::move(Map); } - if (Status.getSize() == 0) { + if (FileSize->Size == 0) { + assert(InitLock.Locked == FileLockRAII::Exclusive); // We are creating a new file; run the constructor. if (Error E = NewFileConstructor(Result)) return std::move(E); @@ -176,6 +199,16 @@ Expected MappedFileRegionBumpPtr::create( Result.initializeBumpPtr(BumpPtrOffset); } + if (FileSize->Size < Capacity && FileSize->AllocatedSize < Capacity) { + // We are initializing the file; sync the allocated size in case it + // changed when truncating or during construction. + FileSize = FileSizeInfo::get(File); + if (!FileSize) + return createFileError(Result.Path, FileSize.getError()); + assert(InitLock.Locked == FileLockRAII::Exclusive); + Result.H->AllocatedSize.exchange(FileSize->AllocatedSize); + } + return Result; } @@ -189,7 +222,7 @@ void MappedFileRegionBumpPtr::destroyImpl() { // Attempt to truncate the file if we can get exclusive access. Ignore any // errors. - if (BumpPtr) { + if (H) { assert(SharedLockFD && "Must have shared lock file open"); if (tryLockFileThreadSafe(*SharedLockFD) == std::error_code()) { size_t Size = size(); @@ -223,15 +256,15 @@ void MappedFileRegionBumpPtr::destroyImpl() { void MappedFileRegionBumpPtr::initializeBumpPtr(int64_t BumpPtrOffset) { assert(capacity() < (uint64_t)INT64_MAX && "capacity must fit in int64_t"); - int64_t BumpPtrEndOffset = BumpPtrOffset + sizeof(decltype(*BumpPtr)); + int64_t BumpPtrEndOffset = BumpPtrOffset + sizeof(decltype(*H)); assert(BumpPtrEndOffset <= (int64_t)capacity() && "Expected end offset to be pre-allocated"); - assert(isAligned(Align::Of(), BumpPtrOffset) && + assert(isAligned(Align::Of(), BumpPtrOffset) && "Expected end offset to be aligned"); - BumpPtr = reinterpret_cast(data() + BumpPtrOffset); + H = reinterpret_cast(data() + BumpPtrOffset); int64_t ExistingValue = 0; - if (!BumpPtr->compare_exchange_strong(ExistingValue, BumpPtrEndOffset)) + if (!H->BumpPtr.compare_exchange_strong(ExistingValue, BumpPtrEndOffset)) assert(ExistingValue >= BumpPtrEndOffset && "Expected 0, or past the end of the BumpPtr itself"); @@ -247,7 +280,7 @@ static Error createAllocatorOutOfSpaceError() { Expected MappedFileRegionBumpPtr::allocateOffset(uint64_t AllocSize) { AllocSize = alignTo(AllocSize, getAlign()); - int64_t OldEnd = BumpPtr->fetch_add(AllocSize); + int64_t OldEnd = H->BumpPtr.fetch_add(AllocSize); int64_t NewEnd = OldEnd + AllocSize; if (LLVM_UNLIKELY(NewEnd > (int64_t)capacity())) { // Return the allocation. If the start already passed the end, that means @@ -257,7 +290,7 @@ Expected MappedFileRegionBumpPtr::allocateOffset(uint64_t AllocSize) { // All other allocation afterwards must have failed and current allocation // is in charge of return the allocation back to a valid value. if (OldEnd <= (int64_t)capacity()) - (void)BumpPtr->exchange(OldEnd); + (void)H->BumpPtr.exchange(OldEnd); if (Logger) Logger->log_MappedFileRegionBumpPtr_oom(Path, capacity(), OldEnd, @@ -266,8 +299,43 @@ Expected MappedFileRegionBumpPtr::allocateOffset(uint64_t AllocSize) { return createAllocatorOutOfSpaceError(); } + int64_t DiskSize = H->AllocatedSize; + if (LLVM_UNLIKELY(NewEnd > DiskSize)) { + int64_t NewSize; + // The minimum increment is a page, but allocate more to amortize the cost. + constexpr int64_t Increment = 1 * 1024 * 1024; // 1 MB + if (Error E = preallocateFileTail(*FD, DiskSize, DiskSize + Increment).moveInto(NewSize)) + return std::move(E); + assert(NewSize >= DiskSize + Increment); + // FIXME: on Darwin this can under-count the size if there is a race to + // preallocate disk, because the semantics of F_PREALLOCATE are to add bytes + // to the end of the file, not to allocate up to a fixed size. + // Any discrepancy will be resolved the next time the file is truncated and + // then reopend. + while (DiskSize < NewSize) + H->AllocatedSize.compare_exchange_strong(DiskSize, NewSize); + } + if (Logger) Logger->log_MappedFileRegionBumpPtr_allocate(data(), OldEnd, AllocSize); return OldEnd; } + +ErrorOr FileSizeInfo::get(sys::fs::file_t File) { +#if LLVM_ON_UNIX && defined(MAPPED_FILE_BSIZE) + struct stat Status; + int StatRet = ::fstat(File, &Status); + if (StatRet) + return errnoAsErrorCode(); + uint64_t AllocatedSize = uint64_t(Status.st_blksize) * MAPPED_FILE_BSIZE; + return FileSizeInfo{uint64_t(Status.st_size), AllocatedSize}; +#else + // Fallback: assume the file is fully allocated. Note: this may result in + // data loss on out-of-space. + sys::fs::file_status Status; + if (std::error_code EC = sys::fs::status(File, Status)) + return EC; + return FileSizeInfo{Status.getSize(), Status.getSize()}; +#endif +} \ No newline at end of file diff --git a/llvm/lib/CAS/OnDiskCommon.cpp b/llvm/lib/CAS/OnDiskCommon.cpp index 77eb73f810bb3..3465500b128c4 100644 --- a/llvm/lib/CAS/OnDiskCommon.cpp +++ b/llvm/lib/CAS/OnDiskCommon.cpp @@ -8,6 +8,7 @@ #include "OnDiskCommon.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Config/config.h" #include "llvm/Support/Error.h" #include "llvm/Support/Process.h" #include @@ -23,6 +24,10 @@ #endif #endif +#if __has_include() +#include +#endif + using namespace llvm; static uint64_t OnDiskCASMaxMappingSize = 0; @@ -107,4 +112,32 @@ cas::ondisk::tryLockFileThreadSafe(int FD, std::chrono::milliseconds Timeout, #else return make_error_code(std::errc::no_lock_available); #endif -} \ No newline at end of file +} + +Expected cas::ondisk::preallocateFileTail(int FD, size_t CurrentSize, size_t NewSize) { + auto CreateErrorFromErrno = [&]() -> Expected { + std::error_code EC = errnoAsErrorCode(); + if (EC == std::errc::not_supported) + // Ignore ENOTSUP in case the filesystem cannot preallocate. + return NewSize; + return createStringError(EC, "failed to allocate to CAS file: " + EC.message()); + }; +#if defined(HAVE_POSIX_FALLOCATE) + if (posix_fallocate(FD, CurrentSize, NewSize - CurrentSize)) + return CreateErrorFromErrno(); + return NewSize; +#elif defined(__APPLE__) + fstore_t FAlloc; + FAlloc.fst_flags = F_ALLOCATEALL | F_ALLOCATEPERSIST; + FAlloc.fst_posmode = F_PEOFPOSMODE; + FAlloc.fst_offset = 0; + FAlloc.fst_length = NewSize - CurrentSize; + FAlloc.fst_bytesalloc = 0; + if (fcntl(FD, F_PREALLOCATE, &FAlloc)) + return CreateErrorFromErrno(); + assert(CurrentSize + FAlloc.fst_bytesalloc >= NewSize); + return CurrentSize + FAlloc.fst_bytesalloc; +#else + return NewSize; // Pretend it worked. +#endif +} diff --git a/llvm/lib/CAS/OnDiskCommon.h b/llvm/lib/CAS/OnDiskCommon.h index c8dc0c998a95e..56f5a0ed2ab78 100644 --- a/llvm/lib/CAS/OnDiskCommon.h +++ b/llvm/lib/CAS/OnDiskCommon.h @@ -43,6 +43,14 @@ std::error_code tryLockFileThreadSafe( int FD, std::chrono::milliseconds Timeout = std::chrono::milliseconds(0), bool Exclusive = true); +/// Allocate space for the file \p FD on disk, if the filesystem supports it. +/// +/// On filesystems that support this operation, this ensures errors such as +/// \c std::errc::no_space_on_device are detected before we write data. +/// +/// \returns the new size of the file, or an \c Error. +Expected preallocateFileTail(int FD, size_t CurrentSize, size_t NewSize); + } // namespace llvm::cas::ondisk #endif // LLVM_LIB_CAS_ONDISKCOMMON_H diff --git a/llvm/lib/CAS/OnDiskGraphDB.cpp b/llvm/lib/CAS/OnDiskGraphDB.cpp index 9db86ba9d6fe1..99c9dd5f681c6 100644 --- a/llvm/lib/CAS/OnDiskGraphDB.cpp +++ b/llvm/lib/CAS/OnDiskGraphDB.cpp @@ -81,7 +81,7 @@ static constexpr StringLiteral DataPoolTableName = "llvm.cas.data"; static constexpr StringLiteral IndexFile = "index"; static constexpr StringLiteral DataPoolFile = "data"; -static constexpr StringLiteral FilePrefix = "v8."; +static constexpr StringLiteral FilePrefix = "v9."; static constexpr StringLiteral FileSuffixData = ".data"; static constexpr StringLiteral FileSuffixLeaf = ".leaf"; static constexpr StringLiteral FileSuffixLeaf0 = ".leaf+0"; @@ -1311,6 +1311,9 @@ OnDiskGraphDB::createTempFile(StringRef FinalPath, uint64_t Size) { if (!File) return File.takeError(); + if (Error E = preallocateFileTail(File->FD, 0, Size).takeError()) + return createFileError(File->TmpName, std::move(E)); + if (auto EC = sys::fs::resize_file_before_mapping_readwrite(File->FD, Size)) return createFileError(File->TmpName, EC); diff --git a/llvm/lib/CAS/OnDiskKeyValueDB.cpp b/llvm/lib/CAS/OnDiskKeyValueDB.cpp index f3a0e863d1dab..ba248281c414a 100644 --- a/llvm/lib/CAS/OnDiskKeyValueDB.cpp +++ b/llvm/lib/CAS/OnDiskKeyValueDB.cpp @@ -19,7 +19,7 @@ using namespace llvm::cas; using namespace llvm::cas::ondisk; static constexpr StringLiteral ActionCacheFile = "actions"; -static constexpr StringLiteral FilePrefix = "v3."; +static constexpr StringLiteral FilePrefix = "v4."; Expected> OnDiskKeyValueDB::put(ArrayRef Key, ArrayRef Value) { diff --git a/llvm/test/CAS/logging.test b/llvm/test/CAS/logging.test index 98dd6cd9ff001..e1fee9920479c 100644 --- a/llvm/test/CAS/logging.test +++ b/llvm/test/CAS/logging.test @@ -6,24 +6,24 @@ RUN: env LLVM_CAS_LOG=2 llvm-cas --cas %t/cas --validate-if-needed -force -allow RUN: FileCheck %s --input-file %t/cas/v1.log -// CHECK: resize mapped file '{{.*}}v8.index' -// CHECK: mmap '{{.*}}v8.index' [[INDEX:0x[0-9a-f]+]] -// CHECK: resize mapped file '{{.*}}v8.data' -// CHECK: mmap '{{.*}}v8.data' [[DATA:0x[0-9a-f]+]] -// CHECK: resize mapped file '{{.*}}v3.actions' -// CHECK: mmap '{{.*}}v3.actions' [[ACTIONS:0x[0-9a-f]+]] +// CHECK: resize mapped file '{{.*}}v9.index' +// CHECK: mmap '{{.*}}v9.index' [[INDEX:0x[0-9a-f]+]] +// CHECK: resize mapped file '{{.*}}v9.data' +// CHECK: mmap '{{.*}}v9.data' [[DATA:0x[0-9a-f]+]] +// CHECK: resize mapped file '{{.*}}v4.actions' +// CHECK: mmap '{{.*}}v4.actions' [[ACTIONS:0x[0-9a-f]+]] // store input/a contents into the datapool // CHECK: create record region=[[INDEX]] offset=[[INPUT_A_OFF:0x[0-9a-f]+]] hash=9b096cd140f119 // CHECK: cmpxcgh subtrie region=[[INDEX]] offset={{.*}} slot={{.*}} expected=0x0 new=[[INPUT_A_OFF]] prev=0x0 // CHECK: alloc [[DATA]] -// CHECK: resize mapped file '{{.*}}v3.actions' -// CHECK: close mmap '{{.*}}v3.actions' -// CHECK: resize mapped file '{{.*}}v8.data' -// CHECK: close mmap '{{.*}}v8.data' -// CHECK: resize mapped file '{{.*}}v8.index' -// CHECK: close mmap '{{.*}}v8.index' +// CHECK: resize mapped file '{{.*}}v4.actions' +// CHECK: close mmap '{{.*}}v4.actions' +// CHECK: resize mapped file '{{.*}}v9.data' +// CHECK: close mmap '{{.*}}v9.data' +// CHECK: resize mapped file '{{.*}}v9.index' +// CHECK: close mmap '{{.*}}v9.index' // CHECK: validate-if-needed '{{.*}}cas' boot=[[BOOT:[0-9]+]] last-valid=0 check-hash=1 allow-recovery=0 force=0 llvm-cas={{.*}}llvm-cas // CHECK: validate-if-needed '{{.*}}cas' boot=[[BOOT]] last-valid=[[BOOT]] check-hash=0 allow-recovery=1 force=1 llvm-cas={{.*}}llvm-cas diff --git a/llvm/test/CAS/validate-if-needed.test b/llvm/test/CAS/validate-if-needed.test index c2d359c153849..0e8c28a23a36e 100644 --- a/llvm/test/CAS/validate-if-needed.test +++ b/llvm/test/CAS/validate-if-needed.test @@ -1,6 +1,6 @@ RUN: rm -rf %t && mkdir %t RUN: llvm-cas --cas %t/cas --ingest %S/Inputs > %t/cas.id -RUN: mv %t/cas/v1.1/v8.data %t/cas/v1.1/v8.data.bak +RUN: mv %t/cas/v1.1/v9.data %t/cas/v1.1/v9.data.bak # INVALID: bad record # VALID: validated successfully @@ -12,7 +12,7 @@ RUN: not llvm-cas --cas %t/cas --validate-if-needed 2>&1 | FileCheck %s -check-p RUN: not llvm-cas --cas %t/cas --validate-if-needed 2>&1 | FileCheck %s -check-prefix=INVALID # Validation happens once per boot. -RUN: mv %t/cas/v1.1/v8.data.bak %t/cas/v1.1/v8.data +RUN: mv %t/cas/v1.1/v9.data.bak %t/cas/v1.1/v9.data RUN: llvm-cas --cas %t/cas --validate-if-needed | FileCheck %s -check-prefix=VALID RUN: llvm-cas --cas %t/cas --validate-if-needed | FileCheck %s -check-prefix=SKIPPED # Wrong timestamp triggers re-validation. @@ -20,7 +20,7 @@ RUN: echo '123' > %t/cas/v1.validation RUN: llvm-cas --cas %t/cas --validate-if-needed | FileCheck %s -check-prefix=VALID RUN: llvm-cas --cas %t/cas --validate-if-needed | FileCheck %s -check-prefix=SKIPPED # Skipped validation does not catch errors. -RUN: mv %t/cas/v1.1/v8.data %t/cas/v1.1/v8.data.bak +RUN: mv %t/cas/v1.1/v9.data %t/cas/v1.1/v9.data.bak RUN: llvm-cas --cas %t/cas --validate-if-needed | FileCheck %s -check-prefix=SKIPPED # Unless forced. @@ -33,7 +33,7 @@ RUN: llvm-cas --cas %t/cas --validate-if-needed --allow-recovery | FileCheck %s RUN: llvm-cas --cas %t/cas --validate-if-needed --force | FileCheck %s -check-prefix=VALID RUN: rm -rf %t/cas/v1.1 RUN: cp -r %t/cas/corrupt.0.v1.1 %t/cas/v1.1 -RUN: mv %t/cas/v1.1/v8.data %t/cas/v1.1/v8.data.bak +RUN: mv %t/cas/v1.1/v9.data %t/cas/v1.1/v9.data.bak RUN: llvm-cas --cas %t/cas --validate-if-needed --allow-recovery --force | FileCheck %s -check-prefix=RECOVERED RUN: ls %t/cas/corrupt.1.v1.1 diff --git a/llvm/test/tools/llvm-cas/validation.test b/llvm/test/tools/llvm-cas/validation.test index 7385fc37a4132..6faed5293098c 100644 --- a/llvm/test/tools/llvm-cas/validation.test +++ b/llvm/test/tools/llvm-cas/validation.test @@ -12,7 +12,7 @@ RUN: llvm-cas --cas %t/cas --ingest %S/Inputs > %t/cas.id RUN: llvm-cas --cas %t/cas --validate RUN: llvm-cas --cas %t/cas --validate --check-hash -RUN: rm %t/cas/v1.1/v8.data +RUN: rm %t/cas/v1.1/v9.data RUN: not llvm-cas --cas %t/cas --validate RUN: not llvm-cas --cas %t/cas --validate --check-hash @@ -28,5 +28,5 @@ RUN: llvm-cas --cas %t/ac --put-cache-key @%t/abc.casid @%t/empty.casid RUN: llvm-cas --cas %t/ac --validate # Note: records are 40 bytes (32 hash bytes + 8 byte value), so trim the last # allocated record, leaving it invalid. -RUN: truncate -s -40 %t/ac/v1.1/v3.actions +RUN: truncate -s -40 %t/ac/v1.1/v4.actions RUN: not llvm-cas --cas %t/ac --validate diff --git a/llvm/tools/llvm-cas/llvm-cas.cpp b/llvm/tools/llvm-cas/llvm-cas.cpp index 40ccf5c1ec57a..73721a40e961b 100644 --- a/llvm/tools/llvm-cas/llvm-cas.cpp +++ b/llvm/tools/llvm-cas/llvm-cas.cpp @@ -703,7 +703,7 @@ static int checkLockFiles(StringRef CASPath) { ExitOnError ExitOnErr("llvm-cas: check-lock-files: "); SmallString<128> DataPoolPath(CASPath); - sys::path::append(DataPoolPath, "v1.1/v8.data"); + sys::path::append(DataPoolPath, "v1.1/v9.data"); auto OpenCASAndGetDataPoolSize = [&]() -> Expected { auto Result = createOnDiskUnifiedCASDatabases(CASPath); diff --git a/llvm/unittests/CAS/OnDiskHashMappedTrieTest.cpp b/llvm/unittests/CAS/OnDiskHashMappedTrieTest.cpp index 6434913248889..ab9f0b09aff53 100644 --- a/llvm/unittests/CAS/OnDiskHashMappedTrieTest.cpp +++ b/llvm/unittests/CAS/OnDiskHashMappedTrieTest.cpp @@ -160,7 +160,7 @@ TEST(OnDiskHashMappedTrieTest, OutOfSpace) { // Just enough for root node but not enough for any insertion. ASSERT_THAT_ERROR(OnDiskHashMappedTrie::create( Temp.path("NoSpace2").str(), "index", - /*NumHashBits=*/8, /*DataSize=*/8, /*MaxFileSize=*/100, + /*NumHashBits=*/8, /*DataSize=*/8, /*MaxFileSize=*/108, /*NewInitialFileSize=*/std::nullopt, /*Logger=*/nullptr, /*NewTableNumRootBits=*/1, /*NewTableNumSubtrieBits=*/1) .moveInto(Trie),