Skip to content

fix(dotc1z): add fsync and size validation to prevent c1z corruption #627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
arreyder wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(dotc1z): add fsync and size validation to prevent c1z corruption #627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6e24cb0
fix(dotc1z): add fsync and size validation to prevent c1z corruption
arreyder Jan 9, 2026
670333b
fix: use O_RDWR for fsync to fix Windows test failure
arreyder Jan 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions pkg/dotc1z/c1file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,25 @@ func (c *C1File) CloseContext(ctx context.Context) error {
if c.readOnly {
return cleanupDbDir(c.dbFilePath, ErrReadOnly)
}

// CRITICAL: Ensure database file is synced to disk before reading for compression.
// On filesystems with aggressive caching (like ZFS with ARC), data written by
// sqlite during Close() may still be in kernel buffers. Without this explicit
// fsync, saveC1z() could read stale/incomplete data, producing a truncated
// zstd stream that appears valid but is missing the end-of-stream marker.
// Note: O_RDWR is required because Sync() on read-only fd fails on Windows.
dbFile, err := os.OpenFile(c.dbFilePath, os.O_RDWR, 0)
if err != nil {
return cleanupDbDir(c.dbFilePath, fmt.Errorf("open db for sync: %w", err))
}
if err := dbFile.Sync(); err != nil {
dbFile.Close()
return cleanupDbDir(c.dbFilePath, fmt.Errorf("sync db before compress: %w", err))
}
if err := dbFile.Close(); err != nil {
return cleanupDbDir(c.dbFilePath, fmt.Errorf("close db after sync: %w", err))
}

err = saveC1z(c.dbFilePath, c.outputFilePath, c.encoderConcurrency)
if err != nil {
return cleanupDbDir(c.dbFilePath, err)
Expand Down
13 changes: 13 additions & 0 deletions pkg/dotc1z/manager/local/local.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,24 @@ func (l *localManager) SaveC1Z(ctx context.Context) error {
}
defer dstFile.Close()

// Get source file size before copy for validation
srcStat, err := tmpFile.Stat()
if err != nil {
return fmt.Errorf("failed to stat source file: %w", err)
}
expectedSize := srcStat.Size()

size, err := io.Copy(dstFile, tmpFile)
if err != nil {
return err
}

// CRITICAL: Validate copy was complete. A truncated copy would result in
// a corrupt c1z file missing the zstd end-of-stream marker.
if size != expectedSize {
return fmt.Errorf("copy truncated: copied %d bytes, expected %d bytes", size, expectedSize)
}

// CRITICAL: Sync to ensure data is written before function returns.
// This is especially important on ZFS ARC where writes may be cached.
if err := dstFile.Sync(); err != nil {
Expand Down
Loading