From 575d03f2eae44ff37733ca2ddaab674f1a0c8fd9 Mon Sep 17 00:00:00 2001 From: Wael Nasreddine Date: Wed, 11 Feb 2026 21:00:51 -0800 Subject: [PATCH 1/2] test: add end-to-end test for CDC chunk compression This commit adds a new test case 'testCDCChunksAreCompressed' to the CDC test suite in pkg/cache/cdc_test.go. This test verifies that when Content-Defined Chunking (CDC) is enabled, chunks are transparently compressed using ZSTD before being stored in the chunk store. Key validations: - Chunks are stored with the expected compression. - The 'compressed_size' is correctly tracked in the database. - Data reassembly (GetNar) is transparent and returns the original decompressed content. This ensures that the recent ZSTD compression features for Local and S3 chunk stores are working correctly in an end-to-end workflow. --- pkg/cache/cdc_test.go | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pkg/cache/cdc_test.go b/pkg/cache/cdc_test.go index 54591808..f01a5225 100644 --- a/pkg/cache/cdc_test.go +++ b/pkg/cache/cdc_test.go @@ -51,6 +51,7 @@ func runCDCTestSuite(t *testing.T, factory cacheFactory) { t.Run("Mixed Mode", testCDCMixedMode(factory)) t.Run("GetNarInfo with CDC chunks", testCDCGetNarInfo(factory)) t.Run("Client Disconnect No Goroutine Leak", testCDCClientDisconnectNoGoroutineLeak(factory)) + t.Run("chunks are stored compressed", testCDCChunksAreCompressed(factory)) } func testCDCPutAndGet(factory cacheFactory) func(*testing.T) { @@ -311,3 +312,47 @@ func testCDCClientDisconnectNoGoroutineLeak(factory cacheFactory) func(*testing. "Goroutine leak detected: baseline=%d, final=%d", baselineGoroutines, finalGoroutines) } } + +func testCDCChunksAreCompressed(factory cacheFactory) func(*testing.T) { + return func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + c, db, _, dir, _, cleanup := factory(t) + t.Cleanup(cleanup) + + // Initialize chunk store + chunkStoreDir := filepath.Join(dir, "chunks-store") + chunkStore, err := chunk.NewLocalStore(chunkStoreDir) + require.NoError(t, err) + + c.SetChunkStore(chunkStore) + err = c.SetCDCConfiguration(true, 1024, 4096, 8192) // Small sizes for testing + require.NoError(t, err) + + // Use highly compressible data (repeated bytes) + content := strings.Repeat("compressible", 1000) + nu := nar.URL{Hash: "testnar-compress", Compression: nar.CompressionTypeNone} + + r := io.NopCloser(strings.NewReader(content)) + err = c.PutNar(ctx, nu, r) + require.NoError(t, err) + + // Verify chunks exist in DB and have compressed_size set + chunkCount, err := db.GetChunkCount(ctx) + require.NoError(t, err) + assert.Positive(t, chunkCount) + + // Verify reassembly to ensure compression is transparent + size, rc, err := c.GetNar(ctx, nu) + require.NoError(t, err) + + defer rc.Close() + + data, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, content, string(data), "decompressed data should match original") + assert.Equal(t, int64(len(content)), size, "size should match original content size") + } +} From a6f74006a26faa439021ce7045a75fdc727b3cbf Mon Sep 17 00:00:00 2001 From: Wael Nasreddine Date: Wed, 11 Feb 2026 23:48:44 -0800 Subject: [PATCH 2/2] fix: address PR comment regarding chunk compression validation --- db/query.mysql.sql | 6 ++-- db/query.postgres.sql | 6 ++-- db/query.sqlite.sql | 6 ++-- pkg/cache/cache.go | 9 +++-- pkg/cache/cdc_test.go | 23 +++++++++++-- pkg/chunker/chunker.go | 9 ++--- pkg/database/generated_models.go | 13 ++----- pkg/database/generated_querier.go | 8 ++--- pkg/database/generated_wrapper_mysql.go | 13 ++++--- pkg/database/generated_wrapper_postgres.go | 13 ++++--- pkg/database/generated_wrapper_sqlite.go | 13 ++++--- pkg/database/mysqldb/querier.go | 8 ++--- pkg/database/mysqldb/query.mysql.sql.go | 34 ++++++++----------- pkg/database/postgresdb/querier.go | 8 ++--- pkg/database/postgresdb/query.postgres.sql.go | 34 ++++++++----------- pkg/database/sqlitedb/querier.go | 8 ++--- pkg/database/sqlitedb/query.sqlite.sql.go | 34 ++++++++----------- 17 files changed, 126 insertions(+), 119 deletions(-) diff --git a/db/query.mysql.sql b/db/query.mysql.sql index b9a17312..cfa2b97d 100644 --- a/db/query.mysql.sql +++ b/db/query.mysql.sql @@ -280,7 +280,7 @@ FROM chunks WHERE id = ?; -- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? @@ -288,9 +288,9 @@ ORDER BY nfc.chunk_index; -- name: CreateChunk :execresult INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id), diff --git a/db/query.postgres.sql b/db/query.postgres.sql index f4666057..af164eea 100644 --- a/db/query.postgres.sql +++ b/db/query.postgres.sql @@ -300,7 +300,7 @@ FROM chunks WHERE id = $1; -- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = $1 @@ -308,9 +308,9 @@ ORDER BY nfc.chunk_index; -- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - $1, $2 + $1, $2, $3 ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP diff --git a/db/query.sqlite.sql b/db/query.sqlite.sql index a6373895..9e00ce87 100644 --- a/db/query.sqlite.sql +++ b/db/query.sqlite.sql @@ -286,7 +286,7 @@ FROM chunks WHERE id = ?; -- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? @@ -294,9 +294,9 @@ ORDER BY nfc.chunk_index; -- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index c6da7209..65424bdc 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1346,7 +1346,7 @@ func (c *Cache) storeNarWithCDC(ctx context.Context, tempPath string, narURL *na } // Store in chunkStore if new - _, _, err = chunkStore.PutChunk(ctx, chunkMetadata.Hash, chunkMetadata.Data) + _, compressedSize, err := chunkStore.PutChunk(ctx, chunkMetadata.Hash, chunkMetadata.Data) if err != nil { chunkMetadata.Free() @@ -1354,6 +1354,8 @@ func (c *Cache) storeNarWithCDC(ctx context.Context, tempPath string, narURL *na } chunkMetadata.Free() + //nolint:gosec // G115: Chunk size is small enough to fit in uint32 + chunkMetadata.CompressedSize = uint32(compressedSize) totalSize += int64(chunkMetadata.Size) @@ -1415,8 +1417,9 @@ func (c *Cache) recordChunkBatch(ctx context.Context, narFileID int64, startInde for i, chunkMetadata := range batch { // Create or increment ref count. ch, err := qtx.CreateChunk(ctx, database.CreateChunkParams{ - Hash: chunkMetadata.Hash, - Size: chunkMetadata.Size, + Hash: chunkMetadata.Hash, + Size: chunkMetadata.Size, + CompressedSize: chunkMetadata.CompressedSize, }) if err != nil { return fmt.Errorf("error creating chunk record: %w", err) diff --git a/pkg/cache/cdc_test.go b/pkg/cache/cdc_test.go index f01a5225..72b2982c 100644 --- a/pkg/cache/cdc_test.go +++ b/pkg/cache/cdc_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/kalbasit/ncps/pkg/database" "github.com/kalbasit/ncps/pkg/nar" "github.com/kalbasit/ncps/pkg/storage/chunk" ) @@ -340,9 +341,27 @@ func testCDCChunksAreCompressed(factory cacheFactory) func(*testing.T) { require.NoError(t, err) // Verify chunks exist in DB and have compressed_size set - chunkCount, err := db.GetChunkCount(ctx) + narFile, err := db.GetNarFileByHashAndCompressionAndQuery(ctx, database.GetNarFileByHashAndCompressionAndQueryParams{ + Hash: nu.Hash, + Compression: nu.Compression.String(), + Query: nu.Query.Encode(), + }) + require.NoError(t, err) + + chunks, err := db.GetChunksByNarFileID(ctx, narFile.ID) require.NoError(t, err) - assert.Positive(t, chunkCount) + require.NotEmpty(t, chunks, "should have chunks in the database") + + var totalSize, totalCompressedSize int64 + for _, chunk := range chunks { + totalSize += int64(chunk.Size) + totalCompressedSize += int64(chunk.CompressedSize) + assert.Positive(t, chunk.CompressedSize, "compressed size should be positive") + } + + assert.Equal(t, int64(len(content)), totalSize, "sum of chunk sizes should equal original content size") + assert.Less(t, totalCompressedSize, totalSize, + "total compressed size should be less than total original size for compressible data") // Verify reassembly to ensure compression is transparent size, rc, err := c.GetNar(ctx, nu) diff --git a/pkg/chunker/chunker.go b/pkg/chunker/chunker.go index 10f67712..4278a0b6 100644 --- a/pkg/chunker/chunker.go +++ b/pkg/chunker/chunker.go @@ -13,10 +13,11 @@ import ( // Chunk represents a single content-defined chunk. type Chunk struct { - Hash string // BLAKE3 hash of chunk content - Offset int64 // Offset in original stream - Size uint32 // Chunk size in bytes - Data []byte // Chunk data + Hash string // BLAKE3 hash of chunk content + Offset int64 // Offset in original stream + Size uint32 // Chunk size in bytes + CompressedSize uint32 // Compressed chunk size in bytes + Data []byte // Chunk data free func() // function to return Data to the pool } diff --git a/pkg/database/generated_models.go b/pkg/database/generated_models.go index 9db36584..798b852b 100644 --- a/pkg/database/generated_models.go +++ b/pkg/database/generated_models.go @@ -44,8 +44,9 @@ type Config struct { } type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } type CreateConfigParams struct { @@ -94,14 +95,6 @@ type GetChunkByNarFileIDAndIndexRow struct { UpdatedAt sql.NullTime } -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - type GetCompressedNarInfosParams struct { Limit int32 Offset int32 diff --git a/pkg/database/generated_querier.go b/pkg/database/generated_querier.go index 5571551a..1ce21e9c 100644 --- a/pkg/database/generated_querier.go +++ b/pkg/database/generated_querier.go @@ -42,9 +42,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // $1, $2 + // $1, $2, $3 // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP @@ -159,12 +159,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = $1 // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, system, ca diff --git a/pkg/database/generated_wrapper_mysql.go b/pkg/database/generated_wrapper_mysql.go index 22583bf7..ebf71d60 100644 --- a/pkg/database/generated_wrapper_mysql.go +++ b/pkg/database/generated_wrapper_mysql.go @@ -74,8 +74,9 @@ func (w *mysqlWrapper) CreateChunk(ctx context.Context, arg CreateChunkParams) ( // MySQL does not support RETURNING for INSERTs. // We insert, get LastInsertId, and then fetch the object. res, err := w.adapter.CreateChunk(ctx, mysqldb.CreateChunkParams{ - Hash: arg.Hash, - Size: arg.Size, + Hash: arg.Hash, + Size: arg.Size, + CompressedSize: arg.CompressedSize, }) if err != nil { return Chunk{}, err @@ -356,7 +357,7 @@ func (w *mysqlWrapper) GetChunkCount(ctx context.Context) (int64, error) { return res, nil } -func (w *mysqlWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (w *mysqlWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.GetChunksByNarFileID(ctx, narFileID) @@ -365,15 +366,17 @@ func (w *mysqlWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64 } // Convert Slice of Domain Structs - items := make([]GetChunksByNarFileIDRow, len(res)) + items := make([]Chunk, len(res)) for i, v := range res { - items[i] = GetChunksByNarFileIDRow{ + items[i] = Chunk{ ID: v.ID, Hash: v.Hash, Size: v.Size, + CompressedSize: v.CompressedSize, + CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, diff --git a/pkg/database/generated_wrapper_postgres.go b/pkg/database/generated_wrapper_postgres.go index 307e2b3f..6551546e 100644 --- a/pkg/database/generated_wrapper_postgres.go +++ b/pkg/database/generated_wrapper_postgres.go @@ -54,8 +54,9 @@ func (w *postgresWrapper) CreateChunk(ctx context.Context, arg CreateChunkParams /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.CreateChunk(ctx, postgresdb.CreateChunkParams{ - Hash: arg.Hash, - Size: arg.Size, + Hash: arg.Hash, + Size: arg.Size, + CompressedSize: arg.CompressedSize, }) if err != nil { @@ -408,7 +409,7 @@ func (w *postgresWrapper) GetChunkCount(ctx context.Context) (int64, error) { return res, nil } -func (w *postgresWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (w *postgresWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.GetChunksByNarFileID(ctx, narFileID) @@ -417,15 +418,17 @@ func (w *postgresWrapper) GetChunksByNarFileID(ctx context.Context, narFileID in } // Convert Slice of Domain Structs - items := make([]GetChunksByNarFileIDRow, len(res)) + items := make([]Chunk, len(res)) for i, v := range res { - items[i] = GetChunksByNarFileIDRow{ + items[i] = Chunk{ ID: v.ID, Hash: v.Hash, Size: v.Size, + CompressedSize: v.CompressedSize, + CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, diff --git a/pkg/database/generated_wrapper_sqlite.go b/pkg/database/generated_wrapper_sqlite.go index 8762a4cb..ca5cc286 100644 --- a/pkg/database/generated_wrapper_sqlite.go +++ b/pkg/database/generated_wrapper_sqlite.go @@ -72,8 +72,9 @@ func (w *sqliteWrapper) CreateChunk(ctx context.Context, arg CreateChunkParams) /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.CreateChunk(ctx, sqlitedb.CreateChunkParams{ - Hash: arg.Hash, - Size: arg.Size, + Hash: arg.Hash, + Size: arg.Size, + CompressedSize: arg.CompressedSize, }) if err != nil { @@ -426,7 +427,7 @@ func (w *sqliteWrapper) GetChunkCount(ctx context.Context) (int64, error) { return res, nil } -func (w *sqliteWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (w *sqliteWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.GetChunksByNarFileID(ctx, narFileID) @@ -435,15 +436,17 @@ func (w *sqliteWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int6 } // Convert Slice of Domain Structs - items := make([]GetChunksByNarFileIDRow, len(res)) + items := make([]Chunk, len(res)) for i, v := range res { - items[i] = GetChunksByNarFileIDRow{ + items[i] = Chunk{ ID: v.ID, Hash: v.Hash, Size: v.Size, + CompressedSize: v.CompressedSize, + CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, diff --git a/pkg/database/mysqldb/querier.go b/pkg/database/mysqldb/querier.go index 311013b4..3d3e464d 100644 --- a/pkg/database/mysqldb/querier.go +++ b/pkg/database/mysqldb/querier.go @@ -29,9 +29,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // ?, ? + // ?, ?, ? // ) // ON DUPLICATE KEY UPDATE // id = LAST_INSERT_ID(id), @@ -144,12 +144,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, `system`, ca diff --git a/pkg/database/mysqldb/query.mysql.sql.go b/pkg/database/mysqldb/query.mysql.sql.go index fda3d70b..d683fa7b 100644 --- a/pkg/database/mysqldb/query.mysql.sql.go +++ b/pkg/database/mysqldb/query.mysql.sql.go @@ -63,9 +63,9 @@ func (q *Queries) AddNarInfoSignature(ctx context.Context, arg AddNarInfoSignatu const createChunk = `-- name: CreateChunk :execresult INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id), @@ -73,22 +73,23 @@ ON DUPLICATE KEY UPDATE ` type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } // CreateChunk // // INSERT INTO chunks ( -// hash, size +// hash, size, compressed_size // ) VALUES ( -// ?, ? +// ?, ?, ? // ) // ON DUPLICATE KEY UPDATE // id = LAST_INSERT_ID(id), // updated_at = CURRENT_TIMESTAMP func (q *Queries) CreateChunk(ctx context.Context, arg CreateChunkParams) (sql.Result, error) { - return q.db.ExecContext(ctx, createChunk, arg.Hash, arg.Size) + return q.db.ExecContext(ctx, createChunk, arg.Hash, arg.Size, arg.CompressedSize) } const createConfig = `-- name: CreateConfig :execresult @@ -465,41 +466,34 @@ func (q *Queries) GetChunkCount(ctx context.Context) (int64, error) { } const getChunksByNarFileID = `-- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? ORDER BY nfc.chunk_index ` -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - // GetChunksByNarFileID // -// SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +// SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index -func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { rows, err := q.db.QueryContext(ctx, getChunksByNarFileID, narFileID) if err != nil { return nil, err } defer rows.Close() - var items []GetChunksByNarFileIDRow + var items []Chunk for rows.Next() { - var i GetChunksByNarFileIDRow + var i Chunk if err := rows.Scan( &i.ID, &i.Hash, &i.Size, + &i.CompressedSize, &i.CreatedAt, &i.UpdatedAt, ); err != nil { diff --git a/pkg/database/postgresdb/querier.go b/pkg/database/postgresdb/querier.go index 931a2875..68255805 100644 --- a/pkg/database/postgresdb/querier.go +++ b/pkg/database/postgresdb/querier.go @@ -45,9 +45,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // $1, $2 + // $1, $2, $3 // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP @@ -162,12 +162,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = $1 // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, system, ca diff --git a/pkg/database/postgresdb/query.postgres.sql.go b/pkg/database/postgresdb/query.postgres.sql.go index 72f2a81a..d10edf0e 100644 --- a/pkg/database/postgresdb/query.postgres.sql.go +++ b/pkg/database/postgresdb/query.postgres.sql.go @@ -115,9 +115,9 @@ func (q *Queries) AddNarInfoSignatures(ctx context.Context, arg AddNarInfoSignat const createChunk = `-- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - $1, $2 + $1, $2, $3 ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP @@ -125,22 +125,23 @@ RETURNING id, hash, size, compressed_size, created_at, updated_at ` type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } // CreateChunk // // INSERT INTO chunks ( -// hash, size +// hash, size, compressed_size // ) VALUES ( -// $1, $2 +// $1, $2, $3 // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP // RETURNING id, hash, size, compressed_size, created_at, updated_at func (q *Queries) CreateChunk(ctx context.Context, arg CreateChunkParams) (Chunk, error) { - row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size) + row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size, arg.CompressedSize) var i Chunk err := row.Scan( &i.ID, @@ -572,41 +573,34 @@ func (q *Queries) GetChunkCount(ctx context.Context) (int64, error) { } const getChunksByNarFileID = `-- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = $1 ORDER BY nfc.chunk_index ` -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - // GetChunksByNarFileID // -// SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +// SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = $1 // ORDER BY nfc.chunk_index -func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { rows, err := q.db.QueryContext(ctx, getChunksByNarFileID, narFileID) if err != nil { return nil, err } defer rows.Close() - var items []GetChunksByNarFileIDRow + var items []Chunk for rows.Next() { - var i GetChunksByNarFileIDRow + var i Chunk if err := rows.Scan( &i.ID, &i.Hash, &i.Size, + &i.CompressedSize, &i.CreatedAt, &i.UpdatedAt, ); err != nil { diff --git a/pkg/database/sqlitedb/querier.go b/pkg/database/sqlitedb/querier.go index d34d590b..907e1281 100644 --- a/pkg/database/sqlitedb/querier.go +++ b/pkg/database/sqlitedb/querier.go @@ -31,9 +31,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // ?, ? + // ?, ?, ? // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP @@ -148,12 +148,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, system, ca diff --git a/pkg/database/sqlitedb/query.sqlite.sql.go b/pkg/database/sqlitedb/query.sqlite.sql.go index 0f5b89d2..7887a323 100644 --- a/pkg/database/sqlitedb/query.sqlite.sql.go +++ b/pkg/database/sqlitedb/query.sqlite.sql.go @@ -67,9 +67,9 @@ func (q *Queries) AddNarInfoSignature(ctx context.Context, arg AddNarInfoSignatu const createChunk = `-- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP @@ -77,22 +77,23 @@ RETURNING id, hash, size, compressed_size, created_at, updated_at ` type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } // CreateChunk // // INSERT INTO chunks ( -// hash, size +// hash, size, compressed_size // ) VALUES ( -// ?, ? +// ?, ?, ? // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP // RETURNING id, hash, size, compressed_size, created_at, updated_at func (q *Queries) CreateChunk(ctx context.Context, arg CreateChunkParams) (Chunk, error) { - row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size) + row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size, arg.CompressedSize) var i Chunk err := row.Scan( &i.ID, @@ -524,41 +525,34 @@ func (q *Queries) GetChunkCount(ctx context.Context) (int64, error) { } const getChunksByNarFileID = `-- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? ORDER BY nfc.chunk_index ` -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - // GetChunksByNarFileID // -// SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +// SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index -func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { rows, err := q.db.QueryContext(ctx, getChunksByNarFileID, narFileID) if err != nil { return nil, err } defer rows.Close() - var items []GetChunksByNarFileIDRow + var items []Chunk for rows.Next() { - var i GetChunksByNarFileIDRow + var i Chunk if err := rows.Scan( &i.ID, &i.Hash, &i.Size, + &i.CompressedSize, &i.CreatedAt, &i.UpdatedAt, ); err != nil {