diff --git a/db/query.mysql.sql b/db/query.mysql.sql index b9a17312..cfa2b97d 100644 --- a/db/query.mysql.sql +++ b/db/query.mysql.sql @@ -280,7 +280,7 @@ FROM chunks WHERE id = ?; -- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? @@ -288,9 +288,9 @@ ORDER BY nfc.chunk_index; -- name: CreateChunk :execresult INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id), diff --git a/db/query.postgres.sql b/db/query.postgres.sql index f4666057..af164eea 100644 --- a/db/query.postgres.sql +++ b/db/query.postgres.sql @@ -300,7 +300,7 @@ FROM chunks WHERE id = $1; -- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = $1 @@ -308,9 +308,9 @@ ORDER BY nfc.chunk_index; -- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - $1, $2 + $1, $2, $3 ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP diff --git a/db/query.sqlite.sql b/db/query.sqlite.sql index a6373895..9e00ce87 100644 --- a/db/query.sqlite.sql +++ b/db/query.sqlite.sql @@ -286,7 +286,7 @@ FROM chunks WHERE id = ?; -- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? @@ -294,9 +294,9 @@ ORDER BY nfc.chunk_index; -- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index c6da7209..65424bdc 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1346,7 +1346,7 @@ func (c *Cache) storeNarWithCDC(ctx context.Context, tempPath string, narURL *na } // Store in chunkStore if new - _, _, err = chunkStore.PutChunk(ctx, chunkMetadata.Hash, chunkMetadata.Data) + _, compressedSize, err := chunkStore.PutChunk(ctx, chunkMetadata.Hash, chunkMetadata.Data) if err != nil { chunkMetadata.Free() @@ -1354,6 +1354,8 @@ func (c *Cache) storeNarWithCDC(ctx context.Context, tempPath string, narURL *na } chunkMetadata.Free() + //nolint:gosec // G115: Chunk size is small enough to fit in uint32 + chunkMetadata.CompressedSize = uint32(compressedSize) totalSize += int64(chunkMetadata.Size) @@ -1415,8 +1417,9 @@ func (c *Cache) recordChunkBatch(ctx context.Context, narFileID int64, startInde for i, chunkMetadata := range batch { // Create or increment ref count. ch, err := qtx.CreateChunk(ctx, database.CreateChunkParams{ - Hash: chunkMetadata.Hash, - Size: chunkMetadata.Size, + Hash: chunkMetadata.Hash, + Size: chunkMetadata.Size, + CompressedSize: chunkMetadata.CompressedSize, }) if err != nil { return fmt.Errorf("error creating chunk record: %w", err) diff --git a/pkg/cache/cdc_test.go b/pkg/cache/cdc_test.go index 54591808..72b2982c 100644 --- a/pkg/cache/cdc_test.go +++ b/pkg/cache/cdc_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/kalbasit/ncps/pkg/database" "github.com/kalbasit/ncps/pkg/nar" "github.com/kalbasit/ncps/pkg/storage/chunk" ) @@ -51,6 +52,7 @@ func runCDCTestSuite(t *testing.T, factory cacheFactory) { t.Run("Mixed Mode", testCDCMixedMode(factory)) t.Run("GetNarInfo with CDC chunks", testCDCGetNarInfo(factory)) t.Run("Client Disconnect No Goroutine Leak", testCDCClientDisconnectNoGoroutineLeak(factory)) + t.Run("chunks are stored compressed", testCDCChunksAreCompressed(factory)) } func testCDCPutAndGet(factory cacheFactory) func(*testing.T) { @@ -311,3 +313,65 @@ func testCDCClientDisconnectNoGoroutineLeak(factory cacheFactory) func(*testing. "Goroutine leak detected: baseline=%d, final=%d", baselineGoroutines, finalGoroutines) } } + +func testCDCChunksAreCompressed(factory cacheFactory) func(*testing.T) { + return func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + c, db, _, dir, _, cleanup := factory(t) + t.Cleanup(cleanup) + + // Initialize chunk store + chunkStoreDir := filepath.Join(dir, "chunks-store") + chunkStore, err := chunk.NewLocalStore(chunkStoreDir) + require.NoError(t, err) + + c.SetChunkStore(chunkStore) + err = c.SetCDCConfiguration(true, 1024, 4096, 8192) // Small sizes for testing + require.NoError(t, err) + + // Use highly compressible data (repeated bytes) + content := strings.Repeat("compressible", 1000) + nu := nar.URL{Hash: "testnar-compress", Compression: nar.CompressionTypeNone} + + r := io.NopCloser(strings.NewReader(content)) + err = c.PutNar(ctx, nu, r) + require.NoError(t, err) + + // Verify chunks exist in DB and have compressed_size set + narFile, err := db.GetNarFileByHashAndCompressionAndQuery(ctx, database.GetNarFileByHashAndCompressionAndQueryParams{ + Hash: nu.Hash, + Compression: nu.Compression.String(), + Query: nu.Query.Encode(), + }) + require.NoError(t, err) + + chunks, err := db.GetChunksByNarFileID(ctx, narFile.ID) + require.NoError(t, err) + require.NotEmpty(t, chunks, "should have chunks in the database") + + var totalSize, totalCompressedSize int64 + for _, chunk := range chunks { + totalSize += int64(chunk.Size) + totalCompressedSize += int64(chunk.CompressedSize) + assert.Positive(t, chunk.CompressedSize, "compressed size should be positive") + } + + assert.Equal(t, int64(len(content)), totalSize, "sum of chunk sizes should equal original content size") + assert.Less(t, totalCompressedSize, totalSize, + "total compressed size should be less than total original size for compressible data") + + // Verify reassembly to ensure compression is transparent + size, rc, err := c.GetNar(ctx, nu) + require.NoError(t, err) + + defer rc.Close() + + data, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, content, string(data), "decompressed data should match original") + assert.Equal(t, int64(len(content)), size, "size should match original content size") + } +} diff --git a/pkg/chunker/chunker.go b/pkg/chunker/chunker.go index 10f67712..4278a0b6 100644 --- a/pkg/chunker/chunker.go +++ b/pkg/chunker/chunker.go @@ -13,10 +13,11 @@ import ( // Chunk represents a single content-defined chunk. type Chunk struct { - Hash string // BLAKE3 hash of chunk content - Offset int64 // Offset in original stream - Size uint32 // Chunk size in bytes - Data []byte // Chunk data + Hash string // BLAKE3 hash of chunk content + Offset int64 // Offset in original stream + Size uint32 // Chunk size in bytes + CompressedSize uint32 // Compressed chunk size in bytes + Data []byte // Chunk data free func() // function to return Data to the pool } diff --git a/pkg/database/generated_models.go b/pkg/database/generated_models.go index 9db36584..798b852b 100644 --- a/pkg/database/generated_models.go +++ b/pkg/database/generated_models.go @@ -44,8 +44,9 @@ type Config struct { } type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } type CreateConfigParams struct { @@ -94,14 +95,6 @@ type GetChunkByNarFileIDAndIndexRow struct { UpdatedAt sql.NullTime } -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - type GetCompressedNarInfosParams struct { Limit int32 Offset int32 diff --git a/pkg/database/generated_querier.go b/pkg/database/generated_querier.go index 5571551a..1ce21e9c 100644 --- a/pkg/database/generated_querier.go +++ b/pkg/database/generated_querier.go @@ -42,9 +42,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // $1, $2 + // $1, $2, $3 // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP @@ -159,12 +159,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = $1 // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, system, ca diff --git a/pkg/database/generated_wrapper_mysql.go b/pkg/database/generated_wrapper_mysql.go index 22583bf7..ebf71d60 100644 --- a/pkg/database/generated_wrapper_mysql.go +++ b/pkg/database/generated_wrapper_mysql.go @@ -74,8 +74,9 @@ func (w *mysqlWrapper) CreateChunk(ctx context.Context, arg CreateChunkParams) ( // MySQL does not support RETURNING for INSERTs. // We insert, get LastInsertId, and then fetch the object. res, err := w.adapter.CreateChunk(ctx, mysqldb.CreateChunkParams{ - Hash: arg.Hash, - Size: arg.Size, + Hash: arg.Hash, + Size: arg.Size, + CompressedSize: arg.CompressedSize, }) if err != nil { return Chunk{}, err @@ -356,7 +357,7 @@ func (w *mysqlWrapper) GetChunkCount(ctx context.Context) (int64, error) { return res, nil } -func (w *mysqlWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (w *mysqlWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.GetChunksByNarFileID(ctx, narFileID) @@ -365,15 +366,17 @@ func (w *mysqlWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64 } // Convert Slice of Domain Structs - items := make([]GetChunksByNarFileIDRow, len(res)) + items := make([]Chunk, len(res)) for i, v := range res { - items[i] = GetChunksByNarFileIDRow{ + items[i] = Chunk{ ID: v.ID, Hash: v.Hash, Size: v.Size, + CompressedSize: v.CompressedSize, + CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, diff --git a/pkg/database/generated_wrapper_postgres.go b/pkg/database/generated_wrapper_postgres.go index 307e2b3f..6551546e 100644 --- a/pkg/database/generated_wrapper_postgres.go +++ b/pkg/database/generated_wrapper_postgres.go @@ -54,8 +54,9 @@ func (w *postgresWrapper) CreateChunk(ctx context.Context, arg CreateChunkParams /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.CreateChunk(ctx, postgresdb.CreateChunkParams{ - Hash: arg.Hash, - Size: arg.Size, + Hash: arg.Hash, + Size: arg.Size, + CompressedSize: arg.CompressedSize, }) if err != nil { @@ -408,7 +409,7 @@ func (w *postgresWrapper) GetChunkCount(ctx context.Context) (int64, error) { return res, nil } -func (w *postgresWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (w *postgresWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.GetChunksByNarFileID(ctx, narFileID) @@ -417,15 +418,17 @@ func (w *postgresWrapper) GetChunksByNarFileID(ctx context.Context, narFileID in } // Convert Slice of Domain Structs - items := make([]GetChunksByNarFileIDRow, len(res)) + items := make([]Chunk, len(res)) for i, v := range res { - items[i] = GetChunksByNarFileIDRow{ + items[i] = Chunk{ ID: v.ID, Hash: v.Hash, Size: v.Size, + CompressedSize: v.CompressedSize, + CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, diff --git a/pkg/database/generated_wrapper_sqlite.go b/pkg/database/generated_wrapper_sqlite.go index 8762a4cb..ca5cc286 100644 --- a/pkg/database/generated_wrapper_sqlite.go +++ b/pkg/database/generated_wrapper_sqlite.go @@ -72,8 +72,9 @@ func (w *sqliteWrapper) CreateChunk(ctx context.Context, arg CreateChunkParams) /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.CreateChunk(ctx, sqlitedb.CreateChunkParams{ - Hash: arg.Hash, - Size: arg.Size, + Hash: arg.Hash, + Size: arg.Size, + CompressedSize: arg.CompressedSize, }) if err != nil { @@ -426,7 +427,7 @@ func (w *sqliteWrapper) GetChunkCount(ctx context.Context) (int64, error) { return res, nil } -func (w *sqliteWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (w *sqliteWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { /* --- Auto-Loop for Bulk Insert on Non-Postgres --- */ res, err := w.adapter.GetChunksByNarFileID(ctx, narFileID) @@ -435,15 +436,17 @@ func (w *sqliteWrapper) GetChunksByNarFileID(ctx context.Context, narFileID int6 } // Convert Slice of Domain Structs - items := make([]GetChunksByNarFileIDRow, len(res)) + items := make([]Chunk, len(res)) for i, v := range res { - items[i] = GetChunksByNarFileIDRow{ + items[i] = Chunk{ ID: v.ID, Hash: v.Hash, Size: v.Size, + CompressedSize: v.CompressedSize, + CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, diff --git a/pkg/database/mysqldb/querier.go b/pkg/database/mysqldb/querier.go index 311013b4..3d3e464d 100644 --- a/pkg/database/mysqldb/querier.go +++ b/pkg/database/mysqldb/querier.go @@ -29,9 +29,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // ?, ? + // ?, ?, ? // ) // ON DUPLICATE KEY UPDATE // id = LAST_INSERT_ID(id), @@ -144,12 +144,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, `system`, ca diff --git a/pkg/database/mysqldb/query.mysql.sql.go b/pkg/database/mysqldb/query.mysql.sql.go index fda3d70b..d683fa7b 100644 --- a/pkg/database/mysqldb/query.mysql.sql.go +++ b/pkg/database/mysqldb/query.mysql.sql.go @@ -63,9 +63,9 @@ func (q *Queries) AddNarInfoSignature(ctx context.Context, arg AddNarInfoSignatu const createChunk = `-- name: CreateChunk :execresult INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON DUPLICATE KEY UPDATE id = LAST_INSERT_ID(id), @@ -73,22 +73,23 @@ ON DUPLICATE KEY UPDATE ` type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } // CreateChunk // // INSERT INTO chunks ( -// hash, size +// hash, size, compressed_size // ) VALUES ( -// ?, ? +// ?, ?, ? // ) // ON DUPLICATE KEY UPDATE // id = LAST_INSERT_ID(id), // updated_at = CURRENT_TIMESTAMP func (q *Queries) CreateChunk(ctx context.Context, arg CreateChunkParams) (sql.Result, error) { - return q.db.ExecContext(ctx, createChunk, arg.Hash, arg.Size) + return q.db.ExecContext(ctx, createChunk, arg.Hash, arg.Size, arg.CompressedSize) } const createConfig = `-- name: CreateConfig :execresult @@ -465,41 +466,34 @@ func (q *Queries) GetChunkCount(ctx context.Context) (int64, error) { } const getChunksByNarFileID = `-- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? ORDER BY nfc.chunk_index ` -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - // GetChunksByNarFileID // -// SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +// SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index -func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { rows, err := q.db.QueryContext(ctx, getChunksByNarFileID, narFileID) if err != nil { return nil, err } defer rows.Close() - var items []GetChunksByNarFileIDRow + var items []Chunk for rows.Next() { - var i GetChunksByNarFileIDRow + var i Chunk if err := rows.Scan( &i.ID, &i.Hash, &i.Size, + &i.CompressedSize, &i.CreatedAt, &i.UpdatedAt, ); err != nil { diff --git a/pkg/database/postgresdb/querier.go b/pkg/database/postgresdb/querier.go index 931a2875..68255805 100644 --- a/pkg/database/postgresdb/querier.go +++ b/pkg/database/postgresdb/querier.go @@ -45,9 +45,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // $1, $2 + // $1, $2, $3 // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP @@ -162,12 +162,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = $1 // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, system, ca diff --git a/pkg/database/postgresdb/query.postgres.sql.go b/pkg/database/postgresdb/query.postgres.sql.go index 72f2a81a..d10edf0e 100644 --- a/pkg/database/postgresdb/query.postgres.sql.go +++ b/pkg/database/postgresdb/query.postgres.sql.go @@ -115,9 +115,9 @@ func (q *Queries) AddNarInfoSignatures(ctx context.Context, arg AddNarInfoSignat const createChunk = `-- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - $1, $2 + $1, $2, $3 ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP @@ -125,22 +125,23 @@ RETURNING id, hash, size, compressed_size, created_at, updated_at ` type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } // CreateChunk // // INSERT INTO chunks ( -// hash, size +// hash, size, compressed_size // ) VALUES ( -// $1, $2 +// $1, $2, $3 // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP // RETURNING id, hash, size, compressed_size, created_at, updated_at func (q *Queries) CreateChunk(ctx context.Context, arg CreateChunkParams) (Chunk, error) { - row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size) + row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size, arg.CompressedSize) var i Chunk err := row.Scan( &i.ID, @@ -572,41 +573,34 @@ func (q *Queries) GetChunkCount(ctx context.Context) (int64, error) { } const getChunksByNarFileID = `-- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = $1 ORDER BY nfc.chunk_index ` -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - // GetChunksByNarFileID // -// SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +// SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = $1 // ORDER BY nfc.chunk_index -func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { rows, err := q.db.QueryContext(ctx, getChunksByNarFileID, narFileID) if err != nil { return nil, err } defer rows.Close() - var items []GetChunksByNarFileIDRow + var items []Chunk for rows.Next() { - var i GetChunksByNarFileIDRow + var i Chunk if err := rows.Scan( &i.ID, &i.Hash, &i.Size, + &i.CompressedSize, &i.CreatedAt, &i.UpdatedAt, ); err != nil { diff --git a/pkg/database/sqlitedb/querier.go b/pkg/database/sqlitedb/querier.go index d34d590b..907e1281 100644 --- a/pkg/database/sqlitedb/querier.go +++ b/pkg/database/sqlitedb/querier.go @@ -31,9 +31,9 @@ type Querier interface { //CreateChunk // // INSERT INTO chunks ( - // hash, size + // hash, size, compressed_size // ) VALUES ( - // ?, ? + // ?, ?, ? // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP @@ -148,12 +148,12 @@ type Querier interface { GetChunkCount(ctx context.Context) (int64, error) //GetChunksByNarFileID // - // SELECT c.id, c.hash, c.size, c.created_at, c.updated_at + // SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index - GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) + GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) //GetCompressedNarInfos // // SELECT id, hash, created_at, updated_at, last_accessed_at, store_path, url, compression, file_hash, file_size, nar_hash, nar_size, deriver, system, ca diff --git a/pkg/database/sqlitedb/query.sqlite.sql.go b/pkg/database/sqlitedb/query.sqlite.sql.go index 0f5b89d2..7887a323 100644 --- a/pkg/database/sqlitedb/query.sqlite.sql.go +++ b/pkg/database/sqlitedb/query.sqlite.sql.go @@ -67,9 +67,9 @@ func (q *Queries) AddNarInfoSignature(ctx context.Context, arg AddNarInfoSignatu const createChunk = `-- name: CreateChunk :one INSERT INTO chunks ( - hash, size + hash, size, compressed_size ) VALUES ( - ?, ? + ?, ?, ? ) ON CONFLICT(hash) DO UPDATE SET updated_at = CURRENT_TIMESTAMP @@ -77,22 +77,23 @@ RETURNING id, hash, size, compressed_size, created_at, updated_at ` type CreateChunkParams struct { - Hash string - Size uint32 + Hash string + Size uint32 + CompressedSize uint32 } // CreateChunk // // INSERT INTO chunks ( -// hash, size +// hash, size, compressed_size // ) VALUES ( -// ?, ? +// ?, ?, ? // ) // ON CONFLICT(hash) DO UPDATE SET // updated_at = CURRENT_TIMESTAMP // RETURNING id, hash, size, compressed_size, created_at, updated_at func (q *Queries) CreateChunk(ctx context.Context, arg CreateChunkParams) (Chunk, error) { - row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size) + row := q.db.QueryRowContext(ctx, createChunk, arg.Hash, arg.Size, arg.CompressedSize) var i Chunk err := row.Scan( &i.ID, @@ -524,41 +525,34 @@ func (q *Queries) GetChunkCount(ctx context.Context) (int64, error) { } const getChunksByNarFileID = `-- name: GetChunksByNarFileID :many -SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at FROM chunks c INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id WHERE nfc.nar_file_id = ? ORDER BY nfc.chunk_index ` -type GetChunksByNarFileIDRow struct { - ID int64 - Hash string - Size uint32 - CreatedAt time.Time - UpdatedAt sql.NullTime -} - // GetChunksByNarFileID // -// SELECT c.id, c.hash, c.size, c.created_at, c.updated_at +// SELECT c.id, c.hash, c.size, c.compressed_size, c.created_at, c.updated_at // FROM chunks c // INNER JOIN nar_file_chunks nfc ON c.id = nfc.chunk_id // WHERE nfc.nar_file_id = ? // ORDER BY nfc.chunk_index -func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]GetChunksByNarFileIDRow, error) { +func (q *Queries) GetChunksByNarFileID(ctx context.Context, narFileID int64) ([]Chunk, error) { rows, err := q.db.QueryContext(ctx, getChunksByNarFileID, narFileID) if err != nil { return nil, err } defer rows.Close() - var items []GetChunksByNarFileIDRow + var items []Chunk for rows.Next() { - var i GetChunksByNarFileIDRow + var i Chunk if err := rows.Scan( &i.ID, &i.Hash, &i.Size, + &i.CompressedSize, &i.CreatedAt, &i.UpdatedAt, ); err != nil {