writing in batches instead of one at a time

database/dbtest/benchmark.go

@@ -19,8 +19,8 @@ var (
 		"Get":            BenchmarkGet,
 		"Put":            BenchmarkPut,
 		"Delete":         BenchmarkDelete,
-		"BatchPut":       BenchmarkBatchPut,
-		"BatchDelete":    BenchmarkBatchDelete,
+		// "BatchPut":       BenchmarkBatchPut,
+		// "BatchDelete":    BenchmarkBatchDelete,
 		"BatchWrite":     BenchmarkBatchWrite,
 		"ParallelGet":    BenchmarkParallelGet,
 		"ParallelPut":    BenchmarkParallelPut,

database/dbtest/dbtest.go

@@ -52,9 +52,9 @@ var Tests = map[string]func(t *testing.T, db database.Database){
 	"MemorySafetyDatabase": TestMemorySafetyDatabase,
 	"MemorySafetyBatch": TestMemorySafetyBatch,
 	// "AtomicClear":       TestAtomicClear,
-	//"Clear": TestClear,
-	//"AtomicClearPrefix": TestAtomicClearPrefix,
-	//"ClearPrefix":                      TestClearPrefix,
+	// "Clear": TestClear,
+	// "AtomicClearPrefix": TestAtomicClearPrefix,
+	// "ClearPrefix":                      TestClearPrefix,
 	"ModifyValueAfterPut":            TestModifyValueAfterPut,
 	"ModifyValueAfterBatchPut":       TestModifyValueAfterBatchPut,
 	"ModifyValueAfterBatchPutReplay": TestModifyValueAfterBatchPutReplay,

go.mod

@@ -95,11 +95,13 @@ require (
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
+	github.com/dgraph-io/ristretto v0.2.0 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/frankban/quicktest v1.14.4 // indirect

go.sum

@@ -162,6 +162,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
 github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE=
+github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
@@ -178,6 +180,8 @@ github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14
 github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
 github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=

x/merkledb/codec.go

@@ -351,7 +351,7 @@ func (r *codecReader) Key() (Key, error) {
 	}
 	byteLen := bytesNeeded(result.length)
 	if byteLen > len(r.b) {
-		return Key{}, io.ErrUnexpectedEOF
+ 		return Key{}, io.ErrUnexpectedEOF
 	}
 	if result.hasPartialByte() {
 		// Confirm that the padding bits in the partial byte are 0.

x/merkledb/db_test.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/database/dbtest"
+
 	// "github.com/ava-labs/avalanchego/database/memdb"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/trace"
@@ -128,16 +129,21 @@ func Test_MerkleDB_DB_Interface(t *testing.T) {
 func TestDelete(t *testing.T) {
 	require := require.New(t)
 	keys, values := dbtest.SetupBenchmark(t, 1024, 32, 32)
+	for _, bf := range validBranchFactors {
 
-	db, err := getBasicDB(t)
-	require.NoError(err)
+		db, err := getBasicDBWithBranchFactor(bf)
+		require.NoError(err)
 
-	for i, key := range keys {
-		value := values[i]
-		require.NoError(db.Put(key, value))
-	}
+		for i, key := range keys {
+			value := values[i]
+			require.NoError(db.Put(key, value))
+		}
 
-	require.NoError(db.Delete(keys[0]))
+		require.NoError(db.Delete(keys[0]))
+		t.Cleanup(func() {
+			db.Close()
+		})
+	}
 
 }
 
@@ -159,6 +165,36 @@ func Benchmark_MerkleDB_DBInterface(b *testing.B) {
 	}
 }
 
+func Benchmark_MerkleDB_DBInterface_Timed(b *testing.B) {
+	totalTime := time.Duration(0)
+	for _, size := range dbtest.BenchmarkSizes {
+		keys, values := dbtest.SetupBenchmark(b, size[0], size[1], size[2])
+		for _, bf := range validBranchFactors {
+			for name, bench := range dbtest.Benchmarks {
+				// Run the benchmark and track its time
+				b.Run(fmt.Sprintf("merkledb_%d_%d_pairs_%d_keys_%d_values_%s", bf, size[0], size[1], size[2], name), func(b *testing.B) {
+					db, err := getBasicDBWithBranchFactor(bf)
+					require.NoError(b, err)
+					b.Cleanup(func() {
+						db.Close()
+					})
+
+					// Record the start time for each individual benchmark
+					start := time.Now()
+					bench(b, db, keys, values)
+					// Calculate the time taken for the specific benchmark
+					duration := time.Since(start)
+					totalTime += duration
+					// Log the time taken for the specific benchmark
+				})
+			}
+		}
+	}
+	// Log the total time taken for all benchmarks
+	fmt.Printf("Total time taken for all benchmarks: %v\n", totalTime)
+	b.Log("Total time taken for all benchmarks: ", totalTime)
+}
+
 // PASSES
 func Test_MerkleDB_DB_Load_Root_From_DB(t *testing.T) {
 	require := require.New(t)
@@ -886,7 +922,7 @@ func TestMerkleDBClear(t *testing.T) {
 		require,
 		r,
 		[]database.Database{db},
-		1_000,
+		95,
 		0.25,
 	)
 

x/merkledb/disk_manager.go

@@ -85,7 +85,7 @@ func newDiskManager(metaData []byte, dir string, fileName string) (*diskMgr, err
 	}
 
 	// start freelist
-	maxSize := 4096 * 4 * 4
+	maxSize := 4096 * 4 * 4 * 4
 	f := newFreeList(maxSize)
 	f.load(dir)
 
@@ -119,8 +119,6 @@ func (dm *diskMgr) putBack(addr diskAddress) error {
 }
 
 func (dm *diskMgr) writeRoot(rootNode dbNode) (diskAddress, error) {
-
-
 	// first check the size of rootNode without the disk address
 	bytes := encodeDBNode_disk(&rootNode)
 	freeSpace, ok := dm.free.get(int64(len(bytes)) + 16)
@@ -190,6 +188,22 @@ func (dm *diskMgr) writeRoot(rootNode dbNode) (diskAddress, error) {
 
 }
 
+func (dm *diskMgr) fetch(byteLength int64) (diskAddress, error) {
+	freeSpace, ok := dm.free.get(int64(byteLength))
+	if !ok {
+		endOffset, err := dm.endOfFile()
+		if err != nil {
+			log.Fatalf("failed to get end of file: %v", err)
+			return diskAddress{}, err
+		}
+		return diskAddress{offset: endOffset, size: int64(byteLength)}, nil
+	} else {
+		return diskAddress{offset: freeSpace.offset, size: int64(byteLength)}, nil
+	}
+
+
+}
+
 // returning diskaddress that it wrote to
 // if we write to freelist: diskaddress would be the size of freespace
 // if we dont write to freelist: append bytes to end, return endoffset and size

x/merkledb/freelist.go

@@ -49,8 +49,15 @@ func (f *freeList) get(size int64) (diskAddress, bool) {
 	if f.closed {
 		return diskAddress{}, false
 	}*/
+	if size == 0{
+		return diskAddress{}, false
+	}
 	bucket := f.bucketIndex(size)
 
+	// first check whether or not the bucket would be out of bounds
+	if bucket >= len(f.buckets) && size >0 {
+		return diskAddress{}, false
+	}
 	if len(f.buckets[bucket]) > 0 {
 		space := f.buckets[bucket][len(f.buckets[bucket])-1]
 		f.buckets[bucket] = f.buckets[bucket][:len(f.buckets[bucket])-1]
@@ -72,7 +79,7 @@ func (f *freeList) put(space diskAddress) {
 
 // bucketIndex returns the index of the bucket that the size belongs to.
 func (f *freeList) bucketIndex(size int64) int {
-	return int(math.Ceil(math.Log2(float64(size))))
+	return int(math.Floor(math.Log2(float64(size))))
 }
 
 // close writes the remaining diskAddresses in the freeList to a file and closes the file.

x/merkledb/helpers_test.go

@@ -11,11 +11,18 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/ava-labs/avalanchego/database/memdb"
+
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/utils/hashing"
 	"github.com/ava-labs/avalanchego/utils/maybe"
+
+
+	"github.com/ava-labs/avalanchego/database/leveldb"
+	"github.com/ava-labs/avalanchego/utils/logging"
+	"github.com/prometheus/client_golang/prometheus"
+
 )
-const disk = true
+const disk = false
 func getBasicDB(tb testing.TB) (*merkleDB, error) {
 	if disk{
 		return getBasicDB_disk(tb)
@@ -29,21 +36,24 @@ func getBasicDB(tb testing.TB) (*merkleDB, error) {
 }
 
 func getBasicDBWithBranchFactor(bf BranchFactor) (*merkleDB, error) {
-	config := newDefaultConfig()
-	config.BranchFactor = bf
-	t := &testing.T{}
-	dir := t.TempDir()
-	if disk{
-		return getBasicDBWithBranchFactor_disk(bf,dir)
-	}
-	return newDatabase(
-		context.Background(),
-		memdb.New(),
-		config,
-		&mockMetrics{},
-	)
-}
+    config := newDefaultConfig()
+    config.BranchFactor = bf
+    t := &testing.T{}
+
+    folder := t.TempDir()
+    db, err := leveldb.New(folder, nil, logging.NoLog{}, prometheus.NewRegistry())
+    require.NoError(t, err)
 
+    if disk {
+        return getBasicDBWithBranchFactor_disk(bf, folder)
+    }
+    return newDatabase(
+        context.Background(),
+        db,
+        config,
+        &mockMetrics{},
+    )
+}
 // Writes []byte{i} -> []byte{i} for i in [0, 4]
 func writeBasicBatch(t *testing.T, db *merkleDB) {
 	require := require.New(t)

x/merkledb/key.go

@@ -37,7 +37,7 @@ var (
 		BranchFactor2,
 		BranchFactor4,
 		BranchFactor16,
-		// BranchFactor256,
+		BranchFactor256,
 	}
 )