Skip to content

Commit 31d76ff

Browse files
roman-khimovroman-khimov
authored
Add more tests for FSTree (#3473)
Ref #3451.
2 parents 6f271cd + fc0f210 commit 31d76ff

File tree

5 files changed

+247
-106
lines changed

5 files changed

+247
-106
lines changed

‎pkg/local_object_storage/blobstor/fstree/bench_test.go‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
package fstree_test
2+
3+
import (
4+
"io"
5+
"testing"
6+
7+
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
8+
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
9+
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
10+
"github.com/stretchr/testify/require"
11+
)
12+
13+
func BenchmarkFSTree_Head(b *testing.B) {
14+
for _, size := range payloadSizes {
15+
b.Run(generateSizeLabel(size), func(b *testing.B) {
16+
runReadBenchmark(b, "Head", size)
17+
})
18+
}
19+
}
20+
21+
func BenchmarkFSTree_Get(b *testing.B) {
22+
for _, size := range payloadSizes {
23+
b.Run(generateSizeLabel(size), func(b *testing.B) {
24+
runReadBenchmark(b, "Get", size)
25+
})
26+
}
27+
}
28+
29+
func BenchmarkFSTree_GetStream(b *testing.B) {
30+
for _, size := range payloadSizes {
31+
b.Run(generateSizeLabel(size), func(b *testing.B) {
32+
runReadBenchmark(b, "GetStream", size)
33+
34+
b.Run("GetStream_with_payload_read", func(b *testing.B) {
35+
freshFSTree := setupFSTree(b)
36+
addr := prepareSingleObject(b, freshFSTree, size)
37+
38+
b.ReportAllocs()
39+
b.ResetTimer()
40+
for range b.N {
41+
header, reader, err := freshFSTree.GetStream(addr)
42+
if err != nil {
43+
b.Fatal(err)
44+
}
45+
if header == nil {
46+
b.Fatal("header is nil")
47+
}
48+
if reader != nil {
49+
// Read all payload to simulate real usage
50+
_, err := io.ReadAll(reader)
51+
if err != nil {
52+
b.Fatal(err)
53+
}
54+
require.NoError(b, reader.Close())
55+
}
56+
}
57+
})
58+
})
59+
}
60+
}
61+
62+
func runReadBenchmark(b *testing.B, methodName string, payloadSize int) {
63+
testRead := func(fsTree *fstree.FSTree, addr oid.Address) {
64+
var err error
65+
switch methodName {
66+
case "Head":
67+
_, err = fsTree.Head(addr)
68+
case "Get":
69+
_, err = fsTree.Get(addr)
70+
case "GetStream":
71+
var (
72+
header *objectSDK.Object
73+
reader io.ReadCloser
74+
)
75+
header, reader, err = fsTree.GetStream(addr)
76+
if header == nil {
77+
b.Fatal("header is nil")
78+
}
79+
if reader != nil {
80+
require.NoError(b, reader.Close())
81+
}
82+
}
83+
if err != nil {
84+
b.Fatal(err)
85+
}
86+
}
87+
88+
b.Run(methodName+"_regular", func(b *testing.B) {
89+
fsTree := setupFSTree(b)
90+
addr := prepareSingleObject(b, fsTree, payloadSize)
91+
92+
b.ReportAllocs()
93+
b.ResetTimer()
94+
for range b.N {
95+
testRead(fsTree, addr)
96+
}
97+
})
98+
99+
b.Run(methodName+"_combined", func(b *testing.B) {
100+
fsTree := setupFSTree(b)
101+
addrs := prepareMultipleObjects(b, fsTree, payloadSize)
102+
103+
b.ReportAllocs()
104+
b.ResetTimer()
105+
for k := range b.N {
106+
testRead(fsTree, addrs[k%len(addrs)])
107+
}
108+
})
109+
110+
b.Run(methodName+"_compressed", func(b *testing.B) {
111+
fsTree := setupFSTree(b)
112+
setupCompressor(b, fsTree)
113+
addr := prepareSingleObject(b, fsTree, payloadSize)
114+
115+
b.ReportAllocs()
116+
b.ResetTimer()
117+
for range b.N {
118+
testRead(fsTree, addr)
119+
}
120+
})
121+
122+
b.Run(methodName+"_compressed_combined", func(b *testing.B) {
123+
fsTree := setupFSTree(b)
124+
setupCompressor(b, fsTree)
125+
addrs := prepareMultipleObjects(b, fsTree, payloadSize)
126+
127+
b.ReportAllocs()
128+
b.ResetTimer()
129+
for k := range b.N {
130+
testRead(fsTree, addrs[k%len(addrs)])
131+
}
132+
})
133+
}

‎pkg/local_object_storage/blobstor/fstree/common_test.go‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package fstree_test
2+
3+
import (
4+
"crypto/rand"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/nspcc-dev/neofs-node/pkg/core/object"
9+
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
10+
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
11+
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
12+
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
13+
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
var payloadSizes = []int{
18+
0, // Empty payload
19+
100, // 100 bytes
20+
4 * 1024, // 4 KB
21+
16 * 1024, // 16 KB
22+
32 * 1024, // 32 KB
23+
100 * 1024, // 100 KB
24+
1024 * 1024, // 1 MB
25+
}
26+
27+
func setupFSTree(tb testing.TB) *fstree.FSTree {
28+
fsTree := fstree.New(fstree.WithPath(tb.TempDir()))
29+
require.NoError(tb, fsTree.Open(false))
30+
require.NoError(tb, fsTree.Init())
31+
return fsTree
32+
}
33+
34+
func setupCompressor(tb testing.TB, fsTree *fstree.FSTree) {
35+
compressConfig := &compression.Config{
36+
Enabled: true,
37+
}
38+
require.NoError(tb, compressConfig.Init())
39+
fsTree.SetCompressor(compressConfig)
40+
}
41+
42+
func prepareSingleObject(tb testing.TB, fsTree *fstree.FSTree, payloadSize int) oid.Address {
43+
obj := generateTestObject(payloadSize)
44+
addr := object.AddressOf(obj)
45+
require.NoError(tb, fsTree.Put(addr, obj.Marshal()))
46+
return addr
47+
}
48+
49+
func addAttribute(obj *objectSDK.Object, key, value string) {
50+
var attr objectSDK.Attribute
51+
attr.SetKey(key)
52+
attr.SetValue(value)
53+
54+
attrs := obj.Attributes()
55+
attrs = append(attrs, attr)
56+
obj.SetAttributes(attrs...)
57+
}
58+
59+
func generateTestObject(payloadSize int) *objectSDK.Object {
60+
obj := objecttest.Object()
61+
if payloadSize > 0 {
62+
payload := make([]byte, payloadSize)
63+
_, _ = rand.Read(payload)
64+
obj.SetPayload(payload)
65+
} else {
66+
obj.SetPayload(nil)
67+
}
68+
obj.SetPayloadSize(uint64(payloadSize))
69+
70+
return &obj
71+
}
72+
73+
func generateSizeLabel(size int) string {
74+
switch {
75+
case size == 0:
76+
return "Empty"
77+
case size < 1024:
78+
return fmt.Sprintf("%dB", size)
79+
case size < 1024*1024:
80+
return fmt.Sprintf("%dKB", size/1024)
81+
default:
82+
return fmt.Sprintf("%dMB", size/(1024*1024))
83+
}
84+
}
85+
86+
func prepareMultipleObjects(tb testing.TB, fsTree *fstree.FSTree, payloadSize int) []oid.Address {
87+
const numObjects = 10
88+
objMap := make(map[oid.Address][]byte, numObjects)
89+
addrs := make([]oid.Address, numObjects)
90+
91+
for i := range numObjects {
92+
obj := generateTestObject(payloadSize)
93+
addr := object.AddressOf(obj)
94+
objMap[addr] = obj.Marshal()
95+
addrs[i] = addr
96+
}
97+
98+
require.NoError(tb, fsTree.PutBatch(objMap))
99+
return addrs
100+
}

‎pkg/local_object_storage/blobstor/fstree/head_bench_test.go‎

Lines changed: 3 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,9 @@
11
package fstree_test
22

33
import (
4-
"crypto/rand"
5-
"fmt"
64
"testing"
7-
8-
"github.com/nspcc-dev/neofs-node/pkg/core/object"
9-
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
10-
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
11-
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
12-
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
13-
"github.com/stretchr/testify/require"
145
)
156

16-
var payloadSizes = []int{
17-
0, // Empty payload
18-
100, // 100 bytes
19-
4 * 1024, // 4 KB
20-
16 * 1024, // 16 KB
21-
32 * 1024, // 32 KB
22-
100 * 1024, // 100 KB
23-
1024 * 1024, // 1 MB
24-
}
25-
267
func BenchmarkFSTree_HeadVsGet(b *testing.B) {
278
for _, size := range payloadSizes {
289
b.Run(generateSizeLabel(size), func(b *testing.B) {
@@ -39,39 +20,16 @@ func BenchmarkFSTree_HeadVsGet_Compressed(b *testing.B) {
3920
}
4021
}
4122

42-
func generateSizeLabel(size int) string {
43-
switch {
44-
case size == 0:
45-
return "Empty"
46-
case size < 1024:
47-
return fmt.Sprintf("%dB", size)
48-
case size < 1024*1024:
49-
return fmt.Sprintf("%dKB", size/1024)
50-
default:
51-
return fmt.Sprintf("%dMB", size/(1024*1024))
52-
}
53-
}
54-
5523
func runHeadVsGetBenchmark(b *testing.B, payloadSize int, compressed bool) {
56-
fsTree := fstree.New(fstree.WithPath(b.TempDir()))
24+
fsTree := setupFSTree(b)
5725

5826
suffix := ""
5927
if compressed {
60-
compressConfig := &compression.Config{
61-
Enabled: true,
62-
}
63-
require.NoError(b, compressConfig.Init())
64-
fsTree.SetCompressor(compressConfig)
28+
setupCompressor(b, fsTree)
6529
suffix = "_Compressed"
6630
}
6731

68-
require.NoError(b, fsTree.Open(false))
69-
require.NoError(b, fsTree.Init())
70-
71-
obj := generateTestObject(payloadSize)
72-
addr := object.AddressOf(obj)
73-
74-
require.NoError(b, fsTree.Put(addr, obj.Marshal()))
32+
addr := prepareSingleObject(b, fsTree, payloadSize)
7533

7634
b.Run("Head"+suffix, func(b *testing.B) {
7735
b.ResetTimer()
@@ -95,17 +53,3 @@ func runHeadVsGetBenchmark(b *testing.B, payloadSize int, compressed bool) {
9553
}
9654
})
9755
}
98-
99-
func generateTestObject(payloadSize int) *objectSDK.Object {
100-
obj := objecttest.Object()
101-
if payloadSize > 0 {
102-
payload := make([]byte, payloadSize)
103-
_, _ = rand.Read(payload)
104-
obj.SetPayload(payload)
105-
} else {
106-
obj.SetPayload(nil)
107-
}
108-
obj.SetPayloadSize(uint64(payloadSize))
109-
110-
return &obj
111-
}

‎pkg/local_object_storage/blobstor/fstree/head_test.go‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,14 @@ import (
55
"testing"
66

77
"github.com/nspcc-dev/neofs-node/pkg/core/object"
8-
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
98
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
109
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
1110
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
1211
"github.com/stretchr/testify/require"
1312
)
1413

1514
func TestHeadStorage(t *testing.T) {
16-
fsTree := fstree.New(fstree.WithPath(t.TempDir()))
17-
require.NoError(t, fsTree.Open(false))
18-
require.NoError(t, fsTree.Init())
15+
fsTree := setupFSTree(t)
1916

2017
testObjects := func(t *testing.T, fsTree *fstree.FSTree, size int) {
2118
obj := generateTestObject(size)
@@ -109,16 +106,8 @@ func TestHeadStorage(t *testing.T) {
109106
})
110107

111108
t.Run("with compression", func(t *testing.T) {
112-
compressConfig := &compression.Config{
113-
Enabled: true,
114-
}
115-
require.NoError(t, compressConfig.Init())
116-
117-
fsComp := fstree.New(fstree.WithPath(t.TempDir()))
118-
fsComp.SetCompressor(compressConfig)
119-
120-
require.NoError(t, fsComp.Open(false))
121-
require.NoError(t, fsComp.Init())
109+
fsComp := setupFSTree(t)
110+
setupCompressor(t, fsComp)
122111

123112
for _, size := range payloadSizes {
124113
t.Run("compressed_"+generateSizeLabel(size), func(t *testing.T) {
@@ -131,13 +120,3 @@ func TestHeadStorage(t *testing.T) {
131120
}
132121
})
133122
}
134-
135-
func addAttribute(obj *objectSDK.Object, key, value string) {
136-
var attr objectSDK.Attribute
137-
attr.SetKey(key)
138-
attr.SetValue(value)
139-
140-
attrs := obj.Attributes()
141-
attrs = append(attrs, attr)
142-
obj.SetAttributes(attrs...)
143-
}

0 commit comments

Comments
 (0)