Skip to content

Commit caf540e

Browse files
iamlinjunhongiamlinjunhong
committed
multi vec
1 parent b305c86 commit caf540e

File tree

6 files changed

+193
-322
lines changed

6 files changed

+193
-322
lines changed

‎pkg/objectio/types.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ import (
2020
"github.com/matrixorigin/matrixone/pkg/common/mpool"
2121
"github.com/matrixorigin/matrixone/pkg/container/batch"
2222
"github.com/matrixorigin/matrixone/pkg/container/types"
23-
"github.com/matrixorigin/matrixone/pkg/container/vector"
2423
"github.com/matrixorigin/matrixone/pkg/fileservice"
2524
"github.com/matrixorigin/matrixone/pkg/vectorindex/metric"
25+
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
2626
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index"
2727
)
2828

@@ -42,7 +42,7 @@ type ColumnMetaFetcher interface {
4242
MustGetColumn(seqnum uint16) ColumnMeta
4343
}
4444

45-
type ReadFilterSearchFuncType func(*vector.Vector) []int64
45+
type ReadFilterSearchFuncType func(containers.Vectors) []int64
4646

4747
type BlockReadFilter struct {
4848
HasFakePK bool

‎pkg/vm/engine/disttae/txn_table.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2252,15 +2252,21 @@ func (tbl *txnTable) PKPersistedBetween(
22522252
tbl.tableDef.Pkey.PkeyColName,
22532253
catalog.IsFakePkName(tbl.tableDef.Pkey.PkeyColName),
22542254
basePKFilter,
2255-
nil, // txn path 这里只用静态 PK 过滤，不参与 ivf BloomFilter
2255+
nil,
22562256
tbl.proc.Load().Mp(),
22572257
)
22582258
if err != nil {
22592259
return false, err
22602260
}
22612261

22622262
buildUnsortedFilter := func() objectio.ReadFilterSearchFuncType {
2263-
return LinearSearchOffsetByValFactory(keys)
2263+
inner := LinearSearchOffsetByValFactory(keys)
2264+
return func(cacheVectors containers.Vectors) []int64 {
2265+
if len(cacheVectors) == 0 || cacheVectors[0].Length() == 0 {
2266+
return nil
2267+
}
2268+
return inner(&cacheVectors[0])
2269+
}
22642270
}
22652271

22662272
cacheVectors := containers.NewVectors(1)
@@ -2291,7 +2297,7 @@ func (tbl *txnTable) PKPersistedBetween(
22912297
searchFunc = buildUnsortedFilter()
22922298
}
22932299

2294-
sels := searchFunc(&cacheVectors[0])
2300+
sels := searchFunc(cacheVectors)
22952301
release()
22962302
if len(sels) > 0 {
22972303
return true, nil

‎pkg/vm/engine/readutil/filter_test.go‎

Lines changed: 63 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
"github.com/matrixorigin/matrixone/pkg/sql/plan/function"
3737
"github.com/matrixorigin/matrixone/pkg/testutil"
3838
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
39+
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
3940
"github.com/stretchr/testify/require"
4041
)
4142

@@ -1249,8 +1250,9 @@ func TestConstructBlockPKFilter(t *testing.T) {
12491250
basePKFilter.String(),
12501251
common.MoVectorToString(inputVec, 100))
12511252

1252-
sel1 := blkPKFilter.SortedSearchFunc(inputVec)
1253-
sel2 := blkPKFilter.UnSortedSearchFunc(inputVec)
1253+
cacheVectors := containers.Vectors{*inputVec}
1254+
sel1 := blkPKFilter.SortedSearchFunc(cacheVectors)
1255+
sel2 := blkPKFilter.UnSortedSearchFunc(cacheVectors)
12541256

12551257
require.Equal(t, sel1, sel2, msg)
12561258
}
@@ -1657,7 +1659,9 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
16571659
require.NotNil(t, readFilter.UnSortedSearchFunc)
16581660

16591661
// Test the search function
1660-
result := readFilter.UnSortedSearchFunc(inputVec)
1662+
// For non-composite PK without optimization, use single vector
1663+
cacheVectors := containers.Vectors{*inputVec}
1664+
result := readFilter.UnSortedSearchFunc(cacheVectors)
16611665
// Should return indices of values that exist in BF: [10, 20, 30] -> indices [1, 3, 5]
16621666
require.Equal(t, []int64{1, 3, 5}, result)
16631667
})
@@ -1695,8 +1699,16 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
16951699
require.True(t, readFilter.Valid)
16961700
require.NotNil(t, readFilter.UnSortedSearchFunc)
16971701

1698-
// Test the search function
1699-
result := readFilter.UnSortedSearchFunc(compositePKVec)
1702+
// Test the search function with optimization (len(cacheVectors) >= 2)
1703+
// Create last column vector for optimization
1704+
lastColVec := vector.NewVec(types.T_int64.ToType())
1705+
for _, tuple := range tuples {
1706+
lastVal := tuple[len(tuple)-1].(int64)
1707+
vector.AppendFixed(lastColVec, lastVal, false, mp)
1708+
}
1709+
defer lastColVec.Free(mp)
1710+
cacheVectors := containers.Vectors{*compositePKVec, *lastColVec}
1711+
result := readFilter.UnSortedSearchFunc(cacheVectors)
17001712
// Should return all indices [0, 1, 2] since all last columns match
17011713
require.Equal(t, []int64{0, 1, 2}, result)
17021714
})
@@ -1731,7 +1743,15 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
17311743
)
17321744
require.NoError(t, err)
17331745

1734-
result := readFilter.UnSortedSearchFunc(compositePKVec)
1746+
// Create last column vector for optimization
1747+
lastColVec := vector.NewVec(types.T_int64.ToType())
1748+
for _, tuple := range tuples {
1749+
lastVal := tuple[len(tuple)-1].(int64)
1750+
vector.AppendFixed(lastColVec, lastVal, false, mp)
1751+
}
1752+
defer lastColVec.Free(mp)
1753+
cacheVectors := containers.Vectors{*compositePKVec, *lastColVec}
1754+
result := readFilter.UnSortedSearchFunc(cacheVectors)
17351755
// Should return indices containing [0, 2] for values 100 and 300
17361756
// Note: BloomFilter may have false positives, so we check that it contains the expected indices
17371757
require.Contains(t, result, int64(0), "Should contain index 0 (value 100)")
@@ -1766,7 +1786,15 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
17661786
)
17671787
require.NoError(t, err)
17681788

1769-
result := readFilter.UnSortedSearchFunc(compositePKVec)
1789+
// Create last column vector for optimization
1790+
lastColVec := vector.NewVec(types.T_int64.ToType())
1791+
for _, tuple := range tuples {
1792+
lastVal := tuple[len(tuple)-1].(int64)
1793+
vector.AppendFixed(lastColVec, lastVal, false, mp)
1794+
}
1795+
defer lastColVec.Free(mp)
1796+
cacheVectors := containers.Vectors{*compositePKVec, *lastColVec}
1797+
result := readFilter.UnSortedSearchFunc(cacheVectors)
17701798
// Should return empty slice since no matches (not nil, but empty slice)
17711799
require.Empty(t, result)
17721800
})
@@ -1812,7 +1840,15 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
18121840
require.True(t, readFilter.Valid)
18131841

18141842
// With only BloomFilter, should return indices 0, 1, 2 (values 100, 200, 300)
1815-
result := readFilter.UnSortedSearchFunc(compositePKVec)
1843+
// Create last column vector for optimization
1844+
lastColVec := vector.NewVec(types.T_int64.ToType())
1845+
for _, tuple := range tuples {
1846+
lastVal := tuple[len(tuple)-1].(int64)
1847+
vector.AppendFixed(lastColVec, lastVal, false, mp)
1848+
}
1849+
defer lastColVec.Free(mp)
1850+
cacheVectors := containers.Vectors{*compositePKVec, *lastColVec}
1851+
result := readFilter.UnSortedSearchFunc(cacheVectors)
18161852
require.NotNil(t, result)
18171853
require.Contains(t, result, int64(0), "Should contain index 0 (value 100)")
18181854
require.Contains(t, result, int64(1), "Should contain index 1 (value 200)")
@@ -1847,7 +1883,15 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
18471883
)
18481884
require.NoError(t, err)
18491885

1850-
result := readFilter.UnSortedSearchFunc(compositePKVec)
1886+
// Create last column vector for optimization
1887+
lastColVec := vector.NewVec(types.T_int32.ToType())
1888+
for _, tuple := range tuples {
1889+
lastVal := tuple[len(tuple)-1].(int32)
1890+
vector.AppendFixed(lastColVec, lastVal, false, mp)
1891+
}
1892+
defer lastColVec.Free(mp)
1893+
cacheVectors := containers.Vectors{*compositePKVec, *lastColVec}
1894+
result := readFilter.UnSortedSearchFunc(cacheVectors)
18511895
require.Equal(t, []int64{0, 1}, result)
18521896
})
18531897

@@ -1872,7 +1916,11 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
18721916
)
18731917
require.NoError(t, err)
18741918

1875-
result := readFilter.UnSortedSearchFunc(compositePKVec)
1919+
// For empty vector, optimization path should return nil
1920+
emptyLastColVec := vector.NewVec(types.T_int64.ToType())
1921+
defer emptyLastColVec.Free(mp)
1922+
cacheVectors := containers.Vectors{*compositePKVec, *emptyLastColVec}
1923+
result := readFilter.UnSortedSearchFunc(cacheVectors)
18761924
require.Nil(t, result)
18771925
})
18781926

@@ -1901,7 +1949,11 @@ func TestConstructBlockPKFilterWithBloomFilter(t *testing.T) {
19011949
require.NoError(t, err)
19021950

19031951
// Should handle invalid data gracefully
1904-
result := readFilter.UnSortedSearchFunc(vec)
1952+
// For invalid data, optimization path should return nil
1953+
invalidLastColVec := vector.NewVec(types.T_int64.ToType())
1954+
defer invalidLastColVec.Free(mp)
1955+
cacheVectors := containers.Vectors{*vec, *invalidLastColVec}
1956+
result := readFilter.UnSortedSearchFunc(cacheVectors)
19051957
// Should return nil or empty since extraction fails
19061958
require.Nil(t, result)
19071959
})

0 commit comments

Comments
 (0)