Skip to content

Commit b8f2f8f

Browse files
olegrokolegrok
committed
utils: implement flatten function using codegeneration
This patch introduces new approach to "flatten" function. Before thus patch we dynamically process every object using cycles, temporary tables, etc. This patch implements "flatten" function using codegeneration. No temporary tables, no redundant loops, simple linear code that just assign input object fields to corresponding tuple fields. My simple benchmarks (8 fields + bucket_id) show that this implementation at least 3 times faster than previous. Example of code is following: ```lua local object, bucket_id = ... for k in pairs(object) do if fieldmap[k] == nil then return nil, format('Unknown field %q is specified', k) end end local result = {NULL,NULL,NULL,NULL,} if object["id"] == nil then return nil, 'Field "id" isn\'t nullable' end result[1] = object["id"] if bucket_id ~= nil then result[2] = bucket_id else result[2] = object["bucket_id"] end if object["name"] == nil then return nil, 'Field "name" isn\'t nullable' end result[3] = object["name"] result[4] = object["age"] return result ``` This patch also prevent tuple serialization to map. An error "Tuple/Key must be MsgPack array" could happen. The reason that lua table could be serialized to map instead of array. Example: ``` tarantool> { [1] = 1, [5] = 5, [10] = 10, [100] = 100 } --- - 1: 1 100: 100 5: 5 10: 10 ... ``` Case is quite relevant if user uses "*_object" functions and object could have huge amount of nullable fields. This patch fixes codegeneration to prevent removing "box.NULL" values because of "nil" values. Also it could fix cases of update through absent fields (see tarantool/tarantool#3378). Closes #119
1 parent 5fa62ae commit b8f2f8f

File tree

4 files changed

+199
-27
lines changed

4 files changed

+199
-27
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
99

1010
### Fixed
1111

12+
* Fixed select crash when dropping indexes
1213
* Using outdated schema on router-side
14+
* Sparsed tuples generation that led to "Tuple/Key must be MsgPack array" error
1315

1416
### Added
1517

1618
* Support for UUID field types and UUID values
1719

18-
### Fixed
19-
20-
* Fixed select crash when dropping indexes.
21-
2220
## [0.4.0] - 2020-12-02
2321

2422
### Fixed

‎crud/common/utils.lua‎

Lines changed: 54 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -52,40 +52,71 @@ function utils.get_space_format(space_name, replicasets)
5252
return space_format
5353
end
5454

55-
local system_fields = { bucket_id = true }
55+
local function append(lines, s, ...)
56+
table.insert(lines, string.format(s, ...))
57+
end
58+
59+
local flatten_functions_cache = setmetatable({}, {__mode = 'k'})
5660

5761
function utils.flatten(object, space_format, bucket_id)
58-
if object == nil then return nil end
62+
local flatten_func = flatten_functions_cache[space_format]
63+
if flatten_func ~= nil then
64+
local data, err = flatten_func(object, bucket_id)
65+
if err ~= nil then
66+
return nil, FlattenError:new(err)
67+
end
68+
return data
69+
end
5970

60-
local tuple = {}
71+
local lines = {}
72+
append(lines, 'local object, bucket_id = ...')
6173

62-
local fieldnames = {}
74+
append(lines, 'for k in pairs(object) do')
75+
append(lines, ' if fieldmap[k] == nil then')
76+
append(lines, ' return nil, format(\'Unknown field %%q is specified\', k)')
77+
append(lines, ' end')
78+
append(lines, 'end')
6379

64-
for fieldno, field_format in ipairs(space_format) do
65-
local fieldname = field_format.name
66-
local value = object[fieldname]
80+
local len = #space_format
81+
append(lines, 'local result = {%s}', string.rep('NULL,', len))
6782

68-
if not system_fields[fieldname] then
69-
if not field_format.is_nullable and value == nil then
70-
return nil, FlattenError:new("Field %q isn't nullable", fieldname)
71-
end
72-
end
83+
local fieldmap = {}
7384

74-
if bucket_id ~= nil and fieldname == 'bucket_id' then
75-
value = bucket_id
85+
for i, field in ipairs(space_format) do
86+
fieldmap[field.name] = true
87+
if field.name ~= 'bucket_id' then
88+
append(lines, 'if object[%q] ~= nil then', field.name)
89+
append(lines, ' result[%d] = object[%q]', i, field.name)
90+
if field.is_nullable ~= true then
91+
append(lines, 'else')
92+
append(lines, ' return nil, \'Field %q isn\\\'t nullable\'', field.name)
93+
end
94+
append(lines, 'end')
95+
else
96+
append(lines, 'if bucket_id ~= nil then')
97+
append(lines, ' result[%d] = bucket_id', i, field.name)
98+
append(lines, 'else')
99+
append(lines, ' result[%d] = object[%q]', i, field.name)
100+
append(lines, 'end')
76101
end
77-
78-
tuple[fieldno] = value
79-
fieldnames[fieldname] = true
80102
end
103+
append(lines, 'return result')
104+
105+
local code = table.concat(lines, '\n')
106+
local env = {
107+
pairs = pairs,
108+
format = string.format,
109+
fieldmap = fieldmap,
110+
NULL = box.NULL,
111+
}
112+
flatten_func = assert(load(code, '@flatten', 't', env))
81113

82-
for fieldname in pairs(object) do
83-
if not fieldnames[fieldname] then
84-
return nil, FlattenError:new("Unknown field %q is specified", fieldname)
85-
end
114+
flatten_functions_cache[space_format] = flatten_func
115+
local data, err = flatten_func(object, bucket_id)
116+
if err ~= nil then
117+
return nil, FlattenError:new(err)
86118
end
87-
88-
return tuple
119+
return data
89120
end
90121

91122
function utils.unflatten(tuple, space_format)

‎test/entrypoint/srv_simple_operations.lua‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,39 @@ package.preload['customers-storage'] = function()
3131
unique = false,
3232
if_not_exists = true,
3333
})
34+
35+
-- Space with huge amount of nullable fields
36+
-- an object that inserted in such space should get
37+
-- explicit nulls in absence fields otherwise
38+
-- Tarantool serializers could consider such object as map (not array).
39+
local tags_space = box.schema.space.create('tags', {
40+
format = {
41+
{name = 'id', type = 'unsigned'},
42+
{name = 'bucket_id', type = 'unsigned'},
43+
{name = 'is_red', type = 'boolean', is_nullable = true},
44+
{name = 'is_green', type = 'boolean', is_nullable = true},
45+
{name = 'is_blue', type = 'boolean', is_nullable = true},
46+
{name = 'is_yellow', type = 'boolean', is_nullable = true},
47+
{name = 'is_sweet', type = 'boolean', is_nullable = true},
48+
{name = 'is_dirty', type = 'boolean', is_nullable = true},
49+
{name = 'is_long', type = 'boolean', is_nullable = true},
50+
{name = 'is_short', type = 'boolean', is_nullable = true},
51+
{name = 'is_useful', type = 'boolean', is_nullable = true},
52+
{name = 'is_correct', type = 'boolean', is_nullable = true},
53+
},
54+
if_not_exists = true,
55+
engine = engine,
56+
})
57+
58+
tags_space:create_index('id', {
59+
parts = { {field = 'id'} },
60+
if_not_exists = true,
61+
})
62+
tags_space:create_index('bucket_id', {
63+
parts = { {field = 'bucket_id'} },
64+
unique = false,
65+
if_not_exists = true,
66+
})
3467
end,
3568
}
3669
end

‎test/integration/simple_operations_test.lua‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,3 +391,113 @@ pgroup:add('test_upsert', function(g)
391391
t.assert_equals(err, nil)
392392
t.assert_equals(result.rows, {{67, 1143, 'Mikhail Saltykov-Shchedrin', 63}})
393393
end)
394+
395+
pgroup:add('test_object_with_nullable_fields', function(g)
396+
-- Insert
397+
local result, err = g.cluster.main_server.net_box:call(
398+
'crud.insert_object', {'tags', {id = 1, is_green = true}})
399+
t.assert_equals(err, nil)
400+
401+
-- {1, 477, NULL, true, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}
402+
local objects = crud.unflatten_rows(result.rows, result.metadata)
403+
t.assert_equals(objects, {
404+
{
405+
bucket_id = 477,
406+
id = 1,
407+
is_blue = box.NULL,
408+
is_correct = box.NULL,
409+
is_dirty = box.NULL,
410+
is_green = true,
411+
is_long = box.NULL,
412+
is_red = box.NULL,
413+
is_short = box.NULL,
414+
is_sweet = box.NULL,
415+
is_useful = box.NULL,
416+
is_yellow = box.NULL,
417+
}
418+
})
419+
420+
-- Update
421+
-- {1, 477, NULL, true, NULL, NULL, true, NULL, NULL, NULL, NULL, NULL}
422+
-- Shouldn't failed because of https://github.com/tarantool/tarantool/issues/3378
423+
result, err = g.cluster.main_server.net_box:call(
424+
'crud.update', {'tags', 1, {{'=', 'is_sweet', true}}})
425+
t.assert_equals(err, nil)
426+
objects = crud.unflatten_rows(result.rows, result.metadata)
427+
t.assert_equals(objects, {
428+
{
429+
bucket_id = 477,
430+
id = 1,
431+
is_blue = box.NULL,
432+
is_correct = box.NULL,
433+
is_dirty = box.NULL,
434+
is_green = true,
435+
is_long = box.NULL,
436+
is_red = box.NULL,
437+
is_short = box.NULL,
438+
is_sweet = true,
439+
is_useful = box.NULL,
440+
is_yellow = box.NULL,
441+
}
442+
})
443+
444+
-- Replace
445+
-- {2, 401, NULL, true, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}
446+
result, err = g.cluster.main_server.net_box:call(
447+
'crud.replace_object', {'tags', {id = 2, is_green = true}})
448+
t.assert_equals(err, nil)
449+
objects = crud.unflatten_rows(result.rows, result.metadata)
450+
t.assert_equals(objects, {
451+
{
452+
bucket_id = 401,
453+
id = 2,
454+
is_blue = box.NULL,
455+
is_correct = box.NULL,
456+
is_dirty = box.NULL,
457+
is_green = true,
458+
is_long = box.NULL,
459+
is_red = box.NULL,
460+
is_short = box.NULL,
461+
is_sweet = box.NULL,
462+
is_useful = box.NULL,
463+
is_yellow = box.NULL,
464+
}
465+
})
466+
467+
-- Upsert: first is insert then update
468+
-- {3, 2804, NULL, NULL, NULL, NULL, NULL, true, NULL, NULL, NULL, NULL}
469+
local _, err = g.cluster.main_server.net_box:call(
470+
'crud.upsert_object', {'tags', {id = 3, is_dirty = true}, {
471+
{'=', 'is_dirty', true},
472+
}})
473+
t.assert_equals(err, nil)
474+
475+
-- {3, 2804, NULL, NULL, NULL, NULL, NULL, true, NULL, true, true, NULL}
476+
-- Shouldn't failed because of https://github.com/tarantool/tarantool/issues/3378
477+
_, err = g.cluster.main_server.net_box:call(
478+
'crud.upsert_object', {'tags', {id = 3, is_dirty = true}, {
479+
{'=', 'is_useful', true},
480+
}})
481+
t.assert_equals(err, nil)
482+
483+
-- Get
484+
result, err = g.cluster.main_server.net_box:call('crud.get', {'tags', 3})
485+
t.assert_equals(err, nil)
486+
objects = crud.unflatten_rows(result.rows, result.metadata)
487+
t.assert_equals(objects, {
488+
{
489+
bucket_id = 2804,
490+
id = 3,
491+
is_blue = box.NULL,
492+
is_correct = box.NULL,
493+
is_dirty = true,
494+
is_green = box.NULL,
495+
is_long = box.NULL,
496+
is_red = box.NULL,
497+
is_short = box.NULL,
498+
is_sweet = box.NULL,
499+
is_useful = true,
500+
is_yellow = box.NULL,
501+
}
502+
})
503+
end)

0 commit comments

Comments
 (0)