Select: support jsonpath indexes (#158)

Author: mRrvz

CHANGELOG.md

@@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * Ignoring `opts.first` on `crud.pairs` call
 * External `keydef` compatibility with built-in `merger`
 
+### Added
+
+* Added jsonpath indexes support for queries.
+
 ## [0.7.0] - 2021-05-27
 
 ### Fixed

crud/common/compat.lua

@@ -30,4 +30,21 @@ function compat.require(module_name, builtin_module_name)
     return module
 end
 
+function compat.exists(module_name, builtin_module_name)
+    local module_cached = rawget(_G, string.format('__crud_%s_cached', module_name))
+    if module_cached ~= nil then
+        return true
+    end
+
+    if package.search(module_name) then
+        return true
+    end
+
+    if package.loaded[builtin_module_name] ~= nil then
+        return true
+    end
+
+    return false
+end
+
 return compat

crud/common/utils.lua

@@ -205,6 +205,10 @@ local function determine_enabled_features()
 
     -- since Tarantool 2.4
     enabled_tarantool_features.uuids = major >= 2 and (minor > 4 or minor == 4 and patch >= 1)
+
+    -- since Tarantool 2.6.3 / 2.7.2 / 2.8.1
+    enabled_tarantool_features.jsonpath_indexes = major >= 3 or (major >= 2 and ((minor >= 6 and patch >= 3)
+        or (minor >= 7 and patch >= 2) or (minor >= 8 and patch >= 1) or minor >= 9))
 end
 
 function utils.tarantool_supports_fieldpaths()
@@ -223,6 +227,14 @@ function utils.tarantool_supports_uuids()
     return enabled_tarantool_features.uuids
 end
 
+function utils.tarantool_supports_jsonpath_indexes()
+    if enabled_tarantool_features.jsonpath_indexes == nil then
+        determine_enabled_features()
+    end
+
+    return enabled_tarantool_features.jsonpath_indexes
+end
+
 local function add_nullable_fields_recursive(operations, operations_map, space_format, tuple, id)
     if id < 2 or tuple[id - 1] ~= box.NULL then
         return operations

crud/compare/comparators.lua

@@ -2,10 +2,18 @@ local errors = require('errors')
 
 local compare_conditions = require('crud.compare.conditions')
 local type_comparators = require('crud.compare.type_comparators')
-local operators = compare_conditions.operators
-
 local utils = require('crud.common.utils')
 
+local compat = require('crud.common.compat')
+local has_keydef = compat.exists('tuple.keydef', 'key_def')
+
+local keydef_lib
+if has_keydef then
+    keydef_lib = compat.require('tuple.keydef', 'key_def')
+end
+
+local operators = compare_conditions.operators
+
 local ComparatorsError = errors.new_class('ComparatorsError')
 
 local comparators = {}
@@ -102,7 +110,8 @@ function comparators.update_key_parts_by_field_names(space_format, field_names,
         local field_name = space_format[part.fieldno].name
         local updated_part = {type = part.type,
                               fieldno = fields_positions[field_name],
-                              is_nullable = part.is_nullable}
+                              is_nullable = part.is_nullable,
+                              path = part.path}
         table.insert(updated_key_parts, updated_part)
     end
 
@@ -150,13 +159,22 @@ function comparators.gen_tuples_comparator(cmp_operator, key_parts, field_names,
     local updated_key_parts = comparators.update_key_parts_by_field_names(
             space_format, field_names, key_parts
     )
-    local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts)
 
-    return function(lhs, rhs)
-        local lhs_key = utils.extract_key(lhs, updated_key_parts)
-        local rhs_key = utils.extract_key(rhs, updated_key_parts)
+    local keys_comparator = comparators.gen_func(cmp_operator, updated_key_parts)
 
-        return keys_comparator(lhs_key, rhs_key)
+    if has_keydef then
+        local key_def = keydef_lib.new(updated_key_parts)
+        return function(lhs, rhs)
+            local lhs_key = key_def:extract_key(lhs)
+            local rhs_key = key_def:extract_key(rhs)
+            return keys_comparator(lhs_key, rhs_key)
+        end
+    else
+        return function(lhs, rhs)
+            local lhs_key = utils.extract_key(lhs, updated_key_parts)
+            local rhs_key = utils.extract_key(rhs, updated_key_parts)
+            return keys_comparator(lhs_key, rhs_key)
+        end
     end
 end
 

crud/select/executor.lua

@@ -2,6 +2,13 @@ local errors = require('errors')
 
 local dev_checks = require('crud.common.dev_checks')
 local select_comparators = require('crud.compare.comparators')
+local compat = require('crud.common.compat')
+local has_keydef = compat.exists('tuple.keydef', 'key_def')
+
+local keydef_lib
+if has_keydef then
+    keydef_lib = compat.require('tuple.keydef', 'key_def')
+end
 
 local utils = require('crud.common.utils')
 
@@ -31,6 +38,26 @@ local function scroll_to_after_tuple(gen, space, scan_index, tarantool_iter, aft
     end
 end
 
+local generate_value
+
+if has_keydef then
+    generate_value = function(after_tuple, scan_value, index_parts)
+        local key_def = keydef_lib.new(index_parts)
+        if key_def:compare_with_key(after_tuple, scan_value) < 0 then
+            return key_def:extract_key(after_tuple)
+        end
+    end
+else
+    generate_value = function(after_tuple, scan_value, index_parts, tarantool_iter)
+        local cmp_operator = select_comparators.get_cmp_operator(tarantool_iter)
+        local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index_parts)
+        local after_tuple_key = utils.extract_key(after_tuple, index_parts)
+        if scan_comparator(after_tuple_key, scan_value) then
+            return after_tuple_key
+        end
+    end
+end
+
 function executor.execute(space, index, filter_func, opts)
     dev_checks('table', 'table', 'function', {
         scan_value = 'table',
@@ -53,11 +80,9 @@ function executor.execute(space, index, filter_func, opts)
         if value == nil then
             value = opts.after_tuple
         else
-            local cmp_operator = select_comparators.get_cmp_operator(opts.tarantool_iter)
-            local scan_comparator = select_comparators.gen_tuples_comparator(cmp_operator, index.parts)
-            local after_tuple_key = utils.extract_key(opts.after_tuple, index.parts)
-            if scan_comparator(after_tuple_key, opts.scan_value) then
-                value = after_tuple_key
+            local new_value = generate_value(opts.after_tuple, opts.scan_value, index.parts, opts.tarantool_iter)
+            if new_value ~= nil then
+                value = new_value
             end
         end
     end

crud/select/filters.lua

@@ -44,7 +44,11 @@ local function get_index_fieldnos(index)
     local index_fieldnos = {}
 
     for _, part in ipairs(index.parts) do
-        table.insert(index_fieldnos, part.fieldno)
+        if part.path ~= nil then
+            table.insert(index_fieldnos, string.format("[%d]%s", part.fieldno, part.path))
+        else
+            table.insert(index_fieldnos, part.fieldno)
+        end
     end
 
     return index_fieldnos
@@ -91,7 +95,6 @@ local function parse(space, conditions, opts)
     end
 
     local filter_conditions = {}
-
     for i, condition in ipairs(conditions) do
         if i ~= opts.scan_condition_num then
             -- Index check (including one and multicolumn)

crud/select/merger.lua

@@ -136,7 +136,6 @@ local reverse_tarantool_iters = {
 local function new(replicasets, space, index_id, func_name, func_args, opts)
     opts = opts or {}
     local call_opts = opts.call_opts
-
     local mode = call_opts.mode or 'read'
     local vshard_call_name = call.get_vshard_call_name(mode, call_opts.prefer_replica, call_opts.balance)
 

crud/select/plan.lua

@@ -4,6 +4,14 @@ local compare_conditions = require('crud.compare.conditions')
 local utils = require('crud.common.utils')
 local dev_checks = require('crud.common.dev_checks')
 
+local compat = require('crud.common.compat')
+local has_keydef = compat.exists('tuple.keydef', 'key_def')
+
+local keydef_lib
+if has_keydef then
+    keydef_lib = compat.require('tuple.keydef', 'key_def')
+end
+
 local select_plan = {}
 
 local IndexTypeError = errors.new_class('IndexTypeError', {capture_stack = false})
@@ -184,7 +192,12 @@ function select_plan.new(space, conditions, opts)
             scan_condition_num = nil
 
             if scan_after_tuple ~= nil then
-                scan_value = utils.extract_key(scan_after_tuple, scan_index.parts)
+                if has_keydef then
+                    local key_def = keydef_lib.new(scan_index.parts)
+                    scan_value = key_def:extract_key(scan_after_tuple)
+                else
+                    scan_value = utils.extract_key(scan_after_tuple, scan_index.parts)
+                end
             else
                 scan_value = nil
             end

test/entrypoint/srv_select.lua

@@ -151,6 +151,43 @@ package.preload['customers-storage'] = function()
                 unique = false,
                 if_not_exists = true,
             })
+
+            if crud_utils.tarantool_supports_jsonpath_indexes() then
+                local cars_space = box.schema.space.create('cars', {
+                    format = {
+                        {name = 'id', type = 'map'},
+                        {name = 'bucket_id', type = 'unsigned'},
+                        {name = 'age', type = 'number'},
+                        {name = 'manufacturer', type = 'string'},
+                        {name = 'data', type = 'map'}
+                    },
+                    if_not_exists = true,
+                    engine = engine,
+                })
+
+                -- primary index
+                cars_space:create_index('id_ind', {
+                    parts = {
+                        {1, 'unsigned', path = 'car_id.signed'},
+                    },
+                    if_not_exists = true,
+                })
+
+                cars_space:create_index('bucket_id', {
+                    parts = { 'bucket_id' },
+                    unique = false,
+                    if_not_exists = true,
+                })
+
+                cars_space:create_index('data_index', {
+                    parts = {
+                        {5, 'str', path = 'car.color'},
+                        {5, 'str', path = 'car.model'},
+                    },
+                    unique = false,
+                    if_not_exists = true,
+                })
+            end
         end,
     }
 end