From e1c47d25476ba72fc64742b5ba20b6b1afeedbb9 Mon Sep 17 00:00:00 2001 From: Sean Zellmer Date: Sat, 22 Nov 2025 15:13:10 -0500 Subject: [PATCH 1/2] Support using arrays as index encoder key component --- builder/codegen.js | 6 + test/basic.js | 31 ++++ test/fixtures/builders/8.js | 51 ++++++ test/fixtures/generate.js | 1 + test/fixtures/generated/8/hyperdb/db.json | 37 ++++ test/fixtures/generated/8/hyperdb/index.js | 161 ++++++++++++++++++ test/fixtures/generated/8/hyperdb/messages.js | 117 +++++++++++++ .../fixtures/generated/8/hyperschema/index.js | 99 +++++++++++ .../generated/8/hyperschema/schema.json | 31 ++++ 9 files changed, 534 insertions(+) create mode 100644 test/fixtures/builders/8.js create mode 100644 test/fixtures/generated/8/hyperdb/db.json create mode 100644 test/fixtures/generated/8/hyperdb/index.js create mode 100644 test/fixtures/generated/8/hyperdb/messages.js create mode 100644 test/fixtures/generated/8/hyperschema/index.js create mode 100644 test/fixtures/generated/8/hyperschema/schema.json diff --git a/builder/codegen.js b/builder/codegen.js index da552ef..3bdd479 100644 --- a/builder/codegen.js +++ b/builder/codegen.js @@ -437,9 +437,15 @@ function generateIndexKeyEncoding(type) { const component = type.keyEncoding[i] const keyType = type.builder.schema.types.get(`@${type.namespace}/${component}`) + + if (keyType?.isArray) str += ' c.array(' + if (keyType?.isEnum) str += ' IndexEncoder.UINT' + else if (keyType?.isArray) str += IndexTypeMap.get(keyType.type.fqn) else str += ' ' + IndexTypeMap.get(component) + if (keyType?.isArray) str += ')' + if (i !== type.keyEncoding.length - 1) str += ',\n' else str += '\n' } diff --git a/test/basic.js b/test/basic.js index 4b4421d..4553b9c 100644 --- a/test/basic.js +++ b/test/basic.js @@ -548,3 +548,34 @@ test('enum as key type', async function ({ create, bee }, t) { await db.close() }) + +test('array as key type', async function ({ create, bee }, t) { + const db = await create({ fixture: 8 }) + + await db.insert('@db/books', { + title: 'Brave New World', + tags: ['science fiction', 'dystopian', 'ficton'] + }) + await db.flush() + + { + const book = await db.get('@db/books', { + title: 'Brave New World', + tags: ['science fiction', 'dystopian', 'ficton'] + }) + t.alike(book, { title: 'Brave New World', tags: ['science fiction', 'dystopian', 'ficton'] }) + } + { + const distopianScifi = await db + .find('@db/books-by-tag', { + lte: { tags: ['science fiction', 'dystopian', 'ficton'] }, + gte: { tags: ['science fiction', 'dystopian', 'ficton'] } + }) + .toArray() + t.alike(distopianScifi, [ + { title: 'Brave New World', tags: ['science fiction', 'dystopian', 'ficton'] } + ]) + } + + await db.close() +}) diff --git a/test/fixtures/builders/8.js b/test/fixtures/builders/8.js new file mode 100644 index 0000000..0e98c02 --- /dev/null +++ b/test/fixtures/builders/8.js @@ -0,0 +1,51 @@ +const HyperDB = require('../../../builder') +const Hyperschema = require('hyperschema') +const path = require('path') + +const SCHEMA_DIR = path.join(__dirname, '../generated/8/hyperschema') +const DB_DIR = path.join(__dirname, '../generated/8/hyperdb') + +const schema = Hyperschema.from(SCHEMA_DIR) + +const dbSchema = schema.namespace('db') + +dbSchema.register({ + name: 'tags', + type: 'string', + array: true +}) + +dbSchema.register({ + name: 'book', + fields: [ + { + name: 'title', + type: 'string', + required: true + }, + { + name: 'tags', + type: '@db/tags', + required: true + } + ] +}) + +Hyperschema.toDisk(schema) + +const db = HyperDB.from(SCHEMA_DIR, DB_DIR) +const testDb = db.namespace('db') + +testDb.collections.register({ + name: 'books', + schema: '@db/book', + key: ['title', 'tags'] +}) + +testDb.indexes.register({ + name: 'books-by-tag', + collection: '@db/books', + key: ['tags'] +}) + +HyperDB.toDisk(db) diff --git a/test/fixtures/generate.js b/test/fixtures/generate.js index fece4cd..50006a6 100644 --- a/test/fixtures/generate.js +++ b/test/fixtures/generate.js @@ -5,3 +5,4 @@ require('./builders/4.js') require('./builders/5.js') require('./builders/6.js') require('./builders/7.js') +require('./builders/8.js') diff --git a/test/fixtures/generated/8/hyperdb/db.json b/test/fixtures/generated/8/hyperdb/db.json new file mode 100644 index 0000000..5b154bf --- /dev/null +++ b/test/fixtures/generated/8/hyperdb/db.json @@ -0,0 +1,37 @@ +{ + "version": 1, + "offset": 0, + "schema": [ + { + "name": "books", + "namespace": "db", + "id": 0, + "type": 1, + "version": 1, + "versionField": null, + "indexes": [ + "@db/books-by-tag" + ], + "schema": "@db/book", + "derived": false, + "key": [ + "title", + "tags" + ], + "trigger": null + }, + { + "name": "books-by-tag", + "namespace": "db", + "id": 1, + "type": 2, + "version": 1, + "collection": "@db/books", + "unique": false, + "deprecated": false, + "key": [ + "tags" + ] + } + ] +} \ No newline at end of file diff --git a/test/fixtures/generated/8/hyperdb/index.js b/test/fixtures/generated/8/hyperdb/index.js new file mode 100644 index 0000000..23c990f --- /dev/null +++ b/test/fixtures/generated/8/hyperdb/index.js @@ -0,0 +1,161 @@ +// This file is autogenerated by the hyperdb compiler +/* eslint-disable camelcase */ + +const { IndexEncoder, c, b4a } = require('hyperdb/runtime') +const { version, getEncoding, setVersion } = require('./messages.js') + +const versions = { schema: version, db: 1 } + +// '@db/books' collection key +const collection0_key = new IndexEncoder([ + IndexEncoder.STRING, + c.array(IndexEncoder.STRING) +], { prefix: 0 }) + +function collection0_indexify (record) { + const arr = [] + + const a0 = record.title + if (a0 === undefined) return arr + arr.push(a0) + + const a1 = record.tags + if (a1 === undefined) return arr + arr.push(a1) + + return arr +} + +// '@db/books' value encoding +const collection0_enc = getEncoding('@db/book/hyperdb#0') + +// '@db/books' reconstruction function +function collection0_reconstruct (schemaVersion, keyBuf, valueBuf) { + const key = collection0_key.decode(keyBuf) + setVersion(schemaVersion) + const state = { start: 0, end: valueBuf.byteLength, buffer: valueBuf } + const type = c.uint.decode(state) + if (type !== 0) throw new Error('Unknown collection type: ' + type) + collection0.decodedVersion = c.uint.decode(state) + const record = collection0_enc.decode(state) + record.title = key[0] + record.tags = key[1] + return record +} +// '@db/books' key reconstruction function +function collection0_reconstruct_key (keyBuf) { + const key = collection0_key.decode(keyBuf) + return { + title: key[0], + tags: key[1] + } +} + +// '@db/books' +const collection0 = { + name: '@db/books', + id: 0, + version: 1, + encodeKey (record) { + const key = [record.title, record.tags] + return collection0_key.encode(key) + }, + encodeKeyRange ({ gt, lt, gte, lte } = {}) { + return collection0_key.encodeRange({ + gt: gt ? collection0_indexify(gt) : null, + lt: lt ? collection0_indexify(lt) : null, + gte: gte ? collection0_indexify(gte) : null, + lte: lte ? collection0_indexify(lte) : null + }) + }, + encodeValue (schemaVersion, collectionVersion, record) { + setVersion(schemaVersion) + const state = { start: 0, end: 2, buffer: null } + collection0_enc.preencode(state, record) + state.buffer = b4a.allocUnsafe(state.end) + state.buffer[state.start++] = 0 + state.buffer[state.start++] = collectionVersion + collection0_enc.encode(state, record) + return state.buffer + }, + trigger: null, + reconstruct: collection0_reconstruct, + reconstructKey: collection0_reconstruct_key, + indexes: [], + decodedVersion: 0 +} + +// '@db/books-by-tag' collection key +const index1_key = new IndexEncoder([ + c.array(IndexEncoder.STRING), + IndexEncoder.STRING, + c.array(IndexEncoder.STRING) +], { prefix: 1 }) + +function index1_indexify (record) { + const arr = [] + + const a0 = record.tags + if (a0 === undefined) return arr + arr.push(a0) + + const a1 = record.title + if (a1 === undefined) return arr + arr.push(a1) + + const a2 = record.tags + if (a2 === undefined) return arr + arr.push(a2) + + return arr +} + +// '@db/books-by-tag' +const index1 = { + name: '@db/books-by-tag', + version: 1, + id: 1, + encodeKey (record) { + return index1_key.encode(index1_indexify(record)) + }, + encodeKeyRange ({ gt, lt, gte, lte } = {}) { + return index1_key.encodeRange({ + gt: gt ? index1_indexify(gt) : null, + lt: lt ? index1_indexify(lt) : null, + gte: gte ? index1_indexify(gte) : null, + lte: lte ? index1_indexify(lte) : null + }) + }, + encodeValue: (record) => index1.collection.encodeKey(record), + encodeIndexKeys (record, context) { + return [index1_key.encode([record.tags, record.title, record.tags])] + }, + reconstruct: (keyBuf, valueBuf) => valueBuf, + offset: collection0.indexes.length, + collection: collection0 +} +collection0.indexes.push(index1) + +const collections = [ + collection0 +] + +const indexes = [ + index1 +] + +module.exports = { versions, collections, indexes, resolveCollection, resolveIndex } + +function resolveCollection (name) { + switch (name) { + case '@db/books': return collection0 + default: return null + } +} + +function resolveIndex (name) { + switch (name) { + case '@db/books-by-tag': return index1 + default: return null + } +} diff --git a/test/fixtures/generated/8/hyperdb/messages.js b/test/fixtures/generated/8/hyperdb/messages.js new file mode 100644 index 0000000..47858b0 --- /dev/null +++ b/test/fixtures/generated/8/hyperdb/messages.js @@ -0,0 +1,117 @@ +// This file is autogenerated by the hyperschema compiler +// Schema Version: 1 +/* eslint-disable camelcase */ +/* eslint-disable quotes */ +/* eslint-disable space-before-function-paren */ + +const { c } = require('hyperschema/runtime') + +const VERSION = 1 + +// eslint-disable-next-line no-unused-vars +let version = VERSION + +// @db/tags +const encoding0 = c.array(c.string) + +// @db/book +const encoding1 = { + preencode(state, m) { + c.string.preencode(state, m.title) + encoding0.preencode(state, m.tags) + }, + encode(state, m) { + c.string.encode(state, m.title) + encoding0.encode(state, m.tags) + }, + decode(state) { + const r0 = c.string.decode(state) + const r1 = encoding0.decode(state) + + return { + title: r0, + tags: r1 + } + } +} + +// @db/book/hyperdb#0 +const encoding2 = { + preencode(state, m) { + + }, + encode(state, m) { + + }, + decode(state) { + return { + title: null, + tags: null + } + } +} + +function setVersion(v) { + version = v +} + +function encode(name, value, v = VERSION) { + version = v + return c.encode(getEncoding(name), value) +} + +function decode(name, buffer, v = VERSION) { + version = v + return c.decode(getEncoding(name), buffer) +} + +function getEnum(name) { + switch (name) { + default: + throw new Error('Enum not found ' + name) + } +} + +function getEncoding(name) { + switch (name) { + case '@db/tags': + return encoding0 + case '@db/book': + return encoding1 + case '@db/book/hyperdb#0': + return encoding2 + default: + throw new Error('Encoder not found ' + name) + } +} + +function getStruct(name, v = VERSION) { + const enc = getEncoding(name) + return { + preencode(state, m) { + version = v + enc.preencode(state, m) + }, + encode(state, m) { + version = v + enc.encode(state, m) + }, + decode(state) { + version = v + return enc.decode(state) + } + } +} + +const resolveStruct = getStruct // compat + +module.exports = { + resolveStruct, + getStruct, + getEnum, + getEncoding, + encode, + decode, + setVersion, + version +} diff --git a/test/fixtures/generated/8/hyperschema/index.js b/test/fixtures/generated/8/hyperschema/index.js new file mode 100644 index 0000000..024c3ff --- /dev/null +++ b/test/fixtures/generated/8/hyperschema/index.js @@ -0,0 +1,99 @@ +// This file is autogenerated by the hyperschema compiler +// Schema Version: 1 +/* eslint-disable camelcase */ +/* eslint-disable quotes */ +/* eslint-disable space-before-function-paren */ + +const { c } = require('hyperschema/runtime') + +const VERSION = 1 + +// eslint-disable-next-line no-unused-vars +let version = VERSION + +// @db/tags +const encoding0 = c.array(c.string) + +// @db/book +const encoding1 = { + preencode(state, m) { + c.string.preencode(state, m.title) + encoding0.preencode(state, m.tags) + }, + encode(state, m) { + c.string.encode(state, m.title) + encoding0.encode(state, m.tags) + }, + decode(state) { + const r0 = c.string.decode(state) + const r1 = encoding0.decode(state) + + return { + title: r0, + tags: r1 + } + } +} + +function setVersion(v) { + version = v +} + +function encode(name, value, v = VERSION) { + version = v + return c.encode(getEncoding(name), value) +} + +function decode(name, buffer, v = VERSION) { + version = v + return c.decode(getEncoding(name), buffer) +} + +function getEnum(name) { + switch (name) { + default: + throw new Error('Enum not found ' + name) + } +} + +function getEncoding(name) { + switch (name) { + case '@db/tags': + return encoding0 + case '@db/book': + return encoding1 + default: + throw new Error('Encoder not found ' + name) + } +} + +function getStruct(name, v = VERSION) { + const enc = getEncoding(name) + return { + preencode(state, m) { + version = v + enc.preencode(state, m) + }, + encode(state, m) { + version = v + enc.encode(state, m) + }, + decode(state) { + version = v + return enc.decode(state) + } + } +} + +const resolveStruct = getStruct // compat + +module.exports = { + resolveStruct, + getStruct, + getEnum, + getEncoding, + encode, + decode, + setVersion, + version +} diff --git a/test/fixtures/generated/8/hyperschema/schema.json b/test/fixtures/generated/8/hyperschema/schema.json new file mode 100644 index 0000000..6fe917e --- /dev/null +++ b/test/fixtures/generated/8/hyperschema/schema.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "schema": [ + { + "name": "tags", + "namespace": "db", + "array": true, + "type": "string" + }, + { + "name": "book", + "namespace": "db", + "compact": false, + "flagsPosition": -1, + "fields": [ + { + "name": "title", + "required": true, + "type": "string", + "version": 1 + }, + { + "name": "tags", + "required": true, + "type": "@db/tags", + "version": 1 + } + ] + } + ] +} From 68e2c6fff8ece86d701c858b9d58a5dbb0514142 Mon Sep 17 00:00:00 2001 From: Sean Zellmer Date: Sat, 22 Nov 2025 15:55:10 -0500 Subject: [PATCH 2/2] Showcase awkwardness of querying because of array length in test --- test/basic.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/basic.js b/test/basic.js index 4553b9c..0d20b82 100644 --- a/test/basic.js +++ b/test/basic.js @@ -556,6 +556,10 @@ test('array as key type', async function ({ create, bee }, t) { title: 'Brave New World', tags: ['science fiction', 'dystopian', 'ficton'] }) + await db.insert('@db/books', { + title: 'Anathem', + tags: ['science fiction', 'philosophy', 'math', 'ficton'] + }) await db.flush() { @@ -576,6 +580,18 @@ test('array as key type', async function ({ create, bee }, t) { { title: 'Brave New World', tags: ['science fiction', 'dystopian', 'ficton'] } ]) } + { + const scifi = await db + .find('@db/books-by-tag', { + lte: { tags: ['science fiction', 'z', null, null] }, + gte: { tags: ['science fiction'] } + }) + .toArray() + t.alike(scifi, [ + { title: 'Brave New World', tags: ['science fiction', 'dystopian', 'ficton'] }, + { title: 'Anathem', tags: ['science fiction', 'philosophy', 'math', 'ficton'] } + ]) + } await db.close() })