From 570fdc88b3b72b61c4b2d657c01cbdfb6c95568a Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 16:17:13 +0300 Subject: [PATCH 01/15] add experiment --- src/jsonpak/builder.nim | 17 ++++++----- src/jsonpak/dollar.nim | 15 ++++++++-- src/jsonpak/mapper.nim | 5 +++- src/jsonpak/parser.nim | 8 +++-- src/jsonpak/private/jsonnode.nim | 39 ++++++++++++++---------- src/jsonpak/private/jsontree.nim | 44 ++++++++++++++++++--------- src/jsonpak/private/rawops.nim | 51 +++++++++++++++++++++++++------- src/jsonpak/sorted.nim | 10 +++++-- 8 files changed, 135 insertions(+), 54 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index ae54d8e..9dde379 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -29,14 +29,17 @@ proc initFromJson*(dst: var JsonTree; tree: JsonTree; n: NodePos) = proc initFromJson*[T: SomeInteger](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JInt}) - when T is BiggestUInt: - dst = parseBiggestUInt n.str - elif T is BiggestInt: - dst = parseBiggestInt n.str - elif T is SomeSignedInt: - dst = T(parseInt n.str) + if n.isShort: + dst = cast[T](n.operand) else: - dst = T(parseUInt n.str) + when T is BiggestUInt: + dst = parseBiggestUInt n.str + elif T is BiggestInt: + dst = parseBiggestInt n.str + elif T is SomeSignedInt: + dst = T(parseInt n.str) + else: + dst = T(parseUInt n.str) proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JInt, JFloat}) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 2be5c9d..5ed38a7 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -80,7 +80,13 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "{" it.push child pendingComma = false - of opcodeInt, opcodeFloat: + of opcodeInt: + if child.isShort: + result.addInt cast[int64](child.operand) + else: + result.add child.str + pendingComma = true + of opcodeFloat: result.add child.str pendingComma = true of opcodeString: @@ -99,7 +105,12 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "}" of opcodeString: escapeJson(n.str, result) - of opcodeInt, opcodeFloat: + of opcodeInt: + if n.isShort: + result.addInt cast[int64](n.operand) + else: + result.add n.str + of opcodeFloat: result.add n.str of opcodeBool: result.add(if n.bval: "true" else: "false") diff --git a/src/jsonpak/mapper.nim b/src/jsonpak/mapper.nim index 7424026..468ca96 100644 --- a/src/jsonpak/mapper.nim +++ b/src/jsonpak/mapper.nim @@ -6,7 +6,10 @@ proc toJson*(s: string; tree: var JsonTree) = storeAtom(tree, opcodeString, s) proc toJson*[T: SomeInteger](n: T; tree: var JsonTree) = - storeAtom(tree, opcodeInt, $n) + if n >= shortIntLow and n <= shortIntHigh: + storeShortAtom(tree, opcodeInt, n) + else: + storeAtom(tree, opcodeInt, $n) proc toJson*[T: SomeFloat](n: T; tree: var JsonTree) = storeAtom(tree, opcodeFloat, $n) diff --git a/src/jsonpak/parser.nim b/src/jsonpak/parser.nim index b82dd7f..4e89a4a 100644 --- a/src/jsonpak/parser.nim +++ b/src/jsonpak/parser.nim @@ -1,4 +1,4 @@ -import std/[parsejson, streams], private/[jsontree, jsonnode] +import std/[parsejson, streams, strutils], private/[jsontree, jsonnode] export JsonParsingError proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) = @@ -7,7 +7,11 @@ proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) = storeAtom(tree, opcodeString, p.a) discard getTok(p) of tkInt: - storeAtom(tree, opcodeInt, p.a) + let n = parseInt(p.a) + if n >= shortIntLow and n <= shortIntHigh: + storeShortAtom(tree, opcodeInt, n) + else: + storeAtom(tree, opcodeInt, p.a) discard getTok(p) of tkFloat: storeAtom(tree, opcodeFloat, p.a) diff --git a/src/jsonpak/private/jsonnode.nim b/src/jsonpak/private/jsonnode.nim index 7730a7f..06827a2 100644 --- a/src/jsonpak/private/jsonnode.nim +++ b/src/jsonpak/private/jsonnode.nim @@ -1,5 +1,5 @@ type - Node* = distinct uint32 + Node* = distinct uint64 JsonNodeKind* = enum ## possible JSON node types JNull, JBool, @@ -10,24 +10,33 @@ type JArray const - opcodeBits = 3'u32 + opcodeBits = 4 + payloadBits* = sizeof(uint64) - opcodeBits + shortBit = 0b0000_1000 - opcodeNull* = uint32 JNull - opcodeBool* = uint32 JBool + shortIntLow* = -(1 shl payloadBits) + shortIntHigh* = (1 shl payloadBits) - 1 + + opcodeNull* = uint64 JNull + opcodeBool* = uint64 JBool opcodeFalse* = opcodeBool - opcodeTrue* = opcodeBool or 0b0000_1000 - opcodeInt* = uint32 JInt - opcodeFloat* = uint32 JFloat - opcodeString* = uint32 JString - opcodeObject* = uint32 JObject - opcodeArray* = uint32 JArray + opcodeTrue* = opcodeBool or shortBit + opcodeInt* = uint64 JInt + opcodeFloat* = uint64 JFloat + opcodeString* = uint64 JString + opcodeObject* = uint64 JObject + opcodeArray* = uint64 JArray + + opcodeMask = (1 shl (opcodeBits - 1)) - 1 - opcodeMask = (1'u32 shl opcodeBits) - 1'u32 +template kind*(n: Node): uint64 = n.uint64 and opcodeMask +template operand*(n: Node): uint64 = n.uint64 shr opcodeBits.uint64 +template isShort*(n: Node): bool = (n.uint64 and shortBit) != 0 -template kind*(n: Node): uint32 = n.uint32 and opcodeMask -template operand*(n: Node): uint32 = n.uint32 shr opcodeBits.uint32 +template toShortNode*(kind, operand: uint64): Node = + Node(operand shl opcodeBits.uint64 or kind or shortBit.uint64) -template toNode*(kind, operand: uint32): Node = - Node(operand shl opcodeBits.uint32 or kind) +template toNode*(kind, operand: uint64): Node = + Node(operand shl opcodeBits.uint64 or kind) proc `==`*(a, b: Node): bool {.borrow.} diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 4854f49..baf6e31 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -23,8 +23,8 @@ proc nextChild*(tree: JsonTree; pos: var int) {.inline.} = else: inc pos -proc toAtomNode*(tree: var JsonTree; kind: uint32, str: string): Node {.inline.} = - toNode(kind, uint32 getOrIncl(tree.atoms, str)) +proc toAtomNode*(tree: var JsonTree; kind: uint64, str: string): Node {.inline.} = + toNode(kind, uint64 getOrIncl(tree.atoms, str)) type NodePos* = distinct int @@ -76,11 +76,18 @@ proc parentImpl*(tree: JsonTree; n: NodePos): NodePos = template parent*(n: NodePos): NodePos = parentImpl(tree, n) -template kind*(n: NodePos): uint32 = tree.nodes[n.int].kind +template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort +template kind*(n: NodePos): uint64 = tree.nodes[n.int].kind template litId*(n: NodePos): LitId = LitId operand(tree.nodes[n.int]) -template operand*(n: NodePos): uint32 = tree.nodes[n.int].operand - -template str*(n: NodePos): string = tree.atoms[litId(n)] +template operand*(n: NodePos): uint64 = tree.nodes[n.int].operand +template str*(n: NodePos): string = + if isShort(tree.nodes[n.int]): + var data = newString(payloadBits div 8) + for i in 0 ..< data.len: + data[i] = chr(n.operand shr (i * 8) and 0xFF) + data + else: + tree.atoms[litId(n)] template bval*(n: NodePos): bool = n.operand == 1 type @@ -90,21 +97,30 @@ proc `<`*(a, b: PatchPos): bool {.borrow.} proc `<=`*(a, b: PatchPos): bool {.borrow.} proc `==`*(a, b: PatchPos): bool {.borrow.} -proc prepare*(tree: var JsonTree; kind: uint32): PatchPos = +proc prepare*(tree: var JsonTree; kind: uint64): PatchPos = result = PatchPos tree.nodes.len tree.nodes.add Node kind proc patch*(tree: var JsonTree; pos: PatchPos) = let pos = pos.int assert tree.nodes[pos].kind > opcodeString - let distance = uint32(tree.nodes.len - pos) - tree.nodes[pos] = toNode(tree.nodes[pos].uint32, distance) + let distance = uint64(tree.nodes.len - pos) + tree.nodes[pos] = toNode(tree.nodes[pos].uint64, distance) + +proc storeEmpty*(tree: var JsonTree; kind: uint64) {.inline.} = + tree.nodes.add toNode(kind, 1) -proc storeAtom*(tree: var JsonTree; kind: uint32) {.inline.} = +proc storeAtom*(tree: var JsonTree; kind: uint64) {.inline.} = tree.nodes.add Node(kind) -proc storeAtom*(tree: var JsonTree; kind: uint32; data: string) {.inline.} = - tree.nodes.add toAtomNode(tree, kind, data) +proc storeShortAtom*[T: SomeInteger](tree: var JsonTree; kind: uint64, data: T) {.inline.} = + tree.nodes.add toShortNode(kind, cast[uint64](data)) -proc storeEmpty*(tree: var JsonTree; kind: uint32) {.inline.} = - tree.nodes.add toNode(kind, 1) +proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = + if data.len <= payloadBits div 8: + var payload: uint64 = 0 + for i in 0 ..< data.len: + payload = payload or (data[i].uint64 shl (i * 8)) + tree.nodes.add toShortNode(kind, payload) + else: + tree.nodes.add toAtomNode(tree, kind, data) diff --git a/src/jsonpak/private/rawops.nim b/src/jsonpak/private/rawops.nim index 2cfd52e..b1d56e5 100644 --- a/src/jsonpak/private/rawops.nim +++ b/src/jsonpak/private/rawops.nim @@ -1,15 +1,28 @@ import bitabs, jsonnode, jsontree, std/importutils -proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = +proc rawGetShort*(tree: JsonTree, n: NodePos, name: uint64): NodePos = privateAccess(JsonTree) - let litId = tree.atoms.getKeyId(name) - if litId == LitId(0): - return nilNodeId for x in keys(tree, n): - if x.litId == litId: + if x.operand == name: return x.firstSon return nilNodeId +proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = + privateAccess(JsonTree) + if name.len <= payloadBits div 8: + var payload: uint64 = 0 + for i in 0 ..< name.len: + payload = payload or (name[i].uint64 shl (i * 8)) + return rawGetShort(tree, n, payload) + else: + let litId = tree.atoms.getKeyId(name) + if litId == LitId(0): + return nilNodeId + for x in keys(tree, n): + if x.litId == litId: + return x.firstSon + return nilNodeId + proc rawUpdateParents*(tree: var JsonTree, parents: seq[PatchPos], diff: int) = privateAccess(JsonTree) for parent in parents: @@ -24,7 +37,10 @@ proc rawExtract*(result: var JsonTree, tree: JsonTree, n: NodePos) = let n = NodePos(i+n.int) # careful case n.kind of opcodeInt, opcodeFloat, opcodeString: - result.nodes[i] = toAtomNode(result, n.kind, n.str) + if n.isShort: + result.nodes[i] = tree.nodes[n.int] + else: + result.nodes[i] = toAtomNode(result, n.kind, n.str) else: result.nodes[i] = tree.nodes[n.int] @@ -51,7 +67,10 @@ proc rawAdd*(result: var JsonTree, tree: JsonTree, n: NodePos) = let m = NodePos(i) case m.kind of opcodeInt, opcodeFloat, opcodeString: - result.nodes[i+n.int] = toAtomNode(result, m.kind, m.str) + if m.isShort: + result.nodes[i+n.int] = tree.nodes[i] + else: + result.nodes[i+n.int] = toAtomNode(result, m.kind, m.str) else: result.nodes[i+n.int] = tree.nodes[i] @@ -80,7 +99,10 @@ proc rawAddKeyValuePair*(result: var JsonTree, tree: JsonTree, n: NodePos, key: let m = NodePos(i) case m.kind of opcodeInt, opcodeFloat, opcodeString: - result.nodes[i+n.int+1] = toAtomNode(result, m.kind, m.str) + if m.isShort: + result.nodes[i+n.int+1] = tree.nodes[i] + else: + result.nodes[i+n.int+1] = toAtomNode(result, m.kind, m.str) else: result.nodes[i+n.int+1] = tree.nodes[i] @@ -146,7 +168,10 @@ proc rawTest*(a, b: JsonTree, na, nb: NodePos): bool = of opcodeBool: return a.nodes[na.int].operand == b.nodes[nb.int].operand of opcodeInt, opcodeFloat, opcodeString: - return a.atoms[LitId a.nodes[na.int].operand] == b.atoms[LitId b.nodes[nb.int].operand] + if a.nodes[na.int].isShort: + return a.nodes[na.int].operand == b.nodes[nb.int].operand + else: + return a.atoms[LitId a.nodes[na.int].operand] == b.atoms[LitId b.nodes[nb.int].operand] of opcodeArray: let lenA = len(a, na) let lenB = len(b, nb) @@ -167,8 +192,12 @@ proc rawTest*(a, b: JsonTree, na, nb: NodePos): bool = return false for keyA in keys(a, na): let valA = keyA.firstSon - let keyStrA = a.atoms[LitId a.nodes[keyA.int].operand] - let valB = b.rawGet(nb, keyStrA) + var valB: NodePos + if a.nodes[keyA.int].isShort: + valB = b.rawGetShort(nb, a.nodes[keyA.int].operand) + else: + let keyStrA = a.atoms[LitId a.nodes[keyA.int].operand] + valB = b.rawGet(nb, keyStrA) if valB.isNil or not rawTest(a, b, valA, valB): return false return true diff --git a/src/jsonpak/sorted.nim b/src/jsonpak/sorted.nim index ee7194c..584af09 100644 --- a/src/jsonpak/sorted.nim +++ b/src/jsonpak/sorted.nim @@ -29,7 +29,10 @@ proc sorted*(tree: JsonTree, n: NodePos): SortedJsonTree = for i in countdown(items.high, 0): stack.add items[i].NodePos of opcodeInt, opcodeFloat, opcodeString: - nodes.add toNode(curr.kind, uint32 getOrIncl(atoms, curr.str)) + if curr.isShort: + nodes.add tree.nodes[curr.int] + else: + nodes.add toNode(curr.kind, uint32 getOrIncl(atoms, curr.str)) else: nodes.add tree.nodes[curr.int] result = JsonTree(nodes: nodes, atoms: atoms).SortedJsonTree @@ -47,7 +50,10 @@ proc rawTest*(tree, value: JsonTree, n: NodePos): bool = let n = NodePos(i+n.int) # careful case n.kind of opcodeInt, opcodeFloat, opcodeString: - if value.atoms[LitId value.nodes[i].operand] != n.str: return false + if n.isShort: + return n.operand == value.nodes[i.int].operand + else: + return n.str == value.atoms[LitId value.nodes[i.int].operand] else: if value.nodes[i] != tree.nodes[n.int]: return false return true From 70fd1df2f9ff4fdf3c82e5c62c65745df90e605d Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 17:02:33 +0300 Subject: [PATCH 02/15] improvements --- src/jsonpak/builder.nim | 33 ++++++++++++++++++---- src/jsonpak/dollar.nim | 48 ++++++++++++++++++++++++-------- src/jsonpak/private/jsonnode.nim | 5 ++++ src/jsonpak/private/jsontree.nim | 20 ++++++------- src/jsonpak/private/rawops.nim | 10 +++---- 5 files changed, 82 insertions(+), 34 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index 9dde379..12e9adc 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -18,7 +18,10 @@ proc initFromJson*(dst: var string; tree: JsonTree; n: NodePos) = if n.kind == opcodeNull: dst = "" else: - dst = n.str + if n.isShort: + dst = n.shortStr + else: + dst = n.str proc initFromJson*(dst: var bool; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JBool}) @@ -44,13 +47,22 @@ proc initFromJson*[T: SomeInteger](dst: var T; tree: JsonTree; n: NodePos) = proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JInt, JFloat}) if n.kind == opcodeFloat: - dst = T(parseFloat n.str) + if n.isShort: + dst = T(parseFloat n.shortStr) + else: + dst = T(parseFloat n.str) else: - dst = T(parseBiggestInt n.str) + if n.isShort: + dst = T(cast[int64](n.operand)) + else: + dst = T(parseBiggestInt n.str) proc initFromJson*[T: enum](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JString}) - dst = parseEnum[T](n.str) + if n.isShort: + dst = parseEnum[T](n.shortStr) + else: + dst = parseEnum[T](n.str) proc initFromJson*[T](dst: var seq[T]; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JArray}) @@ -70,7 +82,10 @@ proc initFromJson*[S, T](dst: var array[S, T]; tree: JsonTree; n: NodePos) = proc initFromJson*[T](dst: var (Table[string, T]|OrderedTable[string, T]); tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject}) for x in keys(tree, n): - initFromJson(mgetOrPut(dst, x.str, default(T)), tree, x.firstSon) + if x.isShort: + initFromJson(mgetOrPut(dst, x.shortStr, default(T)), tree, x.firstSon) + else: + initFromJson(mgetOrPut(dst, x.str, default(T)), tree, x.firstSon) proc initFromJson*[T](dst: var ref T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject, JNull}) @@ -121,6 +136,12 @@ iterator pairs*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): (lent string, raisePathError(path.string) assert n.kind == opcodeObject var item = default(T) + var buf = newString(payloadBits div 8) for x in keys(tree, n): initFromJson(item, tree, x.firstSon) - yield (x.str, item) + if x.isShort: + for i in 0 ..< buf.len: + buf[i] = chr(n.operand shr (i * 8) and 0xFF) + yield (buf, item) + else: + yield (x.str, item) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 5ed38a7..0dc4b82 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -25,27 +25,33 @@ type Action = enum actionElem, actionKeyVal, actionPop, actionEnd -proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, LitId, Action) = +proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, uint64, Action) = if it.pos < it.tosEnd: if it.tos.kind == opcodeArray: - result = (NodePos it.pos, LitId(0), actionElem) + result = (NodePos it.pos, 0, actionElem) else: - let litId = (NodePos it.pos).litId - result = (firstSon(NodePos it.pos), litId, actionKeyVal) + let nodeId = (NodePos it.pos).operand + result = (firstSon(NodePos it.pos), nodeId, actionKeyVal) inc it.pos nextChild tree, it.pos elif it.stack.len > 0: - result = (it.tos, LitId(0), actionPop) + result = (it.tos, 0, actionPop) let tmp = it.stack.pop() it.tos = tmp[0].NodePos it.pos = tmp[1] it.tosEnd = it.tos.tosEnd else: - result = (nilNodeId, LitId(0), actionEnd) + result = (nilNodeId, 0, actionEnd) proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = privateAccess(JsonTree) - template key: string = tree.atoms[keyId] + var buf = newString(payloadBits div 8) + template key: string = + if (NodePos keyId).isShort: + for i in 0 ..< buf.len: + buf[i] = chr(n.operand shr (i * 8) and 0xFF) + buf + else: tree.atoms[keyId.LitId] case n.kind of opcodeArray, opcodeObject: if n.kind == opcodeArray: @@ -87,10 +93,20 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add child.str pendingComma = true of opcodeFloat: - result.add child.str + if child.isShort: + for i in 0 ..< buf.len: + buf[i] = chr(n.operand shr (i * 8) and 0xFF) + result.add buf + else: + result.add child.str pendingComma = true of opcodeString: - escapeJson(child.str, result) + if child.isShort: + for i in 0 ..< buf.len: + buf[i] = chr(n.operand shr (i * 8) and 0xFF) + escapeJson(buf, result) + else: + escapeJson(child.str, result) pendingComma = true of opcodeBool: result.add(if child.bval: "true" else: "false") @@ -104,14 +120,24 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = else: result.add "}" of opcodeString: - escapeJson(n.str, result) + if n.isShort: + for i in 0 ..< buf.len: + buf[i] = chr(n.operand shr (i * 8) and 0xFF) + escapeJson(buf, result) + else: + escapeJson(n.str, result) of opcodeInt: if n.isShort: result.addInt cast[int64](n.operand) else: result.add n.str of opcodeFloat: - result.add n.str + if n.isShort: + for i in 0 ..< buf.len: + buf[i] = chr(n.operand shr (i * 8) and 0xFF) + result.add buf + else: + result.add n.str of opcodeBool: result.add(if n.bval: "true" else: "false") of opcodeNull: diff --git a/src/jsonpak/private/jsonnode.nim b/src/jsonpak/private/jsonnode.nim index 06827a2..6dd0363 100644 --- a/src/jsonpak/private/jsonnode.nim +++ b/src/jsonpak/private/jsonnode.nim @@ -40,3 +40,8 @@ template toNode*(kind, operand: uint64): Node = Node(operand shl opcodeBits.uint64 or kind) proc `==`*(a, b: Node): bool {.borrow.} + +proc createPayload*(data: string): uint64 = + result = 0 + for i in 0 ..< data.len: + result = result or (data[i].uint64 shl (i * 8)) diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index baf6e31..b1b329e 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -80,14 +80,13 @@ template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template kind*(n: NodePos): uint64 = tree.nodes[n.int].kind template litId*(n: NodePos): LitId = LitId operand(tree.nodes[n.int]) template operand*(n: NodePos): uint64 = tree.nodes[n.int].operand -template str*(n: NodePos): string = - if isShort(tree.nodes[n.int]): - var data = newString(payloadBits div 8) - for i in 0 ..< data.len: - data[i] = chr(n.operand shr (i * 8) and 0xFF) - data - else: - tree.atoms[litId(n)] +template str*(n: NodePos): string = tree.atoms[litId(n)] +template shortStr*(n: NodePos): string = + var data = newString(payloadBits div 8) + for i in 0 ..< data.len: + data[i] = chr(n.operand shr (i * 8) and 0xFF) + data + template bval*(n: NodePos): bool = n.operand == 1 type @@ -118,9 +117,6 @@ proc storeShortAtom*[T: SomeInteger](tree: var JsonTree; kind: uint64, data: T) proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = if data.len <= payloadBits div 8: - var payload: uint64 = 0 - for i in 0 ..< data.len: - payload = payload or (data[i].uint64 shl (i * 8)) - tree.nodes.add toShortNode(kind, payload) + tree.nodes.add toShortNode(kind, createPayload(data)) else: tree.nodes.add toAtomNode(tree, kind, data) diff --git a/src/jsonpak/private/rawops.nim b/src/jsonpak/private/rawops.nim index b1d56e5..7327e13 100644 --- a/src/jsonpak/private/rawops.nim +++ b/src/jsonpak/private/rawops.nim @@ -10,10 +10,7 @@ proc rawGetShort*(tree: JsonTree, n: NodePos, name: uint64): NodePos = proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = privateAccess(JsonTree) if name.len <= payloadBits div 8: - var payload: uint64 = 0 - for i in 0 ..< name.len: - payload = payload or (name[i].uint64 shl (i * 8)) - return rawGetShort(tree, n, payload) + return rawGetShort(tree, n, createPayload(name)) else: let litId = tree.atoms.getKeyId(name) if litId == LitId(0): @@ -154,7 +151,10 @@ proc rawReplace*(result: var JsonTree, tree: JsonTree, n: NodePos) = let m = NodePos(i) case m.kind of opcodeInt, opcodeFloat, opcodeString: - result.nodes[i+n.int] = toAtomNode(result, m.kind, m.str) + if m.isShort: + result.nodes[i+n.int] = tree.nodes[i] + else: + result.nodes[i+n.int] = toAtomNode(result, m.kind, m.str) else: result.nodes[i+n.int] = tree.nodes[i] From 30331b8036810974eefc48d93e28def3aef5f844 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 17:10:40 +0300 Subject: [PATCH 03/15] refactor --- src/jsonpak/builder.nim | 7 ++++--- src/jsonpak/dollar.nim | 15 +++++---------- src/jsonpak/private/jsontree.nim | 3 +++ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index 12e9adc..ebaf87f 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -81,9 +81,11 @@ proc initFromJson*[S, T](dst: var array[S, T]; tree: JsonTree; n: NodePos) = proc initFromJson*[T](dst: var (Table[string, T]|OrderedTable[string, T]); tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject}) + var buf = newString(payloadBits div 8) for x in keys(tree, n): if x.isShort: - initFromJson(mgetOrPut(dst, x.shortStr, default(T)), tree, x.firstSon) + copyShortStr(buf, x) + initFromJson(mgetOrPut(dst, buf, default(T)), tree, x.firstSon) else: initFromJson(mgetOrPut(dst, x.str, default(T)), tree, x.firstSon) @@ -140,8 +142,7 @@ iterator pairs*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): (lent string, for x in keys(tree, n): initFromJson(item, tree, x.firstSon) if x.isShort: - for i in 0 ..< buf.len: - buf[i] = chr(n.operand shr (i * 8) and 0xFF) + copyShortStr(buf, x) yield (buf, item) else: yield (x.str, item) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 0dc4b82..1fca62c 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -48,8 +48,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = var buf = newString(payloadBits div 8) template key: string = if (NodePos keyId).isShort: - for i in 0 ..< buf.len: - buf[i] = chr(n.operand shr (i * 8) and 0xFF) + copyShortStr(buf, keyId.NodePos) buf else: tree.atoms[keyId.LitId] case n.kind @@ -94,16 +93,14 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = pendingComma = true of opcodeFloat: if child.isShort: - for i in 0 ..< buf.len: - buf[i] = chr(n.operand shr (i * 8) and 0xFF) + copyShortStr(buf, n) result.add buf else: result.add child.str pendingComma = true of opcodeString: if child.isShort: - for i in 0 ..< buf.len: - buf[i] = chr(n.operand shr (i * 8) and 0xFF) + copyShortStr(buf, child) escapeJson(buf, result) else: escapeJson(child.str, result) @@ -121,8 +118,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "}" of opcodeString: if n.isShort: - for i in 0 ..< buf.len: - buf[i] = chr(n.operand shr (i * 8) and 0xFF) + copyShortStr(buf, n) escapeJson(buf, result) else: escapeJson(n.str, result) @@ -133,8 +129,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add n.str of opcodeFloat: if n.isShort: - for i in 0 ..< buf.len: - buf[i] = chr(n.operand shr (i * 8) and 0xFF) + copyShortStr(buf, n) result.add buf else: result.add n.str diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index b1b329e..771ac92 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -86,6 +86,9 @@ template shortStr*(n: NodePos): string = for i in 0 ..< data.len: data[i] = chr(n.operand shr (i * 8) and 0xFF) data +template copyShortStr*(data: untyped, n: NodePos) = + for i in 0 ..< data.len: + data[i] = chr(n.operand shr (i * 8) and 0xFF) template bval*(n: NodePos): bool = n.operand == 1 From 5e6fbfd95ce787ba58f60eb4a8b27fac2b1d85ee Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 18:02:08 +0300 Subject: [PATCH 04/15] fixes bugs --- src/jsonpak/private/jsontree.nim | 34 +++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 771ac92..61950eb 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -76,21 +76,41 @@ proc parentImpl*(tree: JsonTree; n: NodePos): NodePos = template parent*(n: NodePos): NodePos = parentImpl(tree, n) -template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template kind*(n: NodePos): uint64 = tree.nodes[n.int].kind template litId*(n: NodePos): LitId = LitId operand(tree.nodes[n.int]) template operand*(n: NodePos): uint64 = tree.nodes[n.int].operand template str*(n: NodePos): string = tree.atoms[litId(n)] +template bval*(n: NodePos): bool = n.operand == 1 + +template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort +template get(n: NodePos; i: int): char = char(n.operand shr (i * 8) and 0xff) + +proc shortLenImpl*(tree: JsonTree; n: NodePos): int {.inline.} = + {.push rangeChecks: off.} + template doIndex(i: int) = + if get(n, i) == '\0': + return i + doIndex 6 + doIndex 5 + doIndex 4 + doIndex 3 + doIndex 2 + doIndex 1 + doIndex 0 + {.pop.} + +template shortLen*(n: NodePos): int = shortLenImpl(tree, n) + template shortStr*(n: NodePos): string = - var data = newString(payloadBits div 8) + var data = newString(n.shortLen) for i in 0 ..< data.len: - data[i] = chr(n.operand shr (i * 8) and 0xFF) + data[i] = n.get(i) data + template copyShortStr*(data: untyped, n: NodePos) = + data.setLen(n.shortLen) for i in 0 ..< data.len: - data[i] = chr(n.operand shr (i * 8) and 0xFF) - -template bval*(n: NodePos): bool = n.operand == 1 + data[i] = n.get(i) type PatchPos* = distinct int32 @@ -119,7 +139,7 @@ proc storeShortAtom*[T: SomeInteger](tree: var JsonTree; kind: uint64, data: T) tree.nodes.add toShortNode(kind, cast[uint64](data)) proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = - if data.len <= payloadBits div 8: + if data[^1] != '\0' and data.len <= payloadBits div 8: tree.nodes.add toShortNode(kind, createPayload(data)) else: tree.nodes.add toAtomNode(tree, kind, data) From 399925a2692168c2bdd52147bf0bcc3975896ea0 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 18:08:54 +0300 Subject: [PATCH 05/15] refactor --- src/jsonpak/private/jsonnode.nim | 5 ----- src/jsonpak/private/jsontree.nim | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/jsonpak/private/jsonnode.nim b/src/jsonpak/private/jsonnode.nim index 6dd0363..06827a2 100644 --- a/src/jsonpak/private/jsonnode.nim +++ b/src/jsonpak/private/jsonnode.nim @@ -40,8 +40,3 @@ template toNode*(kind, operand: uint64): Node = Node(operand shl opcodeBits.uint64 or kind) proc `==`*(a, b: Node): bool {.borrow.} - -proc createPayload*(data: string): uint64 = - result = 0 - for i in 0 ..< data.len: - result = result or (data[i].uint64 shl (i * 8)) diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 61950eb..827283d 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -84,6 +84,12 @@ template bval*(n: NodePos): bool = n.operand == 1 template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template get(n: NodePos; i: int): char = char(n.operand shr (i * 8) and 0xff) +template set(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8)) + +proc createPayload*(data: string): uint64 = + result = 0 + for i in 0 ..< data.len: + set(result, i, data[i]) proc shortLenImpl*(tree: JsonTree; n: NodePos): int {.inline.} = {.push rangeChecks: off.} From 4d0af8bb3a657871d41d9ea6e98aadf894cf6f1d Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 18:27:08 +0300 Subject: [PATCH 06/15] fix bugs --- src/jsonpak/builder.nim | 2 +- src/jsonpak/dollar.nim | 13 +++++++------ src/jsonpak/parser.nim | 2 +- src/jsonpak/private/jsontree.nim | 4 ++-- src/jsonpak/private/rawops.nim | 16 +++++++++++----- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index ebaf87f..e6af2f5 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -53,7 +53,7 @@ proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) = dst = T(parseFloat n.str) else: if n.isShort: - dst = T(cast[int64](n.operand)) + dst = T(int64(n.operand)) else: dst = T(parseBiggestInt n.str) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 1fca62c..2b9a39c 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -30,8 +30,7 @@ proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, uint64, Action) if it.tos.kind == opcodeArray: result = (NodePos it.pos, 0, actionElem) else: - let nodeId = (NodePos it.pos).operand - result = (firstSon(NodePos it.pos), nodeId, actionKeyVal) + result = (firstSon(NodePos it.pos), (NodePos it.pos).operand, actionKeyVal) inc it.pos nextChild tree, it.pos elif it.stack.len > 0: @@ -50,7 +49,9 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = if (NodePos keyId).isShort: copyShortStr(buf, keyId.NodePos) buf - else: tree.atoms[keyId.LitId] + else: + tree.atoms[keyId.LitId] + case n.kind of opcodeArray, opcodeObject: if n.kind == opcodeArray: @@ -87,13 +88,13 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = pendingComma = false of opcodeInt: if child.isShort: - result.addInt cast[int64](child.operand) + result.addInt int64(child.operand) else: result.add child.str pendingComma = true of opcodeFloat: if child.isShort: - copyShortStr(buf, n) + copyShortStr(buf, child) result.add buf else: result.add child.str @@ -124,7 +125,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = escapeJson(n.str, result) of opcodeInt: if n.isShort: - result.addInt cast[int64](n.operand) + result.addInt int64(n.operand) else: result.add n.str of opcodeFloat: diff --git a/src/jsonpak/parser.nim b/src/jsonpak/parser.nim index 4e89a4a..2160b42 100644 --- a/src/jsonpak/parser.nim +++ b/src/jsonpak/parser.nim @@ -7,7 +7,7 @@ proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) = storeAtom(tree, opcodeString, p.a) discard getTok(p) of tkInt: - let n = parseInt(p.a) + let n = parseBiggestInt(p.a) if n >= shortIntLow and n <= shortIntHigh: storeShortAtom(tree, opcodeInt, n) else: diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 827283d..724c777 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -86,7 +86,7 @@ template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template get(n: NodePos; i: int): char = char(n.operand shr (i * 8) and 0xff) template set(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8)) -proc createPayload*(data: string): uint64 = +proc toPayload*(data: string): uint64 = result = 0 for i in 0 ..< data.len: set(result, i, data[i]) @@ -146,6 +146,6 @@ proc storeShortAtom*[T: SomeInteger](tree: var JsonTree; kind: uint64, data: T) proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = if data[^1] != '\0' and data.len <= payloadBits div 8: - tree.nodes.add toShortNode(kind, createPayload(data)) + tree.nodes.add toShortNode(kind, toPayload(data)) else: tree.nodes.add toAtomNode(tree, kind, data) diff --git a/src/jsonpak/private/rawops.nim b/src/jsonpak/private/rawops.nim index 7327e13..dadf013 100644 --- a/src/jsonpak/private/rawops.nim +++ b/src/jsonpak/private/rawops.nim @@ -9,8 +9,8 @@ proc rawGetShort*(tree: JsonTree, n: NodePos, name: uint64): NodePos = proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = privateAccess(JsonTree) - if name.len <= payloadBits div 8: - return rawGetShort(tree, n, createPayload(name)) + if name[^1] != '\0' and name.len <= payloadBits div 8: + return rawGetShort(tree, n, toPayload(name)) else: let litId = tree.atoms.getKeyId(name) if litId == LitId(0): @@ -22,7 +22,7 @@ proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = proc rawUpdateParents*(tree: var JsonTree, parents: seq[PatchPos], diff: int) = privateAccess(JsonTree) - for parent in parents: + for parent in items(parents): let distance = tree.nodes[parent.int].rawSpan + diff tree.nodes[parent.int] = toNode(tree.nodes[parent.int].kind, distance.uint32) @@ -78,7 +78,10 @@ proc rawAddKeyValuePair*(result: var JsonTree, src, dest: NodePos, key: string) setLen(result.nodes, oldfull+L) for i in countdown(oldfull-1, dest.int): result.nodes[i+L] = result.nodes[i] - result.nodes[dest.int] = toAtomNode(result, opcodeString, key) + if key[^1] != '\0' and key.len <= payloadBits div 8: + result.nodes[dest.int] = toShortNode(opcodeString, toPayload(key)) + else: + result.nodes[dest.int] = toAtomNode(result, opcodeString, key) let src = if src >= dest: NodePos(src.int+L) else: src for i in 0.. Date: Sat, 13 Apr 2024 18:48:49 +0300 Subject: [PATCH 07/15] I am killing it --- bench/benchmark.nim | 68 ++++++++++++++++---------------- src/jsonpak/private/jsonnode.nim | 5 ++- src/jsonpak/private/jsontree.nim | 24 ++++------- src/jsonpak/private/rawops.nim | 6 +-- 4 files changed, 48 insertions(+), 55 deletions(-) diff --git a/bench/benchmark.nim b/bench/benchmark.nim index 49cb40d..1e88fa7 100644 --- a/bench/benchmark.nim +++ b/bench/benchmark.nim @@ -73,40 +73,40 @@ proc main() = bench "move", tree: move(t, JsonPtr"/records/500/city", JsonPtr"/records/0/location") - # Benchmarks for std/json module - bench "stdlib - extract", JsonNode(): - t = stdTree.copy() - - bench "stdlib - parse", JsonNode(): - t = json.parseJson(JsonData) - - bench "stdlib - toString", stdTree: - discard $t - - bench "stdlib - fromJson", stdTree: - discard t["records"][500].to(UserRecord) - - bench "stdlib - toJson", JsonNode(): - t = %UserRecord(id:1,name:"User1",email:"user1@example.com",age:65,city:"Sydney",balance:37341,active:false) - - bench "stdlib - test", stdTree: - discard t["records"][500]["age"] == %30 - - bench "stdlib - replace", stdTree: - t["records"][500]["age"] = %31 - - bench "stdlib - remove", stdTree: - t["records"][500].delete("city") - - bench "stdlib - add", stdTree: - t["records"][500]["email"] = %"john@example.com" - - bench "stdlib - copy", stdTree: - t["records"][500]["newAge"] = t["records"][0]["age"] - - bench "stdlib - move", stdTree: - t["records"][500]["location"] = t["records"][0]["city"] - t["records"][0].delete("city") + # # Benchmarks for std/json module + # bench "stdlib - extract", JsonNode(): + # t = stdTree.copy() + # + # bench "stdlib - parse", JsonNode(): + # t = json.parseJson(JsonData) + # + # bench "stdlib - toString", stdTree: + # discard $t + # + # bench "stdlib - fromJson", stdTree: + # discard t["records"][500].to(UserRecord) + # + # bench "stdlib - toJson", JsonNode(): + # t = %UserRecord(id:1,name:"User1",email:"user1@example.com",age:65,city:"Sydney",balance:37341,active:false) + # + # bench "stdlib - test", stdTree: + # discard t["records"][500]["age"] == %30 + # + # bench "stdlib - replace", stdTree: + # t["records"][500]["age"] = %31 + # + # bench "stdlib - remove", stdTree: + # t["records"][500].delete("city") + # + # bench "stdlib - add", stdTree: + # t["records"][500]["email"] = %"john@example.com" + # + # bench "stdlib - copy", stdTree: + # t["records"][500]["newAge"] = t["records"][0]["age"] + # + # bench "stdlib - move", stdTree: + # t["records"][500]["location"] = t["records"][0]["city"] + # t["records"][0].delete("city") echo "used Mem: ", formatSize getOccupiedMem() diff --git a/src/jsonpak/private/jsonnode.nim b/src/jsonpak/private/jsonnode.nim index 06827a2..42fa7f1 100644 --- a/src/jsonpak/private/jsonnode.nim +++ b/src/jsonpak/private/jsonnode.nim @@ -10,10 +10,12 @@ type JArray const - opcodeBits = 4 + opcodeBits* = 4 payloadBits* = sizeof(uint64) - opcodeBits shortBit = 0b0000_1000 + shortLenMask = (1 shl opcodeBits) - 1 + shortIntLow* = -(1 shl payloadBits) shortIntHigh* = (1 shl payloadBits) - 1 @@ -32,6 +34,7 @@ const template kind*(n: Node): uint64 = n.uint64 and opcodeMask template operand*(n: Node): uint64 = n.uint64 shr opcodeBits.uint64 template isShort*(n: Node): bool = (n.uint64 and shortBit) != 0 +template shortLen*(n: Node): int = int(n.uint64 shr opcodeBits.uint64 and shortLenMask) template toShortNode*(kind, operand: uint64): Node = Node(operand shl opcodeBits.uint64 or kind or shortBit.uint64) diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 724c777..a07e33b 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -83,6 +83,11 @@ template str*(n: NodePos): string = tree.atoms[litId(n)] template bval*(n: NodePos): bool = n.operand == 1 template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort +template shortLen*(n: NodePos): int = tree.nodes[n.int].shortLen + +template setShortLen*(p: uint64; i: int) = + p = p or i.uint64 shl opcodeBits + template get(n: NodePos; i: int): char = char(n.operand shr (i * 8) and 0xff) template set(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8)) @@ -90,22 +95,7 @@ proc toPayload*(data: string): uint64 = result = 0 for i in 0 ..< data.len: set(result, i, data[i]) - -proc shortLenImpl*(tree: JsonTree; n: NodePos): int {.inline.} = - {.push rangeChecks: off.} - template doIndex(i: int) = - if get(n, i) == '\0': - return i - doIndex 6 - doIndex 5 - doIndex 4 - doIndex 3 - doIndex 2 - doIndex 1 - doIndex 0 - {.pop.} - -template shortLen*(n: NodePos): int = shortLenImpl(tree, n) + setShortLen(result, data.len) template shortStr*(n: NodePos): string = var data = newString(n.shortLen) @@ -145,7 +135,7 @@ proc storeShortAtom*[T: SomeInteger](tree: var JsonTree; kind: uint64, data: T) tree.nodes.add toShortNode(kind, cast[uint64](data)) proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = - if data[^1] != '\0' and data.len <= payloadBits div 8: + if data.len <= payloadBits div 8: tree.nodes.add toShortNode(kind, toPayload(data)) else: tree.nodes.add toAtomNode(tree, kind, data) diff --git a/src/jsonpak/private/rawops.nim b/src/jsonpak/private/rawops.nim index dadf013..75956c9 100644 --- a/src/jsonpak/private/rawops.nim +++ b/src/jsonpak/private/rawops.nim @@ -9,7 +9,7 @@ proc rawGetShort*(tree: JsonTree, n: NodePos, name: uint64): NodePos = proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = privateAccess(JsonTree) - if name[^1] != '\0' and name.len <= payloadBits div 8: + if name.len <= payloadBits div 8: return rawGetShort(tree, n, toPayload(name)) else: let litId = tree.atoms.getKeyId(name) @@ -78,7 +78,7 @@ proc rawAddKeyValuePair*(result: var JsonTree, src, dest: NodePos, key: string) setLen(result.nodes, oldfull+L) for i in countdown(oldfull-1, dest.int): result.nodes[i+L] = result.nodes[i] - if key[^1] != '\0' and key.len <= payloadBits div 8: + if key.len <= payloadBits div 8: result.nodes[dest.int] = toShortNode(opcodeString, toPayload(key)) else: result.nodes[dest.int] = toAtomNode(result, opcodeString, key) @@ -94,7 +94,7 @@ proc rawAddKeyValuePair*(result: var JsonTree, tree: JsonTree, n: NodePos, key: setLen(result.nodes, oldfull+L) for i in countdown(oldfull-1, n.int): result.nodes[i+L] = result.nodes[i] - if key[^1] != '\0' and key.len <= payloadBits div 8: + if key.len <= payloadBits div 8: result.nodes[n.int] = toShortNode(opcodeString, toPayload(key)) else: result.nodes[n.int] = toAtomNode(result, opcodeString, key) From 1dee4c12fbfe5144b4a5061e1056ce3a055a2832 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 20:20:46 +0300 Subject: [PATCH 08/15] bugs fixed --- src/jsonpak/dollar.nim | 25 +++++++++++-------------- src/jsonpak/private/jsonnode.nim | 2 +- src/jsonpak/private/jsontree.nim | 10 ++++++---- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 2b9a39c..1fbadd2 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -25,33 +25,26 @@ type Action = enum actionElem, actionKeyVal, actionPop, actionEnd -proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, uint64, Action) = +proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, NodePos, Action) = if it.pos < it.tosEnd: if it.tos.kind == opcodeArray: - result = (NodePos it.pos, 0, actionElem) + result = (NodePos it.pos, NodePos 0, actionElem) else: - result = (firstSon(NodePos it.pos), (NodePos it.pos).operand, actionKeyVal) + result = (firstSon(NodePos it.pos), NodePos it.pos, actionKeyVal) inc it.pos nextChild tree, it.pos elif it.stack.len > 0: - result = (it.tos, 0, actionPop) + result = (it.tos, NodePos 0, actionPop) let tmp = it.stack.pop() it.tos = tmp[0].NodePos it.pos = tmp[1] it.tosEnd = it.tos.tosEnd else: - result = (nilNodeId, 0, actionEnd) + result = (nilNodeId, NodePos 0, actionEnd) proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = privateAccess(JsonTree) var buf = newString(payloadBits div 8) - template key: string = - if (NodePos keyId).isShort: - copyShortStr(buf, keyId.NodePos) - buf - else: - tree.atoms[keyId.LitId] - case n.kind of opcodeArray, opcodeObject: if n.kind == opcodeArray: @@ -61,7 +54,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = var it = initJsonIter(tree, n) var pendingComma = false while true: - let (child, keyId, action) = currentAndNext(it, tree) + let (child, key, action) = currentAndNext(it, tree) case action of actionPop: if child.kind == opcodeArray: @@ -75,7 +68,11 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "," pendingComma = false if action == actionKeyVal: - key.escapeJson(result) + if key.isShort: + copyShortStr(buf, key) + buf.escapeJson(result) + else: + escapeJson(key.str, result) result.add ":" case child.kind of opcodeArray: diff --git a/src/jsonpak/private/jsonnode.nim b/src/jsonpak/private/jsonnode.nim index 42fa7f1..52565d3 100644 --- a/src/jsonpak/private/jsonnode.nim +++ b/src/jsonpak/private/jsonnode.nim @@ -11,7 +11,7 @@ type const opcodeBits* = 4 - payloadBits* = sizeof(uint64) - opcodeBits + payloadBits* = sizeof(uint64)*8 - opcodeBits shortBit = 0b0000_1000 shortLenMask = (1 shl opcodeBits) - 1 diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index a07e33b..00bdb02 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -85,11 +85,11 @@ template bval*(n: NodePos): bool = n.operand == 1 template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template shortLen*(n: NodePos): int = tree.nodes[n.int].shortLen -template setShortLen*(p: uint64; i: int) = - p = p or i.uint64 shl opcodeBits +template setShortLen*(p: uint64; n: int) = + p = p or n.uint64 -template get(n: NodePos; i: int): char = char(n.operand shr (i * 8) and 0xff) -template set(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8)) +template get(n: NodePos; i: int): char = char(n.operand shr (i * 8 + opcodeBits) and 0xff) +template set(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8 + opcodeBits)) proc toPayload*(data: string): uint64 = result = 0 @@ -137,5 +137,7 @@ proc storeShortAtom*[T: SomeInteger](tree: var JsonTree; kind: uint64, data: T) proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = if data.len <= payloadBits div 8: tree.nodes.add toShortNode(kind, toPayload(data)) + # if data == "email": + # echo (tree.nodes[^1].shortLen, NodePos(tree.nodes.high).shortStr, tree.nodes[^1].isShort) else: tree.nodes.add toAtomNode(tree, kind, data) From e524dd8e67cd045519dbd995046a07e82b964a14 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 20:26:18 +0300 Subject: [PATCH 09/15] last bug --- src/jsonpak/builder.nim | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index e6af2f5..5443e5a 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -107,11 +107,18 @@ proc initFromJson*[T](dst: var Option[T]; tree: JsonTree; n: NodePos) = proc initFromJson*[T: object|tuple](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject}) + var buf = newString(payloadBits div 8) for x in keys(tree, n): for k, v in dst.fieldPairs: - if x.str == k: - initFromJson(v, tree, x.firstSon) - break # emulate elif + if x.isShort: + copyShortStr(buf, x) + if buf == k: + initFromJson(v, tree, x.firstSon) + break + else: + if x.str == k: + initFromJson(v, tree, x.firstSon) + break # emulate elif proc fromJson*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): T = let n = findNode(tree, path.string) From c3625f84debdbf19645214cfd10b431c62acfac3 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 20:53:04 +0300 Subject: [PATCH 10/15] rename --- bench/benchmark.nim | 68 ++++++++++++++++---------------- src/jsonpak/mapper.nim | 2 +- src/jsonpak/parser.nim | 4 +- src/jsonpak/private/jsontree.nim | 10 ++--- 4 files changed, 40 insertions(+), 44 deletions(-) diff --git a/bench/benchmark.nim b/bench/benchmark.nim index 1e88fa7..49cb40d 100644 --- a/bench/benchmark.nim +++ b/bench/benchmark.nim @@ -73,40 +73,40 @@ proc main() = bench "move", tree: move(t, JsonPtr"/records/500/city", JsonPtr"/records/0/location") - # # Benchmarks for std/json module - # bench "stdlib - extract", JsonNode(): - # t = stdTree.copy() - # - # bench "stdlib - parse", JsonNode(): - # t = json.parseJson(JsonData) - # - # bench "stdlib - toString", stdTree: - # discard $t - # - # bench "stdlib - fromJson", stdTree: - # discard t["records"][500].to(UserRecord) - # - # bench "stdlib - toJson", JsonNode(): - # t = %UserRecord(id:1,name:"User1",email:"user1@example.com",age:65,city:"Sydney",balance:37341,active:false) - # - # bench "stdlib - test", stdTree: - # discard t["records"][500]["age"] == %30 - # - # bench "stdlib - replace", stdTree: - # t["records"][500]["age"] = %31 - # - # bench "stdlib - remove", stdTree: - # t["records"][500].delete("city") - # - # bench "stdlib - add", stdTree: - # t["records"][500]["email"] = %"john@example.com" - # - # bench "stdlib - copy", stdTree: - # t["records"][500]["newAge"] = t["records"][0]["age"] - # - # bench "stdlib - move", stdTree: - # t["records"][500]["location"] = t["records"][0]["city"] - # t["records"][0].delete("city") + # Benchmarks for std/json module + bench "stdlib - extract", JsonNode(): + t = stdTree.copy() + + bench "stdlib - parse", JsonNode(): + t = json.parseJson(JsonData) + + bench "stdlib - toString", stdTree: + discard $t + + bench "stdlib - fromJson", stdTree: + discard t["records"][500].to(UserRecord) + + bench "stdlib - toJson", JsonNode(): + t = %UserRecord(id:1,name:"User1",email:"user1@example.com",age:65,city:"Sydney",balance:37341,active:false) + + bench "stdlib - test", stdTree: + discard t["records"][500]["age"] == %30 + + bench "stdlib - replace", stdTree: + t["records"][500]["age"] = %31 + + bench "stdlib - remove", stdTree: + t["records"][500].delete("city") + + bench "stdlib - add", stdTree: + t["records"][500]["email"] = %"john@example.com" + + bench "stdlib - copy", stdTree: + t["records"][500]["newAge"] = t["records"][0]["age"] + + bench "stdlib - move", stdTree: + t["records"][500]["location"] = t["records"][0]["city"] + t["records"][0].delete("city") echo "used Mem: ", formatSize getOccupiedMem() diff --git a/src/jsonpak/mapper.nim b/src/jsonpak/mapper.nim index 468ca96..d9303ba 100644 --- a/src/jsonpak/mapper.nim +++ b/src/jsonpak/mapper.nim @@ -7,7 +7,7 @@ proc toJson*(s: string; tree: var JsonTree) = proc toJson*[T: SomeInteger](n: T; tree: var JsonTree) = if n >= shortIntLow and n <= shortIntHigh: - storeShortAtom(tree, opcodeInt, n) + storeShortInt(tree, n) else: storeAtom(tree, opcodeInt, $n) diff --git a/src/jsonpak/parser.nim b/src/jsonpak/parser.nim index 2160b42..e76b291 100644 --- a/src/jsonpak/parser.nim +++ b/src/jsonpak/parser.nim @@ -7,9 +7,9 @@ proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) = storeAtom(tree, opcodeString, p.a) discard getTok(p) of tkInt: - let n = parseBiggestInt(p.a) + let n = parseInt(p.a) if n >= shortIntLow and n <= shortIntHigh: - storeShortAtom(tree, opcodeInt, n) + storeShortInt(tree, n) else: storeAtom(tree, opcodeInt, p.a) discard getTok(p) diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 00bdb02..020517c 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -85,9 +85,7 @@ template bval*(n: NodePos): bool = n.operand == 1 template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template shortLen*(n: NodePos): int = tree.nodes[n.int].shortLen -template setShortLen*(p: uint64; n: int) = - p = p or n.uint64 - +template setShortLen*(p: uint64; n: int) = p = p or n.uint64 template get(n: NodePos; i: int): char = char(n.operand shr (i * 8 + opcodeBits) and 0xff) template set(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8 + opcodeBits)) @@ -131,13 +129,11 @@ proc storeEmpty*(tree: var JsonTree; kind: uint64) {.inline.} = proc storeAtom*(tree: var JsonTree; kind: uint64) {.inline.} = tree.nodes.add Node(kind) -proc storeShortAtom*[T: SomeInteger](tree: var JsonTree; kind: uint64, data: T) {.inline.} = - tree.nodes.add toShortNode(kind, cast[uint64](data)) +proc storeShortInt*[T: SomeInteger](tree: var JsonTree; data: T) {.inline.} = + tree.nodes.add toShortNode(opcodeInt, cast[uint64](data)) proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = if data.len <= payloadBits div 8: tree.nodes.add toShortNode(kind, toPayload(data)) - # if data == "email": - # echo (tree.nodes[^1].shortLen, NodePos(tree.nodes.high).shortStr, tree.nodes[^1].isShort) else: tree.nodes.add toAtomNode(tree, kind, data) From de7a00df425e44420a88883f5c16687771e3b622 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 21:28:18 +0300 Subject: [PATCH 11/15] refactor 2 --- src/jsonpak/builder.nim | 8 ++++---- src/jsonpak/dollar.nim | 2 +- src/jsonpak/mapper.nim | 2 +- src/jsonpak/parser.nim | 2 +- src/jsonpak/private/jsonnode.nim | 23 ++++++++++++++++++----- src/jsonpak/private/jsontree.nim | 17 ++++------------- src/jsonpak/private/rawops.nim | 8 ++++---- 7 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index 5443e5a..240d95a 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -53,7 +53,7 @@ proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) = dst = T(parseFloat n.str) else: if n.isShort: - dst = T(int64(n.operand)) + dst = T(cast[int64](n.operand)) else: dst = T(parseBiggestInt n.str) @@ -81,7 +81,7 @@ proc initFromJson*[S, T](dst: var array[S, T]; tree: JsonTree; n: NodePos) = proc initFromJson*[T](dst: var (Table[string, T]|OrderedTable[string, T]); tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject}) - var buf = newString(payloadBits div 8) + var buf = "" for x in keys(tree, n): if x.isShort: copyShortStr(buf, x) @@ -107,7 +107,7 @@ proc initFromJson*[T](dst: var Option[T]; tree: JsonTree; n: NodePos) = proc initFromJson*[T: object|tuple](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject}) - var buf = newString(payloadBits div 8) + var buf = "" for x in keys(tree, n): for k, v in dst.fieldPairs: if x.isShort: @@ -145,7 +145,7 @@ iterator pairs*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): (lent string, raisePathError(path.string) assert n.kind == opcodeObject var item = default(T) - var buf = newString(payloadBits div 8) + var buf = "" for x in keys(tree, n): initFromJson(item, tree, x.firstSon) if x.isShort: diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 1fbadd2..df6e4d7 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -44,7 +44,7 @@ proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, NodePos, Action proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = privateAccess(JsonTree) - var buf = newString(payloadBits div 8) + var buf = "" case n.kind of opcodeArray, opcodeObject: if n.kind == opcodeArray: diff --git a/src/jsonpak/mapper.nim b/src/jsonpak/mapper.nim index d9303ba..553b43f 100644 --- a/src/jsonpak/mapper.nim +++ b/src/jsonpak/mapper.nim @@ -6,7 +6,7 @@ proc toJson*(s: string; tree: var JsonTree) = storeAtom(tree, opcodeString, s) proc toJson*[T: SomeInteger](n: T; tree: var JsonTree) = - if n >= shortIntLow and n <= shortIntHigh: + if inShortIntRange(n): storeShortInt(tree, n) else: storeAtom(tree, opcodeInt, $n) diff --git a/src/jsonpak/parser.nim b/src/jsonpak/parser.nim index e76b291..a7dd01a 100644 --- a/src/jsonpak/parser.nim +++ b/src/jsonpak/parser.nim @@ -8,7 +8,7 @@ proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) = discard getTok(p) of tkInt: let n = parseInt(p.a) - if n >= shortIntLow and n <= shortIntHigh: + if inShortIntRange(n): storeShortInt(tree, n) else: storeAtom(tree, opcodeInt, p.a) diff --git a/src/jsonpak/private/jsonnode.nim b/src/jsonpak/private/jsonnode.nim index 52565d3..9cf4b26 100644 --- a/src/jsonpak/private/jsonnode.nim +++ b/src/jsonpak/private/jsonnode.nim @@ -10,14 +10,14 @@ type JArray const - opcodeBits* = 4 - payloadBits* = sizeof(uint64)*8 - opcodeBits + opcodeBits = 4 + payloadBits = sizeof(uint64)*8 - opcodeBits shortBit = 0b0000_1000 shortLenMask = (1 shl opcodeBits) - 1 - shortIntLow* = -(1 shl payloadBits) - shortIntHigh* = (1 shl payloadBits) - 1 + shortIntLow = -(1 shl payloadBits) + shortIntHigh = (1 shl payloadBits) - 1 opcodeNull* = uint64 JNull opcodeBool* = uint64 JBool @@ -31,6 +31,8 @@ const opcodeMask = (1 shl (opcodeBits - 1)) - 1 +proc `==`*(a, b: Node): bool {.borrow.} + template kind*(n: Node): uint64 = n.uint64 and opcodeMask template operand*(n: Node): uint64 = n.uint64 shr opcodeBits.uint64 template isShort*(n: Node): bool = (n.uint64 and shortBit) != 0 @@ -42,4 +44,15 @@ template toShortNode*(kind, operand: uint64): Node = template toNode*(kind, operand: uint64): Node = Node(operand shl opcodeBits.uint64 or kind) -proc `==`*(a, b: Node): bool {.borrow.} +template get*(n: Node; i: int): char = char(n.operand shr (i * 8 + opcodeBits) and 0xff) +template set*(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8 + opcodeBits)) +template setShortLen*(p: uint64; n: int) = p = p or n.uint64 + +template inShortIntRange*(x: int64): bool = x >= shortIntLow and x <= shortIntHigh +template inShortStrRange*(x: string): bool = x.len <= payloadBits div 8 + +proc toPayload*(data: string): uint64 = + result = 0 + for i in 0 ..< data.len: + set(result, i, data[i]) + setShortLen(result, data.len) diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 020517c..7bfab75 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -84,27 +84,18 @@ template bval*(n: NodePos): bool = n.operand == 1 template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template shortLen*(n: NodePos): int = tree.nodes[n.int].shortLen - -template setShortLen*(p: uint64; n: int) = p = p or n.uint64 -template get(n: NodePos; i: int): char = char(n.operand shr (i * 8 + opcodeBits) and 0xff) -template set(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8 + opcodeBits)) - -proc toPayload*(data: string): uint64 = - result = 0 - for i in 0 ..< data.len: - set(result, i, data[i]) - setShortLen(result, data.len) +template get*(n: NodePos; i: int): char = get(tree.nodes[n.int], i) template shortStr*(n: NodePos): string = var data = newString(n.shortLen) for i in 0 ..< data.len: - data[i] = n.get(i) + data[i] = get(n, i) data template copyShortStr*(data: untyped, n: NodePos) = data.setLen(n.shortLen) for i in 0 ..< data.len: - data[i] = n.get(i) + data[i] = get(n, i) type PatchPos* = distinct int32 @@ -133,7 +124,7 @@ proc storeShortInt*[T: SomeInteger](tree: var JsonTree; data: T) {.inline.} = tree.nodes.add toShortNode(opcodeInt, cast[uint64](data)) proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = - if data.len <= payloadBits div 8: + if inShortStrRange(data): tree.nodes.add toShortNode(kind, toPayload(data)) else: tree.nodes.add toAtomNode(tree, kind, data) diff --git a/src/jsonpak/private/rawops.nim b/src/jsonpak/private/rawops.nim index 75956c9..bd4ef09 100644 --- a/src/jsonpak/private/rawops.nim +++ b/src/jsonpak/private/rawops.nim @@ -9,8 +9,8 @@ proc rawGetShort*(tree: JsonTree, n: NodePos, name: uint64): NodePos = proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = privateAccess(JsonTree) - if name.len <= payloadBits div 8: - return rawGetShort(tree, n, toPayload(name)) + if inShortStrRange(name): + rawGetShort(tree, n, toPayload(name)) else: let litId = tree.atoms.getKeyId(name) if litId == LitId(0): @@ -78,7 +78,7 @@ proc rawAddKeyValuePair*(result: var JsonTree, src, dest: NodePos, key: string) setLen(result.nodes, oldfull+L) for i in countdown(oldfull-1, dest.int): result.nodes[i+L] = result.nodes[i] - if key.len <= payloadBits div 8: + if inShortStrRange(key): result.nodes[dest.int] = toShortNode(opcodeString, toPayload(key)) else: result.nodes[dest.int] = toAtomNode(result, opcodeString, key) @@ -94,7 +94,7 @@ proc rawAddKeyValuePair*(result: var JsonTree, tree: JsonTree, n: NodePos, key: setLen(result.nodes, oldfull+L) for i in countdown(oldfull-1, n.int): result.nodes[i+L] = result.nodes[i] - if key.len <= payloadBits div 8: + if inShortStrRange(key): result.nodes[n.int] = toShortNode(opcodeString, toPayload(key)) else: result.nodes[n.int] = toAtomNode(result, opcodeString, key) From 4e7c70914b7a8ecdd3411589a74faf8bc648ad42 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 21:36:42 +0300 Subject: [PATCH 12/15] fix $ --- src/jsonpak/dollar.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index df6e4d7..b6a8c64 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -70,7 +70,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = if action == actionKeyVal: if key.isShort: copyShortStr(buf, key) - buf.escapeJson(result) + escapeJson(buf, result) else: escapeJson(key.str, result) result.add ":" @@ -85,7 +85,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = pendingComma = false of opcodeInt: if child.isShort: - result.addInt int64(child.operand) + result.addInt cast[int64](child.operand) else: result.add child.str pendingComma = true @@ -122,7 +122,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = escapeJson(n.str, result) of opcodeInt: if n.isShort: - result.addInt int64(n.operand) + result.addInt cast[int64](n.operand) else: result.add n.str of opcodeFloat: From 01a764fb4efc64481466e5ee4c30acaaa2c8c26b Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 21:50:33 +0300 Subject: [PATCH 13/15] remove craft --- src/jsonpak/builder.nim | 39 +++++++------------------------- src/jsonpak/dollar.nim | 30 ++++-------------------- src/jsonpak/private/jsontree.nim | 8 ++++++- 3 files changed, 20 insertions(+), 57 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index 240d95a..c52546d 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -18,10 +18,7 @@ proc initFromJson*(dst: var string; tree: JsonTree; n: NodePos) = if n.kind == opcodeNull: dst = "" else: - if n.isShort: - dst = n.shortStr - else: - dst = n.str + dst = n.anyStr proc initFromJson*(dst: var bool; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JBool}) @@ -47,10 +44,7 @@ proc initFromJson*[T: SomeInteger](dst: var T; tree: JsonTree; n: NodePos) = proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JInt, JFloat}) if n.kind == opcodeFloat: - if n.isShort: - dst = T(parseFloat n.shortStr) - else: - dst = T(parseFloat n.str) + dst = T(parseFloat n.anyStr) else: if n.isShort: dst = T(cast[int64](n.operand)) @@ -59,10 +53,7 @@ proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) = proc initFromJson*[T: enum](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JString}) - if n.isShort: - dst = parseEnum[T](n.shortStr) - else: - dst = parseEnum[T](n.str) + dst = parseEnum[T](n.anyStr) proc initFromJson*[T](dst: var seq[T]; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JArray}) @@ -83,11 +74,7 @@ proc initFromJson*[T](dst: var (Table[string, T]|OrderedTable[string, T]); tree: verifyJsonKind(tree, n, {JObject}) var buf = "" for x in keys(tree, n): - if x.isShort: - copyShortStr(buf, x) - initFromJson(mgetOrPut(dst, buf, default(T)), tree, x.firstSon) - else: - initFromJson(mgetOrPut(dst, x.str, default(T)), tree, x.firstSon) + initFromJson(mgetOrPut(dst, x.anyStrBuffered, default(T)), tree, x.firstSon) proc initFromJson*[T](dst: var ref T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject, JNull}) @@ -110,15 +97,9 @@ proc initFromJson*[T: object|tuple](dst: var T; tree: JsonTree; n: NodePos) = var buf = "" for x in keys(tree, n): for k, v in dst.fieldPairs: - if x.isShort: - copyShortStr(buf, x) - if buf == k: - initFromJson(v, tree, x.firstSon) - break - else: - if x.str == k: - initFromJson(v, tree, x.firstSon) - break # emulate elif + if x.anyStrBuffered == k: + initFromJson(v, tree, x.firstSon) + break # emulate elif proc fromJson*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): T = let n = findNode(tree, path.string) @@ -148,8 +129,4 @@ iterator pairs*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): (lent string, var buf = "" for x in keys(tree, n): initFromJson(item, tree, x.firstSon) - if x.isShort: - copyShortStr(buf, x) - yield (buf, item) - else: - yield (x.str, item) + yield (x.anyStrBuffered, item) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index b6a8c64..9a1af5d 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -68,11 +68,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "," pendingComma = false if action == actionKeyVal: - if key.isShort: - copyShortStr(buf, key) - escapeJson(buf, result) - else: - escapeJson(key.str, result) + escapeJson(key.anyStrBuffered, result) result.add ":" case child.kind of opcodeArray: @@ -90,18 +86,10 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add child.str pendingComma = true of opcodeFloat: - if child.isShort: - copyShortStr(buf, child) - result.add buf - else: - result.add child.str + result.add child.anyStrBuffered pendingComma = true of opcodeString: - if child.isShort: - copyShortStr(buf, child) - escapeJson(buf, result) - else: - escapeJson(child.str, result) + escapeJson(child.anyStrBuffered, result) pendingComma = true of opcodeBool: result.add(if child.bval: "true" else: "false") @@ -115,22 +103,14 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = else: result.add "}" of opcodeString: - if n.isShort: - copyShortStr(buf, n) - escapeJson(buf, result) - else: - escapeJson(n.str, result) + escapeJson(n.anyStrBuffered, result) of opcodeInt: if n.isShort: result.addInt cast[int64](n.operand) else: result.add n.str of opcodeFloat: - if n.isShort: - copyShortStr(buf, n) - result.add buf - else: - result.add n.str + result.add n.anyStrBuffered of opcodeBool: result.add(if n.bval: "true" else: "false") of opcodeNull: diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 7bfab75..ba4453c 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -92,11 +92,17 @@ template shortStr*(n: NodePos): string = data[i] = get(n, i) data -template copyShortStr*(data: untyped, n: NodePos) = +template copyShortStr*(data: string, n: NodePos) = data.setLen(n.shortLen) for i in 0 ..< data.len: data[i] = get(n, i) +template anyStr*(n: NodePos): untyped = + (if n.isShort: n.shortStr else: n.str) + +template anyStrBuffered*(x: NodePos): untyped = + (if n.isShort: (copyShortStr(buf, n); buf) else: n.str) + type PatchPos* = distinct int32 From 61f0e7866f5c44855ceb91ac94a6f3a998c8dfe5 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sat, 13 Apr 2024 22:25:29 +0300 Subject: [PATCH 14/15] refactor 3 --- src/jsonpak/builder.nim | 6 +++--- src/jsonpak/dollar.nim | 22 +++++++++++----------- src/jsonpak/private/jsontree.nim | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index c52546d..10fbcab 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -74,7 +74,7 @@ proc initFromJson*[T](dst: var (Table[string, T]|OrderedTable[string, T]); tree: verifyJsonKind(tree, n, {JObject}) var buf = "" for x in keys(tree, n): - initFromJson(mgetOrPut(dst, x.anyStrBuffered, default(T)), tree, x.firstSon) + initFromJson(mgetOrPut(dst, x.anyStrBuffer, default(T)), tree, x.firstSon) proc initFromJson*[T](dst: var ref T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject, JNull}) @@ -97,7 +97,7 @@ proc initFromJson*[T: object|tuple](dst: var T; tree: JsonTree; n: NodePos) = var buf = "" for x in keys(tree, n): for k, v in dst.fieldPairs: - if x.anyStrBuffered == k: + if x.anyStrBuffer == k: initFromJson(v, tree, x.firstSon) break # emulate elif @@ -129,4 +129,4 @@ iterator pairs*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): (lent string, var buf = "" for x in keys(tree, n): initFromJson(item, tree, x.firstSon) - yield (x.anyStrBuffered, item) + yield (x.anyStrBuffer, item) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 9a1af5d..6b762e6 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -25,22 +25,22 @@ type Action = enum actionElem, actionKeyVal, actionPop, actionEnd -proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, NodePos, Action) = +proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, Action) = if it.pos < it.tosEnd: if it.tos.kind == opcodeArray: - result = (NodePos it.pos, NodePos 0, actionElem) + result = (NodePos it.pos, actionElem) else: - result = (firstSon(NodePos it.pos), NodePos it.pos, actionKeyVal) + result = (firstSon(NodePos it.pos), actionKeyVal) inc it.pos nextChild tree, it.pos elif it.stack.len > 0: - result = (it.tos, NodePos 0, actionPop) + result = (it.tos, actionPop) let tmp = it.stack.pop() it.tos = tmp[0].NodePos it.pos = tmp[1] it.tosEnd = it.tos.tosEnd else: - result = (nilNodeId, NodePos 0, actionEnd) + result = (nilNodeId, actionEnd) proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = privateAccess(JsonTree) @@ -54,7 +54,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = var it = initJsonIter(tree, n) var pendingComma = false while true: - let (child, key, action) = currentAndNext(it, tree) + let (child, action) = currentAndNext(it, tree) case action of actionPop: if child.kind == opcodeArray: @@ -68,7 +68,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "," pendingComma = false if action == actionKeyVal: - escapeJson(key.anyStrBuffered, result) + escapeJson((NodePos child.int-1).anyStrBuffer, result) result.add ":" case child.kind of opcodeArray: @@ -86,10 +86,10 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add child.str pendingComma = true of opcodeFloat: - result.add child.anyStrBuffered + result.add child.anyStrBuffer pendingComma = true of opcodeString: - escapeJson(child.anyStrBuffered, result) + escapeJson(child.anyStrBuffer, result) pendingComma = true of opcodeBool: result.add(if child.bval: "true" else: "false") @@ -103,14 +103,14 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = else: result.add "}" of opcodeString: - escapeJson(n.anyStrBuffered, result) + escapeJson(n.anyStrBuffer, result) of opcodeInt: if n.isShort: result.addInt cast[int64](n.operand) else: result.add n.str of opcodeFloat: - result.add n.anyStrBuffered + result.add n.anyStrBuffer of opcodeBool: result.add(if n.bval: "true" else: "false") of opcodeNull: diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index ba4453c..a1d4552 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -100,7 +100,7 @@ template copyShortStr*(data: string, n: NodePos) = template anyStr*(n: NodePos): untyped = (if n.isShort: n.shortStr else: n.str) -template anyStrBuffered*(x: NodePos): untyped = +template anyStrBuffer*(x: NodePos): untyped = (if n.isShort: (copyShortStr(buf, n); buf) else: n.str) type From 019f7094a41a75455e6bcfb1c820e6d0e1dbaa42 Mon Sep 17 00:00:00 2001 From: Antonis Geralis Date: Sun, 14 Apr 2024 00:00:25 +0300 Subject: [PATCH 15/15] minor --- src/jsonpak/dollar.nim | 2 +- src/jsonpak/private/jsontree.nim | 12 ++++++------ src/jsonpak/private/rawops.nim | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 6b762e6..b0325b5 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -68,7 +68,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "," pendingComma = false if action == actionKeyVal: - escapeJson((NodePos child.int-1).anyStrBuffer, result) + escapeJson(anyStrBuffer(NodePos child.int-1), result) result.add ":" case child.kind of opcodeArray: diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index a1d4552..07c64dd 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -86,22 +86,22 @@ template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort template shortLen*(n: NodePos): int = tree.nodes[n.int].shortLen template get*(n: NodePos; i: int): char = get(tree.nodes[n.int], i) -template shortStr*(n: NodePos): string = - var data = newString(n.shortLen) +template copyShortStrToBuffer*(data: string, n: NodePos) = + data.setLen(n.shortLen) for i in 0 ..< data.len: data[i] = get(n, i) - data -template copyShortStr*(data: string, n: NodePos) = - data.setLen(n.shortLen) +template shortStr*(n: NodePos): string = + var data = newString(n.shortLen) for i in 0 ..< data.len: data[i] = get(n, i) + data template anyStr*(n: NodePos): untyped = (if n.isShort: n.shortStr else: n.str) template anyStrBuffer*(x: NodePos): untyped = - (if n.isShort: (copyShortStr(buf, n); buf) else: n.str) + (if n.isShort: (copyShortStrToBuffer(buf, n); buf) else: n.str) type PatchPos* = distinct int32 diff --git a/src/jsonpak/private/rawops.nim b/src/jsonpak/private/rawops.nim index bd4ef09..58f9d9c 100644 --- a/src/jsonpak/private/rawops.nim +++ b/src/jsonpak/private/rawops.nim @@ -198,12 +198,11 @@ proc rawTest*(a, b: JsonTree, na, nb: NodePos): bool = return false for keyA in keys(a, na): let valA = keyA.firstSon - var valB: NodePos - if a.nodes[keyA.int].isShort: - valB = b.rawGetShort(nb, a.nodes[keyA.int].operand) - else: - let keyStrA = a.atoms[LitId a.nodes[keyA.int].operand] - valB = b.rawGet(nb, keyStrA) + let valB = if a.nodes[keyA.int].isShort: + b.rawGetShort(nb, a.nodes[keyA.int].operand) + else: + let keyStrA = a.atoms[LitId a.nodes[keyA.int].operand] + b.rawGet(nb, keyStrA) if valB.isNil or not rawTest(a, b, valA, valB): return false return true