diff --git a/src/jsonpak/builder.nim b/src/jsonpak/builder.nim index ae54d8e..10fbcab 100644 --- a/src/jsonpak/builder.nim +++ b/src/jsonpak/builder.nim @@ -18,7 +18,7 @@ proc initFromJson*(dst: var string; tree: JsonTree; n: NodePos) = if n.kind == opcodeNull: dst = "" else: - dst = n.str + dst = n.anyStr proc initFromJson*(dst: var bool; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JBool}) @@ -29,25 +29,31 @@ proc initFromJson*(dst: var JsonTree; tree: JsonTree; n: NodePos) = proc initFromJson*[T: SomeInteger](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JInt}) - when T is BiggestUInt: - dst = parseBiggestUInt n.str - elif T is BiggestInt: - dst = parseBiggestInt n.str - elif T is SomeSignedInt: - dst = T(parseInt n.str) + if n.isShort: + dst = cast[T](n.operand) else: - dst = T(parseUInt n.str) + when T is BiggestUInt: + dst = parseBiggestUInt n.str + elif T is BiggestInt: + dst = parseBiggestInt n.str + elif T is SomeSignedInt: + dst = T(parseInt n.str) + else: + dst = T(parseUInt n.str) proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JInt, JFloat}) if n.kind == opcodeFloat: - dst = T(parseFloat n.str) + dst = T(parseFloat n.anyStr) else: - dst = T(parseBiggestInt n.str) + if n.isShort: + dst = T(cast[int64](n.operand)) + else: + dst = T(parseBiggestInt n.str) proc initFromJson*[T: enum](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JString}) - dst = parseEnum[T](n.str) + dst = parseEnum[T](n.anyStr) proc initFromJson*[T](dst: var seq[T]; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JArray}) @@ -66,8 +72,9 @@ proc initFromJson*[S, T](dst: var array[S, T]; tree: JsonTree; n: NodePos) = proc initFromJson*[T](dst: var (Table[string, T]|OrderedTable[string, T]); tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject}) + var buf = "" for x in keys(tree, n): - initFromJson(mgetOrPut(dst, x.str, default(T)), tree, x.firstSon) + initFromJson(mgetOrPut(dst, x.anyStrBuffer, default(T)), tree, x.firstSon) proc initFromJson*[T](dst: var ref T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject, JNull}) @@ -87,9 +94,10 @@ proc initFromJson*[T](dst: var Option[T]; tree: JsonTree; n: NodePos) = proc initFromJson*[T: object|tuple](dst: var T; tree: JsonTree; n: NodePos) = verifyJsonKind(tree, n, {JObject}) + var buf = "" for x in keys(tree, n): for k, v in dst.fieldPairs: - if x.str == k: + if x.anyStrBuffer == k: initFromJson(v, tree, x.firstSon) break # emulate elif @@ -118,6 +126,7 @@ iterator pairs*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): (lent string, raisePathError(path.string) assert n.kind == opcodeObject var item = default(T) + var buf = "" for x in keys(tree, n): initFromJson(item, tree, x.firstSon) - yield (x.str, item) + yield (x.anyStrBuffer, item) diff --git a/src/jsonpak/dollar.nim b/src/jsonpak/dollar.nim index 2be5c9d..b0325b5 100644 --- a/src/jsonpak/dollar.nim +++ b/src/jsonpak/dollar.nim @@ -25,27 +25,26 @@ type Action = enum actionElem, actionKeyVal, actionPop, actionEnd -proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, LitId, Action) = +proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, Action) = if it.pos < it.tosEnd: if it.tos.kind == opcodeArray: - result = (NodePos it.pos, LitId(0), actionElem) + result = (NodePos it.pos, actionElem) else: - let litId = (NodePos it.pos).litId - result = (firstSon(NodePos it.pos), litId, actionKeyVal) + result = (firstSon(NodePos it.pos), actionKeyVal) inc it.pos nextChild tree, it.pos elif it.stack.len > 0: - result = (it.tos, LitId(0), actionPop) + result = (it.tos, actionPop) let tmp = it.stack.pop() it.tos = tmp[0].NodePos it.pos = tmp[1] it.tosEnd = it.tos.tosEnd else: - result = (nilNodeId, LitId(0), actionEnd) + result = (nilNodeId, actionEnd) proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = privateAccess(JsonTree) - template key: string = tree.atoms[keyId] + var buf = "" case n.kind of opcodeArray, opcodeObject: if n.kind == opcodeArray: @@ -55,7 +54,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = var it = initJsonIter(tree, n) var pendingComma = false while true: - let (child, keyId, action) = currentAndNext(it, tree) + let (child, action) = currentAndNext(it, tree) case action of actionPop: if child.kind == opcodeArray: @@ -69,7 +68,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "," pendingComma = false if action == actionKeyVal: - key.escapeJson(result) + escapeJson(anyStrBuffer(NodePos child.int-1), result) result.add ":" case child.kind of opcodeArray: @@ -80,11 +79,17 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = result.add "{" it.push child pendingComma = false - of opcodeInt, opcodeFloat: - result.add child.str + of opcodeInt: + if child.isShort: + result.addInt cast[int64](child.operand) + else: + result.add child.str + pendingComma = true + of opcodeFloat: + result.add child.anyStrBuffer pendingComma = true of opcodeString: - escapeJson(child.str, result) + escapeJson(child.anyStrBuffer, result) pendingComma = true of opcodeBool: result.add(if child.bval: "true" else: "false") @@ -98,9 +103,14 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) = else: result.add "}" of opcodeString: - escapeJson(n.str, result) - of opcodeInt, opcodeFloat: - result.add n.str + escapeJson(n.anyStrBuffer, result) + of opcodeInt: + if n.isShort: + result.addInt cast[int64](n.operand) + else: + result.add n.str + of opcodeFloat: + result.add n.anyStrBuffer of opcodeBool: result.add(if n.bval: "true" else: "false") of opcodeNull: diff --git a/src/jsonpak/mapper.nim b/src/jsonpak/mapper.nim index 7424026..553b43f 100644 --- a/src/jsonpak/mapper.nim +++ b/src/jsonpak/mapper.nim @@ -6,7 +6,10 @@ proc toJson*(s: string; tree: var JsonTree) = storeAtom(tree, opcodeString, s) proc toJson*[T: SomeInteger](n: T; tree: var JsonTree) = - storeAtom(tree, opcodeInt, $n) + if inShortIntRange(n): + storeShortInt(tree, n) + else: + storeAtom(tree, opcodeInt, $n) proc toJson*[T: SomeFloat](n: T; tree: var JsonTree) = storeAtom(tree, opcodeFloat, $n) diff --git a/src/jsonpak/parser.nim b/src/jsonpak/parser.nim index b82dd7f..a7dd01a 100644 --- a/src/jsonpak/parser.nim +++ b/src/jsonpak/parser.nim @@ -1,4 +1,4 @@ -import std/[parsejson, streams], private/[jsontree, jsonnode] +import std/[parsejson, streams, strutils], private/[jsontree, jsonnode] export JsonParsingError proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) = @@ -7,7 +7,11 @@ proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) = storeAtom(tree, opcodeString, p.a) discard getTok(p) of tkInt: - storeAtom(tree, opcodeInt, p.a) + let n = parseInt(p.a) + if inShortIntRange(n): + storeShortInt(tree, n) + else: + storeAtom(tree, opcodeInt, p.a) discard getTok(p) of tkFloat: storeAtom(tree, opcodeFloat, p.a) diff --git a/src/jsonpak/private/jsonnode.nim b/src/jsonpak/private/jsonnode.nim index 7730a7f..9cf4b26 100644 --- a/src/jsonpak/private/jsonnode.nim +++ b/src/jsonpak/private/jsonnode.nim @@ -1,5 +1,5 @@ type - Node* = distinct uint32 + Node* = distinct uint64 JsonNodeKind* = enum ## possible JSON node types JNull, JBool, @@ -10,24 +10,49 @@ type JArray const - opcodeBits = 3'u32 + opcodeBits = 4 + payloadBits = sizeof(uint64)*8 - opcodeBits + shortBit = 0b0000_1000 - opcodeNull* = uint32 JNull - opcodeBool* = uint32 JBool - opcodeFalse* = opcodeBool - opcodeTrue* = opcodeBool or 0b0000_1000 - opcodeInt* = uint32 JInt - opcodeFloat* = uint32 JFloat - opcodeString* = uint32 JString - opcodeObject* = uint32 JObject - opcodeArray* = uint32 JArray + shortLenMask = (1 shl opcodeBits) - 1 - opcodeMask = (1'u32 shl opcodeBits) - 1'u32 + shortIntLow = -(1 shl payloadBits) + shortIntHigh = (1 shl payloadBits) - 1 -template kind*(n: Node): uint32 = n.uint32 and opcodeMask -template operand*(n: Node): uint32 = n.uint32 shr opcodeBits.uint32 + opcodeNull* = uint64 JNull + opcodeBool* = uint64 JBool + opcodeFalse* = opcodeBool + opcodeTrue* = opcodeBool or shortBit + opcodeInt* = uint64 JInt + opcodeFloat* = uint64 JFloat + opcodeString* = uint64 JString + opcodeObject* = uint64 JObject + opcodeArray* = uint64 JArray -template toNode*(kind, operand: uint32): Node = - Node(operand shl opcodeBits.uint32 or kind) + opcodeMask = (1 shl (opcodeBits - 1)) - 1 proc `==`*(a, b: Node): bool {.borrow.} + +template kind*(n: Node): uint64 = n.uint64 and opcodeMask +template operand*(n: Node): uint64 = n.uint64 shr opcodeBits.uint64 +template isShort*(n: Node): bool = (n.uint64 and shortBit) != 0 +template shortLen*(n: Node): int = int(n.uint64 shr opcodeBits.uint64 and shortLenMask) + +template toShortNode*(kind, operand: uint64): Node = + Node(operand shl opcodeBits.uint64 or kind or shortBit.uint64) + +template toNode*(kind, operand: uint64): Node = + Node(operand shl opcodeBits.uint64 or kind) + +template get*(n: Node; i: int): char = char(n.operand shr (i * 8 + opcodeBits) and 0xff) +template set*(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8 + opcodeBits)) +template setShortLen*(p: uint64; n: int) = p = p or n.uint64 + +template inShortIntRange*(x: int64): bool = x >= shortIntLow and x <= shortIntHigh +template inShortStrRange*(x: string): bool = x.len <= payloadBits div 8 + +proc toPayload*(data: string): uint64 = + result = 0 + for i in 0 ..< data.len: + set(result, i, data[i]) + setShortLen(result, data.len) diff --git a/src/jsonpak/private/jsontree.nim b/src/jsonpak/private/jsontree.nim index 4854f49..07c64dd 100644 --- a/src/jsonpak/private/jsontree.nim +++ b/src/jsonpak/private/jsontree.nim @@ -23,8 +23,8 @@ proc nextChild*(tree: JsonTree; pos: var int) {.inline.} = else: inc pos -proc toAtomNode*(tree: var JsonTree; kind: uint32, str: string): Node {.inline.} = - toNode(kind, uint32 getOrIncl(tree.atoms, str)) +proc toAtomNode*(tree: var JsonTree; kind: uint64, str: string): Node {.inline.} = + toNode(kind, uint64 getOrIncl(tree.atoms, str)) type NodePos* = distinct int @@ -76,13 +76,33 @@ proc parentImpl*(tree: JsonTree; n: NodePos): NodePos = template parent*(n: NodePos): NodePos = parentImpl(tree, n) -template kind*(n: NodePos): uint32 = tree.nodes[n.int].kind +template kind*(n: NodePos): uint64 = tree.nodes[n.int].kind template litId*(n: NodePos): LitId = LitId operand(tree.nodes[n.int]) -template operand*(n: NodePos): uint32 = tree.nodes[n.int].operand - +template operand*(n: NodePos): uint64 = tree.nodes[n.int].operand template str*(n: NodePos): string = tree.atoms[litId(n)] template bval*(n: NodePos): bool = n.operand == 1 +template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort +template shortLen*(n: NodePos): int = tree.nodes[n.int].shortLen +template get*(n: NodePos; i: int): char = get(tree.nodes[n.int], i) + +template copyShortStrToBuffer*(data: string, n: NodePos) = + data.setLen(n.shortLen) + for i in 0 ..< data.len: + data[i] = get(n, i) + +template shortStr*(n: NodePos): string = + var data = newString(n.shortLen) + for i in 0 ..< data.len: + data[i] = get(n, i) + data + +template anyStr*(n: NodePos): untyped = + (if n.isShort: n.shortStr else: n.str) + +template anyStrBuffer*(x: NodePos): untyped = + (if n.isShort: (copyShortStrToBuffer(buf, n); buf) else: n.str) + type PatchPos* = distinct int32 @@ -90,21 +110,27 @@ proc `<`*(a, b: PatchPos): bool {.borrow.} proc `<=`*(a, b: PatchPos): bool {.borrow.} proc `==`*(a, b: PatchPos): bool {.borrow.} -proc prepare*(tree: var JsonTree; kind: uint32): PatchPos = +proc prepare*(tree: var JsonTree; kind: uint64): PatchPos = result = PatchPos tree.nodes.len tree.nodes.add Node kind proc patch*(tree: var JsonTree; pos: PatchPos) = let pos = pos.int assert tree.nodes[pos].kind > opcodeString - let distance = uint32(tree.nodes.len - pos) - tree.nodes[pos] = toNode(tree.nodes[pos].uint32, distance) + let distance = uint64(tree.nodes.len - pos) + tree.nodes[pos] = toNode(tree.nodes[pos].uint64, distance) + +proc storeEmpty*(tree: var JsonTree; kind: uint64) {.inline.} = + tree.nodes.add toNode(kind, 1) -proc storeAtom*(tree: var JsonTree; kind: uint32) {.inline.} = +proc storeAtom*(tree: var JsonTree; kind: uint64) {.inline.} = tree.nodes.add Node(kind) -proc storeAtom*(tree: var JsonTree; kind: uint32; data: string) {.inline.} = - tree.nodes.add toAtomNode(tree, kind, data) +proc storeShortInt*[T: SomeInteger](tree: var JsonTree; data: T) {.inline.} = + tree.nodes.add toShortNode(opcodeInt, cast[uint64](data)) -proc storeEmpty*(tree: var JsonTree; kind: uint32) {.inline.} = - tree.nodes.add toNode(kind, 1) +proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} = + if inShortStrRange(data): + tree.nodes.add toShortNode(kind, toPayload(data)) + else: + tree.nodes.add toAtomNode(tree, kind, data) diff --git a/src/jsonpak/private/rawops.nim b/src/jsonpak/private/rawops.nim index 2cfd52e..58f9d9c 100644 --- a/src/jsonpak/private/rawops.nim +++ b/src/jsonpak/private/rawops.nim @@ -1,18 +1,28 @@ import bitabs, jsonnode, jsontree, std/importutils -proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = +proc rawGetShort*(tree: JsonTree, n: NodePos, name: uint64): NodePos = privateAccess(JsonTree) - let litId = tree.atoms.getKeyId(name) - if litId == LitId(0): - return nilNodeId for x in keys(tree, n): - if x.litId == litId: + if x.operand == name: return x.firstSon return nilNodeId +proc rawGet*(tree: JsonTree, n: NodePos, name: string): NodePos = + privateAccess(JsonTree) + if inShortStrRange(name): + rawGetShort(tree, n, toPayload(name)) + else: + let litId = tree.atoms.getKeyId(name) + if litId == LitId(0): + return nilNodeId + for x in keys(tree, n): + if x.litId == litId: + return x.firstSon + return nilNodeId + proc rawUpdateParents*(tree: var JsonTree, parents: seq[PatchPos], diff: int) = privateAccess(JsonTree) - for parent in parents: + for parent in items(parents): let distance = tree.nodes[parent.int].rawSpan + diff tree.nodes[parent.int] = toNode(tree.nodes[parent.int].kind, distance.uint32) @@ -24,7 +34,10 @@ proc rawExtract*(result: var JsonTree, tree: JsonTree, n: NodePos) = let n = NodePos(i+n.int) # careful case n.kind of opcodeInt, opcodeFloat, opcodeString: - result.nodes[i] = toAtomNode(result, n.kind, n.str) + if n.isShort: + result.nodes[i] = tree.nodes[n.int] + else: + result.nodes[i] = toAtomNode(result, n.kind, n.str) else: result.nodes[i] = tree.nodes[n.int] @@ -51,7 +64,10 @@ proc rawAdd*(result: var JsonTree, tree: JsonTree, n: NodePos) = let m = NodePos(i) case m.kind of opcodeInt, opcodeFloat, opcodeString: - result.nodes[i+n.int] = toAtomNode(result, m.kind, m.str) + if m.isShort: + result.nodes[i+n.int] = tree.nodes[i] + else: + result.nodes[i+n.int] = toAtomNode(result, m.kind, m.str) else: result.nodes[i+n.int] = tree.nodes[i] @@ -62,7 +78,10 @@ proc rawAddKeyValuePair*(result: var JsonTree, src, dest: NodePos, key: string) setLen(result.nodes, oldfull+L) for i in countdown(oldfull-1, dest.int): result.nodes[i+L] = result.nodes[i] - result.nodes[dest.int] = toAtomNode(result, opcodeString, key) + if inShortStrRange(key): + result.nodes[dest.int] = toShortNode(opcodeString, toPayload(key)) + else: + result.nodes[dest.int] = toAtomNode(result, opcodeString, key) let src = if src >= dest: NodePos(src.int+L) else: src for i in 0..