Fix #32 Huge speed up when creating copies of JsonTrees and memory usage drop.

src/jsonpak/builder.nim

@@ -18,7 +18,7 @@ proc initFromJson*(dst: var string; tree: JsonTree; n: NodePos) =
   if n.kind == opcodeNull:
     dst = ""
   else:
-    dst = n.str
+    dst = n.anyStr
 
 proc initFromJson*(dst: var bool; tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JBool})
@@ -29,25 +29,31 @@ proc initFromJson*(dst: var JsonTree; tree: JsonTree; n: NodePos) =
 
 proc initFromJson*[T: SomeInteger](dst: var T; tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JInt})
-  when T is BiggestUInt:
-    dst = parseBiggestUInt n.str
-  elif T is BiggestInt:
-    dst = parseBiggestInt n.str
-  elif T is SomeSignedInt:
-    dst = T(parseInt n.str)
+  if n.isShort:
+    dst = cast[T](n.operand)
   else:
-    dst = T(parseUInt n.str)
+    when T is BiggestUInt:
+      dst = parseBiggestUInt n.str
+    elif T is BiggestInt:
+      dst = parseBiggestInt n.str
+    elif T is SomeSignedInt:
+      dst = T(parseInt n.str)
+    else:
+      dst = T(parseUInt n.str)
 
 proc initFromJson*[T: SomeFloat](dst: var T; tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JInt, JFloat})
   if n.kind == opcodeFloat:
-    dst = T(parseFloat n.str)
+    dst = T(parseFloat n.anyStr)
   else:
-    dst = T(parseBiggestInt n.str)
+    if n.isShort:
+      dst = T(cast[int64](n.operand))
+    else:
+      dst = T(parseBiggestInt n.str)
 
 proc initFromJson*[T: enum](dst: var T; tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JString})
-  dst = parseEnum[T](n.str)
+  dst = parseEnum[T](n.anyStr)
 
 proc initFromJson*[T](dst: var seq[T]; tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JArray})
@@ -66,8 +72,9 @@ proc initFromJson*[S, T](dst: var array[S, T]; tree: JsonTree; n: NodePos) =
 
 proc initFromJson*[T](dst: var (Table[string, T]|OrderedTable[string, T]); tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JObject})
+  var buf = ""
   for x in keys(tree, n):
-    initFromJson(mgetOrPut(dst, x.str, default(T)), tree, x.firstSon)
+    initFromJson(mgetOrPut(dst, x.anyStrBuffer, default(T)), tree, x.firstSon)
 
 proc initFromJson*[T](dst: var ref T; tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JObject, JNull})
@@ -87,9 +94,10 @@ proc initFromJson*[T](dst: var Option[T]; tree: JsonTree; n: NodePos) =
 
 proc initFromJson*[T: object|tuple](dst: var T; tree: JsonTree; n: NodePos) =
   verifyJsonKind(tree, n, {JObject})
+  var buf = ""
   for x in keys(tree, n):
     for k, v in dst.fieldPairs:
-      if x.str == k:
+      if x.anyStrBuffer == k:
         initFromJson(v, tree, x.firstSon)
         break # emulate elif
 
@@ -118,6 +126,7 @@ iterator pairs*[T](tree: JsonTree; path: JsonPtr; t: typedesc[T]): (lent string,
     raisePathError(path.string)
   assert n.kind == opcodeObject
   var item = default(T)
+  var buf = ""
   for x in keys(tree, n):
     initFromJson(item, tree, x.firstSon)
-    yield (x.str, item)
+    yield (x.anyStrBuffer, item)

src/jsonpak/dollar.nim

@@ -25,27 +25,26 @@ type
   Action = enum
     actionElem, actionKeyVal, actionPop, actionEnd
 
-proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, LitId, Action) =
+proc currentAndNext(it: var JsonIter, tree: JsonTree): (NodePos, Action) =
   if it.pos < it.tosEnd:
     if it.tos.kind == opcodeArray:
-      result = (NodePos it.pos, LitId(0), actionElem)
+      result = (NodePos it.pos, actionElem)
     else:
-      let litId = (NodePos it.pos).litId
-      result = (firstSon(NodePos it.pos), litId, actionKeyVal)
+      result = (firstSon(NodePos it.pos), actionKeyVal)
       inc it.pos
     nextChild tree, it.pos
   elif it.stack.len > 0:
-    result = (it.tos, LitId(0), actionPop)
+    result = (it.tos, actionPop)
     let tmp = it.stack.pop()
     it.tos = tmp[0].NodePos
     it.pos = tmp[1]
     it.tosEnd = it.tos.tosEnd
   else:
-    result = (nilNodeId, LitId(0), actionEnd)
+    result = (nilNodeId, actionEnd)
 
 proc toUgly*(result: var string, tree: JsonTree, n: NodePos) =
   privateAccess(JsonTree)
-  template key: string = tree.atoms[keyId]
+  var buf = ""
   case n.kind
   of opcodeArray, opcodeObject:
     if n.kind == opcodeArray:
@@ -55,7 +54,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) =
     var it = initJsonIter(tree, n)
     var pendingComma = false
     while true:
-      let (child, keyId, action) = currentAndNext(it, tree)
+      let (child, action) = currentAndNext(it, tree)
       case action
       of actionPop:
         if child.kind == opcodeArray:
@@ -69,7 +68,7 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) =
           result.add ","
           pendingComma = false
         if action == actionKeyVal:
-          key.escapeJson(result)
+          escapeJson(anyStrBuffer(NodePos child.int-1), result)
           result.add ":"
         case child.kind
         of opcodeArray:
@@ -80,11 +79,17 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) =
           result.add "{"
           it.push child
           pendingComma = false
-        of opcodeInt, opcodeFloat:
-          result.add child.str
+        of opcodeInt:
+          if child.isShort:
+            result.addInt cast[int64](child.operand)
+          else:
+            result.add child.str
+          pendingComma = true
+        of opcodeFloat:
+          result.add child.anyStrBuffer
           pendingComma = true
         of opcodeString:
-          escapeJson(child.str, result)
+          escapeJson(child.anyStrBuffer, result)
           pendingComma = true
         of opcodeBool:
           result.add(if child.bval: "true" else: "false")
@@ -98,9 +103,14 @@ proc toUgly*(result: var string, tree: JsonTree, n: NodePos) =
     else:
       result.add "}"
   of opcodeString:
-    escapeJson(n.str, result)
-  of opcodeInt, opcodeFloat:
-    result.add n.str
+    escapeJson(n.anyStrBuffer, result)
+  of opcodeInt:
+    if n.isShort:
+      result.addInt cast[int64](n.operand)
+    else:
+      result.add n.str
+  of opcodeFloat:
+    result.add n.anyStrBuffer
   of opcodeBool:
     result.add(if n.bval: "true" else: "false")
   of opcodeNull:

src/jsonpak/mapper.nim

@@ -6,7 +6,10 @@ proc toJson*(s: string; tree: var JsonTree) =
   storeAtom(tree, opcodeString, s)
 
 proc toJson*[T: SomeInteger](n: T; tree: var JsonTree) =
-  storeAtom(tree, opcodeInt, $n)
+  if inShortIntRange(n):
+    storeShortInt(tree, n)
+  else:
+    storeAtom(tree, opcodeInt, $n)
 
 proc toJson*[T: SomeFloat](n: T; tree: var JsonTree) =
   storeAtom(tree, opcodeFloat, $n)

src/jsonpak/parser.nim

@@ -1,4 +1,4 @@
-import std/[parsejson, streams], private/[jsontree, jsonnode]
+import std/[parsejson, streams, strutils], private/[jsontree, jsonnode]
 export JsonParsingError
 
 proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) =
@@ -7,7 +7,11 @@ proc parseJsonAtom(tree: var JsonTree; p: var JsonParser) =
     storeAtom(tree, opcodeString, p.a)
     discard getTok(p)
   of tkInt:
-    storeAtom(tree, opcodeInt, p.a)
+    let n = parseInt(p.a)
+    if inShortIntRange(n):
+      storeShortInt(tree, n)
+    else:
+      storeAtom(tree, opcodeInt, p.a)
     discard getTok(p)
   of tkFloat:
     storeAtom(tree, opcodeFloat, p.a)

src/jsonpak/private/jsonnode.nim

@@ -1,5 +1,5 @@
 type
-  Node* = distinct uint32
+  Node* = distinct uint64
   JsonNodeKind* = enum ## possible JSON node types
     JNull,
     JBool,
@@ -10,24 +10,49 @@ type
     JArray
 
 const
-  opcodeBits = 3'u32
+  opcodeBits = 4
+  payloadBits = sizeof(uint64)*8 - opcodeBits
+  shortBit = 0b0000_1000
 
-  opcodeNull* = uint32 JNull
-  opcodeBool* = uint32 JBool
-  opcodeFalse* = opcodeBool
-  opcodeTrue* = opcodeBool or 0b0000_1000
-  opcodeInt* = uint32 JInt
-  opcodeFloat* = uint32 JFloat
-  opcodeString* = uint32 JString
-  opcodeObject* = uint32 JObject
-  opcodeArray* = uint32 JArray
+  shortLenMask = (1 shl opcodeBits) - 1
 
-  opcodeMask = (1'u32 shl opcodeBits) - 1'u32
+  shortIntLow = -(1 shl payloadBits)
+  shortIntHigh = (1 shl payloadBits) - 1
 
-template kind*(n: Node): uint32 = n.uint32 and opcodeMask
-template operand*(n: Node): uint32 = n.uint32 shr opcodeBits.uint32
+  opcodeNull* = uint64 JNull
+  opcodeBool* = uint64 JBool
+  opcodeFalse* = opcodeBool
+  opcodeTrue* = opcodeBool or shortBit
+  opcodeInt* = uint64 JInt
+  opcodeFloat* = uint64 JFloat
+  opcodeString* = uint64 JString
+  opcodeObject* = uint64 JObject
+  opcodeArray* = uint64 JArray
 
-template toNode*(kind, operand: uint32): Node =
-  Node(operand shl opcodeBits.uint32 or kind)
+  opcodeMask = (1 shl (opcodeBits - 1)) - 1
 
 proc `==`*(a, b: Node): bool {.borrow.}
+
+template kind*(n: Node): uint64 = n.uint64 and opcodeMask
+template operand*(n: Node): uint64 = n.uint64 shr opcodeBits.uint64
+template isShort*(n: Node): bool = (n.uint64 and shortBit) != 0
+template shortLen*(n: Node): int = int(n.uint64 shr opcodeBits.uint64 and shortLenMask)
+
+template toShortNode*(kind, operand: uint64): Node =
+  Node(operand shl opcodeBits.uint64 or kind or shortBit.uint64)
+
+template toNode*(kind, operand: uint64): Node =
+  Node(operand shl opcodeBits.uint64 or kind)
+
+template get*(n: Node; i: int): char = char(n.operand shr (i * 8 + opcodeBits) and 0xff)
+template set*(p: uint64; i: int; c: char) = p = p or (c.uint64 shl (i * 8 + opcodeBits))
+template setShortLen*(p: uint64; n: int) = p = p or n.uint64
+
+template inShortIntRange*(x: int64): bool = x >= shortIntLow and x <= shortIntHigh
+template inShortStrRange*(x: string): bool = x.len <= payloadBits div 8
+
+proc toPayload*(data: string): uint64 =
+  result = 0
+  for i in 0 ..< data.len:
+    set(result, i, data[i])
+  setShortLen(result, data.len)

src/jsonpak/private/jsontree.nim

@@ -23,8 +23,8 @@ proc nextChild*(tree: JsonTree; pos: var int) {.inline.} =
   else:
     inc pos
 
-proc toAtomNode*(tree: var JsonTree; kind: uint32, str: string): Node {.inline.} =
-  toNode(kind, uint32 getOrIncl(tree.atoms, str))
+proc toAtomNode*(tree: var JsonTree; kind: uint64, str: string): Node {.inline.} =
+  toNode(kind, uint64 getOrIncl(tree.atoms, str))
 
 type
   NodePos* = distinct int
@@ -76,35 +76,61 @@ proc parentImpl*(tree: JsonTree; n: NodePos): NodePos =
 
 template parent*(n: NodePos): NodePos = parentImpl(tree, n)
 
-template kind*(n: NodePos): uint32 = tree.nodes[n.int].kind
+template kind*(n: NodePos): uint64 = tree.nodes[n.int].kind
 template litId*(n: NodePos): LitId = LitId operand(tree.nodes[n.int])
-template operand*(n: NodePos): uint32 = tree.nodes[n.int].operand
-
+template operand*(n: NodePos): uint64 = tree.nodes[n.int].operand
 template str*(n: NodePos): string = tree.atoms[litId(n)]
 template bval*(n: NodePos): bool = n.operand == 1
 
+template isShort*(n: NodePos): bool = tree.nodes[n.int].isShort
+template shortLen*(n: NodePos): int = tree.nodes[n.int].shortLen
+template get*(n: NodePos; i: int): char = get(tree.nodes[n.int], i)
+
+template copyShortStrToBuffer*(data: string, n: NodePos) =
+  data.setLen(n.shortLen)
+  for i in 0 ..< data.len:
+    data[i] = get(n, i)
+
+template shortStr*(n: NodePos): string =
+  var data = newString(n.shortLen)
+  for i in 0 ..< data.len:
+    data[i] = get(n, i)
+  data
+
+template anyStr*(n: NodePos): untyped =
+  (if n.isShort: n.shortStr else: n.str)
+
+template anyStrBuffer*(x: NodePos): untyped =
+  (if n.isShort: (copyShortStrToBuffer(buf, n); buf) else: n.str)
+
 type
   PatchPos* = distinct int32
 
 proc `<`*(a, b: PatchPos): bool {.borrow.}
 proc `<=`*(a, b: PatchPos): bool {.borrow.}
 proc `==`*(a, b: PatchPos): bool {.borrow.}
 
-proc prepare*(tree: var JsonTree; kind: uint32): PatchPos =
+proc prepare*(tree: var JsonTree; kind: uint64): PatchPos =
   result = PatchPos tree.nodes.len
   tree.nodes.add Node kind
 
 proc patch*(tree: var JsonTree; pos: PatchPos) =
   let pos = pos.int
   assert tree.nodes[pos].kind > opcodeString
-  let distance = uint32(tree.nodes.len - pos)
-  tree.nodes[pos] = toNode(tree.nodes[pos].uint32, distance)
+  let distance = uint64(tree.nodes.len - pos)
+  tree.nodes[pos] = toNode(tree.nodes[pos].uint64, distance)
+
+proc storeEmpty*(tree: var JsonTree; kind: uint64) {.inline.} =
+  tree.nodes.add toNode(kind, 1)
 
-proc storeAtom*(tree: var JsonTree; kind: uint32) {.inline.} =
+proc storeAtom*(tree: var JsonTree; kind: uint64) {.inline.} =
   tree.nodes.add Node(kind)
 
-proc storeAtom*(tree: var JsonTree; kind: uint32; data: string) {.inline.} =
-  tree.nodes.add toAtomNode(tree, kind, data)
+proc storeShortInt*[T: SomeInteger](tree: var JsonTree; data: T) {.inline.} =
+  tree.nodes.add toShortNode(opcodeInt, cast[uint64](data))
 
-proc storeEmpty*(tree: var JsonTree; kind: uint32) {.inline.} =
-  tree.nodes.add toNode(kind, 1)
+proc storeAtom*(tree: var JsonTree; kind: uint64; data: string) {.inline.} =
+  if inShortStrRange(data):
+    tree.nodes.add toShortNode(kind, toPayload(data))
+  else:
+    tree.nodes.add toAtomNode(tree, kind, data)