From 3b88895a1f2619a1c47919e947d5308172a5892a Mon Sep 17 00:00:00 2001 From: "illya.shveda" Date: Tue, 22 Aug 2023 18:51:22 +0300 Subject: [PATCH 1/5] add SetBytesOptionsManyByGetResult() --- go.mod | 2 + sjson.go | 237 +++++++++++++++++++++++++++++++++++++++++++++++--- sjson_test.go | 106 ++++++++++++++++++++-- 3 files changed, 326 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index fa6b7e8..6a39f3c 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/tidwall/sjson go 1.14 +replace github.com/tidwall/sjson => github.com/ContinuumLLC/sjson v1.2.5 + require ( github.com/tidwall/gjson v1.14.2 github.com/tidwall/pretty v1.2.0 diff --git a/sjson.go b/sjson.go index a55eef3..4eafb53 100644 --- a/sjson.go +++ b/sjson.go @@ -3,6 +3,7 @@ package sjson import ( jsongo "encoding/json" + "fmt" "sort" "strconv" "unsafe" @@ -427,19 +428,18 @@ func isOptimisticPath(path string) bool { // // A path is a series of keys separated by a dot. // -// { -// "name": {"first": "Tom", "last": "Anderson"}, -// "age":37, -// "children": ["Sara","Alex","Jack"], -// "friends": [ -// {"first": "James", "last": "Murphy"}, -// {"first": "Roger", "last": "Craig"} -// ] -// } -// "name.last" >> "Anderson" -// "age" >> 37 -// "children.1" >> "Alex" -// +// { +// "name": {"first": "Tom", "last": "Anderson"}, +// "age":37, +// "children": ["Sara","Alex","Jack"], +// "friends": [ +// {"first": "James", "last": "Murphy"}, +// {"first": "Roger", "last": "Craig"} +// ] +// } +// "name.last" >> "Anderson" +// "age" >> 37 +// "children.1" >> "Alex" func Set(json, path string, value interface{}) (string, error) { return SetOptions(json, path, value, nil) } @@ -503,6 +503,103 @@ type sliceHeader struct { cap int } +func setByGetResult(jstr, raw string, res gjson.Result, + stringify, del, optimistic, inplace bool) ([]byte, error) { + + sz := len(jstr) - len(res.Raw) + len(raw) + if stringify { + sz += 2 + } + if inplace && sz <= len(jstr) { + if !stringify || !mustMarshalString(raw) { + jsonh := *(*stringHeader)(unsafe.Pointer(&jstr)) + jsonbh := sliceHeader{ + data: jsonh.data, len: jsonh.len, cap: jsonh.len} + jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + if stringify { + jbytes[res.Index] = '"' + copy(jbytes[res.Index+1:], []byte(raw)) + jbytes[res.Index+1+len(raw)] = '"' + copy(jbytes[res.Index+1+len(raw)+1:], + jbytes[res.Index+len(res.Raw):]) + } else { + copy(jbytes[res.Index:], []byte(raw)) + copy(jbytes[res.Index+len(raw):], + jbytes[res.Index+len(res.Raw):]) + } + return jbytes, nil + } + return []byte(jstr), nil + } + buf := make([]byte, 0, sz) + buf = append(buf, jstr[:res.Index]...) + if stringify { + buf = appendStringify(buf, raw) + } else { + buf = append(buf, raw...) + } + buf = append(buf, jstr[res.Index+len(res.Raw):]...) + return buf, nil +} + +func setManyByGetResult(jstr string, raws []interface{}, valueDiff int, ress []gjson.Result, + stringify, inplace bool) ([]byte, error) { + + var blen int + + if stringify { + valueDiff += 2 * len(raws) + } + + blen = len(jstr) + valueDiff + var buf = make([]byte, len(jstr)) + copy(buf, jstr) + if !inplace { + return nil, fmt.Errorf("not supported if replace is not inplace") + } + jsonh := *(*stringHeader)(unsafe.Pointer(&jstr)) + jsonbh := sliceHeader{ + data: jsonh.data, len: blen, cap: blen} + jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh)) + var rwb []byte + var diff int + for i := 0; i < len(ress); i++ { + raw := raws[i] + res := ress[i] + rwb = getBytes(raw) + var currentSz int + + currentSz = len(rwb) - len(res.Raw) + + if stringify { + currentSz += 2 + + jbytes[res.Index+diff] = '"' + copy(jbytes[res.Index+1+diff:], rwb) // 1 index + jbytes[res.Index+1+len(rwb)+diff] = '"' + if i+1 < len(ress) { + copy(jbytes[res.Index+1+len(rwb)+1+diff:], + buf[res.Index+len(res.Raw):ress[i+1].Index]) // next index, copy from index + len + till next index + } else { + copy(jbytes[res.Index+1+len(rwb)+1+diff:], // last index + buf[res.Index+len(res.Raw):]) + } + diff += currentSz + } else { + copy(jbytes[res.Index+diff:], rwb) // 1 index + if i+1 < len(ress) { + copy(jbytes[res.Index+len(rwb)+diff:], + buf[res.Index+len(res.Raw):ress[i+1].Index]) // next index, copy from index + len + till next index + } else { + copy(jbytes[res.Index+len(rwb)+diff:], // last index + buf[res.Index+len(res.Raw):]) + } + diff += currentSz + } + } + return jbytes, nil +} + func set(jstr, path, raw string, stringify, del, optimistic, inplace bool) ([]byte, error) { if path == "" { @@ -646,6 +743,120 @@ func SetOptions(json, path string, value interface{}, return string(res), err } +// SetBytesOptionsByGetResult - if you have already gotten the result, no need to get it in set again +func SetBytesOptionsByGetResult(json []byte, getResult gjson.Result, value interface{}, + opts *Options) ([]byte, error) { + var optimistic, inplace bool + if opts != nil { + optimistic = opts.Optimistic + inplace = opts.ReplaceInPlace + } + jstr := *(*string)(unsafe.Pointer(&json)) + var res []byte + var err error + switch v := value.(type) { + default: + b, merr := jsongo.Marshal(value) + if merr != nil { + return nil, merr + } + raw := *(*string)(unsafe.Pointer(&b)) + res, err = setByGetResult(jstr, raw, getResult, false, false, optimistic, inplace) + case dtype: + res, err = setByGetResult(jstr, "", getResult, false, true, optimistic, inplace) + case string: + res, err = setByGetResult(jstr, v, getResult, true, false, optimistic, inplace) + case []byte: + raw := *(*string)(unsafe.Pointer(&v)) + res, err = setByGetResult(jstr, raw, getResult, true, false, optimistic, inplace) + case bool: + if v { + res, err = setByGetResult(jstr, "true", getResult, false, false, optimistic, inplace) + } else { + res, err = setByGetResult(jstr, "false", getResult, false, false, optimistic, inplace) + } + case int8: + res, err = setByGetResult(jstr, strconv.FormatInt(int64(v), 10), getResult, + false, false, optimistic, inplace) + case int16: + res, err = setByGetResult(jstr, strconv.FormatInt(int64(v), 10), getResult, + false, false, optimistic, inplace) + case int32: + res, err = setByGetResult(jstr, strconv.FormatInt(int64(v), 10), getResult, + false, false, optimistic, inplace) + case int64: + res, err = setByGetResult(jstr, strconv.FormatInt(int64(v), 10), getResult, + false, false, optimistic, inplace) + case uint8: + res, err = setByGetResult(jstr, strconv.FormatUint(uint64(v), 10), getResult, + false, false, optimistic, inplace) + case uint16: + res, err = setByGetResult(jstr, strconv.FormatUint(uint64(v), 10), getResult, + false, false, optimistic, inplace) + case uint32: + res, err = setByGetResult(jstr, strconv.FormatUint(uint64(v), 10), getResult, + false, false, optimistic, inplace) + case uint64: + res, err = setByGetResult(jstr, strconv.FormatUint(uint64(v), 10), getResult, + false, false, optimistic, inplace) + case float32: + res, err = setByGetResult(jstr, strconv.FormatFloat(float64(v), 'f', -1, 64), getResult, + false, false, optimistic, inplace) + case float64: + res, err = setByGetResult(jstr, strconv.FormatFloat(float64(v), 'f', -1, 64), getResult, + false, false, optimistic, inplace) + } + if err == errNoChange { + return json, nil + } + return res, err +} + +func SetBytesOptionsManyByGetResult(json []byte, getResult []gjson.Result, values []interface{}, + opts *Options) ([]byte, error) { + var inplace bool + if opts != nil { + inplace = opts.ReplaceInPlace + } + jstr := *(*string)(unsafe.Pointer(&json)) + var res []byte + var err error + + var valueDiff int + for i := 0; i < len(getResult); i++ { + if values[i] == nil { + return nil, fmt.Errorf("nil value appeared in replacement array that matches [%v] value from original payload", getResult[i].Value()) + } + + v, ok := values[i].(string) + if !ok { + v = fmt.Sprintf("%v", values[i]) + } + + valueDiff += len(v) - len(getResult[i].Raw) + } + + var stringify bool + switch val := values[0].(type) { + case string: + stringify = true + case bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float32, float64: + stringify = false + default: + return nil, fmt.Errorf("value type is not supported %v", val) + } + res, err = setManyByGetResult(jstr, values, valueDiff, getResult, stringify, inplace) + + if err == errNoChange { + return json, nil + } + return res, err +} + +func getBytes(v interface{}) []byte { + return []byte(fmt.Sprintf("%v", v)) +} + // SetBytesOptions sets a json value for the specified path with options. // If working with bytes, this method preferred over // SetOptions(string(data), path, value) diff --git a/sjson_test.go b/sjson_test.go index aa3d968..f6fb421 100644 --- a/sjson_test.go +++ b/sjson_test.go @@ -172,23 +172,23 @@ func TestDeleteIssue21(t *testing.T) { // We change the number of characters in this to make the section of the string before the section that we want to delete a certain length - //--------------------------- + // --------------------------- lenBeforeToDeleteIs307AsBytes := `{"1":"","0":"012345678901234567890123456789012345678901234567890123456789012345678901234567","to_delete":"0","2":""}` expectedForLenBefore307AsBytes := `{"1":"","0":"012345678901234567890123456789012345678901234567890123456789012345678901234567","2":""}` - //--------------------------- + // --------------------------- - //--------------------------- + // --------------------------- lenBeforeToDeleteIs308AsBytes := `{"1":"","0":"0123456789012345678901234567890123456789012345678901234567890123456789012345678","to_delete":"0","2":""}` expectedForLenBefore308AsBytes := `{"1":"","0":"0123456789012345678901234567890123456789012345678901234567890123456789012345678","2":""}` - //--------------------------- + // --------------------------- - //--------------------------- + // --------------------------- lenBeforeToDeleteIs309AsBytes := `{"1":"","0":"01234567890123456789012345678901234567890123456789012345678901234567890123456","to_delete":"0","2":""}` expectedForLenBefore309AsBytes := `{"1":"","0":"01234567890123456789012345678901234567890123456789012345678901234567890123456","2":""}` - //--------------------------- + // --------------------------- var data = []struct { desc string @@ -351,3 +351,97 @@ func TestIssue61(t *testing.T) { t.Fail() } } + +func TestSetBytesOptionsManyByGetResult(t *testing.T) { + tcs := []struct { + name string + jsonPath string + newIDs []interface{} + actual string + expected string + }{ + { + name: "string", + jsonPath: "#.id", + newIDs: []interface{}{"stringid1", "stringid2", "stringid3"}, + actual: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ]`, + expected: `[ + {"id": "stringid1","first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"id": "stringid2","first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"id": "stringid3","first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ]`, + }, + { + name: "bool", + jsonPath: "#.isAdult", + newIDs: []interface{}{false, false, false}, + actual: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 44, "isAdult": true, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 68, "isAdult": true, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 47, "isAdult": true, "nets": ["ig", "tw"]} + ]`, + expected: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 44, "isAdult": false, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 68, "isAdult": false, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 47, "isAdult": false, "nets": ["ig", "tw"]} + ]`, + }, + { + name: "int", + jsonPath: "#.age", + newIDs: []interface{}{10, 20, 30}, + actual: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ]`, + expected: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 10, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 20, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 30, "nets": ["ig", "tw"]} + ]`, + }, + { + name: "float", + jsonPath: "#.age", + newIDs: []interface{}{10.1, 20.1, 30.1}, + actual: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 44.1, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 68.1, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 471., "nets": ["ig", "tw"]} + ]`, + expected: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 10.1, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 20.1, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 30.1, "nets": ["ig", "tw"]} + ]`, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + getResult := gjson.Get(tc.actual, tc.jsonPath) + if !getResult.Exists() || !getResult.IsArray() { + t.Fail() + } + arrayResult := getResult.Array() + + opts := &Options{ + Optimistic: true, + ReplaceInPlace: true, + } + actual, err := SetBytesOptionsManyByGetResult([]byte(tc.actual), arrayResult, tc.newIDs, opts) + if err != nil { + t.Fatal(err) + } + + if sortJSON(tc.expected) != sortJSON(string(actual)) { + t.Fatalf("expected '%v', got '%v'", tc.expected, string(actual)) + } + }) + } +} From 4cdb03ae846844b31c0b54c66b863a5d7fccdabd Mon Sep 17 00:00:00 2001 From: Illya Shveda <32647486+ishveda@users.noreply.github.com> Date: Fri, 25 Aug 2023 13:11:31 +0300 Subject: [PATCH 2/5] Drop replace --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 6a39f3c..fa6b7e8 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/tidwall/sjson go 1.14 -replace github.com/tidwall/sjson => github.com/ContinuumLLC/sjson v1.2.5 - require ( github.com/tidwall/gjson v1.14.2 github.com/tidwall/pretty v1.2.0 From 5b34c4f4845eb1df63d290dd37e031955dead8d8 Mon Sep 17 00:00:00 2001 From: "oksana.zhykina" Date: Tue, 29 Aug 2023 17:45:26 +0300 Subject: [PATCH 3/5] set many result function can have values both stringified and not --- sjson.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/sjson.go b/sjson.go index 4eafb53..4eae7b1 100644 --- a/sjson.go +++ b/sjson.go @@ -542,16 +542,24 @@ func setByGetResult(jstr, raw string, res gjson.Result, return buf, nil } -func setManyByGetResult(jstr string, raws []interface{}, valueDiff int, ress []gjson.Result, - stringify, inplace bool) ([]byte, error) { +func setManyByGetResult(jstr string, raws []interface{}, valueDiff int, ress []gjson.Result, inplace bool) ([]byte, error) { + var stringifiedCount int - var blen int + stringified := make([]bool, len(raws)) - if stringify { - valueDiff += 2 * len(raws) + for i, r := range raws { + switch r.(type) { + case string: + stringified[i] = true + stringifiedCount += 1 + default: + stringified[i] = false + } } - blen = len(jstr) + valueDiff + valueDiff += 2 * len(raws) + blen := len(jstr) + valueDiff + var buf = make([]byte, len(jstr)) copy(buf, jstr) if !inplace { @@ -571,6 +579,7 @@ func setManyByGetResult(jstr string, raws []interface{}, valueDiff int, ress []g currentSz = len(rwb) - len(res.Raw) + stringify := stringified[i] if stringify { currentSz += 2 @@ -836,16 +845,7 @@ func SetBytesOptionsManyByGetResult(json []byte, getResult []gjson.Result, value valueDiff += len(v) - len(getResult[i].Raw) } - var stringify bool - switch val := values[0].(type) { - case string: - stringify = true - case bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float32, float64: - stringify = false - default: - return nil, fmt.Errorf("value type is not supported %v", val) - } - res, err = setManyByGetResult(jstr, values, valueDiff, getResult, stringify, inplace) + res, err = setManyByGetResult(jstr, values, valueDiff, getResult, inplace) if err == errNoChange { return json, nil From 6bddb4998048459b5d7a074c9757cbe96d0c46d7 Mon Sep 17 00:00:00 2001 From: "oksana.zhykina" Date: Tue, 29 Aug 2023 17:57:34 +0300 Subject: [PATCH 4/5] fix len calculation --- sjson.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sjson.go b/sjson.go index 4eae7b1..9fc65b4 100644 --- a/sjson.go +++ b/sjson.go @@ -557,7 +557,7 @@ func setManyByGetResult(jstr string, raws []interface{}, valueDiff int, ress []g } } - valueDiff += 2 * len(raws) + valueDiff += 2 * stringifiedCount blen := len(jstr) + valueDiff var buf = make([]byte, len(jstr)) From 81a29e2f576007508dcf5a5b973f1214c53010fc Mon Sep 17 00:00:00 2001 From: "oksana.zhykina" Date: Mon, 4 Sep 2023 17:34:30 +0300 Subject: [PATCH 5/5] add test-case --- sjson_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sjson_test.go b/sjson_test.go index f6fb421..8906589 100644 --- a/sjson_test.go +++ b/sjson_test.go @@ -418,6 +418,21 @@ func TestSetBytesOptionsManyByGetResult(t *testing.T) { {"id": "id1","first": "Dale", "last": "Murphy", "age": 10.1, "nets": ["ig", "fb", "tw"]}, {"id": "id2","first": "Roger", "last": "Craig", "age": 20.1, "nets": ["fb", "tw"]}, {"id": "id3","first": "Jane", "last": "Murphy", "age": 30.1, "nets": ["ig", "tw"]} + ]`, + }, + { + name: "float_and_string", + jsonPath: "#.age", + newIDs: []interface{}{"forty four", 20.1, "forty seven"}, + actual: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": 44.1, "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 68.1, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": 47.1, "nets": ["ig", "tw"]} + ]`, + expected: `[ + {"id": "id1","first": "Dale", "last": "Murphy", "age": "forty four", "nets": ["ig", "fb", "tw"]}, + {"id": "id2","first": "Roger", "last": "Craig", "age": 20.1, "nets": ["fb", "tw"]}, + {"id": "id3","first": "Jane", "last": "Murphy", "age": "forty seven", "nets": ["ig", "tw"]} ]`, }, }