From d348b9811042aad85a83e8d203e5c4ba9be10ca0 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Thu, 2 Mar 2023 20:14:06 +0000 Subject: [PATCH 01/11] Feature/ Jsonc - Added jsonc support - Unit tests --- index.js | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ spec/index.js | 22 ++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/index.js b/index.js index dbf5860..5ed7dcd 100644 --- a/index.js +++ b/index.js @@ -58,12 +58,60 @@ exports.parse = function (source, _, options) { case '\t': column += 4; break; case '\r': column = 0; break; case '\n': column = 0; line++; break; + case '/': pos++; parseComment(); continue; default: break loop; } pos++; } } + function parseComment() { + var commentStr = '/'; + var nextChar = getChar(); + commentStr += nextChar; + + if (nextChar === '/') { + // read until `\n` + singleLineComment: { + while (true) { + nextChar = getChar(); + + if (nextChar === '\n') { + line++; + break singleLineComment; + } + + commentStr += nextChar; + } + } + } else if (nextChar === '*') { + // read until `*/` + multiLineComment: { + while (true) { + nextChar = getChar(); + + if (nextChar === '\n') + line++; + + if (nextChar === '*') { + commentStr += nextChar; + nextChar = getChar(); + commentStr += nextChar; + + if (nextChar === '/') + break multiLineComment; + } + + commentStr += nextChar; + } + } + } else { + wasUnexpectedToken(); + } + + return commentStr; + } + function parseString() { var str = ''; var char; diff --git a/spec/index.js b/spec/index.js index f254f4e..6a01eb0 100644 --- a/spec/index.js +++ b/spec/index.js @@ -292,6 +292,28 @@ describe('parse', function() { }); }); + describe("jsonc", () => { + const jsonc = +`{ + // Hello World + "prop1": "test", + // Test + "prop2": "test2" + /* Hello World */, + "prop3": [ 123, /* Hello World */ "456", /* Hello World */ 789 ] /* Hello World */, + "prop4" /* Hello World*/ : "test3" +}`; + + const badJsonc = `{ "prop1": "test" / }`; + + it("Should parse jsonc comments as whitespace and execute as normal", () => { + assert.deepStrictEqual(jsonMap.parse(jsonc).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); + }); + + it("Should throw errors on a / not followed by a * or another /", () => { + assert.throws(() => jsonMap.parse(badJsonc), "Unexpected token in JSON at position 18"); + }); + }); function testParse(json, expectedData, skipReverseCheck, whitespace) { var result = jsonMap.parse(json); From 948de1a41d86c70777fb7cb8242649b80eb2fd72 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 00:02:12 +0000 Subject: [PATCH 02/11] Jsonc - Add jsonc as a configurable option --- index.js | 5 ++++- spec/index.js | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 5ed7dcd..d272e5e 100644 --- a/index.js +++ b/index.js @@ -19,6 +19,7 @@ exports.parse = function (source, _, options) { var line = 0; var column = 0; var pos = 0; + var jsonc = !!(options && options.jsonc && typeof options.jsonc != 'undefined'); var bigint = options && options.bigint && typeof BigInt != 'undefined'; return { data: _parse('', true), @@ -58,7 +59,9 @@ exports.parse = function (source, _, options) { case '\t': column += 4; break; case '\r': column = 0; break; case '\n': column = 0; line++; break; - case '/': pos++; parseComment(); continue; + case '/': + if (jsonc) { pos++; parseComment(); continue; } + break loop; default: break loop; } pos++; diff --git a/spec/index.js b/spec/index.js index 6a01eb0..9bf3c84 100644 --- a/spec/index.js +++ b/spec/index.js @@ -307,11 +307,18 @@ describe('parse', function() { const badJsonc = `{ "prop1": "test" / }`; it("Should parse jsonc comments as whitespace and execute as normal", () => { - assert.deepStrictEqual(jsonMap.parse(jsonc).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); + assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); }); it("Should throw errors on a / not followed by a * or another /", () => { - assert.throws(() => jsonMap.parse(badJsonc), "Unexpected token in JSON at position 18"); + assert.throws(() => jsonMap.parse(badJsonc, null, { jsonc: true }), /Unexpected token[ ]{3}in JSON at position 19/, "Didn't throw error for unterminated multiline string"); + }); + + it("Should throw errors for invalid json if jsonc option false or not given and json contains comments", () => { + assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc option false and comments in json."); + assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when empty options given and comments in json."); + assert.throws(() => jsonMap.parse(jsonc), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc not given and comments in json."); + }); }); From 80e5db624def62f37113e5fe459a08ece3be57db Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 00:33:19 +0000 Subject: [PATCH 03/11] Jsonc - Updated readme for jsonc option - jsonc allows trailing commas --- README.md | 1 + index.js | 2 ++ spec/index.js | 12 +++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1edd150..345c38f 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ Location object has properties (zero-based numbers): Options: - _bigint_: parse large integers as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt). +- _jsonc_: parse as [jsonc](https://code.visualstudio.com/docs/languages/json#_json-with-comments). Whitespace: - the only character that increases line number in mappings is line feed ('\n'), so if your JSON string has '\r\n' sequence, it will still be counted as one line, diff --git a/index.js b/index.js index d272e5e..fc12000 100644 --- a/index.js +++ b/index.js @@ -179,6 +179,7 @@ exports.parse = function (source, _, options) { if (char == ']') break; if (char != ',') wasUnexpectedToken(); whitespace(); + if (jsonc && source[pos] == ']') {getChar(); break;} i++; } return arr; @@ -206,6 +207,7 @@ exports.parse = function (source, _, options) { if (char == '}') break; if (char != ',') wasUnexpectedToken(); whitespace(); + if (jsonc && source[pos] == '}') {getChar(); break;} } return obj; } diff --git a/spec/index.js b/spec/index.js index 9bf3c84..f138886 100644 --- a/spec/index.js +++ b/spec/index.js @@ -301,12 +301,18 @@ describe('parse', function() { "prop2": "test2" /* Hello World */, "prop3": [ 123, /* Hello World */ "456", /* Hello World */ 789 ] /* Hello World */, - "prop4" /* Hello World*/ : "test3" + "prop4" /* Hello World*/ : "test3", }`; const badJsonc = `{ "prop1": "test" / }`; - it("Should parse jsonc comments as whitespace and execute as normal", () => { + const jsonWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3,] , }`; + + it("Should parse jsonc comments as whitespace and execute as normal if jsonc true", () => { + assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); + }); + + it("Should parse trailing commas as valid if jsonc true", () => { assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); }); @@ -318,7 +324,7 @@ describe('parse', function() { assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc option false and comments in json."); assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when empty options given and comments in json."); assert.throws(() => jsonMap.parse(jsonc), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc not given and comments in json."); - + assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/, "Didn't throw error when jsonc not given and comments in json."); }); }); From f8d735ba9e4254e1b01c1169e9450368cc4cfbe3 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 00:36:10 +0000 Subject: [PATCH 04/11] Jsonc - Updated test case output message --- spec/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/index.js b/spec/index.js index f138886..b4d176a 100644 --- a/spec/index.js +++ b/spec/index.js @@ -324,7 +324,7 @@ describe('parse', function() { assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc option false and comments in json."); assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when empty options given and comments in json."); assert.throws(() => jsonMap.parse(jsonc), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc not given and comments in json."); - assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/, "Didn't throw error when jsonc not given and comments in json."); + assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/, "Didn't throw error when jsonc not given and trailing commas in json."); }); }); From 4513650e3f99ffcb24a7dfb0ff85483d9cf57746 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 00:49:33 +0000 Subject: [PATCH 05/11] Jsonc - Refactor read comments function - Added additional unit tests for branch coverage --- index.js | 51 ++++++++++++++++++++++----------------------------- spec/index.js | 8 +++++--- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/index.js b/index.js index fc12000..aff5a27 100644 --- a/index.js +++ b/index.js @@ -73,43 +73,36 @@ exports.parse = function (source, _, options) { var nextChar = getChar(); commentStr += nextChar; - if (nextChar === '/') { - // read until `\n` - singleLineComment: { - while (true) { - nextChar = getChar(); - - if (nextChar === '\n') { - line++; - break singleLineComment; - } + var singleLineComment = nextChar === '/'; + var multiLineComment = nextChar === '*'; - commentStr += nextChar; - } - } - } else if (nextChar === '*') { - // read until `*/` - multiLineComment: { - while (true) { - nextChar = getChar(); + if (!singleLineComment && !multiLineComment) + wasUnexpectedToken(); - if (nextChar === '\n') - line++; + if (multiLineComment && source[pos] === '*') + getChar(); - if (nextChar === '*') { - commentStr += nextChar; - nextChar = getChar(); - commentStr += nextChar; + readComment: { + while (true) { + nextChar = getChar(); - if (nextChar === '/') - break multiLineComment; - } + if (nextChar === '\n') { + line++; + if (singleLineComment) + break readComment; + } + if (multiLineComment && nextChar === '*') { + commentStr += nextChar; + nextChar = getChar(); commentStr += nextChar; + + if (nextChar === '/') + break readComment; } + + commentStr += nextChar; } - } else { - wasUnexpectedToken(); } return commentStr; diff --git a/spec/index.js b/spec/index.js index b4d176a..7bf1e39 100644 --- a/spec/index.js +++ b/spec/index.js @@ -306,7 +306,8 @@ describe('parse', function() { const badJsonc = `{ "prop1": "test" / }`; - const jsonWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3,] , }`; + const jsonWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3,] }`; + const jsonObjWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3] , }`; it("Should parse jsonc comments as whitespace and execute as normal if jsonc true", () => { assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); @@ -320,11 +321,12 @@ describe('parse', function() { assert.throws(() => jsonMap.parse(badJsonc, null, { jsonc: true }), /Unexpected token[ ]{3}in JSON at position 19/, "Didn't throw error for unterminated multiline string"); }); - it("Should throw errors for invalid json if jsonc option false or not given and json contains comments", () => { + it("Should throw errors for invalid json if jsonc option false or not given and json contains comments or trailing commas", () => { assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc option false and comments in json."); assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when empty options given and comments in json."); assert.throws(() => jsonMap.parse(jsonc), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc not given and comments in json."); - assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/, "Didn't throw error when jsonc not given and trailing commas in json."); + assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/, "Didn't throw error when jsonc not given and trailing comma in array."); + assert.throws(() => jsonMap.parse(jsonObjWithTrailingComma), /Unexpected token } in JSON at position 45/, "Didn't throw error when jsonc not given and trailing comma in object."); }); }); From 1ad3e5309eea169fbfe20f93b7a72c52b09ce9a3 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 01:18:26 +0000 Subject: [PATCH 06/11] Jsonc - test coverage --- index.js | 6 +----- spec/index.js | 29 +++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index aff5a27..40f54a0 100644 --- a/index.js +++ b/index.js @@ -69,18 +69,14 @@ exports.parse = function (source, _, options) { } function parseComment() { - var commentStr = '/'; var nextChar = getChar(); - commentStr += nextChar; - var singleLineComment = nextChar === '/'; var multiLineComment = nextChar === '*'; if (!singleLineComment && !multiLineComment) wasUnexpectedToken(); - if (multiLineComment && source[pos] === '*') - getChar(); + var commentStr = '/' + nextChar; readComment: { while (true) { diff --git a/spec/index.js b/spec/index.js index 7bf1e39..88a89d0 100644 --- a/spec/index.js +++ b/spec/index.js @@ -302,9 +302,21 @@ describe('parse', function() { /* Hello World */, "prop3": [ 123, /* Hello World */ "456", /* Hello World */ 789 ] /* Hello World */, "prop4" /* Hello World*/ : "test3", + // /********** + /** + * + * Multi line Big Comment + */ + /// Header + /// Subtext }`; const badJsonc = `{ "prop1": "test" / }`; + const badJsoncArr = `[ "test" / ]`; + const tooManyExits = `{ "prop1": "test" /* */ */ }`; + const tooManyExitsArr = `{ "test" /* */ */ }`; + const unopenedComment = `{ Hello World! }`; + const unopenedCommentArr = `[ Hello World! ]`; const jsonWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3,] }`; const jsonObjWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3] , }`; @@ -318,15 +330,20 @@ describe('parse', function() { }); it("Should throw errors on a / not followed by a * or another /", () => { - assert.throws(() => jsonMap.parse(badJsonc, null, { jsonc: true }), /Unexpected token[ ]{3}in JSON at position 19/, "Didn't throw error for unterminated multiline string"); + assert.throws(() => jsonMap.parse(badJsonc, null, { jsonc: true }), /Unexpected token[ ]{3}in JSON at position 19/); + assert.throws(() => jsonMap.parse(badJsoncArr, null, { jsonc: true }), /Unexpected token[ ]{3}in JSON at position 10/); + assert.throws(() => jsonMap.parse(tooManyExits, null, { jsonc: true }), /Unexpected token \* in JSON at position 24/); + assert.throws(() => jsonMap.parse(tooManyExitsArr, null, { jsonc: true }), /Unexpected token \* in JSON at position 15/); + assert.throws(() => jsonMap.parse(unopenedComment, null, { jsonc: true }), /Unexpected token H in JSON at position 2/); + assert.throws(() => jsonMap.parse(unopenedCommentArr, null, { jsonc: true }), /Unexpected token H in JSON at position 2/); }); it("Should throw errors for invalid json if jsonc option false or not given and json contains comments or trailing commas", () => { - assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc option false and comments in json."); - assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when empty options given and comments in json."); - assert.throws(() => jsonMap.parse(jsonc), /Unexpected token [/] in JSON at position 4/, "Didn't throw error when jsonc not given and comments in json."); - assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/, "Didn't throw error when jsonc not given and trailing comma in array."); - assert.throws(() => jsonMap.parse(jsonObjWithTrailingComma), /Unexpected token } in JSON at position 45/, "Didn't throw error when jsonc not given and trailing comma in object."); + assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token \/ in JSON at position 4/); + assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token \/ in JSON at position 4/); + assert.throws(() => jsonMap.parse(jsonc), /Unexpected token \/ in JSON at position 4/); + assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/); + assert.throws(() => jsonMap.parse(jsonObjWithTrailingComma), /Unexpected token } in JSON at position 45/); }); }); From 34e03251e7d1a2719f8df868c3d4c0cb9fec3ed8 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 02:07:29 +0000 Subject: [PATCH 07/11] Jsonc - code refactor --- index.js | 24 +++++++++++++++++------ spec/index.js | 54 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/index.js b/index.js index 40f54a0..328185c 100644 --- a/index.js +++ b/index.js @@ -160,15 +160,21 @@ exports.parse = function (source, _, options) { if (getChar() == ']') return arr; backChar(); - while (true) { + readArray: while (true) { var itemPtr = ptr + '/' + i; arr.push(_parse(itemPtr)); whitespace(); var char = getChar(); - if (char == ']') break; + if (char == ']') break readArray; if (char != ',') wasUnexpectedToken(); whitespace(); - if (jsonc && source[pos] == ']') {getChar(); break;} + readTrailingCommas: while (jsonc) { + char = getChar(); + if (char == ']') break readArray; + if (char == ',') { whitespace(); continue; } + backChar(); + break readTrailingCommas; + } i++; } return arr; @@ -180,7 +186,7 @@ exports.parse = function (source, _, options) { if (getChar() == '}') return obj; backChar(); - while (true) { + readObject: while (true) { var loc = getLoc(); if (getChar() != '"') wasUnexpectedToken(); var key = parseString(); @@ -193,10 +199,16 @@ exports.parse = function (source, _, options) { obj[key] = _parse(propPtr); whitespace(); var char = getChar(); - if (char == '}') break; + if (char == '}') break readObject; if (char != ',') wasUnexpectedToken(); whitespace(); - if (jsonc && source[pos] == '}') {getChar(); break;} + readTrailingCommas: while (jsonc) { + char = getChar(); + if (char == '}') break readObject; + if (char == ',') { whitespace(); continue; } + backChar(); + break readTrailingCommas; + } } return obj; } diff --git a/spec/index.js b/spec/index.js index 88a89d0..525764b 100644 --- a/spec/index.js +++ b/spec/index.js @@ -294,22 +294,34 @@ describe('parse', function() { describe("jsonc", () => { const jsonc = -`{ - // Hello World - "prop1": "test", - // Test - "prop2": "test2" - /* Hello World */, - "prop3": [ 123, /* Hello World */ "456", /* Hello World */ 789 ] /* Hello World */, - "prop4" /* Hello World*/ : "test3", - // /********** - /** - * - * Multi line Big Comment - */ - /// Header - /// Subtext -}`; + `{ + // Hello World + "prop1": "test", + // Test + "prop2": "test2" + /* Hello World */, + "prop3": [ 123, /* Hello World */ "456",,, /* Hello World */ 789 ] /* Hello World */, + "prop4" /* Hello World ,,, * */ : "test3",,,, + "prop5": [ + // Hello World! + /// Header + // /* */ + 0, + 1, /* Hello World! + ,,,,,,,,,, + */ + 2, 3, 4, + ], + // /********** + /** + * + * Multi line Big Comment + */ + /// Header + /// Subtext + ,,,,,,,,,,,,,, + }`; + const expectedJsonc = { prop1: "test", prop2: "test2", prop3: [123, "456", 789], prop4: "test3", prop5: [0, 1, 2, 3, 4] }; const badJsonc = `{ "prop1": "test" / }`; const badJsoncArr = `[ "test" / ]`; @@ -322,11 +334,11 @@ describe('parse', function() { const jsonObjWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3] , }`; it("Should parse jsonc comments as whitespace and execute as normal if jsonc true", () => { - assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); + assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, expectedJsonc); }); it("Should parse trailing commas as valid if jsonc true", () => { - assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, { "prop1": "test", "prop2": "test2", "prop3": [123, "456", 789], "prop4": "test3" }); + assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, expectedJsonc); }); it("Should throw errors on a / not followed by a * or another /", () => { @@ -339,9 +351,9 @@ describe('parse', function() { }); it("Should throw errors for invalid json if jsonc option false or not given and json contains comments or trailing commas", () => { - assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token \/ in JSON at position 4/); - assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token \/ in JSON at position 4/); - assert.throws(() => jsonMap.parse(jsonc), /Unexpected token \/ in JSON at position 4/); + assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token \/ in JSON at position 8/); + assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token \/ in JSON at position 8/); + assert.throws(() => jsonMap.parse(jsonc), /Unexpected token \/ in JSON at position 8/); assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/); assert.throws(() => jsonMap.parse(jsonObjWithTrailingComma), /Unexpected token } in JSON at position 45/); }); From d186ba5e193e91bebab9204c35f95143552d72b5 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 02:42:50 +0000 Subject: [PATCH 08/11] Jsonc - Added test to make sure pointers were still working - Added functionality for earlyCommas as well as trailing commas --- index.js | 62 +++++++++++++++++++++++++++----------- spec/index.js | 83 +++++++++++++++++++++++++++++---------------------- 2 files changed, 92 insertions(+), 53 deletions(-) diff --git a/index.js b/index.js index 328185c..3a86b2e 100644 --- a/index.js +++ b/index.js @@ -19,7 +19,7 @@ exports.parse = function (source, _, options) { var line = 0; var column = 0; var pos = 0; - var jsonc = !!(options && options.jsonc && typeof options.jsonc != 'undefined'); + var jsonc = !!(options && options.jsonc); var bigint = options && options.bigint && typeof BigInt != 'undefined'; return { data: _parse('', true), @@ -82,19 +82,26 @@ exports.parse = function (source, _, options) { while (true) { nextChar = getChar(); - if (nextChar === '\n') { - line++; - if (singleLineComment) - break readComment; - } - - if (multiLineComment && nextChar === '*') { - commentStr += nextChar; - nextChar = getChar(); - commentStr += nextChar; - - if (nextChar === '/') - break readComment; + switch (nextChar) { + case '\t': column += 3; break; + case '\r': column = 0; break; + case '\n': + column = 0; + line++; + + if (singleLineComment) break readComment; + break; + case '*': + if (multiLineComment) { + commentStr += nextChar; + nextChar = getChar(); + commentStr += nextChar; + + if (nextChar === '/') + break readComment; + } + break; + default: break; } commentStr += nextChar; @@ -157,14 +164,23 @@ exports.parse = function (source, _, options) { whitespace(); var arr = []; var i = 0; - if (getChar() == ']') return arr; + var char = getChar(); + if (char == ']') return arr; backChar(); + whitespace(); readArray: while (true) { + readEarlyCommas: while (jsonc) { + char = getChar(); + if (char == ']') break readArray; + if (char == ',') { whitespace(); continue; } + backChar(); + break readEarlyCommas; + } var itemPtr = ptr + '/' + i; arr.push(_parse(itemPtr)); whitespace(); - var char = getChar(); + char = getChar(); if (char == ']') break readArray; if (char != ',') wasUnexpectedToken(); whitespace(); @@ -175,6 +191,7 @@ exports.parse = function (source, _, options) { backChar(); break readTrailingCommas; } + whitespace(); i++; } return arr; @@ -183,10 +200,19 @@ exports.parse = function (source, _, options) { function parseObject(ptr) { whitespace(); var obj = {}; - if (getChar() == '}') return obj; + var char = getChar(); + if (char == '}') return obj; backChar(); + whitespace(); readObject: while (true) { + readEarlyCommas: while (jsonc) { + char = getChar(); + if (char == '}') break readObject; + if (char == ',') { whitespace(); continue; } + backChar(); + break readEarlyCommas; + } var loc = getLoc(); if (getChar() != '"') wasUnexpectedToken(); var key = parseString(); @@ -198,7 +224,7 @@ exports.parse = function (source, _, options) { whitespace(); obj[key] = _parse(propPtr); whitespace(); - var char = getChar(); + char = getChar(); if (char == '}') break readObject; if (char != ',') wasUnexpectedToken(); whitespace(); diff --git a/spec/index.js b/spec/index.js index 525764b..42be0b7 100644 --- a/spec/index.js +++ b/spec/index.js @@ -294,34 +294,50 @@ describe('parse', function() { describe("jsonc", () => { const jsonc = - `{ - // Hello World - "prop1": "test", - // Test - "prop2": "test2" - /* Hello World */, - "prop3": [ 123, /* Hello World */ "456",,, /* Hello World */ 789 ] /* Hello World */, - "prop4" /* Hello World ,,, * */ : "test3",,,, - "prop5": [ - // Hello World! - /// Header - // /* */ - 0, - 1, /* Hello World! - ,,,,,,,,,, - */ - 2, 3, 4, - ], - // /********** - /** - * - * Multi line Big Comment - */ - /// Header - /// Subtext - ,,,,,,,,,,,,,, - }`; +`{ + // Hello World + "prop1": "test", + // Test + "prop2": "test2" + /* Hello World */, + "prop3": [ 123, /* Hello World */ "456",,, /* Hello World */ 789 ] /* Hello World */, + "prop4" /* Hello World ,,, * */ : "test3",,,, + "prop5": [ , , , , \r \r \r + // Hello World! + /// Header \r \r \n \n \r \n + // /* */ + 0, + 1, /* Hello World! + ,,,,,,,,,, + */ + 2, 3, 4, + ], + // /********** + /** + * + * Multi line Big Comment + */ + /// Header + /// Subtext + ,,,,,,,,,,,,,, +}`; const expectedJsonc = { prop1: "test", prop2: "test2", prop3: [123, "456", 789], prop4: "test3", prop5: [0, 1, 2, 3, 4] }; + const expectedPointers = { + "":{"value":{"line":0,"column":0,"pos":0},"valueEnd":{"line":27,"column":1,"pos":503}}, + "/prop1":{"key":{"line":2,"column":2,"pos":21},"keyEnd":{"line":2,"column":9,"pos":28},"value":{"line":2,"column":11,"pos":30},"valueEnd":{"line":2,"column":17,"pos":36}}, + "/prop2":{"key":{"line":4,"column":2,"pos":50},"keyEnd":{"line":4,"column":9,"pos":57},"value":{"line":4,"column":11,"pos":59},"valueEnd":{"line":4,"column":18,"pos":66}}, + "/prop3":{"key":{"line":6,"column":2,"pos":90},"keyEnd":{"line":6,"column":9,"pos":97},"value":{"line":6,"column":11,"pos":99},"valueEnd":{"line":6,"column":66,"pos":156}}, + "/prop3/0":{"value":{"line":6,"column":13,"pos":101},"valueEnd":{"line":6,"column":16,"pos":104}}, + "/prop3/1":{"value":{"line":6,"column":35,"pos":124},"valueEnd":{"line":6,"column":40,"pos":129}}, + "/prop3/2":{"value":{"line":6,"column":61,"pos":151},"valueEnd":{"line":6,"column":64,"pos":154}}, + "/prop4":{"key":{"line":7,"column":2,"pos":178},"keyEnd":{"line":7,"column":9,"pos":185},"value":{"line":7,"column":35,"pos":212},"valueEnd":{"line":7,"column":42,"pos":219}}, + "/prop5":{"key":{"line":8,"column":2,"pos":226},"keyEnd":{"line":8,"column":9,"pos":233},"value":{"line":8,"column":11,"pos":235},"valueEnd":{"line":20,"column":3,"pos":394}}, + "/prop5/0":{"value":{"line":15,"column":4,"pos":332},"valueEnd":{"line":15,"column":5,"pos":333}}, + "/prop5/1":{"value":{"line":16,"column":4,"pos":339},"valueEnd":{"line":16,"column":5,"pos":340}}, + "/prop5/2":{"value":{"line":19,"column":2,"pos":382},"valueEnd":{"line":19,"column":3,"pos":383}}, + "/prop5/3":{"value":{"line":19,"column":5,"pos":385},"valueEnd":{"line":19,"column":6,"pos":386}}, + "/prop5/4":{"value":{"line":19,"column":8,"pos":388},"valueEnd":{"line":19,"column":9,"pos":389}} + }; const badJsonc = `{ "prop1": "test" / }`; const badJsoncArr = `[ "test" / ]`; @@ -333,12 +349,9 @@ describe('parse', function() { const jsonWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3,] }`; const jsonObjWithTrailingComma = `{ "prop1": "test1", "prop2": [1, 2, 3] , }`; - it("Should parse jsonc comments as whitespace and execute as normal if jsonc true", () => { - assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, expectedJsonc); - }); - - it("Should parse trailing commas as valid if jsonc true", () => { + it("Should parse json with comments and trailing commas as whitespace and execute as normal if jsonc true", () => { assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, expectedJsonc); + assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).pointers, expectedPointers); }); it("Should throw errors on a / not followed by a * or another /", () => { @@ -351,9 +364,9 @@ describe('parse', function() { }); it("Should throw errors for invalid json if jsonc option false or not given and json contains comments or trailing commas", () => { - assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token \/ in JSON at position 8/); - assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token \/ in JSON at position 8/); - assert.throws(() => jsonMap.parse(jsonc), /Unexpected token \/ in JSON at position 8/); + assert.throws(() => jsonMap.parse(jsonc, null, { jsonc: false }), /Unexpected token \/ in JSON at position 4/); + assert.throws(() => jsonMap.parse(jsonc, null, {}), /Unexpected token \/ in JSON at position 4/); + assert.throws(() => jsonMap.parse(jsonc), /Unexpected token \/ in JSON at position 4/); assert.throws(() => jsonMap.parse(jsonWithTrailingComma), /Unexpected token ] in JSON at position 38/); assert.throws(() => jsonMap.parse(jsonObjWithTrailingComma), /Unexpected token } in JSON at position 45/); }); From 2d08c09a156535facaf80c4bf710f0567357a083 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 11:42:18 +0000 Subject: [PATCH 09/11] Jsonc - Cleaned up early commas and test cases --- index.js | 30 ++++++++++++++++-------------- spec/index.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index 3a86b2e..7694717 100644 --- a/index.js +++ b/index.js @@ -169,14 +169,15 @@ exports.parse = function (source, _, options) { backChar(); whitespace(); + readEarlyCommas: while (jsonc) { + char = getChar(); + if (char == ']') return arr; + if (char == ',') { whitespace(); continue; } + backChar(); + break readEarlyCommas; + } + readArray: while (true) { - readEarlyCommas: while (jsonc) { - char = getChar(); - if (char == ']') break readArray; - if (char == ',') { whitespace(); continue; } - backChar(); - break readEarlyCommas; - } var itemPtr = ptr + '/' + i; arr.push(_parse(itemPtr)); whitespace(); @@ -205,14 +206,15 @@ exports.parse = function (source, _, options) { backChar(); whitespace(); + readEarlyCommas: while (jsonc) { + char = getChar(); + if (char == '}') return obj; + if (char == ',') { whitespace(); continue; } + backChar(); + break readEarlyCommas; + } + readObject: while (true) { - readEarlyCommas: while (jsonc) { - char = getChar(); - if (char == '}') break readObject; - if (char == ',') { whitespace(); continue; } - backChar(); - break readEarlyCommas; - } var loc = getLoc(); if (getChar() != '"') wasUnexpectedToken(); var key = parseString(); diff --git a/spec/index.js b/spec/index.js index 42be0b7..25d62c4 100644 --- a/spec/index.js +++ b/spec/index.js @@ -339,6 +339,46 @@ describe('parse', function() { "/prop5/4":{"value":{"line":19,"column":8,"pos":388},"valueEnd":{"line":19,"column":9,"pos":389}} }; + const simpleJsonc = + `/* Simple jsonc*/{ "prop1": "test1" }`; + const expectedSimpleJsonc = { prop1: "test1" }; + const pointersSimpleJsonc = { + "": { + "value": { + "column": 16, + "line": 0, + "pos": 17, + }, + "valueEnd": { + "column": 36, + "line": 0, + "pos": 37, + }, + }, + "/prop1": { + "key": { + "column": 18, + "line": 0, + "pos": 19 + }, + "keyEnd": { + "column": 25, + "line": 0, + "pos": 26 + }, + "value": { + "column": 27, + "line": 0, + "pos": 28 + }, + "valueEnd": { + "column": 34, + "line": 0, + "pos": 35 + } + } + }; + const badJsonc = `{ "prop1": "test" / }`; const badJsoncArr = `[ "test" / ]`; const tooManyExits = `{ "prop1": "test" /* */ */ }`; @@ -352,6 +392,8 @@ describe('parse', function() { it("Should parse json with comments and trailing commas as whitespace and execute as normal if jsonc true", () => { assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).data, expectedJsonc); assert.deepStrictEqual(jsonMap.parse(jsonc, null, { jsonc: true }).pointers, expectedPointers); + assert.deepStrictEqual(jsonMap.parse(simpleJsonc, null, { jsonc: true }).data, expectedSimpleJsonc); + assert.deepStrictEqual(jsonMap.parse(simpleJsonc, null, { jsonc: true }).pointers, pointersSimpleJsonc); }); it("Should throw errors on a / not followed by a * or another /", () => { From 60ec13d6f864dc30f514a59b6498cc424af75c59 Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Fri, 3 Mar 2023 16:13:20 +0000 Subject: [PATCH 10/11] Jsonc - Performance improvements --- index.js | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/index.js b/index.js index 7694717..9db8387 100644 --- a/index.js +++ b/index.js @@ -60,8 +60,10 @@ exports.parse = function (source, _, options) { case '\r': column = 0; break; case '\n': column = 0; line++; break; case '/': - if (jsonc) { pos++; parseComment(); continue; } - break loop; + if (!jsonc) break loop; + pos++; + parseComment(); + continue loop; default: break loop; } pos++; @@ -170,11 +172,11 @@ exports.parse = function (source, _, options) { whitespace(); readEarlyCommas: while (jsonc) { - char = getChar(); - if (char == ']') return arr; - if (char == ',') { whitespace(); continue; } - backChar(); - break readEarlyCommas; + switch (getChar()) { + case ']': return arr; + case ',': whitespace(); continue readEarlyCommas; + default: backChar(); break readEarlyCommas; + } } readArray: while (true) { @@ -186,13 +188,12 @@ exports.parse = function (source, _, options) { if (char != ',') wasUnexpectedToken(); whitespace(); readTrailingCommas: while (jsonc) { - char = getChar(); - if (char == ']') break readArray; - if (char == ',') { whitespace(); continue; } - backChar(); - break readTrailingCommas; + switch (getChar()) { + case ']': break readArray; + case ',': whitespace(); continue readTrailingCommas; + default: backChar(); break readTrailingCommas; + } } - whitespace(); i++; } return arr; @@ -207,11 +208,11 @@ exports.parse = function (source, _, options) { whitespace(); readEarlyCommas: while (jsonc) { - char = getChar(); - if (char == '}') return obj; - if (char == ',') { whitespace(); continue; } - backChar(); - break readEarlyCommas; + switch (getChar()) { + case '}': return obj; + case ',': whitespace(); continue readEarlyCommas; + default: backChar(); break readEarlyCommas; + } } readObject: while (true) { @@ -231,11 +232,11 @@ exports.parse = function (source, _, options) { if (char != ',') wasUnexpectedToken(); whitespace(); readTrailingCommas: while (jsonc) { - char = getChar(); - if (char == '}') break readObject; - if (char == ',') { whitespace(); continue; } - backChar(); - break readTrailingCommas; + switch (getChar()) { + case '}': break readObject; + case ',': whitespace(); continue readTrailingCommas; + default: backChar(); break readTrailingCommas; + } } } return obj; From 10a93ddcdaed2217d48a939e2e9f6ecedbbbb71d Mon Sep 17 00:00:00 2001 From: Sykander <26931543+Sykander@users.noreply.github.com> Date: Sat, 4 Mar 2023 02:47:19 +0000 Subject: [PATCH 11/11] Jsonc - fixed issue with column count and comments --- index.js | 2 +- spec/index.js | 92 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/index.js b/index.js index 9db8387..e2d92a5 100644 --- a/index.js +++ b/index.js @@ -61,7 +61,7 @@ exports.parse = function (source, _, options) { case '\n': column = 0; line++; break; case '/': if (!jsonc) break loop; - pos++; + column++; pos++; parseComment(); continue loop; default: break loop; diff --git a/spec/index.js b/spec/index.js index 25d62c4..54bd28a 100644 --- a/spec/index.js +++ b/spec/index.js @@ -323,20 +323,72 @@ describe('parse', function() { }`; const expectedJsonc = { prop1: "test", prop2: "test2", prop3: [123, "456", 789], prop4: "test3", prop5: [0, 1, 2, 3, 4] }; const expectedPointers = { - "":{"value":{"line":0,"column":0,"pos":0},"valueEnd":{"line":27,"column":1,"pos":503}}, - "/prop1":{"key":{"line":2,"column":2,"pos":21},"keyEnd":{"line":2,"column":9,"pos":28},"value":{"line":2,"column":11,"pos":30},"valueEnd":{"line":2,"column":17,"pos":36}}, - "/prop2":{"key":{"line":4,"column":2,"pos":50},"keyEnd":{"line":4,"column":9,"pos":57},"value":{"line":4,"column":11,"pos":59},"valueEnd":{"line":4,"column":18,"pos":66}}, - "/prop3":{"key":{"line":6,"column":2,"pos":90},"keyEnd":{"line":6,"column":9,"pos":97},"value":{"line":6,"column":11,"pos":99},"valueEnd":{"line":6,"column":66,"pos":156}}, - "/prop3/0":{"value":{"line":6,"column":13,"pos":101},"valueEnd":{"line":6,"column":16,"pos":104}}, - "/prop3/1":{"value":{"line":6,"column":35,"pos":124},"valueEnd":{"line":6,"column":40,"pos":129}}, - "/prop3/2":{"value":{"line":6,"column":61,"pos":151},"valueEnd":{"line":6,"column":64,"pos":154}}, - "/prop4":{"key":{"line":7,"column":2,"pos":178},"keyEnd":{"line":7,"column":9,"pos":185},"value":{"line":7,"column":35,"pos":212},"valueEnd":{"line":7,"column":42,"pos":219}}, - "/prop5":{"key":{"line":8,"column":2,"pos":226},"keyEnd":{"line":8,"column":9,"pos":233},"value":{"line":8,"column":11,"pos":235},"valueEnd":{"line":20,"column":3,"pos":394}}, - "/prop5/0":{"value":{"line":15,"column":4,"pos":332},"valueEnd":{"line":15,"column":5,"pos":333}}, - "/prop5/1":{"value":{"line":16,"column":4,"pos":339},"valueEnd":{"line":16,"column":5,"pos":340}}, - "/prop5/2":{"value":{"line":19,"column":2,"pos":382},"valueEnd":{"line":19,"column":3,"pos":383}}, - "/prop5/3":{"value":{"line":19,"column":5,"pos":385},"valueEnd":{"line":19,"column":6,"pos":386}}, - "/prop5/4":{"value":{"line":19,"column":8,"pos":388},"valueEnd":{"line":19,"column":9,"pos":389}} + '': { + value: { line: 0, column: 0, pos: 0 }, + valueEnd: { line: 27, column: 1, pos: 503 } + }, + '/prop1': { + key: { line: 2, column: 2, pos: 21 }, + keyEnd: { line: 2, column: 9, pos: 28 }, + value: { line: 2, column: 11, pos: 30 }, + valueEnd: { line: 2, column: 17, pos: 36 } + }, + '/prop2': { + key: { line: 4, column: 2, pos: 50 }, + keyEnd: { line: 4, column: 9, pos: 57 }, + value: { line: 4, column: 11, pos: 59 }, + valueEnd: { line: 4, column: 18, pos: 66 } + }, + '/prop3': { + key: { line: 6, column: 2, pos: 90 }, + keyEnd: { line: 6, column: 9, pos: 97 }, + value: { line: 6, column: 11, pos: 99 }, + valueEnd: { line: 6, column: 68, pos: 156 } + }, + '/prop3/0': { + value: { line: 6, column: 13, pos: 101 }, + valueEnd: { line: 6, column: 16, pos: 104 } + }, + '/prop3/1': { + value: { line: 6, column: 36, pos: 124 }, + valueEnd: { line: 6, column: 41, pos: 129 } + }, + '/prop3/2': { + value: { line: 6, column: 63, pos: 151 }, + valueEnd: { line: 6, column: 66, pos: 154 } + }, + '/prop4': { + key: { line: 7, column: 2, pos: 178 }, + keyEnd: { line: 7, column: 9, pos: 185 }, + value: { line: 7, column: 36, pos: 212 }, + valueEnd: { line: 7, column: 43, pos: 219 } + }, + '/prop5': { + key: { line: 8, column: 2, pos: 226 }, + keyEnd: { line: 8, column: 9, pos: 233 }, + value: { line: 8, column: 11, pos: 235 }, + valueEnd: { line: 20, column: 3, pos: 394 } + }, + '/prop5/0': { + value: { line: 15, column: 4, pos: 332 }, + valueEnd: { line: 15, column: 5, pos: 333 } + }, + '/prop5/1': { + value: { line: 16, column: 4, pos: 339 }, + valueEnd: { line: 16, column: 5, pos: 340 } + }, + '/prop5/2': { + value: { line: 19, column: 2, pos: 382 }, + valueEnd: { line: 19, column: 3, pos: 383 } + }, + '/prop5/3': { + value: { line: 19, column: 5, pos: 385 }, + valueEnd: { line: 19, column: 6, pos: 386 } + }, + '/prop5/4': { + value: { line: 19, column: 8, pos: 388 }, + valueEnd: { line: 19, column: 9, pos: 389 } + } }; const simpleJsonc = @@ -345,34 +397,34 @@ describe('parse', function() { const pointersSimpleJsonc = { "": { "value": { - "column": 16, + "column": 17, "line": 0, "pos": 17, }, "valueEnd": { - "column": 36, + "column": 37, "line": 0, "pos": 37, }, }, "/prop1": { "key": { - "column": 18, + "column": 19, "line": 0, "pos": 19 }, "keyEnd": { - "column": 25, + "column": 26, "line": 0, "pos": 26 }, "value": { - "column": 27, + "column": 28, "line": 0, "pos": 28 }, "valueEnd": { - "column": 34, + "column": 35, "line": 0, "pos": 35 }