fix: support utf8 string and add some test cases:

1. column number support utf8 string located
2. add some line and column number test cases
3. add utf8_str_len function
4. error object has correct columnNumber and lineNumber properties

Author: ErosZy

include/quickjs/cutils.h

@@ -277,6 +277,7 @@ static inline void dbuf_set_error(DynBuf *s)
 
 int unicode_to_utf8(uint8_t *buf, unsigned int c);
 int unicode_from_utf8(const uint8_t *p, int max_len, const uint8_t **pp);
+int utf8_str_len(const uint8_t *p_start, const uint8_t *p_end);
 
 static inline int from_hex(int c)
 {

qjscalc.c

@@ -129,6 +129,15 @@ static void __attribute((unused)) dump_token(JSParseState *s,
   }
 }
 
+static int calc_column_position(JSParseState *s) {
+  if(s->column_last_ptr > s->column_ptr) {
+    s->column_num_count += utf8_str_len(s->column_ptr, s->column_last_ptr);
+    s->column_ptr = s->column_last_ptr;
+  }
+
+  return s->column_num_count;
+}
+
 int __attribute__((format(printf, 2, 3))) js_parse_error(JSParseState *s, const char *fmt, ...)
 {
   JSContext *ctx = s->ctx;
@@ -142,7 +151,7 @@ int __attribute__((format(printf, 2, 3))) js_parse_error(JSParseState *s, const
   if (s->cur_func && s->cur_func->backtrace_barrier)
     backtrace_flags = JS_BACKTRACE_FLAG_SINGLE_LEVEL;
 
-  int column_num = s->column_last_ptr - s->column_ptr;
+  int column_num = calc_column_position(s);
   build_backtrace(ctx, ctx->rt->current_exception, 
                   s->filename, s->line_num,
                   column_num < 0 ? -1 : column_num, 
@@ -183,7 +192,7 @@ static __exception int js_parse_template_part(JSParseState *s, const uint8_t *p)
 {
   uint32_t c;
   StringBuffer b_s, *b = &b_s;
-  s->token.column_num = s->column_last_ptr - s->column_ptr;
+  s->token.column_num = calc_column_position(s);
 
   /* p points to the first byte of the template part */
   if (string_buffer_init(s->ctx, b, 32))
@@ -216,8 +225,8 @@ static __exception int js_parse_template_part(JSParseState *s, const uint8_t *p)
     }
     if (c == '\n') {
       s->line_num++;
-      s->column_last_ptr = p;
-      s->column_ptr = p;
+      s->column_ptr = s->column_last_ptr = p;
+      s->column_num_count = 0;
     } else if (c >= 0x80) {
       const uint8_t *p_next;
       c = unicode_from_utf8(p - 1, UTF8_CHAR_LEN_MAX, &p_next);
@@ -251,7 +260,7 @@ static __exception int js_parse_string(JSParseState *s, int sep,
   int ret;
   uint32_t c;
   StringBuffer b_s, *b = &b_s;
-  s->token.column_num = s->column_last_ptr - s->column_ptr;
+  s->token.column_num = calc_column_position(s);
 
   /* string */
   if (string_buffer_init(s->ctx, b, 32))
@@ -309,8 +318,8 @@ static __exception int js_parse_string(JSParseState *s, int sep,
           p++;
           if (sep != '`') {
             s->line_num++;
-            s->column_last_ptr = p;
-            s->column_ptr = p;
+            s->column_ptr = s->column_last_ptr = p;
+            s->column_num_count = 0;
           }
           continue;
         default:
@@ -567,7 +576,6 @@ static JSAtom parse_ident(JSParseState *s, const uint8_t **pp,
   return atom;
 }
 
-
 static __exception int next_token(JSParseState *s)
 {
   const uint8_t *p;
@@ -619,6 +627,7 @@ static __exception int next_token(JSParseState *s)
       s->got_lf = TRUE;
       s->line_num++;
       s->column_ptr = p;
+      s->column_num_count = 0;
       goto redo;
     case '\f':
     case '\v':
@@ -643,6 +652,7 @@ static __exception int next_token(JSParseState *s)
             s->line_num++;
             s->got_lf = TRUE; /* considered as LF for ASI */
             s->column_ptr = ++p;
+            s->column_num_count = 0;
           } else if (*p == '\r') {
             s->got_lf = TRUE; /* considered as LF for ASI */
             p++;
@@ -1077,8 +1087,8 @@ static __exception int next_token(JSParseState *s)
   }
 
   s->buf_ptr = p;
-  if(!s->token.column_num && s->column_last_ptr > s->column_ptr) {
-    s->token.column_num = s->column_last_ptr - s->column_ptr;
+  if (!s->token.column_num) {
+    s->token.column_num = calc_column_position(s);
   }
 
   // dump_token(s, &s->token);
@@ -1170,6 +1180,7 @@ __exception int json_next_token(JSParseState *s)
       p++;
       s->line_num++;
       s->column_ptr = p;
+      s->column_num_count = 0;
       goto redo;
     case '\f':
     case '\v':
@@ -1202,6 +1213,7 @@ __exception int json_next_token(JSParseState *s)
           if (*p == '\n') {
             s->line_num++;
             s->column_ptr = ++p;
+            s->column_num_count = 0;
           } else if (*p == '\r') {
             p++;
           } else if (*p >= 0x80) {
@@ -1312,8 +1324,8 @@ __exception int json_next_token(JSParseState *s)
   }
 
   s->buf_ptr = p;
-  if(!s->token.column_num && s->column_last_ptr > s->column_ptr){
-    s->token.column_num = s->column_last_ptr - s->column_ptr;
+  if (!s->token.column_num) {
+    s->token.column_num = calc_column_position(s);
   }
 
   // dump_token(s, &s->token);
@@ -2354,16 +2366,20 @@ static int __exception js_parse_property_name(JSParseState *s,
 typedef struct JSParsePos {
   int last_line_num;
   int line_num;
-  int column_num;
   BOOL got_lf;
   const uint8_t *ptr;
+  const uint8_t *column_ptr;
+  const uint8_t *column_last_ptr;
+  int column_num_count;
 } JSParsePos;
 
 static int js_parse_get_pos(JSParseState *s, JSParsePos *sp)
 {
   sp->last_line_num = s->last_line_num;
   sp->line_num = s->token.line_num;
-  sp->column_num = s->token.column_num;
+  sp->column_ptr = s->column_ptr;
+  sp->column_last_ptr = s->column_last_ptr;
+  sp->column_num_count = s->column_num_count;
   sp->ptr = s->token.ptr;
   sp->got_lf = s->got_lf;
   return 0;
@@ -2373,7 +2389,9 @@ static __exception int js_parse_seek_token(JSParseState *s, const JSParsePos *sp
 {
   s->token.line_num = sp->last_line_num;
   s->line_num = sp->line_num;
-  s->column_last_ptr = sp->ptr;
+  s->column_ptr = sp->column_ptr;
+  s->column_last_ptr = sp->column_last_ptr;
+  s->column_num_count = sp->column_num_count;
   s->buf_ptr = sp->ptr;
   s->got_lf = sp->got_lf;
   return next_token(s);
@@ -12259,6 +12277,8 @@ void js_parse_init(JSContext *ctx, JSParseState *s,
   s->filename = filename;
   s->line_num = 1;
   s->column_ptr = (const uint8_t*)input;
+  s->column_last_ptr = s->column_ptr;
+  s->column_num_count = 0;
   s->buf_ptr = (const uint8_t *)input;
   s->buf_end = s->buf_ptr + input_len;
   s->token.val = ' ';

repl.c

@@ -368,6 +368,7 @@ typedef struct JSParseState {
   int line_num;       /* line number of current offset */
   const uint8_t *column_ptr; /* column head pointer on every line */
   const uint8_t *column_last_ptr;
+  int column_num_count;
   const char *filename;
   JSToken token;
   BOOL got_lf; /* true if got line feed before the current token */

src/core/parser.c

@@ -1137,15 +1137,19 @@ void build_backtrace(JSContext* ctx, JSValueConst error_obj, const char* filenam
   const char* str1;
   JSObject* p;
   BOOL backtrace_barrier;
+  int latest_line_num = -1;
+  int latest_column_num = -1;
 
   js_dbuf_init(ctx, &dbuf);
   if (filename) {
     dbuf_printf(&dbuf, "    at %s", filename);
     if (line_num != -1) {
+      latest_line_num = line_num;
       dbuf_printf(&dbuf, ":%d", line_num);
     }
 
     if (column_num != -1) {
+      latest_column_num = column_num;
       dbuf_printf(&dbuf, ":%d", column_num);
     }
 
@@ -1186,6 +1190,14 @@ void build_backtrace(JSContext* ctx, JSValueConst error_obj, const char* filenam
           column_num += 1;
         }
 
+        if (latest_line_num == -1) {
+          latest_line_num = line_num;
+        }
+
+        if (latest_column_num == -1) {
+          latest_column_num = column_num;
+        }
+
         atom_str = JS_AtomToCString(ctx, b->debug.filename);
         dbuf_printf(&dbuf, " (%s", atom_str ? atom_str : "<null>");
         JS_FreeCString(ctx, atom_str);
@@ -1220,9 +1232,22 @@ void build_backtrace(JSContext* ctx, JSValueConst error_obj, const char* filenam
   JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_stack, str, JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE);
 
   if (line_num != -1) {
-    JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_lineNumber, JS_NewInt32(ctx, line_num), JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE);
+    JS_DefinePropertyValue(
+      ctx, 
+      error_obj, 
+      JS_ATOM_lineNumber, 
+      JS_NewInt32(ctx, latest_line_num), 
+      JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE
+    );
+
     if (column_num != -1) {
-      JS_DefinePropertyValue(ctx, error_obj, JS_ATOM_columnNumber, JS_NewInt32(ctx, column_num), JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE);
+      JS_DefinePropertyValue(
+        ctx, 
+        error_obj, 
+        JS_ATOM_columnNumber, 
+        JS_NewInt32(ctx, latest_column_num), 
+        JS_PROP_WRITABLE | JS_PROP_CONFIGURABLE
+      );
     }
   }
 }

src/core/parser.h

@@ -303,6 +303,18 @@ int unicode_from_utf8(const uint8_t *p, int max_len, const uint8_t **pp)
     return c;
 }
 
+int utf8_str_len(const uint8_t *p_start, const uint8_t *p_end) {
+    int count = 0;
+    while (p_start < p_end) {
+        if (!unicode_from_utf8(p_start, UTF8_CHAR_LEN_MAX, &p_start)) {
+            break;
+        }
+        count += 1;
+    }
+    
+    return count;
+}
+
 #if 0
 
 #if defined(EMSCRIPTEN) || defined(__ANDROID__)