fix: lexer parse comment bug

Author: gengjiawen

lexer/lexer_test.rs

@@ -64,6 +64,15 @@ mod tests {
     fn test_lexer_bool() {
         test_lexer_common("bool", "let y=true");
     }
+    
+    #[test]
+    fn test_comments_then_blank_line() {
+        // Ensure comments followed by a blank line are skipped correctly
+        // and do not yield ILLEGAL tokens.
+        test_lexer_common("comments_then_blank_line", "// comment\n\nlet x = 5");
+    }
+
+
 
     #[test]
     fn test_lexer_complex() {

lexer/lib.rs

@@ -47,8 +47,8 @@ impl<'a> Lexer<'a> {
 
     pub fn next_token(&mut self) -> Token {
         // println!("self ch {}, position {} read_position {}", self.ch, self.position, self.read_position);
-        self.skip_whitespace();
-        self.skip_comments();
+        // Skip any whitespace and successive line comments before producing a token.
+        self.skip_ignorable();
         let t = match self.ch {
             '=' => {
                 if self.peek_char() == '=' {
@@ -115,6 +115,18 @@ impl<'a> Lexer<'a> {
         }
     }
 
+    fn skip_ignorable(&mut self) {
+        loop {
+            self.skip_whitespace();
+            if self.ch == '/' && self.peek_char() == '/' {
+                self.skip_comments();
+                // Continue the loop, in case there are more comments or whitespace
+                continue;
+            }
+            break;
+        }
+    }
+
     fn skip_comments(&mut self) {
         if self.ch == '/' && self.peek_char() == '/' {
             self.read_char();

lexer/snapshots/lexer__lexer_test__tests__comments_then_blank_line.snap

@@ -0,0 +1,55 @@
+---
+source: lexer/lexer_test.rs
+expression: "// comment\n\nlet x = 5"
+---
+[
+  {
+    "kind": {
+      "type": "LET"
+    },
+    "span": {
+      "start": 12,
+      "end": 15
+    }
+  },
+  {
+    "kind": {
+      "type": "IDENTIFIER",
+      "value": {
+        "name": "x"
+      }
+    },
+    "span": {
+      "start": 16,
+      "end": 17
+    }
+  },
+  {
+    "kind": {
+      "type": "ASSIGN"
+    },
+    "span": {
+      "start": 18,
+      "end": 19
+    }
+  },
+  {
+    "kind": {
+      "type": "INT",
+      "value": 5
+    },
+    "span": {
+      "start": 20,
+      "end": 21
+    }
+  },
+  {
+    "kind": {
+      "type": "EOF"
+    },
+    "span": {
+      "start": 21,
+      "end": 22
+    }
+  }
+]