Remove 'de lifetime from Token, replace with Span

crates/lexer/tests/lexer_test.rs

@@ -1,5 +1,5 @@
 use insta::assert_yaml_snapshot;
-use lexer::{Lexer, Token};
+use lexer::{Lexer, Token, TokenType};
 use miette::Error;
 use std::fmt::Write;
 
@@ -9,7 +9,11 @@ fn test_identifiers() {
 abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_";
     let output = Lexer::new(input)
         .filter_map(Result::ok)
-        .map(|x| format!("{x:?}"))
+        .filter(|t| t.token_type != TokenType::Eof)
+        .map(|x| {
+            let s = input[x.span.start..=x.span.end].to_string();
+            format!("{x:?} - `{s}`")
+        })
         .collect::<Vec<String>>();
     insta::with_settings!({
         description => input,
@@ -24,7 +28,11 @@ fn test_keywords() {
     let input = "and class else false for fun if nil or return super this true var while print";
     let output = Lexer::new(input)
         .filter_map(Result::ok)
-        .map(|x| format!("{x:?}"))
+        .filter(|t| t.token_type != TokenType::Eof)
+        .map(|x| {
+            let s = input[x.span.start..=x.span.end].to_string();
+            format!("{x:?} - `{s}`")
+        })
         .collect::<Vec<String>>();
     insta::with_settings!({
         description => input,
@@ -42,9 +50,14 @@ fn test_numbers() {
 123.
 90
 523.";
+    println!("len: {}", input.len());
     let out = Lexer::new(input)
         .filter_map(Result::ok)
-        .map(|x| format!("{x:?}"))
+        .filter(|t| t.token_type != TokenType::Eof)
+        .map(|x| {
+            let s = input[x.span.start..=x.span.end].to_string();
+            format!("{x:?} - `{s}`")
+        })
         .collect::<Vec<String>>();
     insta::with_settings!({
         description => input,
@@ -59,7 +72,12 @@ fn test_punctuators() {
     let input = "(){};,+-*!===<=>=!=<>/.=!";
     let out = Lexer::new(input)
         .filter_map(Result::ok)
-        .map(|x| format!("{x:?}"))
+        .filter(|t| t.token_type != TokenType::Eof)
+        .map(|x| {
+            dbg!(&x);
+            let s = input[x.span.start..=x.span.end].to_string();
+            format!("{x:?} - `{s}`")
+        })
         .collect::<Vec<String>>();
     insta::with_settings!({
         description => input,
@@ -73,9 +91,14 @@ fn test_punctuators() {
 fn test_strings() {
     let input = "\"\"
 \"string\"";
-    let out = Lexer::new(input)
+    let lexer = Lexer::new(input);
+    let out = lexer
         .filter_map(Result::ok)
-        .map(|x| format!("{x:?}"))
+        .filter(|t| t.token_type != TokenType::Eof)
+        .map(|x| {
+            let s = input[x.span.start..=x.span.end].to_string();
+            format!("{x:?} - `{s}`")
+        })
         .collect::<Vec<String>>();
     insta::with_settings!({
         description => input,
@@ -91,11 +114,14 @@ fn test_whitespace() {
 
 //
 
-
 end//";
     let out = Lexer::new(input)
         .filter_map(Result::ok)
-        .map(|x| format!("{x:?}"))
+        .filter(|t| t.token_type != TokenType::Eof)
+        .map(|x| {
+            let s = input[x.span.start..=x.span.end].to_string();
+            format!("{x:?} - {s}")
+        })
         .collect::<Vec<String>>();
 
     insta::with_settings!({
@@ -121,20 +147,32 @@ fn test_errors() {
 
 #[test]
 fn test_group_literal() {
-    let out = Lexer::new("((true))")
+    let input = "((true))";
+    let out = Lexer::new(input)
         .filter_map(Result::ok)
-        .map(|x| format!("{x:?}"))
-        .collect::<Vec<String>>()
-        .join("\n");
-    assert_yaml_snapshot!(out);
+        .filter(|t| t.token_type != TokenType::Eof)
+        .map(|x| {
+            let s = input[x.span.start..=x.span.end].to_string();
+            format!("{x:?} - `{s}`")
+        })
+        .collect::<Vec<String>>();
+
+    insta::with_settings!({
+        description => input,
+        omit_expression => true
+    }, {
+        assert_yaml_snapshot!(out);
+    });
 }
 
 #[test]
 fn test_empty_handling() {
-    let out: String = Lexer::new("")
+    let input = "";
+    let out: String = Lexer::new(input)
         .filter_map(Result::ok)
         .fold(String::new(), |mut out, t| {
-            let _ = write!(out, "{t:?}");
+            let s = input[t.span.start..t.span.end].to_string();
+            let _ = write!(out, "{t:?} - `{s}`");
             out
         });
     assert_yaml_snapshot!(out);

crates/lexer/tests/snapshots/lexer_test__empty_handling.snap

@@ -2,4 +2,4 @@
 source: crates/lexer/tests/lexer_test.rs
 expression: out
 ---
-"Token { token_type: Eof, origin: \"\", line: 1 }"
+"Token { token_type: Eof, span: Span { start: 0, end: 0 }, origin: \"\", line: 1 } - ``"

crates/lexer/tests/snapshots/lexer_test__group_literal.snap

@@ -1,5 +1,9 @@
 ---
 source: crates/lexer/tests/lexer_test.rs
-expression: out
+description: ((true))
 ---
-"Token { token_type: LeftParen, origin: \"(\", line: 1 }\nToken { token_type: LeftParen, origin: \"(\", line: 1 }\nToken { token_type: True, origin: \"true\", line: 1 }\nToken { token_type: RightParen, origin: \")\", line: 1 }\nToken { token_type: RightParen, origin: \")\", line: 1 }\nToken { token_type: Eof, origin: \"\", line: 1 }"
+- "Token { token_type: LeftParen, span: Span { start: 0, end: 0 }, origin: \"(\", line: 1 } - `(`"
+- "Token { token_type: LeftParen, span: Span { start: 1, end: 1 }, origin: \"(\", line: 1 } - `(`"
+- "Token { token_type: True, span: Span { start: 2, end: 5 }, origin: \"true\", line: 1 } - `true`"
+- "Token { token_type: RightParen, span: Span { start: 6, end: 6 }, origin: \")\", line: 1 } - `)`"
+- "Token { token_type: RightParen, span: Span { start: 7, end: 7 }, origin: \")\", line: 1 } - `)`"

crates/lexer/tests/snapshots/lexer_test__identifiers.snap

@@ -2,12 +2,11 @@
 source: crates/lexer/tests/lexer_test.rs
 description: "andy formless fo _ _123 _abc ab123\nabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_"
 ---
-- "Token { token_type: Identifier, origin: \"andy\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"formless\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"fo\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"_\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"_123\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"_abc\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"ab123\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\", line: 2 }"
-- "Token { token_type: Eof, origin: \"\", line: 2 }"
+- "Token { token_type: Identifier, span: Span { start: 0, end: 3 }, origin: \"andy\", line: 1 } - `andy`"
+- "Token { token_type: Identifier, span: Span { start: 5, end: 12 }, origin: \"formless\", line: 1 } - `formless`"
+- "Token { token_type: Identifier, span: Span { start: 14, end: 15 }, origin: \"fo\", line: 1 } - `fo`"
+- "Token { token_type: Identifier, span: Span { start: 17, end: 17 }, origin: \"_\", line: 1 } - `_`"
+- "Token { token_type: Identifier, span: Span { start: 19, end: 22 }, origin: \"_123\", line: 1 } - `_123`"
+- "Token { token_type: Identifier, span: Span { start: 24, end: 27 }, origin: \"_abc\", line: 1 } - `_abc`"
+- "Token { token_type: Identifier, span: Span { start: 29, end: 33 }, origin: \"ab123\", line: 1 } - `ab123`"
+- "Token { token_type: Identifier, span: Span { start: 35, end: 97 }, origin: \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\", line: 2 } - `abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_`"

crates/lexer/tests/snapshots/lexer_test__keywords.snap

@@ -2,20 +2,19 @@
 source: crates/lexer/tests/lexer_test.rs
 description: and class else false for fun if nil or return super this true var while print
 ---
-- "Token { token_type: And, origin: \"and\", line: 1 }"
-- "Token { token_type: Class, origin: \"class\", line: 1 }"
-- "Token { token_type: Else, origin: \"else\", line: 1 }"
-- "Token { token_type: False, origin: \"false\", line: 1 }"
-- "Token { token_type: For, origin: \"for\", line: 1 }"
-- "Token { token_type: Fun, origin: \"fun\", line: 1 }"
-- "Token { token_type: If, origin: \"if\", line: 1 }"
-- "Token { token_type: Nil, origin: \"nil\", line: 1 }"
-- "Token { token_type: Or, origin: \"or\", line: 1 }"
-- "Token { token_type: Return, origin: \"return\", line: 1 }"
-- "Token { token_type: Super, origin: \"super\", line: 1 }"
-- "Token { token_type: This, origin: \"this\", line: 1 }"
-- "Token { token_type: True, origin: \"true\", line: 1 }"
-- "Token { token_type: Var, origin: \"var\", line: 1 }"
-- "Token { token_type: While, origin: \"while\", line: 1 }"
-- "Token { token_type: Print, origin: \"print\", line: 1 }"
-- "Token { token_type: Eof, origin: \"\", line: 1 }"
+- "Token { token_type: And, span: Span { start: 0, end: 2 }, origin: \"and\", line: 1 } - `and`"
+- "Token { token_type: Class, span: Span { start: 4, end: 8 }, origin: \"class\", line: 1 } - `class`"
+- "Token { token_type: Else, span: Span { start: 10, end: 13 }, origin: \"else\", line: 1 } - `else`"
+- "Token { token_type: False, span: Span { start: 15, end: 19 }, origin: \"false\", line: 1 } - `false`"
+- "Token { token_type: For, span: Span { start: 21, end: 23 }, origin: \"for\", line: 1 } - `for`"
+- "Token { token_type: Fun, span: Span { start: 25, end: 27 }, origin: \"fun\", line: 1 } - `fun`"
+- "Token { token_type: If, span: Span { start: 29, end: 30 }, origin: \"if\", line: 1 } - `if`"
+- "Token { token_type: Nil, span: Span { start: 32, end: 34 }, origin: \"nil\", line: 1 } - `nil`"
+- "Token { token_type: Or, span: Span { start: 36, end: 37 }, origin: \"or\", line: 1 } - `or`"
+- "Token { token_type: Return, span: Span { start: 39, end: 44 }, origin: \"return\", line: 1 } - `return`"
+- "Token { token_type: Super, span: Span { start: 46, end: 50 }, origin: \"super\", line: 1 } - `super`"
+- "Token { token_type: This, span: Span { start: 52, end: 55 }, origin: \"this\", line: 1 } - `this`"
+- "Token { token_type: True, span: Span { start: 57, end: 60 }, origin: \"true\", line: 1 } - `true`"
+- "Token { token_type: Var, span: Span { start: 62, end: 64 }, origin: \"var\", line: 1 } - `var`"
+- "Token { token_type: While, span: Span { start: 66, end: 70 }, origin: \"while\", line: 1 } - `while`"
+- "Token { token_type: Print, span: Span { start: 72, end: 76 }, origin: \"print\", line: 1 } - `print`"

crates/lexer/tests/snapshots/lexer_test__numbers.snap

@@ -2,13 +2,12 @@
 source: crates/lexer/tests/lexer_test.rs
 description: "123\n123.456\n.457\n123.\n90\n523."
 ---
-- "Token { token_type: Number(123.0), origin: \"123\", line: 1 }"
-- "Token { token_type: Number(123.456), origin: \"123.456\", line: 2 }"
-- "Token { token_type: Dot, origin: \".\", line: 3 }"
-- "Token { token_type: Number(457.0), origin: \"457\", line: 3 }"
-- "Token { token_type: Number(123.0), origin: \"123\", line: 4 }"
-- "Token { token_type: Dot, origin: \".\", line: 4 }"
-- "Token { token_type: Number(90.0), origin: \"90\", line: 5 }"
-- "Token { token_type: Number(523.0), origin: \"523\", line: 6 }"
-- "Token { token_type: Dot, origin: \".\", line: 6 }"
-- "Token { token_type: Eof, origin: \"\", line: 6 }"
+- "Token { token_type: Number(123.0), span: Span { start: 0, end: 2 }, origin: \"123\", line: 1 } - `123`"
+- "Token { token_type: Number(123.456), span: Span { start: 4, end: 10 }, origin: \"123.456\", line: 2 } - `123.456`"
+- "Token { token_type: Dot, span: Span { start: 12, end: 12 }, origin: \".\", line: 3 } - `.`"
+- "Token { token_type: Number(457.0), span: Span { start: 13, end: 15 }, origin: \"457\", line: 3 } - `457`"
+- "Token { token_type: Number(123.0), span: Span { start: 17, end: 19 }, origin: \"123\", line: 4 } - `123`"
+- "Token { token_type: Dot, span: Span { start: 20, end: 20 }, origin: \".\", line: 4 } - `.`"
+- "Token { token_type: Number(90.0), span: Span { start: 22, end: 23 }, origin: \"90\", line: 5 } - `90`"
+- "Token { token_type: Number(523.0), span: Span { start: 25, end: 27 }, origin: \"523\", line: 6 } - `523`"
+- "Token { token_type: Dot, span: Span { start: 28, end: 28 }, origin: \".\", line: 6 } - `.`"

crates/lexer/tests/snapshots/lexer_test__punctuators.snap

@@ -2,24 +2,23 @@
 source: crates/lexer/tests/lexer_test.rs
 description: "(){};,+-*!===<=>=!=<>/.=!"
 ---
-- "Token { token_type: LeftParen, origin: \"(\", line: 1 }"
-- "Token { token_type: RightParen, origin: \")\", line: 1 }"
-- "Token { token_type: LeftBrace, origin: \"{\", line: 1 }"
-- "Token { token_type: RightBrace, origin: \"}\", line: 1 }"
-- "Token { token_type: Semicolon, origin: \";\", line: 1 }"
-- "Token { token_type: Comma, origin: \",\", line: 1 }"
-- "Token { token_type: Plus, origin: \"+\", line: 1 }"
-- "Token { token_type: Minus, origin: \"-\", line: 1 }"
-- "Token { token_type: Star, origin: \"*\", line: 1 }"
-- "Token { token_type: BangEqual, origin: \"!=\", line: 1 }"
-- "Token { token_type: EqualEqual, origin: \"==\", line: 1 }"
-- "Token { token_type: LessEqual, origin: \"<=\", line: 1 }"
-- "Token { token_type: GreaterEqual, origin: \">=\", line: 1 }"
-- "Token { token_type: BangEqual, origin: \"!=\", line: 1 }"
-- "Token { token_type: Less, origin: \"<\", line: 1 }"
-- "Token { token_type: Greater, origin: \">\", line: 1 }"
-- "Token { token_type: Slash, origin: \"/\", line: 1 }"
-- "Token { token_type: Dot, origin: \".\", line: 1 }"
-- "Token { token_type: Equal, origin: \"=\", line: 1 }"
-- "Token { token_type: Bang, origin: \"!\", line: 1 }"
-- "Token { token_type: Eof, origin: \"\", line: 1 }"
+- "Token { token_type: LeftParen, span: Span { start: 0, end: 0 }, origin: \"(\", line: 1 } - `(`"
+- "Token { token_type: RightParen, span: Span { start: 1, end: 1 }, origin: \")\", line: 1 } - `)`"
+- "Token { token_type: LeftBrace, span: Span { start: 2, end: 2 }, origin: \"{\", line: 1 } - `{`"
+- "Token { token_type: RightBrace, span: Span { start: 3, end: 3 }, origin: \"}\", line: 1 } - `}`"
+- "Token { token_type: Semicolon, span: Span { start: 4, end: 4 }, origin: \";\", line: 1 } - `;`"
+- "Token { token_type: Comma, span: Span { start: 5, end: 5 }, origin: \",\", line: 1 } - `,`"
+- "Token { token_type: Plus, span: Span { start: 6, end: 6 }, origin: \"+\", line: 1 } - `+`"
+- "Token { token_type: Minus, span: Span { start: 7, end: 7 }, origin: \"-\", line: 1 } - `-`"
+- "Token { token_type: Star, span: Span { start: 8, end: 8 }, origin: \"*\", line: 1 } - `*`"
+- "Token { token_type: BangEqual, span: Span { start: 9, end: 10 }, origin: \"!\", line: 1 } - `!=`"
+- "Token { token_type: EqualEqual, span: Span { start: 11, end: 12 }, origin: \"=\", line: 1 } - `==`"
+- "Token { token_type: LessEqual, span: Span { start: 13, end: 14 }, origin: \"<\", line: 1 } - `<=`"
+- "Token { token_type: GreaterEqual, span: Span { start: 15, end: 16 }, origin: \">\", line: 1 } - `>=`"
+- "Token { token_type: BangEqual, span: Span { start: 17, end: 18 }, origin: \"!\", line: 1 } - `!=`"
+- "Token { token_type: Less, span: Span { start: 19, end: 19 }, origin: \"<\", line: 1 } - `<`"
+- "Token { token_type: Greater, span: Span { start: 20, end: 20 }, origin: \">\", line: 1 } - `>`"
+- "Token { token_type: Slash, span: Span { start: 21, end: 22 }, origin: \"/\", line: 1 } - `/.`"
+- "Token { token_type: Dot, span: Span { start: 22, end: 22 }, origin: \".\", line: 1 } - `.`"
+- "Token { token_type: Equal, span: Span { start: 23, end: 23 }, origin: \"=\", line: 1 } - `=`"
+- "Token { token_type: Bang, span: Span { start: 24, end: 24 }, origin: \"!\", line: 1 } - `!`"

crates/lexer/tests/snapshots/lexer_test__strings.snap

@@ -2,6 +2,5 @@
 source: crates/lexer/tests/lexer_test.rs
 description: "\"\"\n\"string\""
 ---
-- "Token { token_type: String, origin: \"\", line: 1 }"
-- "Token { token_type: String, origin: \"string\", line: 2 }"
-- "Token { token_type: Eof, origin: \"\", line: 2 }"
+- "Token { token_type: String, span: Span { start: 0, end: 1 }, origin: \"\", line: 1 } - `\"\"`"
+- "Token { token_type: String, span: Span { start: 3, end: 10 }, origin: \"string\", line: 2 } - `\"string\"`"

crates/lexer/tests/snapshots/lexer_test__whitespace.snap

@@ -1,9 +1,8 @@
 ---
 source: crates/lexer/tests/lexer_test.rs
-description: "space    tabs\t\t\t\tnewlines\n\n//\n\n\nend//"
+description: "space    tabs\t\t\t\tnewlines\n\n//\n\nend//"
 ---
-- "Token { token_type: Identifier, origin: \"space\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"tabs\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"newlines\", line: 1 }"
-- "Token { token_type: Identifier, origin: \"end\", line: 6 }"
-- "Token { token_type: Eof, origin: \"\", line: 6 }"
+- "Token { token_type: Identifier, span: Span { start: 0, end: 4 }, origin: \"space\", line: 1 } - space"
+- "Token { token_type: Identifier, span: Span { start: 9, end: 12 }, origin: \"tabs\", line: 1 } - tabs"
+- "Token { token_type: Identifier, span: Span { start: 17, end: 24 }, origin: \"newlines\", line: 1 } - newlines"
+- "Token { token_type: Identifier, span: Span { start: 31, end: 33 }, origin: \"end\", line: 5 } - end"