diff --git a/crates/lexer/src/lib.rs b/crates/lexer/src/lib.rs index 07cb2d4..8fba269 100644 --- a/crates/lexer/src/lib.rs +++ b/crates/lexer/src/lib.rs @@ -5,13 +5,21 @@ use miette::{Error, LabeledSpan, Result, miette}; +#[derive(Clone, Debug)] +pub struct Span { + pub start: usize, + pub end: usize, +} + /// `Token` is formed of a token type (`TokenType`) and a reference to a str in the input string #[derive(Clone, Debug)] -pub struct Token<'de> { +pub struct Token { /// The `TokenType` of `Token` pub token_type: TokenType, /// The text reference from the source code - pub origin: &'de str, // TODO: Replace this with Span to avoid the lifetime + pub span: Span, + /// The text reference from the source code + pub origin: String, /// The line number where the token was parsed from pub line: usize, } @@ -123,8 +131,8 @@ enum Started { CodeComment, } -impl<'de> Iterator for Lexer<'de> { - type Item = Result, Error>; +impl Iterator for Lexer<'_> { + type Item = Result; fn next(&mut self) -> Option { if self.at_eof { @@ -137,49 +145,112 @@ impl<'de> Iterator for Lexer<'de> { let mut c_str = &self.rest[at..at + c.len_utf8()]; let c_onwards = self.rest; self.rest = chars.as_str(); + let start_byte = self.byte; self.byte += c.len_utf8(); - let res = move |t_type: TokenType, line_num: usize| { - Some(Ok(Token { - token_type: t_type, - origin: c_str, - line: line_num, - })) - }; - let started = match c { - '(' => return res(TokenType::LeftParen, self.line_num), - ')' => return res(TokenType::RightParen, self.line_num), - '{' => return res(TokenType::LeftBrace, self.line_num), - '}' => return res(TokenType::RightBrace, self.line_num), - ',' => return res(TokenType::Comma, self.line_num), - '.' => return res(TokenType::Dot, self.line_num), - '-' => return res(TokenType::Minus, self.line_num), - '+' => return res(TokenType::Plus, self.line_num), - '*' => return res(TokenType::Star, self.line_num), - ';' => return res(TokenType::Semicolon, self.line_num), - '"' => Started::String, - '0'..='9' => Started::Number, - 'a'..='z' | 'A'..='Z' | '_' => Started::Identifier, - '<' => Started::IfNextEqual(TokenType::LessEqual, TokenType::Less), - '>' => Started::IfNextEqual(TokenType::GreaterEqual, TokenType::Greater), - '!' => Started::IfNextEqual(TokenType::BangEqual, TokenType::Bang), - '=' => Started::IfNextEqual(TokenType::EqualEqual, TokenType::Equal), - '/' => Started::CodeComment, - c if c.is_whitespace() => { - if c == '\n' { - self.line_num += 1; + '(' => { + return Some(Ok(Token { + token_type: TokenType::LeftParen, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + ')' => { + return Some(Ok(Token { + token_type: TokenType::RightParen, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + '{' => { + return Some(Ok(Token { + token_type: TokenType::LeftBrace, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + '}' => { + return Some(Ok(Token { + token_type: TokenType::RightBrace, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + ',' => { + return Some(Ok(Token { + token_type: TokenType::Comma, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + '.' => { + return Some(Ok(Token { + token_type: TokenType::Dot, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + '-' => { + return Some(Ok(Token { + token_type: TokenType::Minus, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + '+' => { + return Some(Ok(Token { + token_type: TokenType::Plus, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + '*' => { + return Some(Ok(Token { + token_type: TokenType::Star, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) } - continue; - }, - _ => { - return Some(Err( - miette! {labels = vec![LabeledSpan::at(self.byte-c.len_utf8()..self.byte, "this character")], - "[line {}] Error: Unexpected character: {c}", self.line_num} - .with_source_code(self.whole.to_string()), - )) + ';' => { + return Some(Ok(Token { + token_type: TokenType::Semicolon, + span: Span { start: start_byte, end: start_byte}, + origin: c_str.to_string(), + line: self.line_num, + })) + } + '"' => Started::String, + '0'..='9' => Started::Number, + 'a'..='z' | 'A'..='Z' | '_' => Started::Identifier, + '<' => Started::IfNextEqual(TokenType::LessEqual, TokenType::Less), + '>' => Started::IfNextEqual(TokenType::GreaterEqual, TokenType::Greater), + '!' => Started::IfNextEqual(TokenType::BangEqual, TokenType::Bang), + '=' => Started::IfNextEqual(TokenType::EqualEqual, TokenType::Equal), + '/' => Started::CodeComment, + c if c.is_whitespace() => { + if c == '\n' { + self.line_num += 1; + } + continue; + } + _ => { + return Some(Err(miette! { + labels = vec![LabeledSpan::at(self.byte-c.len_utf8()..self.byte, "this character")], + "[line {}] Error: Unexpected character: {c}", self.line_num + } + .with_source_code(self.whole.to_string()))) } - }; + }; match started { Started::CodeComment => { if self.rest.starts_with('/') { @@ -194,7 +265,11 @@ impl<'de> Iterator for Lexer<'de> { } else { return Some(Ok(Token { token_type: TokenType::Slash, - origin: c_str, + span: Span { + start: start_byte, + end: self.byte, + }, + origin: c_str.to_string(), line: self.line_num, })); } @@ -207,55 +282,52 @@ impl<'de> Iterator for Lexer<'de> { self.rest = &self.rest[s.len() + 1..]; return Some(Ok(Token { token_type: TokenType::String, - origin: c_str, + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), line: self.line_num, })); } // Scan to end, we cannot continue to scan for tokens when in an unterminated string self.byte = self.whole.len(); self.rest = &self.rest[self.rest.len()..self.rest.len()]; - return Some(Err( - miette! {labels = vec![LabeledSpan::at(self.byte -c.len_utf8()..self.byte, "this unterminated string")], - "[line {}] Error: Unterminated string.", self.line_num}.with_source_code(self.whole.to_string()) - )); + return Some(Err(miette! { + labels = vec![LabeledSpan::at(start_byte..self.byte, "this unterminated string")], + "[line {}] Error: Unterminated string.", self.line_num + } + .with_source_code(self.whole.to_string()))); } - Started::Number => { - loop { - let next_num = chars.next(); - if let Some((_, cn)) = next_num { - if cn.is_numeric() { - c_str = &c_onwards[at..c_str.len() + cn.len_utf8()]; - self.rest = chars.as_str(); - self.byte += cn.len_utf8(); - } else if cn == '.' { - if let Some((_, c_peek)) = chars.next() { - // 456. != 456.0 but unstead 456 DOT - if c_peek.is_numeric() { - c_str = &c_onwards[at..c_str.len() - + cn.len_utf8() - + c_peek.len_utf8()]; - self.rest = chars.as_str(); - self.byte += cn.len_utf8() + c_peek.len_utf8(); - } else { - let num = c_str.parse().expect( - "We have called is_numeric on each char in `c_str`", - ); - return Some(Ok(Token { - token_type: TokenType::Number(num), - origin: c_str, - line: self.line_num, - })); - } + Started::Number => loop { + let next_num = chars.next(); + if let Some((_, cn)) = next_num { + if cn.is_numeric() { + c_str = &c_onwards[at..c_str.len() + cn.len_utf8()]; + self.rest = chars.as_str(); + self.byte += cn.len_utf8(); + } else if cn == '.' { + if let Some((_, c_peek)) = chars.next() { + // 456. != 456.0 but unstead 456 DOT + if c_peek.is_numeric() { + c_str = &c_onwards + [at..c_str.len() + cn.len_utf8() + c_peek.len_utf8()]; + self.rest = chars.as_str(); + self.byte += cn.len_utf8() + c_peek.len_utf8(); + } else { + let num = c_str.parse().expect( + "We have called is_numeric on each char in `c_str`", + ); + return Some(Ok(Token { + token_type: TokenType::Number(num), + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), + line: self.line_num, + })); } - } else { - let num: f64 = c_str.parse().expect( - "We have called is_numeric on each in char in `c_str`", - ); - return Some(Ok(Token { - token_type: TokenType::Number(num), - origin: c_str, - line: self.line_num, - })); } } else { let num: f64 = c_str @@ -263,12 +335,29 @@ impl<'de> Iterator for Lexer<'de> { .expect("We have called is_numeric on each in char in `c_str`"); return Some(Ok(Token { token_type: TokenType::Number(num), - origin: c_str, + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), line: self.line_num, })); } + } else { + let num: f64 = c_str + .parse() + .expect("We have called is_numeric on each in char in `c_str`"); + return Some(Ok(Token { + token_type: TokenType::Number(num), + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), + line: self.line_num, + })); } - } + }, Started::Identifier => loop { let next_char = chars.next(); @@ -277,7 +366,11 @@ impl<'de> Iterator for Lexer<'de> { let token_type = Self::match_reserved_word(c_str); return Some(Ok(Token { token_type, - origin: c_str, + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), line: self.line_num, })); } @@ -290,32 +383,47 @@ impl<'de> Iterator for Lexer<'de> { let token_type = Self::match_reserved_word(c_str); return Some(Ok(Token { token_type, - origin: c_str, + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), line: self.line_num, })); } let token_type = Self::match_reserved_word(c_str); return Some(Ok(Token { token_type, - origin: c_str, + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), line: self.line_num, })); }, Started::IfNextEqual(then, else_t) => { if self.rest.starts_with('=') { - let c_str = &c_onwards[..2]; self.byte += 1; self.rest = &self.rest[1..]; let token = Token { token_type: then, - origin: c_str, + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), line: self.line_num, }; return Some(Ok(token)); } let token = Token { token_type: else_t, - origin: c_str, + span: Span { + start: start_byte, + end: self.byte - 1, + }, + origin: c_str.to_string(), line: self.line_num, }; return Some(Ok(token)); @@ -325,7 +433,11 @@ impl<'de> Iterator for Lexer<'de> { self.at_eof = true; return Some(Ok(Token { token_type: TokenType::Eof, - origin: "", + span: Span { + start: self.byte, + end: self.byte, + }, + origin: String::new(), line: self.line_num, })); } diff --git a/crates/lexer/tests/lexer_test.rs b/crates/lexer/tests/lexer_test.rs index d52adce..5b72494 100644 --- a/crates/lexer/tests/lexer_test.rs +++ b/crates/lexer/tests/lexer_test.rs @@ -1,5 +1,5 @@ use insta::assert_yaml_snapshot; -use lexer::{Lexer, Token}; +use lexer::{Lexer, Token, TokenType}; use miette::Error; use std::fmt::Write; @@ -9,7 +9,11 @@ fn test_identifiers() { abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_"; let output = Lexer::new(input) .filter_map(Result::ok) - .map(|x| format!("{x:?}")) + .filter(|t| t.token_type != TokenType::Eof) + .map(|x| { + let s = input[x.span.start..=x.span.end].to_string(); + format!("{x:?} - `{s}`") + }) .collect::>(); insta::with_settings!({ description => input, @@ -24,7 +28,11 @@ fn test_keywords() { let input = "and class else false for fun if nil or return super this true var while print"; let output = Lexer::new(input) .filter_map(Result::ok) - .map(|x| format!("{x:?}")) + .filter(|t| t.token_type != TokenType::Eof) + .map(|x| { + let s = input[x.span.start..=x.span.end].to_string(); + format!("{x:?} - `{s}`") + }) .collect::>(); insta::with_settings!({ description => input, @@ -42,9 +50,14 @@ fn test_numbers() { 123. 90 523."; + println!("len: {}", input.len()); let out = Lexer::new(input) .filter_map(Result::ok) - .map(|x| format!("{x:?}")) + .filter(|t| t.token_type != TokenType::Eof) + .map(|x| { + let s = input[x.span.start..=x.span.end].to_string(); + format!("{x:?} - `{s}`") + }) .collect::>(); insta::with_settings!({ description => input, @@ -59,7 +72,12 @@ fn test_punctuators() { let input = "(){};,+-*!===<=>=!=<>/.=!"; let out = Lexer::new(input) .filter_map(Result::ok) - .map(|x| format!("{x:?}")) + .filter(|t| t.token_type != TokenType::Eof) + .map(|x| { + dbg!(&x); + let s = input[x.span.start..=x.span.end].to_string(); + format!("{x:?} - `{s}`") + }) .collect::>(); insta::with_settings!({ description => input, @@ -73,9 +91,14 @@ fn test_punctuators() { fn test_strings() { let input = "\"\" \"string\""; - let out = Lexer::new(input) + let lexer = Lexer::new(input); + let out = lexer .filter_map(Result::ok) - .map(|x| format!("{x:?}")) + .filter(|t| t.token_type != TokenType::Eof) + .map(|x| { + let s = input[x.span.start..=x.span.end].to_string(); + format!("{x:?} - `{s}`") + }) .collect::>(); insta::with_settings!({ description => input, @@ -91,11 +114,14 @@ fn test_whitespace() { // - end//"; let out = Lexer::new(input) .filter_map(Result::ok) - .map(|x| format!("{x:?}")) + .filter(|t| t.token_type != TokenType::Eof) + .map(|x| { + let s = input[x.span.start..=x.span.end].to_string(); + format!("{x:?} - {s}") + }) .collect::>(); insta::with_settings!({ @@ -121,20 +147,32 @@ fn test_errors() { #[test] fn test_group_literal() { - let out = Lexer::new("((true))") + let input = "((true))"; + let out = Lexer::new(input) .filter_map(Result::ok) - .map(|x| format!("{x:?}")) - .collect::>() - .join("\n"); - assert_yaml_snapshot!(out); + .filter(|t| t.token_type != TokenType::Eof) + .map(|x| { + let s = input[x.span.start..=x.span.end].to_string(); + format!("{x:?} - `{s}`") + }) + .collect::>(); + + insta::with_settings!({ + description => input, + omit_expression => true + }, { + assert_yaml_snapshot!(out); + }); } #[test] fn test_empty_handling() { - let out: String = Lexer::new("") + let input = ""; + let out: String = Lexer::new(input) .filter_map(Result::ok) .fold(String::new(), |mut out, t| { - let _ = write!(out, "{t:?}"); + let s = input[t.span.start..t.span.end].to_string(); + let _ = write!(out, "{t:?} - `{s}`"); out }); assert_yaml_snapshot!(out); diff --git a/crates/lexer/tests/snapshots/lexer_test__empty_handling.snap b/crates/lexer/tests/snapshots/lexer_test__empty_handling.snap index cd72672..ac536a9 100644 --- a/crates/lexer/tests/snapshots/lexer_test__empty_handling.snap +++ b/crates/lexer/tests/snapshots/lexer_test__empty_handling.snap @@ -2,4 +2,4 @@ source: crates/lexer/tests/lexer_test.rs expression: out --- -"Token { token_type: Eof, origin: \"\", line: 1 }" +"Token { token_type: Eof, span: Span { start: 0, end: 0 }, origin: \"\", line: 1 } - ``" diff --git a/crates/lexer/tests/snapshots/lexer_test__group_literal.snap b/crates/lexer/tests/snapshots/lexer_test__group_literal.snap index 0528771..4353852 100644 --- a/crates/lexer/tests/snapshots/lexer_test__group_literal.snap +++ b/crates/lexer/tests/snapshots/lexer_test__group_literal.snap @@ -1,5 +1,9 @@ --- source: crates/lexer/tests/lexer_test.rs -expression: out +description: ((true)) --- -"Token { token_type: LeftParen, origin: \"(\", line: 1 }\nToken { token_type: LeftParen, origin: \"(\", line: 1 }\nToken { token_type: True, origin: \"true\", line: 1 }\nToken { token_type: RightParen, origin: \")\", line: 1 }\nToken { token_type: RightParen, origin: \")\", line: 1 }\nToken { token_type: Eof, origin: \"\", line: 1 }" +- "Token { token_type: LeftParen, span: Span { start: 0, end: 0 }, origin: \"(\", line: 1 } - `(`" +- "Token { token_type: LeftParen, span: Span { start: 1, end: 1 }, origin: \"(\", line: 1 } - `(`" +- "Token { token_type: True, span: Span { start: 2, end: 5 }, origin: \"true\", line: 1 } - `true`" +- "Token { token_type: RightParen, span: Span { start: 6, end: 6 }, origin: \")\", line: 1 } - `)`" +- "Token { token_type: RightParen, span: Span { start: 7, end: 7 }, origin: \")\", line: 1 } - `)`" diff --git a/crates/lexer/tests/snapshots/lexer_test__identifiers.snap b/crates/lexer/tests/snapshots/lexer_test__identifiers.snap index fa30f33..d98a1ee 100644 --- a/crates/lexer/tests/snapshots/lexer_test__identifiers.snap +++ b/crates/lexer/tests/snapshots/lexer_test__identifiers.snap @@ -2,12 +2,11 @@ source: crates/lexer/tests/lexer_test.rs description: "andy formless fo _ _123 _abc ab123\nabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_" --- -- "Token { token_type: Identifier, origin: \"andy\", line: 1 }" -- "Token { token_type: Identifier, origin: \"formless\", line: 1 }" -- "Token { token_type: Identifier, origin: \"fo\", line: 1 }" -- "Token { token_type: Identifier, origin: \"_\", line: 1 }" -- "Token { token_type: Identifier, origin: \"_123\", line: 1 }" -- "Token { token_type: Identifier, origin: \"_abc\", line: 1 }" -- "Token { token_type: Identifier, origin: \"ab123\", line: 1 }" -- "Token { token_type: Identifier, origin: \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\", line: 2 }" -- "Token { token_type: Eof, origin: \"\", line: 2 }" +- "Token { token_type: Identifier, span: Span { start: 0, end: 3 }, origin: \"andy\", line: 1 } - `andy`" +- "Token { token_type: Identifier, span: Span { start: 5, end: 12 }, origin: \"formless\", line: 1 } - `formless`" +- "Token { token_type: Identifier, span: Span { start: 14, end: 15 }, origin: \"fo\", line: 1 } - `fo`" +- "Token { token_type: Identifier, span: Span { start: 17, end: 17 }, origin: \"_\", line: 1 } - `_`" +- "Token { token_type: Identifier, span: Span { start: 19, end: 22 }, origin: \"_123\", line: 1 } - `_123`" +- "Token { token_type: Identifier, span: Span { start: 24, end: 27 }, origin: \"_abc\", line: 1 } - `_abc`" +- "Token { token_type: Identifier, span: Span { start: 29, end: 33 }, origin: \"ab123\", line: 1 } - `ab123`" +- "Token { token_type: Identifier, span: Span { start: 35, end: 97 }, origin: \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\", line: 2 } - `abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_`" diff --git a/crates/lexer/tests/snapshots/lexer_test__keywords.snap b/crates/lexer/tests/snapshots/lexer_test__keywords.snap index a91eb94..4920c9b 100644 --- a/crates/lexer/tests/snapshots/lexer_test__keywords.snap +++ b/crates/lexer/tests/snapshots/lexer_test__keywords.snap @@ -2,20 +2,19 @@ source: crates/lexer/tests/lexer_test.rs description: and class else false for fun if nil or return super this true var while print --- -- "Token { token_type: And, origin: \"and\", line: 1 }" -- "Token { token_type: Class, origin: \"class\", line: 1 }" -- "Token { token_type: Else, origin: \"else\", line: 1 }" -- "Token { token_type: False, origin: \"false\", line: 1 }" -- "Token { token_type: For, origin: \"for\", line: 1 }" -- "Token { token_type: Fun, origin: \"fun\", line: 1 }" -- "Token { token_type: If, origin: \"if\", line: 1 }" -- "Token { token_type: Nil, origin: \"nil\", line: 1 }" -- "Token { token_type: Or, origin: \"or\", line: 1 }" -- "Token { token_type: Return, origin: \"return\", line: 1 }" -- "Token { token_type: Super, origin: \"super\", line: 1 }" -- "Token { token_type: This, origin: \"this\", line: 1 }" -- "Token { token_type: True, origin: \"true\", line: 1 }" -- "Token { token_type: Var, origin: \"var\", line: 1 }" -- "Token { token_type: While, origin: \"while\", line: 1 }" -- "Token { token_type: Print, origin: \"print\", line: 1 }" -- "Token { token_type: Eof, origin: \"\", line: 1 }" +- "Token { token_type: And, span: Span { start: 0, end: 2 }, origin: \"and\", line: 1 } - `and`" +- "Token { token_type: Class, span: Span { start: 4, end: 8 }, origin: \"class\", line: 1 } - `class`" +- "Token { token_type: Else, span: Span { start: 10, end: 13 }, origin: \"else\", line: 1 } - `else`" +- "Token { token_type: False, span: Span { start: 15, end: 19 }, origin: \"false\", line: 1 } - `false`" +- "Token { token_type: For, span: Span { start: 21, end: 23 }, origin: \"for\", line: 1 } - `for`" +- "Token { token_type: Fun, span: Span { start: 25, end: 27 }, origin: \"fun\", line: 1 } - `fun`" +- "Token { token_type: If, span: Span { start: 29, end: 30 }, origin: \"if\", line: 1 } - `if`" +- "Token { token_type: Nil, span: Span { start: 32, end: 34 }, origin: \"nil\", line: 1 } - `nil`" +- "Token { token_type: Or, span: Span { start: 36, end: 37 }, origin: \"or\", line: 1 } - `or`" +- "Token { token_type: Return, span: Span { start: 39, end: 44 }, origin: \"return\", line: 1 } - `return`" +- "Token { token_type: Super, span: Span { start: 46, end: 50 }, origin: \"super\", line: 1 } - `super`" +- "Token { token_type: This, span: Span { start: 52, end: 55 }, origin: \"this\", line: 1 } - `this`" +- "Token { token_type: True, span: Span { start: 57, end: 60 }, origin: \"true\", line: 1 } - `true`" +- "Token { token_type: Var, span: Span { start: 62, end: 64 }, origin: \"var\", line: 1 } - `var`" +- "Token { token_type: While, span: Span { start: 66, end: 70 }, origin: \"while\", line: 1 } - `while`" +- "Token { token_type: Print, span: Span { start: 72, end: 76 }, origin: \"print\", line: 1 } - `print`" diff --git a/crates/lexer/tests/snapshots/lexer_test__numbers.snap b/crates/lexer/tests/snapshots/lexer_test__numbers.snap index ce8c3ca..aeb0a24 100644 --- a/crates/lexer/tests/snapshots/lexer_test__numbers.snap +++ b/crates/lexer/tests/snapshots/lexer_test__numbers.snap @@ -2,13 +2,12 @@ source: crates/lexer/tests/lexer_test.rs description: "123\n123.456\n.457\n123.\n90\n523." --- -- "Token { token_type: Number(123.0), origin: \"123\", line: 1 }" -- "Token { token_type: Number(123.456), origin: \"123.456\", line: 2 }" -- "Token { token_type: Dot, origin: \".\", line: 3 }" -- "Token { token_type: Number(457.0), origin: \"457\", line: 3 }" -- "Token { token_type: Number(123.0), origin: \"123\", line: 4 }" -- "Token { token_type: Dot, origin: \".\", line: 4 }" -- "Token { token_type: Number(90.0), origin: \"90\", line: 5 }" -- "Token { token_type: Number(523.0), origin: \"523\", line: 6 }" -- "Token { token_type: Dot, origin: \".\", line: 6 }" -- "Token { token_type: Eof, origin: \"\", line: 6 }" +- "Token { token_type: Number(123.0), span: Span { start: 0, end: 2 }, origin: \"123\", line: 1 } - `123`" +- "Token { token_type: Number(123.456), span: Span { start: 4, end: 10 }, origin: \"123.456\", line: 2 } - `123.456`" +- "Token { token_type: Dot, span: Span { start: 12, end: 12 }, origin: \".\", line: 3 } - `.`" +- "Token { token_type: Number(457.0), span: Span { start: 13, end: 15 }, origin: \"457\", line: 3 } - `457`" +- "Token { token_type: Number(123.0), span: Span { start: 17, end: 19 }, origin: \"123\", line: 4 } - `123`" +- "Token { token_type: Dot, span: Span { start: 20, end: 20 }, origin: \".\", line: 4 } - `.`" +- "Token { token_type: Number(90.0), span: Span { start: 22, end: 23 }, origin: \"90\", line: 5 } - `90`" +- "Token { token_type: Number(523.0), span: Span { start: 25, end: 27 }, origin: \"523\", line: 6 } - `523`" +- "Token { token_type: Dot, span: Span { start: 28, end: 28 }, origin: \".\", line: 6 } - `.`" diff --git a/crates/lexer/tests/snapshots/lexer_test__punctuators.snap b/crates/lexer/tests/snapshots/lexer_test__punctuators.snap index a210670..f00e011 100644 --- a/crates/lexer/tests/snapshots/lexer_test__punctuators.snap +++ b/crates/lexer/tests/snapshots/lexer_test__punctuators.snap @@ -2,24 +2,23 @@ source: crates/lexer/tests/lexer_test.rs description: "(){};,+-*!===<=>=!=<>/.=!" --- -- "Token { token_type: LeftParen, origin: \"(\", line: 1 }" -- "Token { token_type: RightParen, origin: \")\", line: 1 }" -- "Token { token_type: LeftBrace, origin: \"{\", line: 1 }" -- "Token { token_type: RightBrace, origin: \"}\", line: 1 }" -- "Token { token_type: Semicolon, origin: \";\", line: 1 }" -- "Token { token_type: Comma, origin: \",\", line: 1 }" -- "Token { token_type: Plus, origin: \"+\", line: 1 }" -- "Token { token_type: Minus, origin: \"-\", line: 1 }" -- "Token { token_type: Star, origin: \"*\", line: 1 }" -- "Token { token_type: BangEqual, origin: \"!=\", line: 1 }" -- "Token { token_type: EqualEqual, origin: \"==\", line: 1 }" -- "Token { token_type: LessEqual, origin: \"<=\", line: 1 }" -- "Token { token_type: GreaterEqual, origin: \">=\", line: 1 }" -- "Token { token_type: BangEqual, origin: \"!=\", line: 1 }" -- "Token { token_type: Less, origin: \"<\", line: 1 }" -- "Token { token_type: Greater, origin: \">\", line: 1 }" -- "Token { token_type: Slash, origin: \"/\", line: 1 }" -- "Token { token_type: Dot, origin: \".\", line: 1 }" -- "Token { token_type: Equal, origin: \"=\", line: 1 }" -- "Token { token_type: Bang, origin: \"!\", line: 1 }" -- "Token { token_type: Eof, origin: \"\", line: 1 }" +- "Token { token_type: LeftParen, span: Span { start: 0, end: 0 }, origin: \"(\", line: 1 } - `(`" +- "Token { token_type: RightParen, span: Span { start: 1, end: 1 }, origin: \")\", line: 1 } - `)`" +- "Token { token_type: LeftBrace, span: Span { start: 2, end: 2 }, origin: \"{\", line: 1 } - `{`" +- "Token { token_type: RightBrace, span: Span { start: 3, end: 3 }, origin: \"}\", line: 1 } - `}`" +- "Token { token_type: Semicolon, span: Span { start: 4, end: 4 }, origin: \";\", line: 1 } - `;`" +- "Token { token_type: Comma, span: Span { start: 5, end: 5 }, origin: \",\", line: 1 } - `,`" +- "Token { token_type: Plus, span: Span { start: 6, end: 6 }, origin: \"+\", line: 1 } - `+`" +- "Token { token_type: Minus, span: Span { start: 7, end: 7 }, origin: \"-\", line: 1 } - `-`" +- "Token { token_type: Star, span: Span { start: 8, end: 8 }, origin: \"*\", line: 1 } - `*`" +- "Token { token_type: BangEqual, span: Span { start: 9, end: 10 }, origin: \"!\", line: 1 } - `!=`" +- "Token { token_type: EqualEqual, span: Span { start: 11, end: 12 }, origin: \"=\", line: 1 } - `==`" +- "Token { token_type: LessEqual, span: Span { start: 13, end: 14 }, origin: \"<\", line: 1 } - `<=`" +- "Token { token_type: GreaterEqual, span: Span { start: 15, end: 16 }, origin: \">\", line: 1 } - `>=`" +- "Token { token_type: BangEqual, span: Span { start: 17, end: 18 }, origin: \"!\", line: 1 } - `!=`" +- "Token { token_type: Less, span: Span { start: 19, end: 19 }, origin: \"<\", line: 1 } - `<`" +- "Token { token_type: Greater, span: Span { start: 20, end: 20 }, origin: \">\", line: 1 } - `>`" +- "Token { token_type: Slash, span: Span { start: 21, end: 22 }, origin: \"/\", line: 1 } - `/.`" +- "Token { token_type: Dot, span: Span { start: 22, end: 22 }, origin: \".\", line: 1 } - `.`" +- "Token { token_type: Equal, span: Span { start: 23, end: 23 }, origin: \"=\", line: 1 } - `=`" +- "Token { token_type: Bang, span: Span { start: 24, end: 24 }, origin: \"!\", line: 1 } - `!`" diff --git a/crates/lexer/tests/snapshots/lexer_test__strings.snap b/crates/lexer/tests/snapshots/lexer_test__strings.snap index de05826..752898e 100644 --- a/crates/lexer/tests/snapshots/lexer_test__strings.snap +++ b/crates/lexer/tests/snapshots/lexer_test__strings.snap @@ -2,6 +2,5 @@ source: crates/lexer/tests/lexer_test.rs description: "\"\"\n\"string\"" --- -- "Token { token_type: String, origin: \"\", line: 1 }" -- "Token { token_type: String, origin: \"string\", line: 2 }" -- "Token { token_type: Eof, origin: \"\", line: 2 }" +- "Token { token_type: String, span: Span { start: 0, end: 1 }, origin: \"\", line: 1 } - `\"\"`" +- "Token { token_type: String, span: Span { start: 3, end: 10 }, origin: \"string\", line: 2 } - `\"string\"`" diff --git a/crates/lexer/tests/snapshots/lexer_test__whitespace.snap b/crates/lexer/tests/snapshots/lexer_test__whitespace.snap index 7e5ac49..5818549 100644 --- a/crates/lexer/tests/snapshots/lexer_test__whitespace.snap +++ b/crates/lexer/tests/snapshots/lexer_test__whitespace.snap @@ -1,9 +1,8 @@ --- source: crates/lexer/tests/lexer_test.rs -description: "space tabs\t\t\t\tnewlines\n\n//\n\n\nend//" +description: "space tabs\t\t\t\tnewlines\n\n//\n\nend//" --- -- "Token { token_type: Identifier, origin: \"space\", line: 1 }" -- "Token { token_type: Identifier, origin: \"tabs\", line: 1 }" -- "Token { token_type: Identifier, origin: \"newlines\", line: 1 }" -- "Token { token_type: Identifier, origin: \"end\", line: 6 }" -- "Token { token_type: Eof, origin: \"\", line: 6 }" +- "Token { token_type: Identifier, span: Span { start: 0, end: 4 }, origin: \"space\", line: 1 } - space" +- "Token { token_type: Identifier, span: Span { start: 9, end: 12 }, origin: \"tabs\", line: 1 } - tabs" +- "Token { token_type: Identifier, span: Span { start: 17, end: 24 }, origin: \"newlines\", line: 1 } - newlines" +- "Token { token_type: Identifier, span: Span { start: 31, end: 33 }, origin: \"end\", line: 5 } - end" diff --git a/crates/loxide/src/interpreter.rs b/crates/loxide/src/interpreter.rs index eb19c40..3866175 100644 --- a/crates/loxide/src/interpreter.rs +++ b/crates/loxide/src/interpreter.rs @@ -41,16 +41,16 @@ impl NativeFunction { /// A user defined lox function #[derive(Debug, Clone)] -pub struct LoxFunction<'de> { +pub struct LoxFunction { /// The identifier - pub name: Token<'de>, + pub name: Token, /// The parameter values - pub parameters: Vec>, + pub parameters: Vec, /// The body of the function - pub body: Vec>, + pub body: Vec, } -impl<'de> LoxFunction<'de> { +impl LoxFunction { #[allow(dead_code)] fn arity(&self) -> u8 { u8::try_from(self.parameters.len()).expect("arity < 255 is enforced by parser") @@ -58,14 +58,14 @@ impl<'de> LoxFunction<'de> { #[allow(dead_code)] fn call( &self, - interpreter: &mut Interpreter<'de>, + interpreter: &mut Interpreter, args: &[EvaluatedValue], ) -> Result { let _args_env: HashMap<_, _> = self .parameters .iter() .zip(args.iter()) - .map(|(param, arg)| (param.origin, (Some(arg.clone()),))) + .map(|(param, arg)| (param.origin.clone(), (Some(arg.clone()),))) .collect(); let saved_env = interpreter.environment.clone(); let block = Stmt::Block(self.body.clone()); @@ -78,36 +78,36 @@ impl<'de> LoxFunction<'de> { /// `Interpreter` /// responsible for iterating over the rusults of parser /// and evaluating the statements and expressions encountered -pub struct Interpreter<'de> { - parser: Parser<'de>, - environment: Environment<'de>, - globals: Environment<'de>, - lox_functions: HashMap>, +pub struct Interpreter { + parser: Parser, + environment: Environment, + globals: Environment, + lox_functions: HashMap, counter: u64, } #[derive(Debug, Clone)] -struct Environment<'de> { - data: HashMap<&'de str, EvaluatedValue>, - enclosing: Option>>>, +struct Environment { + data: HashMap, + enclosing: Option>>, } -impl<'de> Environment<'de> { +impl Environment { fn new() -> Self { - Environment { + Self { data: HashMap::new(), enclosing: None, } } fn from_parent(enclosing: Self) -> Self { - Environment { + Self { data: HashMap::new(), enclosing: Some(Rc::new(RefCell::new(enclosing))), } } - fn get(&self, key: &'de str) -> Option { + fn get(&self, key: &str) -> Option { // get the given key from the environment // checks the current scope level, then works up through the enclosing env self.data.get(key).map_or_else( @@ -120,8 +120,8 @@ impl<'de> Environment<'de> { ) } - fn assign(&mut self, key: &'de str, value: &EvaluatedValue) -> Result<(), String> { - match self.data.get(key) { + fn assign(&mut self, key: String, value: &EvaluatedValue) -> Result<(), String> { + match self.data.get(&key) { Some(_) => { self.data.insert(key, value.clone()); Ok(()) @@ -138,22 +138,22 @@ impl<'de> Environment<'de> { } } } - fn var_assign(&mut self, key: &'de str, value: &EvaluatedValue) { + fn var_assign(&mut self, key: String, value: &EvaluatedValue) { // var = expr; // Only updates the current environment scope self.data.insert(key, value.clone()); } } -impl<'de> Interpreter<'de> { +impl Interpreter { /// Create a new `Interpreter` to process a given input source code /// # Panics /// Will panic if `SystemTime::now()` returns value before `UNIX_EPOCH` #[must_use] - pub fn new(input: &'de str) -> Self { + pub fn new(input: &str) -> Self { let mut global_data = HashMap::new(); global_data.insert( - "clock", + "clock".to_string(), EvaluatedValue::NativeFunction(NativeFunction { name: "clock".to_string(), arity: 0, @@ -185,7 +185,7 @@ impl<'de> Interpreter<'de> { } } -impl Iterator for Interpreter<'_> { +impl Iterator for Interpreter { type Item = Result<(), u8>; fn next(&mut self) -> Option { @@ -200,10 +200,7 @@ impl Iterator for Interpreter<'_> { } } -fn evaluate_statement<'de>( - stmt: &Stmt<'de>, - interpreter: &mut Interpreter<'de>, -) -> Result<(), String> { +fn evaluate_statement(stmt: &Stmt, interpreter: &mut Interpreter) -> Result<(), String> { match stmt { Stmt::Print(expr) => { let val = evaluate_expression(expr, interpreter)?; @@ -223,12 +220,14 @@ fn evaluate_statement<'de>( Stmt::Var(name, expr) => match expr { Some(v) => { let evalutated_val = evaluate_expression(v, interpreter)?; - interpreter.environment.var_assign(name, &evalutated_val); + interpreter + .environment + .var_assign(name.to_string(), &evalutated_val); } None => { interpreter .environment - .var_assign(name, &EvaluatedValue::Nil); + .var_assign(name.to_string(), &EvaluatedValue::Nil); } }, Stmt::Block(stmts) => { @@ -265,7 +264,7 @@ fn evaluate_statement<'de>( }; interpreter.lox_functions.insert(func_id, lox_fun); interpreter.environment.assign( - name.origin, + name.origin.clone(), &EvaluatedValue::LoxFunction { name: name.origin.to_string(), func_id, @@ -277,9 +276,9 @@ fn evaluate_statement<'de>( Ok(()) } -fn evaluate_expression<'de>( - expr: &Expr<'de>, - interpreter: &mut Interpreter<'de>, +fn evaluate_expression( + expr: &Expr, + interpreter: &mut Interpreter, ) -> Result { match expr { Expr::Binary { @@ -422,9 +421,9 @@ fn evaluate_expression<'de>( LiteralAtom::Bool(b) => Ok(EvaluatedValue::Bool(*b)), }, Expr::Grouping(expr) => evaluate_expression(expr, interpreter), - Expr::Variable(token) => match interpreter.environment.get(token.origin) { + Expr::Variable(token) => match interpreter.environment.get(&token.origin) { Some(v) => Ok(v), - None => interpreter.globals.get(token.origin).map_or_else( + None => interpreter.globals.get(&token.origin).map_or_else( || { eprintln!("Undefined variable `{}`.", token.origin); eprintln!("[line {}]", token.line); @@ -436,7 +435,7 @@ fn evaluate_expression<'de>( Expr::Assign(name, expr) => { if interpreter.environment.get(name).is_some() { let eval_expr = evaluate_expression(expr, interpreter)?; - interpreter.environment.assign(name, &eval_expr)?; + interpreter.environment.assign(name.clone(), &eval_expr)?; Ok(eval_expr) } else { eprintln!("Undefined variable '{name}'."); diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 1c95951..121071f 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -8,17 +8,17 @@ use lexer::{Lexer, Token, TokenType}; /// `Parser` is responsible for iterating over the token stream from `Lexer` /// and converting the lexed `Token` into `Expr` which represent an Abstract Syntax Tree (AST) -pub struct Parser<'de> { - tokens: Vec>, +pub struct Parser { + tokens: Vec, current: usize, parse_failed: bool, } #[derive(Debug, Clone, PartialEq)] /// `LiteralAtom` represents the types of literals supported by Lox -pub enum LiteralAtom<'de> { +pub enum LiteralAtom { /// `String` literal for example `"foo"` - String(&'de str), + String(String), /// Number literal for example `123.1` Number(f64), /// Nil literal @@ -29,82 +29,82 @@ pub enum LiteralAtom<'de> { #[derive(Debug, Clone)] /// `Expr` represents a unit of an AST -pub enum Expr<'de> { +pub enum Expr { /// `Binary` is a binary expression such as `1 * 2` Binary { /// The left item `Expr` in an expression - left: Box>, + left: Box, /// The operator to be applied on the `left` and `right` `Expr` - operator: Token<'de>, + operator: Token, /// The right item `Expr` in an expression. - right: Box>, + right: Box, }, /// `Unary` is a unary expression such as `!true` Unary { /// The operator to be applied on the `right` `Expr` - operator: Token<'de>, + operator: Token, /// The expression the unary operator will be applied to - right: Box>, + right: Box, }, /// `Literal` is a value - Literal(LiteralAtom<'de>), + Literal(LiteralAtom), /// `Grouping` holds other `Expr` such as `(1 * 2)` - Grouping(Box>), + Grouping(Box), /// `Variable` - Variable(Token<'de>), + Variable(Token), /// `Assign` - Assign(&'de str, Box>), + Assign(String, Box), /// `Logical` - `or` and `and` Logical { /// The left expression of a Logical expression - left: Box>, + left: Box, /// The operator of a Logical expression - operator: Token<'de>, + operator: Token, /// The right expression of a Logical expression - right: Box>, + right: Box, }, /// Function `Call` Call { /// function to be called - callee: Box>, + callee: Box, /// paren token - paren: Token<'de>, + paren: Token, /// arguments to be passed to function call - arguments: Vec>, + arguments: Vec, }, } #[derive(Debug, Clone)] /// `Stmt` represents the possible statements supported -pub enum Stmt<'de> { +pub enum Stmt { /// A print statement - Print(Expr<'de>), + Print(Expr), /// An expression statement - ExpressionStatement(Expr<'de>), + ExpressionStatement(Expr), /// Var statement // var = expr; - Var(&'de str, Option>), + Var(String, Option), /// Block - Block(Vec>), + Block(Vec), /// If statement - If(Expr<'de>, Box>, Option>>), + If(Expr, Box, Option>), /// While statement While { /// The condition that must be `true` for the body to be run - condition: Expr<'de>, + condition: Expr, /// The statements that will be executed repreatedly if `condition` - body: Box>, + body: Box, }, /// Func statement Function { - name: Token<'de>, - parameters: Vec>, - body: Vec>, + name: Token, + parameters: Vec, + body: Vec, }, } -impl<'de> Iterator for Parser<'de> { - type Item = Result, String>; +impl Iterator for Parser { + type Item = Result; fn next(&mut self) -> Option { if self.is_at_end() || self.parse_failed { @@ -123,19 +123,19 @@ impl<'de> Iterator for Parser<'de> { } } -impl<'de> Parser<'de> { +impl Parser { /// Create new `Parser` from a lexed token stream #[must_use] - pub fn new(input: &'de str) -> Self { - let tokens: Vec> = Lexer::new(input).flatten().collect(); - Parser { + pub fn new(input: &str) -> Self { + let tokens: Vec = Lexer::new(input).flatten().collect(); + Self { tokens, current: 0, parse_failed: false, } } - fn declaration(&mut self) -> Result, String> { + fn declaration(&mut self) -> Result { if self.match_tokens(&[TokenType::Fun]) { return self.function("function"); } @@ -145,7 +145,7 @@ impl<'de> Parser<'de> { self.statement() } - fn function(&mut self, kind: &str) -> Result, String> { + fn function(&mut self, kind: &str) -> Result { let name = self .consume(&TokenType::Identifier, &format!("Expect {kind} name."))? .clone(); @@ -179,10 +179,9 @@ impl<'de> Parser<'de> { }) } - fn var_declaration(&mut self) -> Result, String> { - let name = self - .consume(&TokenType::Identifier, "Expect variable name.")? - .origin; + fn var_declaration(&mut self) -> Result { + let token = self.consume(&TokenType::Identifier, "Expect variable name.")?; + let name = token.origin.clone(); let intializer = if self.match_tokens(&[TokenType::Equal]) { Some(self.expression()?) } else { @@ -192,7 +191,7 @@ impl<'de> Parser<'de> { Ok(Stmt::Var(name, intializer)) } - fn statement(&mut self) -> Result, String> { + fn statement(&mut self) -> Result { if self.match_tokens(&[TokenType::While]) { return self.while_statement(); } @@ -212,7 +211,7 @@ impl<'de> Parser<'de> { self.expression_statement() } - fn for_statement(&mut self) -> Result, String> { + fn for_statement(&mut self) -> Result { self.consume(&TokenType::LeftParen, "Expect '(' after 'for'")?; let mut initializer: Option = None; @@ -252,7 +251,7 @@ impl<'de> Parser<'de> { Ok(body) } - fn while_statement(&mut self) -> Result, String> { + fn while_statement(&mut self) -> Result { self.consume(&TokenType::LeftParen, "Expect '(' after 'while'")?; let condition = self.expression()?; self.consume(&TokenType::RightParen, "Expect ')' after condition")?; @@ -261,7 +260,7 @@ impl<'de> Parser<'de> { Ok(Stmt::While { condition, body }) } - fn if_statement(&mut self) -> Result, String> { + fn if_statement(&mut self) -> Result { self.consume(&TokenType::LeftParen, "Expect ')' after 'if'")?; let condition = self.expression()?; self.consume(&TokenType::RightParen, "Expect ')' after if condition.")?; @@ -276,8 +275,8 @@ impl<'de> Parser<'de> { Ok(if_stmt) } - fn block(&mut self) -> Result>, String> { - let mut stmts: Vec> = Vec::new(); + fn block(&mut self) -> Result, String> { + let mut stmts: Vec = Vec::new(); while !self.check(&TokenType::RightBrace) && !self.is_at_end() { stmts.push(self.declaration()?); } @@ -285,23 +284,23 @@ impl<'de> Parser<'de> { Ok(stmts) } - fn expression_statement(&mut self) -> Result, String> { + fn expression_statement(&mut self) -> Result { let expr = self.expression()?; self.consume(&TokenType::Semicolon, "Expect ';' after value")?; Ok(Stmt::ExpressionStatement(expr)) } - fn print_statement(&mut self) -> Result, String> { + fn print_statement(&mut self) -> Result { let expr = self.expression()?; self.consume(&TokenType::Semicolon, "Expect ';' after value")?; Ok(Stmt::Print(expr)) } - fn expression(&mut self) -> Result, String> { + fn expression(&mut self) -> Result { self.assignment() } - fn assignment(&mut self) -> Result, String> { + fn assignment(&mut self) -> Result { let expr = self.or()?; if self.match_tokens(&[TokenType::Equal]) { @@ -309,12 +308,11 @@ impl<'de> Parser<'de> { let value = self.assignment()?; if let Expr::Variable(token) = expr { - let name = token.origin; - return Ok(Expr::Assign(name, Box::new(value))); + return Ok(Expr::Assign(token.origin, Box::new(value))); } let err_msg = Self::error_msg( &equals.token_type, - equals.origin, + &equals.origin, equals.line, "Invalid assignment type.", ); @@ -324,7 +322,7 @@ impl<'de> Parser<'de> { Ok(expr) } - fn or(&mut self) -> Result, String> { + fn or(&mut self) -> Result { let mut expr = self.and()?; while self.match_tokens(&[TokenType::Or]) { @@ -340,7 +338,7 @@ impl<'de> Parser<'de> { Ok(expr) } - fn and(&mut self) -> Result, String> { + fn and(&mut self) -> Result { let mut expr = self.equality()?; while self.match_tokens(&[TokenType::And]) { let operator = self.previous().clone(); @@ -354,7 +352,7 @@ impl<'de> Parser<'de> { Ok(expr) } - fn equality(&mut self) -> Result, String> { + fn equality(&mut self) -> Result { let mut expr = self.comparison()?; while self.match_tokens(&[TokenType::BangEqual, TokenType::EqualEqual]) { @@ -370,7 +368,7 @@ impl<'de> Parser<'de> { Ok(expr) } - fn comparison(&mut self) -> Result, String> { + fn comparison(&mut self) -> Result { let mut expr = self.term()?; while self.match_tokens(&[ @@ -391,7 +389,7 @@ impl<'de> Parser<'de> { Ok(expr) } - fn term(&mut self) -> Result, String> { + fn term(&mut self) -> Result { let mut expr = self.factor()?; while self.match_tokens(&[TokenType::Minus, TokenType::Plus]) { @@ -407,7 +405,7 @@ impl<'de> Parser<'de> { Ok(expr) } - fn factor(&mut self) -> Result, String> { + fn factor(&mut self) -> Result { let mut expr = self.unary()?; while self.match_tokens(&[TokenType::Slash, TokenType::Star]) { @@ -423,7 +421,7 @@ impl<'de> Parser<'de> { Ok(expr) } - fn unary(&mut self) -> Result, String> { + fn unary(&mut self) -> Result { if self.match_tokens(&[TokenType::Bang, TokenType::Minus]) { let operator = self.previous().clone(); let right = self.unary()?; @@ -436,7 +434,7 @@ impl<'de> Parser<'de> { self.call() } - fn call(&mut self) -> Result, String> { + fn call(&mut self) -> Result { let mut expr = self.primary()?; loop { if self.match_tokens(&[TokenType::LeftParen]) { @@ -448,8 +446,8 @@ impl<'de> Parser<'de> { Ok(expr) } - fn finish_call(&mut self, callee: Expr<'de>) -> Result, String> { - let mut arguments: Vec> = vec![]; + fn finish_call(&mut self, callee: Expr) -> Result { + let mut arguments: Vec = vec![]; if !self.check(&TokenType::RightParen) { loop { @@ -473,11 +471,11 @@ impl<'de> Parser<'de> { }) } - fn primary(&mut self) -> Result, String> { + fn primary(&mut self) -> Result { let token = self.advance(); match token.token_type { TokenType::Number(n) => Ok(Expr::Literal(LiteralAtom::Number(n))), - TokenType::String => Ok(Expr::Literal(LiteralAtom::String(token.origin))), + TokenType::String => Ok(Expr::Literal(LiteralAtom::String(token.origin.clone()))), TokenType::True => Ok(Expr::Literal(LiteralAtom::Bool(true))), TokenType::False => Ok(Expr::Literal(LiteralAtom::Bool(false))), TokenType::Nil => Ok(Expr::Literal(LiteralAtom::Nil)), @@ -490,7 +488,7 @@ impl<'de> Parser<'de> { _ => { let err_msg = Self::error_msg( &token.token_type, - token.origin, + &token.origin, token.line, "Expect expression", ); @@ -516,20 +514,20 @@ impl<'de> Parser<'de> { std::mem::discriminant(&self.peek().token_type) == std::mem::discriminant(token_type) } - fn advance(&mut self) -> &Token<'de> { + fn advance(&mut self) -> &Token { if !self.is_at_end() { self.current += 1; } self.previous() } - fn peek(&self) -> &Token<'de> { + fn peek(&self) -> &Token { self.tokens .get(self.current) .expect("current should never be outside index") } - fn previous(&self) -> &Token<'de> { + fn previous(&self) -> &Token { self.tokens.get(self.current - 1).unwrap() } @@ -538,14 +536,14 @@ impl<'de> Parser<'de> { matches!(token.token_type, TokenType::Eof) } - fn consume(&mut self, token_type: &TokenType, message: &str) -> Result<&Token<'de>, String> { + fn consume(&mut self, token_type: &TokenType, message: &str) -> Result<&Token, String> { if self.check(token_type) { Ok(self.advance()) } else { let token = self.peek(); Err(Self::error_msg( token_type, - token.origin, + &token.origin, token.line, message, )) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@16 * 38 __ 58;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@16 * 38 __ 58;.snap index 53b79b3..1498b39 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@16 * 38 __ 58;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@16 * 38 __ 58;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: 16 * 38 / 58; --- -ExpressionStatement(Binary { left: Binary { left: Literal(Number(16.0)), operator: Token { token_type: Star, origin: "*", line: 1 }, right: Literal(Number(38.0)) }, operator: Token { token_type: Slash, origin: "/", line: 1 }, right: Literal(Number(58.0)) }) +ExpressionStatement(Binary { left: Binary { left: Literal(Number(16.0)), operator: Token { token_type: Star, span: Span { start: 3, end: 3 }, origin: "*", line: 1 }, right: Literal(Number(38.0)) }, operator: Token { token_type: Slash, span: Span { start: 8, end: 9 }, origin: "/", line: 1 }, right: Literal(Number(58.0)) }) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@52 + 80 - 94;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@52 + 80 - 94;.snap index 0dca37c..b5b1040 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@52 + 80 - 94;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_arimethic@52 + 80 - 94;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: 52 + 80 - 94; --- -ExpressionStatement(Binary { left: Binary { left: Literal(Number(52.0)), operator: Token { token_type: Plus, origin: "+", line: 1 }, right: Literal(Number(80.0)) }, operator: Token { token_type: Minus, origin: "-", line: 1 }, right: Literal(Number(94.0)) }) +ExpressionStatement(Binary { left: Binary { left: Literal(Number(52.0)), operator: Token { token_type: Plus, span: Span { start: 3, end: 3 }, origin: "+", line: 1 }, right: Literal(Number(80.0)) }, operator: Token { token_type: Minus, span: Span { start: 8, end: 8 }, origin: "-", line: 1 }, right: Literal(Number(94.0)) }) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@52 <= 80 >= 94;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@52 <= 80 >= 94;.snap index c7ab012..2dad664 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@52 <= 80 >= 94;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@52 <= 80 >= 94;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: 52 <= 80 >= 94; --- -ExpressionStatement(Binary { left: Binary { left: Literal(Number(52.0)), operator: Token { token_type: LessEqual, origin: "<=", line: 1 }, right: Literal(Number(80.0)) }, operator: Token { token_type: GreaterEqual, origin: ">=", line: 1 }, right: Literal(Number(94.0)) }) +ExpressionStatement(Binary { left: Binary { left: Literal(Number(52.0)), operator: Token { token_type: LessEqual, span: Span { start: 3, end: 4 }, origin: "<", line: 1 }, right: Literal(Number(80.0)) }, operator: Token { token_type: GreaterEqual, span: Span { start: 9, end: 10 }, origin: ">", line: 1 }, right: Literal(Number(94.0)) }) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@83 < 99 > 115;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@83 < 99 > 115;.snap index 1e8fd11..fb71990 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@83 < 99 > 115;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_comparison@83 < 99 > 115;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: 83 < 99 > 115; --- -ExpressionStatement(Binary { left: Binary { left: Literal(Number(83.0)), operator: Token { token_type: Less, origin: "<", line: 1 }, right: Literal(Number(99.0)) }, operator: Token { token_type: Greater, origin: ">", line: 1 }, right: Literal(Number(115.0)) }) +ExpressionStatement(Binary { left: Binary { left: Literal(Number(83.0)), operator: Token { token_type: Less, span: Span { start: 3, end: 3 }, origin: "<", line: 1 }, right: Literal(Number(99.0)) }, operator: Token { token_type: Greater, span: Span { start: 8, end: 8 }, origin: ">", line: 1 }, right: Literal(Number(115.0)) }) diff --git "a/crates/parser/tests/snapshots/eval_parser_test__parser_equality@\"baz\" == \"baz\";.snap" "b/crates/parser/tests/snapshots/eval_parser_test__parser_equality@\"baz\" == \"baz\";.snap" index eba40b5..f3909d7 100644 --- "a/crates/parser/tests/snapshots/eval_parser_test__parser_equality@\"baz\" == \"baz\";.snap" +++ "b/crates/parser/tests/snapshots/eval_parser_test__parser_equality@\"baz\" == \"baz\";.snap" @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: "\"baz\" == \"baz\";" --- -ExpressionStatement(Binary { left: Literal(String("baz")), operator: Token { token_type: EqualEqual, origin: "==", line: 1 }, right: Literal(String("baz")) }) +ExpressionStatement(Binary { left: Literal(String("baz")), operator: Token { token_type: EqualEqual, span: Span { start: 6, end: 7 }, origin: "=", line: 1 }, right: Literal(String("baz")) }) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_equality@10 != 9;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_equality@10 != 9;.snap index b27589c..384c479 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_equality@10 != 9;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_equality@10 != 9;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: 10 != 9; --- -ExpressionStatement(Binary { left: Literal(Number(10.0)), operator: Token { token_type: BangEqual, origin: "!=", line: 1 }, right: Literal(Number(9.0)) }) +ExpressionStatement(Binary { left: Literal(Number(10.0)), operator: Token { token_type: BangEqual, span: Span { start: 3, end: 4 }, origin: "!", line: 1 }, right: Literal(Number(9.0)) }) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!false;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!false;.snap index 22222bf..7abbf2a 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!false;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!false;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: "!false;" --- -ExpressionStatement(Unary { operator: Token { token_type: Bang, origin: "!", line: 1 }, right: Literal(Bool(false)) }) +ExpressionStatement(Unary { operator: Token { token_type: Bang, span: Span { start: 0, end: 0 }, origin: "!", line: 1 }, right: Literal(Bool(false)) }) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!true;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!true;.snap index 6cb7e32..f6669b6 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!true;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_unary@!true;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: "!true;" --- -ExpressionStatement(Unary { operator: Token { token_type: Bang, origin: "!", line: 1 }, right: Literal(Bool(true)) }) +ExpressionStatement(Unary { operator: Token { token_type: Bang, span: Span { start: 0, end: 0 }, origin: "!", line: 1 }, right: Literal(Bool(true)) }) diff --git a/crates/parser/tests/snapshots/eval_parser_test__parser_unary@-10;.snap b/crates/parser/tests/snapshots/eval_parser_test__parser_unary@-10;.snap index 5c09e1b..be7a3d2 100644 --- a/crates/parser/tests/snapshots/eval_parser_test__parser_unary@-10;.snap +++ b/crates/parser/tests/snapshots/eval_parser_test__parser_unary@-10;.snap @@ -2,4 +2,4 @@ source: crates/parser/tests/eval_parser_test.rs description: "-10;" --- -ExpressionStatement(Unary { operator: Token { token_type: Minus, origin: "-", line: 1 }, right: Literal(Number(10.0)) }) +ExpressionStatement(Unary { operator: Token { token_type: Minus, span: Span { start: 0, end: 0 }, origin: "-", line: 1 }, right: Literal(Number(10.0)) })