From fdd1f919847330c48fe60122ad5c9a06a16737c8 Mon Sep 17 00:00:00 2001 From: Luuk Verweij Date: Sun, 30 Jul 2023 19:01:09 +0200 Subject: [PATCH 1/5] [ci skip] wip: parse self --- examples/selfhosting/parser.aaa | 904 ++++++++++++++++++++++++ examples/selfhosting/test_parser.aaa | 638 +++++++++++++++++ examples/selfhosting/test_tokenizer.aaa | 14 +- examples/selfhosting/tokenizer.aaa | 36 +- 4 files changed, 1568 insertions(+), 24 deletions(-) create mode 100644 examples/selfhosting/parser.aaa create mode 100644 examples/selfhosting/test_parser.aaa diff --git a/examples/selfhosting/parser.aaa b/examples/selfhosting/parser.aaa new file mode 100644 index 00000000..b4d0eedf --- /dev/null +++ b/examples/selfhosting/parser.aaa @@ -0,0 +1,904 @@ +// TODO create models file +from "tokenizer" import + FileReadResult, + make_tokenizer, + OptionalToken, + Position, + print_tokens, + read_file, + Token, + Tokenizer, + TokenizeResult, + TokenType, + +struct BooleanLiteral { // TODO rename to Boolean + value as bool, + position as Position, +} + +struct IntegerLiteral { // TODO rename to Integer + value as int, + position as Position, +} + +struct StringLiteral { // TODO rename to String + value as str, + position as Position, +} + +struct Identifier { + value as str, + position as Position, +} + +struct Return { + position as Position, +} + +struct StructFieldQuery { + position as Position, + field_name as str, +} + +struct StructFieldUpdate { + position as Position, + field_name as str, + body as FunctionBody, +} + +struct Branch { + condition as FunctionBody, + if_body as FunctionBody, + else_body as OptionalFunctionBody, +} + +struct WhileLoop { + condition as FunctionBody, + body as FunctionBody, +} + +struct ForeachLoop { + position as Position, + body as FunctionBody, +} + +struct UseBlock { + position as Position, + // TODO +} + +struct MatchBlock { + position as Position, + // TODO +} + +struct Assignment { + position as Position, + // TODO +} + +struct Call { + position as Position, + // TODO +} + +struct FunctionBody { + position as Position, + items as vec[FunctionBodyItem], +} + +enum FunctionBodyItem { + string as StringLiteral, + struct_field_query as StructFieldQuery, + struct_field_update as StructFieldUpdate, + boolean as BooleanLiteral, + integer as IntegerLiteral, + assignment as Assignment, + call_ as Call, + branch as Branch, + while_loop as WhileLoop, + foreach_loop as ForeachLoop, + use_block as UseBlock, + match_block as MatchBlock, + return_ as Return, +} + +fn make_boolean_parse_result args token as Token, next_offset as int return BooleanParseResult { + BooleanLiteral + dup "value" { token "value" ? "true" = } ! + dup "position" { token "position" ? copy swap drop } ! + next_offset + BooleanParseResult:ok +} + +fn make_integer_parse_result args token as Token, next_offset as int return IntegerParseResult { + token "value" ? str:to_int + use value, ok { + if ok not { + IntegerParseResult:err return + } + + IntegerLiteral + dup "value" { value } ! + dup "position" { token "position" ? copy swap drop } ! + next_offset + IntegerParseResult:ok + } +} + +fn make_string_parse_result args token as Token, next_offset as int return StringParseResult { + token "value" ? + + use value { + value <- { value 1 value str:len 1 - str:substr drop } + value <- { value "\\\\" "\\" str:replace } + value <- { value "\\n" "\n" str:replace } + value <- { value "\\r" "\r" str:replace } + value <- { value "\\\"" "\"" str:replace } + + StringLiteral + dup "value" { value } ! + dup "position" { token "position" ? copy swap drop } ! + next_offset + StringParseResult:ok + } +} + +fn make_identifier_parse_result args token as Token, next_offset as int return IdentifierParseResult { + Identifier + dup "value" { token "value" ? } ! + dup "position" { token "position" ? copy swap drop } ! + next_offset + IdentifierParseResult:ok +} + +fn make_return_parse_result args token as Token, next_offset as int return ReturnParseResult { + Return + dup "position" { token "position" ? copy swap drop } ! + next_offset + ReturnParseResult:ok +} + +fn make_function_body args items as vec[FunctionBodyItem], next_offset as int return FunctionBody { + FunctionBody + dup "items" { items } ! +} + +fn make_branch args condition as FunctionBody, if_body as FunctionBody, else_body as OptionalFunctionBody return Branch { + Branch + dup "condition" { condition } ! + dup "if_body" { if_body } ! + dup "else_body" { else_body } ! +} + +fn make_while_loop args condition as FunctionBody, body as FunctionBody return WhileLoop { + WhileLoop + dup "condition" { condition } ! + dup "body" { body } ! +} + +fn make_foreach_loop args body as FunctionBody return ForeachLoop { + ForeachLoop + dup "body" { body } ! +} + +fn make_struct_field_query args field_name as str return StructFieldQuery { + StructFieldQuery + dup "field_name" { field_name } ! +} + +fn make_struct_field_update args field_name as str, body as FunctionBody return StructFieldUpdate { + StructFieldUpdate + dup "field_name" { field_name } ! + dup "body" { body } ! +} + +enum BooleanParseResult { + err, // TODO change to Position + ok as { BooleanLiteral, int }, +} + +enum IntegerParseResult { + err, + ok as { IntegerLiteral, int }, +} + +enum StringParseResult { + err, + ok as { StringLiteral, int }, +} + +enum IdentifierParseResult { + err, + ok as { Identifier, int }, +} + +enum ReturnParseResult { + err, + ok as { Return, int }, +} + +enum AssignmentParseResult { + err, + ok as { Assignment, int }, +} + +enum CallParseResult { + err, + ok as { Call, int }, +} + +enum BranchParseResult { + err, + ok as { Branch, int }, +} + +enum WhileLoopParseResult { + err, + ok as { WhileLoop, int }, +} + +enum ForeachLoopParseResult { + err, + ok as { ForeachLoop, int }, +} + +enum UseBlockParseResult { + err, + ok as { UseBlock, int }, +} + +enum MatchBlockParseResult { + err, + ok as { MatchBlock, int }, +} + +enum FunctionBodyItemParseResult { + err, + ok as { FunctionBodyItem, int }, +} + +enum FunctionBodyParseResult { + err, + ok as { FunctionBody, int }, +} + +enum StructFieldQueryResult { + err, + ok as { StructFieldQuery, int }, +} + +enum StructFieldUpdateResult { + err, + ok as { StructFieldUpdate, int }, +} + +enum OptionalFunctionBody { + none, + some as FunctionBody, +} + +struct Parser { + tokens as vec[Token], +} + +fn make_parser args tokens as vec[Token] return Parser { + Parser + dup "tokens" { tokens } ! +} + +fn Parser:run args parser as Parser { + "running parser!\n" . +} + +fn Parser:get_token args parser as Parser, offset as int return OptionalToken { + parser "tokens" ? + use tokens { + if tokens vec:len offset <= { + OptionalToken:none + } else { + tokens offset vec:get OptionalToken:some + } + } +} + +fn Parser:parse_boolean args parser as Parser, offset as int return BooleanParseResult { + parser offset Parser:get_token + match { + case OptionalToken:none { BooleanParseResult:err return } + case OptionalToken:some { nop } + } + + use token { + token "type_" ? + match { + case TokenType:false_ { nop } + case TokenType:true_ { nop } + default { BooleanParseResult:err return } + } + + token offset 1 + make_boolean_parse_result + } +} + +fn Parser:parse_integer args parser as Parser, offset as int return IntegerParseResult { + parser offset Parser:get_token + match { + case OptionalToken:none { IntegerParseResult:err return } + case OptionalToken:some { nop } + } + + use token { + token "type_" ? + match { + case TokenType:integer { token offset 1 + make_integer_parse_result } + default { IntegerParseResult:err } + } + } +} + +fn Parser:parse_string args parser as Parser, offset as int return StringParseResult { + parser offset Parser:get_token + match { + case OptionalToken:none { StringParseResult:err return } + case OptionalToken:some { nop } + } + + use token { + token "type_" ? + match { + case TokenType:string { token offset 1 + make_string_parse_result } + default { StringParseResult:err } + } + } +} + +fn Parser:parse_identifier args parser as Parser, offset as int return IdentifierParseResult { + parser offset Parser:get_token + match { + case OptionalToken:none { IdentifierParseResult:err return } + case OptionalToken:some { nop } + } + + use token { + token "type_" ? + match { + case TokenType:identifier { token offset 1 + make_identifier_parse_result } + default { IdentifierParseResult:err } + } + } +} + +fn Parser:parse_return args parser as Parser, offset as int, return ReturnParseResult { + parser offset Parser:get_token + match { + case OptionalToken:none { ReturnParseResult:err return } + case OptionalToken:some { nop } + } + + use token { + token "type_" ? + match { + case TokenType:return_ { token offset 1 + make_return_parse_result } + default { ReturnParseResult:err } + } + } +} + +fn Parser:parse_function_body_item args parser as Parser, offset as int return FunctionBodyItemParseResult { + parser offset Parser:get_token + match { + case OptionalToken:none { FunctionBodyItemParseResult:err return } + case OptionalToken:some { nop } + } + + use token { + token "type_" ? + match { + case TokenType:string { + parser offset 1 + Parser:get_token + match { + case OptionalToken:none { nop } + case OptionalToken:some as next_token { + next_token "type_" ? + match { + case TokenType:get_field { + parser offset Parser:parse_struct_field_query + match { + case StructFieldQueryResult:ok as query, next_offset { + query FunctionBodyItem:struct_field_query + next_offset FunctionBodyItemParseResult:ok + } + case StructFieldQueryResult:err { + FunctionBodyItemParseResult:err + } + } + return + } + case TokenType:start { + parser offset Parser:parse_struct_field_update + match { + case StructFieldUpdateResult:ok as update, next_offset { + update FunctionBodyItem:struct_field_update + next_offset FunctionBodyItemParseResult:ok + } + case StructFieldUpdateResult:err { + FunctionBodyItemParseResult:err + } + } + return + } + default { nop } + } + } + } + parser offset Parser:parse_string + match { + case StringParseResult:ok as string, next_offset { + string FunctionBodyItem:string + next_offset FunctionBodyItemParseResult:ok + } + case StringParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:true_ { + parser offset Parser:parse_boolean + match { + case BooleanParseResult:ok as boolean, next_offset { + boolean FunctionBodyItem:boolean + next_offset FunctionBodyItemParseResult:ok + } + case BooleanParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:false_ { + parser offset Parser:parse_boolean + match { + case BooleanParseResult:ok as boolean, next_offset { + boolean FunctionBodyItem:boolean + next_offset FunctionBodyItemParseResult:ok + } + case BooleanParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:integer { + parser offset Parser:parse_integer + match { + case IntegerParseResult:ok as integer, next_offset { + integer FunctionBodyItem:integer + next_offset FunctionBodyItemParseResult:ok + } + case IntegerParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:identifier { + parser offset 1 + Parser:get_token + match { + case OptionalToken:none { nop } + case OptionalToken:some as next_token { + next_token "type_" ? + match { + case TokenType:comma { + parser offset Parser:parse_assignment + match { + case AssignmentParseResult:ok as assignment, next_offset { + assignment FunctionBodyItem:assignment + next_offset FunctionBodyItemParseResult:ok + } + case AssignmentParseResult:err { + FunctionBodyItemParseResult:err + } + } + return + } + case TokenType:assign { + parser offset Parser:parse_assignment + match { + case AssignmentParseResult:ok as assignment, next_offset { + assignment FunctionBodyItem:assignment + next_offset FunctionBodyItemParseResult:ok + } + case AssignmentParseResult:err { + FunctionBodyItemParseResult:err + } + } + return + } + default { nop } + } + } + } + + parser offset Parser:parse_call + match { + case CallParseResult:ok as call_, next_offset { + call_ FunctionBodyItem:call_ + next_offset FunctionBodyItemParseResult:ok + } + case CallParseResult:err { + FunctionBodyItemParseResult:err + } + } + } + case TokenType:if_ { + parser offset Parser:parse_branch + match { + case BranchParseResult:ok as branch, next_offset { + branch FunctionBodyItem:branch + next_offset FunctionBodyItemParseResult:ok + } + case BranchParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:while_ { + parser offset Parser:parse_while_loop + match { + case WhileLoopParseResult:ok as loop, next_offset { + loop FunctionBodyItem:while_loop + next_offset FunctionBodyItemParseResult:ok + } + case WhileLoopParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:foreach_ { + parser offset Parser:parse_foreach_loop + match { + case ForeachLoopParseResult:ok as loop, next_offset { + loop FunctionBodyItem:foreach_loop + next_offset FunctionBodyItemParseResult:ok + } + case ForeachLoopParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:use_ { + parser offset Parser:parse_use_block + match { + case UseBlockParseResult:ok as use_block, next_offset { + use_block FunctionBodyItem:use_block + next_offset FunctionBodyItemParseResult:ok + } + case UseBlockParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:match_ { + parser offset Parser:parse_match_block + match { + case MatchBlockParseResult:ok as match_block, next_offset { + match_block FunctionBodyItem:match_block + next_offset FunctionBodyItemParseResult:ok + } + case MatchBlockParseResult:err { FunctionBodyItemParseResult:err } + } + } + case TokenType:return_ { + parser offset Parser:parse_return + match { + case ReturnParseResult:ok as return_, next_offset { + return_ FunctionBodyItem:return_ + next_offset FunctionBodyItemParseResult:ok + } + case ReturnParseResult:err { FunctionBodyItemParseResult:err } + } + } + default { FunctionBodyItemParseResult:err } + } + } +} + +fn Parser:parse_function_body args parser as Parser, offset as int return FunctionBodyParseResult { + vec[FunctionBodyItem] + use items { + while true { + parser offset Parser:parse_function_body_item + match { + case FunctionBodyItemParseResult:err { + if items vec:empty { + FunctionBodyParseResult:err + } else { + items offset make_function_body offset FunctionBodyParseResult:ok + } + return + } + case FunctionBodyItemParseResult:ok as item, item_offset { + offset <- { item_offset } + items item vec:push + } + } + } + } +} + +fn Parser:has_token_type args parser as Parser, offset as int, token_type as str return bool { + parser offset Parser:get_token + match { + case OptionalToken:none { false } + case OptionalToken:some as token { + token "type_" ? TokenType:to_str token_type = + } + } +} + +fn Parser:parse_branch args parser as Parser, offset as int return BranchParseResult { + if parser offset "IF" Parser:has_token_type not { + BranchParseResult:err return + } + + offset <- { offset 1 + } + + parser offset Parser:parse_function_body + match { + case FunctionBodyParseResult:ok { nop } + default { BranchParseResult:err return } + } + + use condition, condition_offset { + offset <- { condition_offset } + + if parser offset "BLOCK_START" Parser:has_token_type not { + BranchParseResult:err return + } + + offset <- { offset 1 + } + + parser offset Parser:parse_function_body + match { + case FunctionBodyParseResult:ok { nop } + default { BranchParseResult:err return } + } + + OptionalFunctionBody:none + use if_body, if_body_offset, else_body { + offset <- { if_body_offset } + + if parser offset "BLOCK_END" Parser:has_token_type not { + BranchParseResult:err return + } + + offset <- { offset 1 + } + + if parser offset "ELSE" Parser:has_token_type { + offset <- { offset 1 + } + + if parser offset "BLOCK_START" Parser:has_token_type not { + BranchParseResult:err return + } + + offset <- { offset 1 + } + + parser offset Parser:parse_function_body + + match { + case FunctionBodyParseResult:ok as body, body_offset { + else_body <- { body OptionalFunctionBody:some } + offset <- { body_offset } + } + default { BranchParseResult:err return } + } + + if parser offset "BLOCK_END" Parser:has_token_type not { + BranchParseResult:err return + } + + offset <- { offset 1 + } + } + + condition if_body else_body make_branch + offset + BranchParseResult:ok + } + } +} + +fn Parser:parse_while_loop args parser as Parser, offset as int return WhileLoopParseResult { + if parser offset "WHILE" Parser:has_token_type not { + WhileLoopParseResult:err return + } + + offset <- { offset 1 + } + + parser offset Parser:parse_function_body + match { + case FunctionBodyParseResult:ok { nop } + default { WhileLoopParseResult:err return } + } + + use condition, condition_offset { + offset <- { condition_offset } + + if parser offset "BLOCK_START" Parser:has_token_type not { + WhileLoopParseResult:err return + } + + offset <- { offset 1 + } + + parser offset Parser:parse_function_body + match { + case FunctionBodyParseResult:ok { nop } + default { WhileLoopParseResult:err return } + } + + use body, body_offset { + offset <- { body_offset } + + if parser offset "BLOCK_END" Parser:has_token_type not { + WhileLoopParseResult:err return + } + + offset <- { offset 1 + } + + condition body make_while_loop offset WhileLoopParseResult:ok + } + } +} + +fn Parser:parse_foreach_loop args parser as Parser, offset as int return ForeachLoopParseResult { + if parser offset "FOREACH" Parser:has_token_type not { + ForeachLoopParseResult:err return + } + + offset <- { offset 1 + } + + if parser offset "BLOCK_START" Parser:has_token_type not { + ForeachLoopParseResult:err return + } + + offset <- { offset 1 + } + + parser offset Parser:parse_function_body + match { + case FunctionBodyParseResult:ok { nop } + default { ForeachLoopParseResult:err return } + } + + use body, body_offset { + offset <- { body_offset } + + if parser offset "BLOCK_END" Parser:has_token_type not { + ForeachLoopParseResult:err return + } + + offset <- { offset 1 + } + + body make_foreach_loop offset ForeachLoopParseResult:ok + } +} + +fn Parser:parse_struct_field_query args parser as Parser, offset as int return StructFieldQueryResult { + parser offset Parser:parse_string + match { + case StringParseResult:err { StructFieldQueryResult:err return } + case StringParseResult:ok { nop } + } + + use field_name_literal, field_name_offset { + offset <- { field_name_offset } + + if parser offset "GET_FIELD" Parser:has_token_type not { + StructFieldQueryResult:err return + } + + offset <- { offset 1 + } + + field_name_literal "value" ? + use field_name { + field_name make_struct_field_query offset StructFieldQueryResult:ok + } + } +} + +fn Parser:parse_struct_field_update args parser as Parser, offset as int return StructFieldUpdateResult { + parser offset Parser:parse_string + match { + case StringParseResult:err { StructFieldUpdateResult:err return } + case StringParseResult:ok { nop } + } + + use field_name_literal, field_name_offset { + offset <- { field_name_offset } + + if parser offset "BLOCK_START" Parser:has_token_type not { + StructFieldUpdateResult:err return + } + + offset <- { offset 1 + } + + parser offset Parser:parse_function_body + match { + case FunctionBodyParseResult:err { StructFieldUpdateResult:err return } + case FunctionBodyParseResult:ok { nop } + } + + use body, body_offset { + offset <- { body_offset } + + if parser offset "BLOCK_END" Parser:has_token_type not { + StructFieldUpdateResult:err return + } + + offset <- { offset 1 + } + + if parser offset "SET_FIELD" Parser:has_token_type not { + StructFieldUpdateResult:err return + } + + offset <- { offset 1 + } + + field_name_literal "value" ? + use field_name { + field_name body make_struct_field_update offset StructFieldUpdateResult:ok + } + } + } +} + +fn Parser:parse_case_block args parser as Parser, offset as int { todo } +fn Parser:parse_default_block args parser as Parser, offset as int { todo } +fn Parser:parse_match_block args parser as Parser, offset as int return MatchBlockParseResult { todo } +fn Parser:parse_enum_variant args parser as Parser, offset as int { todo } +fn Parser:parse_enum_variants args parser as Parser, offset as int { todo } +fn Parser:parse_enum_definition args parser as Parser, offset as int { todo } +fn Parser:parse_variables args parser as Parser, offset as int { todo } +fn Parser:parse_use_block args parser as Parser, offset as int return UseBlockParseResult { todo } +fn Parser:parse_assignment args parser as Parser, offset as int return AssignmentParseResult { todo } +fn Parser:parse_call args parser as Parser, offset as int return CallParseResult { todo } +fn Parser:parse_function_name args parser as Parser, offset as int { todo } +fn Parser:parse_function_declaration args parser as Parser, offset as int { todo } +fn Parser:parse_function_definition args parser as Parser, offset as int { todo } + +fn Parser:parse_type_literal args parser as Parser, offset as int { todo } +fn Parser:parse_type_params args parser as Parser, offset as int { todo } +fn Parser:parse_argument args parser as Parser, offset as int { todo } +fn Parser:parse_arguments args parser as Parser, offset as int { todo } +fn Parser:parse_return_types args parser as Parser, offset as int { todo } + +fn Parser:parse_import_item args parser as Parser, offset as int { todo } +fn Parser:parse_import_items args parser as Parser, offset as int { todo } +fn Parser:parse_import_statement args parser as Parser, offset as int { todo } + +fn Parser:parse_struct_definition args parser as Parser, offset as int { todo } +fn Parser:parse_flat_type_params args parser as Parser, offset as int { todo } +fn Parser:parse_flat_type_literal args parser as Parser, offset as int { todo } +fn Parser:parse_type_declaration args parser as Parser, offset as int { todo } +fn Parser:parse_builtins_file_root args parser as Parser, offset as int { todo } +fn Parser:parse_regular_file_root args parser as Parser, offset as int { todo } + +fn main args argv as vec[str] return int { + // TODO move duplicate code out, it's similar to tokenizer main + + if argv vec:len 2 = not { + "Usage: " . + argv 0 vec:get . + " \n" . + 1 return + } + + argv 1 vec:get + + use source_path { + source_path read_file + match { + case FileReadResult:open_error { + "Could not open " . source_path . "\n" . + 1 return + } + case FileReadResult:read_error { + "Could not read " . source_path . "\n" . + 1 return + } + case FileReadResult:ok as content { + source_path content make_tokenizer Tokenizer:run + match { + case TokenizeResult:ok as tokens { + tokens make_parser Parser:run + } + case TokenizeResult:error { + "Tokenization failed.\n" . + 1 return + } + } + } + } + } + + 0 +} diff --git a/examples/selfhosting/test_parser.aaa b/examples/selfhosting/test_parser.aaa new file mode 100644 index 00000000..920b39f5 --- /dev/null +++ b/examples/selfhosting/test_parser.aaa @@ -0,0 +1,638 @@ +from "tokenizer" import + make_tokenizer, + Tokenizer, + TokenizeResult, + +from "parser" import + BooleanParseResult, + BranchParseResult, + ForeachLoopParseResult, + FunctionBodyItem, + FunctionBodyItemParseResult, + FunctionBodyParseResult, + IdentifierParseResult, + IntegerParseResult, + make_parser, + Parser, + ReturnParseResult, + StringParseResult, + StructFieldQueryResult, + StructFieldUpdateResult, + WhileLoopParseResult, + +fn code_to_parser args code as str return Parser { + "" code make_tokenizer Tokenizer:run + match { + case TokenizeResult:ok { nop } + default { unreachable } + } + + use tokens { + tokens make_parser + } +} + +fn test_parse_boolean_ok_true { + "true" code_to_parser 0 Parser:parse_boolean + match { + case BooleanParseResult:ok { + use boolean, next_offset { + boolean "value" ? assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_boolean_ok_false { + "false" code_to_parser 0 Parser:parse_boolean + match { + case BooleanParseResult:ok { + use boolean, next_offset { + boolean "value" ? not assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_boolean_unreachable_wrong_token_type { + "123" code_to_parser 0 Parser:parse_boolean + match { + case BooleanParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_boolean_unreachable_no_tokens { + "" code_to_parser 0 Parser:parse_boolean + match { + case BooleanParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_integer_ok { + "123" code_to_parser 0 Parser:parse_integer + match { + case IntegerParseResult:ok { + use integer, next_offset { + integer "value" ? 123 = assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_integer_ok_negative { + "-123" code_to_parser 0 Parser:parse_integer + match { + case IntegerParseResult:ok { + use integer, next_offset { + integer "value" ? -123 = assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_integer_unreachable_too_big { + "123456789012345678901234567890" code_to_parser 0 Parser:parse_integer + match { + case IntegerParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_integer_unreachable_wrong_token_type { + "true" code_to_parser 0 Parser:parse_integer + match { + case IntegerParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_integer_unreachable_no_tokens { + "" code_to_parser 0 Parser:parse_integer + match { + case IntegerParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_string_ok_empty { + "\"\"" code_to_parser 0 Parser:parse_string + match { + case StringParseResult:ok { + use string, next_offset { + string "value" ? "" = assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_string_ok_non_empty { + "\"hello world\"" code_to_parser 0 Parser:parse_string + match { + case StringParseResult:ok { + use string, next_offset { + string "value" ? "hello world" = assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_string_ok_escaped_chars { + "\"backslash \\\\ newline \\n carriage return \\r double quote \\\"\"" + code_to_parser 0 Parser:parse_string + match { + case StringParseResult:ok { + use string, next_offset { + string "value" ? + "backslash \\ newline \n carriage return \r double quote \"" + = assert + + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_string_unreachable_wrong_token_type { + "true" code_to_parser 0 Parser:parse_string + match { + case StringParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_string_unreachable_no_tokens { + "" code_to_parser 0 Parser:parse_string + match { + case StringParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_identifier_ok { + "something" code_to_parser 0 Parser:parse_identifier + match { + case IdentifierParseResult:ok { + use identifier, next_offset { + identifier "value" ? "something" = assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_identifier_unreachable_wrong_token_type { + "true" code_to_parser 0 Parser:parse_identifier + match { + case IdentifierParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_identifier_unreachable_no_tokens { + "" code_to_parser 0 Parser:parse_identifier + match { + case IdentifierParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_return_ok { + "return" code_to_parser 0 Parser:parse_return + match { + case ReturnParseResult:ok { + use return_, next_offset { + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_return_unreachable_wrong_token_type { + "true" code_to_parser 0 Parser:parse_return + match { + case ReturnParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_return_unreachable_no_tokens { + "" code_to_parser 0 Parser:parse_return + match { + case ReturnParseResult:err { nop } + default { unreachable } + } +} + +// TODO +// fn test_parse_function_body_item_ok_struct_field_query { +// "\"field\" ?" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:struct_field_query { +// "next_offset" ? 2 = assert +// } +// default { unreachable } +// } +// } + +// TODO +//fn test_parse_function_body_item_ok_struct_field_update { +// "\"field\" { 3 }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:struct_field_update { +// "next_offset" ? 4 = assert +// } +// default { unreachable } +// } +//} + +fn test_parse_function_body_item_ok_string { + "\"something\"" code_to_parser 0 Parser:parse_function_body_item + match { + case FunctionBodyItemParseResult:ok { + use item, next_offset { + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_function_body_item_ok_true { + "true" code_to_parser 0 Parser:parse_function_body_item + match { + case FunctionBodyItemParseResult:ok { + use item, next_offset { + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_function_body_item_ok_false { + "false" code_to_parser 0 Parser:parse_function_body_item + match { + case FunctionBodyItemParseResult:ok { + use item, next_offset { + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_function_body_item_ok_integer { + "69" code_to_parser 0 Parser:parse_function_body_item + match { + case FunctionBodyItemParseResult:ok { + use item, next_offset { + next_offset 1 = assert + } + } + default { unreachable } + } +} + +// TODO +// fn test_parse_function_body_item_ok_assign { +// "foo <- { 3 }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:assignment { +// "next_offset" ? 5 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_assign_with_comma { +// "foo, bar <- { 3 6 }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:assignment { +// "next_offset" ? 8 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_call { +// "foo" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:call { +// "next_offset" ? 1 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_branch { +// "if false { nop }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:branch { +// "next_offset" ? 5 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_while_loop { +// "while false { nop }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:while_loop { +// "next_offset" ? 5 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_foreach_loop { +// "foreach { drop }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:foreach_loop { +// "next_offset" ? 4 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_use_block { +// "use x { drop }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:use_block { +// "next_offset" ? 5 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_match_block { +// "match { default { nop } }" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:match_block { +// "next_offset" ? 7 = assert +// } +// default { unreachable } +// } +// } + +// TODO +// fn test_parse_function_body_item_ok_return { +// "return" code_to_parser 0 Parser:parse_function_body_item +// match { +// case FunctionBodyItemParseResult:return_ { +// "next_offset" ? 1 = assert +// } +// default { unreachable } +// } +// } + +fn test_parse_function_body_item_unreachable { + "123456789012345678901234567890" // too big integer + code_to_parser 0 Parser:parse_function_body_item + match { + case FunctionBodyItemParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_function_body_ok_one_item { + "true" code_to_parser 0 Parser:parse_function_body + match { + case FunctionBodyParseResult:ok { + use items, next_offset { + items "items" ? vec:len 1 = assert + next_offset 1 = assert + } + } + default { unreachable } + } +} + +fn test_parse_function_body_ok_two_items { + "true 69 \"foo\"" code_to_parser 0 Parser:parse_function_body + match { + case FunctionBodyParseResult:ok { + use items, next_offset { + items "items" ? vec:len 3 = assert + next_offset 3 = assert + } + } + default { unreachable } + } +} + +fn test_parse_function_body_unreachable_no_item { + "fn" code_to_parser 0 Parser:parse_function_body + match { + case FunctionBodyParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_function_body_unreachable_no_tokens { + "" code_to_parser 0 Parser:parse_function_body + match { + case FunctionBodyParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_branch_ok_without_else { + "if true { 69 }" code_to_parser 0 Parser:parse_branch + match { + case BranchParseResult:ok { + use branch, next_offset { + next_offset 5 = assert + } + } + default { unreachable } + } +} + +fn test_parse_branch_ok_with_else { + "if true { 69 } else { 420 }" code_to_parser 0 Parser:parse_branch + match { + case BranchParseResult:ok { + use branch, next_offset { + next_offset 9 = assert + } + } + default { unreachable } + } +} + +fn test_parse_branch_ok_with_else_and_more_items { + "if 1 2 3 { 4 5 6 true } else { 7 8 9 \"\" }" code_to_parser 0 Parser:parse_branch + match { + case BranchParseResult:ok { + use branch, next_offset { + next_offset 17 = assert + } + } + default { unreachable } + } +} + +fn check_parse_branch_unreachables args code as str { + code code_to_parser 0 Parser:parse_branch + match { + case BranchParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_branch_unreachable_without_else { + "if true { 69 fn" check_parse_branch_unreachables + "if true { fn" check_parse_branch_unreachables + "if true fn" check_parse_branch_unreachables + "if fn" check_parse_branch_unreachables + "fn" check_parse_branch_unreachables +} + +fn test_parse_branch_unreachable_with_else { + "if true { 69 } else { 420 fn" check_parse_branch_unreachables + "if true { 69 } else { fn" check_parse_branch_unreachables + "if true { 69 } else fn" check_parse_branch_unreachables +} + +fn test_parse_while_loop_ok { + "while true { 69 }" code_to_parser 0 Parser:parse_while_loop + match { + case WhileLoopParseResult:ok { + use loop, next_offset { + next_offset 5 = assert } + } + default { unreachable } + } +} + +fn test_parse_while_loop_ok_more_items { + "while true 1 2 3 \"\" 4 { 69 3 false }" code_to_parser 0 Parser:parse_while_loop + match { + case WhileLoopParseResult:ok { + use loop, next_offset { + next_offset 12 = assert + } + } + default { unreachable } + } +} + +fn check_parse_while_loop_unreachable args code as str { + code code_to_parser 0 Parser:parse_while_loop + match { + case WhileLoopParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_while_loop_unreachable { + "while true { 69 fn" check_parse_while_loop_unreachable + "while true { fn" check_parse_while_loop_unreachable + "while true fn" check_parse_while_loop_unreachable + "while fn" check_parse_while_loop_unreachable + "fn" check_parse_while_loop_unreachable +} + +fn test_parse_foreach_loop_ok { + "foreach { 69 }" code_to_parser 0 Parser:parse_foreach_loop + match { + case ForeachLoopParseResult:ok { + use loop, next_offset { + next_offset 4 = assert + } + } + default { unreachable } + } +} + +fn check_parse_foreach_loop_unreachable args code as str { + code code_to_parser 0 Parser:parse_foreach_loop + match { + case ForeachLoopParseResult:err { nop } + default { unreachable } + } +} + +fn test_parse_foreach_loop_unreachable { + "foreach { 69 fn" check_parse_foreach_loop_unreachable + "foreach { fn" check_parse_foreach_loop_unreachable + "foreach fn" check_parse_foreach_loop_unreachable + "fn" check_parse_foreach_loop_unreachable +} + +fn test_parse_struct_field_query_ok { + "\"foo\" ?" code_to_parser 0 Parser:parse_struct_field_query + match { + case StructFieldQueryResult:ok { + use loop, next_offset { + next_offset 2 = assert + } + } + default { unreachable } + } +} + +fn check_parse_struct_field_query_unreachable args code as str { + code code_to_parser 0 Parser:parse_struct_field_query + match { + case StructFieldQueryResult:err { nop } + default { unreachable } + } +} + +fn test_parse_struct_field_query_unreachable { + "\"foo\" fn" check_parse_struct_field_query_unreachable + "fn" check_parse_struct_field_query_unreachable +} + +fn test_parse_struct_field_update_ok { + "\"foo\" { 69 } !" code_to_parser 0 Parser:parse_struct_field_update + match { + case StructFieldUpdateResult:ok { + use loop, next_offset { + next_offset 5 = assert + } + } + default { unreachable } + } +} + +fn check_parse_struct_field_update_unreachable args code as str { + code code_to_parser 0 Parser:parse_struct_field_update + match { + case StructFieldUpdateResult:err { nop } + default { unreachable } + } +} + +fn test_parse_struct_field_update_unreachable { + "\"foo\" { 69 } fn" check_parse_struct_field_update_unreachable + "\"foo\" { 69 fn" check_parse_struct_field_update_unreachable + "\"foo\" { fn" check_parse_struct_field_update_unreachable + "\"foo\" fn" check_parse_struct_field_update_unreachable + "fn" check_parse_struct_field_update_unreachable +} diff --git a/examples/selfhosting/test_tokenizer.aaa b/examples/selfhosting/test_tokenizer.aaa index 20b0a9b0..dc765f25 100644 --- a/examples/selfhosting/test_tokenizer.aaa +++ b/examples/selfhosting/test_tokenizer.aaa @@ -174,16 +174,14 @@ fn check_tokenize_fixed_size args input as str, offset as int, expected_type as dup "offset" { offset } ! Tokenizer:tokenize_fixed_size match { - case OptionalToken:some { - use token_ { - token_ + case OptionalToken:some as token { + token - Token - dup "type_" { expected_type } ! - dup "value" { expected_value } ! + Token + dup "type_" { expected_type } ! + dup "value" { expected_value } ! - = assert - } + = assert } case OptionalToken:none { unreachable } } diff --git a/examples/selfhosting/tokenizer.aaa b/examples/selfhosting/tokenizer.aaa index 35443772..2df1088f 100644 --- a/examples/selfhosting/tokenizer.aaa +++ b/examples/selfhosting/tokenizer.aaa @@ -394,11 +394,9 @@ fn Tokenizer:tokenize_fixed_size args tokenizer as Tokenizer return OptionalToke use fixed_sized_token { tokenizer fixed_sized_token Tokenizer:match_fixed_size match { - case OptionalToken:some { - use optional_token { - drop - optional_token OptionalToken:some return - } + case OptionalToken:some as token { + drop + token OptionalToken:some return } default { nop @@ -536,18 +534,24 @@ fn Tokenizer:run args tokenizer as Tokenizer return TokenizeResult { case OptionalToken:none { TokenizeResult:error return } - case OptionalToken:some { - use token { - token "position" { tokenizer "offset_position" ? copy swap drop } ! - - tokens token vec:push - tokenizer "offset" { - tokenizer "offset" ? - token "value" ? str:len - + - } ! - tokenizer "offset_position" ? token Position:update + case OptionalToken:some as token { + token "position" { tokenizer "offset_position" ? copy swap drop } ! + + token "type_" ? + match { + case TokenType:whitespace { nop } + case TokenType:comment { nop } + default { + tokens token vec:push + } } + + tokenizer "offset" { + tokenizer "offset" ? + token "value" ? str:len + + + } ! + tokenizer "offset_position" ? token Position:update } } } From add0ba73e81503b87acb756c0f8843547ec88399 Mon Sep 17 00:00:00 2001 From: Luuk Verweij Date: Sun, 17 Dec 2023 20:19:00 +0100 Subject: [PATCH 2/5] wip --- aaa/parser/models.py | 16 +- examples/selfhosting/main.aaa | 6 + examples/selfhosting/parser.aaa | 904 ------------------------ examples/selfhosting/syntax_loader.aaa | 26 + examples/selfhosting/test_parser.aaa | 638 ----------------- examples/selfhosting/test_tokenizer.aaa | 387 ---------- examples/selfhosting/tokenizer.aaa | 632 ----------------- tests/selfhosting/test_tokenizer.py | 1 + 8 files changed, 44 insertions(+), 2566 deletions(-) create mode 100644 examples/selfhosting/main.aaa delete mode 100644 examples/selfhosting/parser.aaa create mode 100644 examples/selfhosting/syntax_loader.aaa delete mode 100644 examples/selfhosting/test_parser.aaa delete mode 100644 examples/selfhosting/test_tokenizer.aaa delete mode 100644 examples/selfhosting/tokenizer.aaa diff --git a/aaa/parser/models.py b/aaa/parser/models.py index cdd77195..089c2ac1 100644 --- a/aaa/parser/models.py +++ b/aaa/parser/models.py @@ -341,12 +341,18 @@ def __init__( def get_source_file(self) -> Path: source_path = Path(self.source.value) - if source_path.is_file() and self.source.value.endswith(".aaa"): - return source_path + if source_path.is_absolute(): + if source_path.is_file() and self.source.value.endswith(".aaa"): + return source_path + else: - return self.position.file.parent / ( - self.source.value.replace(".", os.sep) + ".aaa" - ) + source_path = (self.position.file.parent / source_path).resolve() + if source_path.is_file() and self.source.value.endswith(".aaa"): + return source_path + + return self.position.file.parent / ( + self.source.value.replace(".", os.sep) + ".aaa" + ) @classmethod def load(cls, children: list[AaaParseModel | Token]) -> Import: diff --git a/examples/selfhosting/main.aaa b/examples/selfhosting/main.aaa new file mode 100644 index 00000000..f9dc6fb4 --- /dev/null +++ b/examples/selfhosting/main.aaa @@ -0,0 +1,6 @@ +from "syntax_loader" import syntax_loader_new_from_str // TODO remove + +fn main { + nop + // TODO do something useful +} diff --git a/examples/selfhosting/parser.aaa b/examples/selfhosting/parser.aaa deleted file mode 100644 index b4d0eedf..00000000 --- a/examples/selfhosting/parser.aaa +++ /dev/null @@ -1,904 +0,0 @@ -// TODO create models file -from "tokenizer" import - FileReadResult, - make_tokenizer, - OptionalToken, - Position, - print_tokens, - read_file, - Token, - Tokenizer, - TokenizeResult, - TokenType, - -struct BooleanLiteral { // TODO rename to Boolean - value as bool, - position as Position, -} - -struct IntegerLiteral { // TODO rename to Integer - value as int, - position as Position, -} - -struct StringLiteral { // TODO rename to String - value as str, - position as Position, -} - -struct Identifier { - value as str, - position as Position, -} - -struct Return { - position as Position, -} - -struct StructFieldQuery { - position as Position, - field_name as str, -} - -struct StructFieldUpdate { - position as Position, - field_name as str, - body as FunctionBody, -} - -struct Branch { - condition as FunctionBody, - if_body as FunctionBody, - else_body as OptionalFunctionBody, -} - -struct WhileLoop { - condition as FunctionBody, - body as FunctionBody, -} - -struct ForeachLoop { - position as Position, - body as FunctionBody, -} - -struct UseBlock { - position as Position, - // TODO -} - -struct MatchBlock { - position as Position, - // TODO -} - -struct Assignment { - position as Position, - // TODO -} - -struct Call { - position as Position, - // TODO -} - -struct FunctionBody { - position as Position, - items as vec[FunctionBodyItem], -} - -enum FunctionBodyItem { - string as StringLiteral, - struct_field_query as StructFieldQuery, - struct_field_update as StructFieldUpdate, - boolean as BooleanLiteral, - integer as IntegerLiteral, - assignment as Assignment, - call_ as Call, - branch as Branch, - while_loop as WhileLoop, - foreach_loop as ForeachLoop, - use_block as UseBlock, - match_block as MatchBlock, - return_ as Return, -} - -fn make_boolean_parse_result args token as Token, next_offset as int return BooleanParseResult { - BooleanLiteral - dup "value" { token "value" ? "true" = } ! - dup "position" { token "position" ? copy swap drop } ! - next_offset - BooleanParseResult:ok -} - -fn make_integer_parse_result args token as Token, next_offset as int return IntegerParseResult { - token "value" ? str:to_int - use value, ok { - if ok not { - IntegerParseResult:err return - } - - IntegerLiteral - dup "value" { value } ! - dup "position" { token "position" ? copy swap drop } ! - next_offset - IntegerParseResult:ok - } -} - -fn make_string_parse_result args token as Token, next_offset as int return StringParseResult { - token "value" ? - - use value { - value <- { value 1 value str:len 1 - str:substr drop } - value <- { value "\\\\" "\\" str:replace } - value <- { value "\\n" "\n" str:replace } - value <- { value "\\r" "\r" str:replace } - value <- { value "\\\"" "\"" str:replace } - - StringLiteral - dup "value" { value } ! - dup "position" { token "position" ? copy swap drop } ! - next_offset - StringParseResult:ok - } -} - -fn make_identifier_parse_result args token as Token, next_offset as int return IdentifierParseResult { - Identifier - dup "value" { token "value" ? } ! - dup "position" { token "position" ? copy swap drop } ! - next_offset - IdentifierParseResult:ok -} - -fn make_return_parse_result args token as Token, next_offset as int return ReturnParseResult { - Return - dup "position" { token "position" ? copy swap drop } ! - next_offset - ReturnParseResult:ok -} - -fn make_function_body args items as vec[FunctionBodyItem], next_offset as int return FunctionBody { - FunctionBody - dup "items" { items } ! -} - -fn make_branch args condition as FunctionBody, if_body as FunctionBody, else_body as OptionalFunctionBody return Branch { - Branch - dup "condition" { condition } ! - dup "if_body" { if_body } ! - dup "else_body" { else_body } ! -} - -fn make_while_loop args condition as FunctionBody, body as FunctionBody return WhileLoop { - WhileLoop - dup "condition" { condition } ! - dup "body" { body } ! -} - -fn make_foreach_loop args body as FunctionBody return ForeachLoop { - ForeachLoop - dup "body" { body } ! -} - -fn make_struct_field_query args field_name as str return StructFieldQuery { - StructFieldQuery - dup "field_name" { field_name } ! -} - -fn make_struct_field_update args field_name as str, body as FunctionBody return StructFieldUpdate { - StructFieldUpdate - dup "field_name" { field_name } ! - dup "body" { body } ! -} - -enum BooleanParseResult { - err, // TODO change to Position - ok as { BooleanLiteral, int }, -} - -enum IntegerParseResult { - err, - ok as { IntegerLiteral, int }, -} - -enum StringParseResult { - err, - ok as { StringLiteral, int }, -} - -enum IdentifierParseResult { - err, - ok as { Identifier, int }, -} - -enum ReturnParseResult { - err, - ok as { Return, int }, -} - -enum AssignmentParseResult { - err, - ok as { Assignment, int }, -} - -enum CallParseResult { - err, - ok as { Call, int }, -} - -enum BranchParseResult { - err, - ok as { Branch, int }, -} - -enum WhileLoopParseResult { - err, - ok as { WhileLoop, int }, -} - -enum ForeachLoopParseResult { - err, - ok as { ForeachLoop, int }, -} - -enum UseBlockParseResult { - err, - ok as { UseBlock, int }, -} - -enum MatchBlockParseResult { - err, - ok as { MatchBlock, int }, -} - -enum FunctionBodyItemParseResult { - err, - ok as { FunctionBodyItem, int }, -} - -enum FunctionBodyParseResult { - err, - ok as { FunctionBody, int }, -} - -enum StructFieldQueryResult { - err, - ok as { StructFieldQuery, int }, -} - -enum StructFieldUpdateResult { - err, - ok as { StructFieldUpdate, int }, -} - -enum OptionalFunctionBody { - none, - some as FunctionBody, -} - -struct Parser { - tokens as vec[Token], -} - -fn make_parser args tokens as vec[Token] return Parser { - Parser - dup "tokens" { tokens } ! -} - -fn Parser:run args parser as Parser { - "running parser!\n" . -} - -fn Parser:get_token args parser as Parser, offset as int return OptionalToken { - parser "tokens" ? - use tokens { - if tokens vec:len offset <= { - OptionalToken:none - } else { - tokens offset vec:get OptionalToken:some - } - } -} - -fn Parser:parse_boolean args parser as Parser, offset as int return BooleanParseResult { - parser offset Parser:get_token - match { - case OptionalToken:none { BooleanParseResult:err return } - case OptionalToken:some { nop } - } - - use token { - token "type_" ? - match { - case TokenType:false_ { nop } - case TokenType:true_ { nop } - default { BooleanParseResult:err return } - } - - token offset 1 + make_boolean_parse_result - } -} - -fn Parser:parse_integer args parser as Parser, offset as int return IntegerParseResult { - parser offset Parser:get_token - match { - case OptionalToken:none { IntegerParseResult:err return } - case OptionalToken:some { nop } - } - - use token { - token "type_" ? - match { - case TokenType:integer { token offset 1 + make_integer_parse_result } - default { IntegerParseResult:err } - } - } -} - -fn Parser:parse_string args parser as Parser, offset as int return StringParseResult { - parser offset Parser:get_token - match { - case OptionalToken:none { StringParseResult:err return } - case OptionalToken:some { nop } - } - - use token { - token "type_" ? - match { - case TokenType:string { token offset 1 + make_string_parse_result } - default { StringParseResult:err } - } - } -} - -fn Parser:parse_identifier args parser as Parser, offset as int return IdentifierParseResult { - parser offset Parser:get_token - match { - case OptionalToken:none { IdentifierParseResult:err return } - case OptionalToken:some { nop } - } - - use token { - token "type_" ? - match { - case TokenType:identifier { token offset 1 + make_identifier_parse_result } - default { IdentifierParseResult:err } - } - } -} - -fn Parser:parse_return args parser as Parser, offset as int, return ReturnParseResult { - parser offset Parser:get_token - match { - case OptionalToken:none { ReturnParseResult:err return } - case OptionalToken:some { nop } - } - - use token { - token "type_" ? - match { - case TokenType:return_ { token offset 1 + make_return_parse_result } - default { ReturnParseResult:err } - } - } -} - -fn Parser:parse_function_body_item args parser as Parser, offset as int return FunctionBodyItemParseResult { - parser offset Parser:get_token - match { - case OptionalToken:none { FunctionBodyItemParseResult:err return } - case OptionalToken:some { nop } - } - - use token { - token "type_" ? - match { - case TokenType:string { - parser offset 1 + Parser:get_token - match { - case OptionalToken:none { nop } - case OptionalToken:some as next_token { - next_token "type_" ? - match { - case TokenType:get_field { - parser offset Parser:parse_struct_field_query - match { - case StructFieldQueryResult:ok as query, next_offset { - query FunctionBodyItem:struct_field_query - next_offset FunctionBodyItemParseResult:ok - } - case StructFieldQueryResult:err { - FunctionBodyItemParseResult:err - } - } - return - } - case TokenType:start { - parser offset Parser:parse_struct_field_update - match { - case StructFieldUpdateResult:ok as update, next_offset { - update FunctionBodyItem:struct_field_update - next_offset FunctionBodyItemParseResult:ok - } - case StructFieldUpdateResult:err { - FunctionBodyItemParseResult:err - } - } - return - } - default { nop } - } - } - } - parser offset Parser:parse_string - match { - case StringParseResult:ok as string, next_offset { - string FunctionBodyItem:string - next_offset FunctionBodyItemParseResult:ok - } - case StringParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:true_ { - parser offset Parser:parse_boolean - match { - case BooleanParseResult:ok as boolean, next_offset { - boolean FunctionBodyItem:boolean - next_offset FunctionBodyItemParseResult:ok - } - case BooleanParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:false_ { - parser offset Parser:parse_boolean - match { - case BooleanParseResult:ok as boolean, next_offset { - boolean FunctionBodyItem:boolean - next_offset FunctionBodyItemParseResult:ok - } - case BooleanParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:integer { - parser offset Parser:parse_integer - match { - case IntegerParseResult:ok as integer, next_offset { - integer FunctionBodyItem:integer - next_offset FunctionBodyItemParseResult:ok - } - case IntegerParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:identifier { - parser offset 1 + Parser:get_token - match { - case OptionalToken:none { nop } - case OptionalToken:some as next_token { - next_token "type_" ? - match { - case TokenType:comma { - parser offset Parser:parse_assignment - match { - case AssignmentParseResult:ok as assignment, next_offset { - assignment FunctionBodyItem:assignment - next_offset FunctionBodyItemParseResult:ok - } - case AssignmentParseResult:err { - FunctionBodyItemParseResult:err - } - } - return - } - case TokenType:assign { - parser offset Parser:parse_assignment - match { - case AssignmentParseResult:ok as assignment, next_offset { - assignment FunctionBodyItem:assignment - next_offset FunctionBodyItemParseResult:ok - } - case AssignmentParseResult:err { - FunctionBodyItemParseResult:err - } - } - return - } - default { nop } - } - } - } - - parser offset Parser:parse_call - match { - case CallParseResult:ok as call_, next_offset { - call_ FunctionBodyItem:call_ - next_offset FunctionBodyItemParseResult:ok - } - case CallParseResult:err { - FunctionBodyItemParseResult:err - } - } - } - case TokenType:if_ { - parser offset Parser:parse_branch - match { - case BranchParseResult:ok as branch, next_offset { - branch FunctionBodyItem:branch - next_offset FunctionBodyItemParseResult:ok - } - case BranchParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:while_ { - parser offset Parser:parse_while_loop - match { - case WhileLoopParseResult:ok as loop, next_offset { - loop FunctionBodyItem:while_loop - next_offset FunctionBodyItemParseResult:ok - } - case WhileLoopParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:foreach_ { - parser offset Parser:parse_foreach_loop - match { - case ForeachLoopParseResult:ok as loop, next_offset { - loop FunctionBodyItem:foreach_loop - next_offset FunctionBodyItemParseResult:ok - } - case ForeachLoopParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:use_ { - parser offset Parser:parse_use_block - match { - case UseBlockParseResult:ok as use_block, next_offset { - use_block FunctionBodyItem:use_block - next_offset FunctionBodyItemParseResult:ok - } - case UseBlockParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:match_ { - parser offset Parser:parse_match_block - match { - case MatchBlockParseResult:ok as match_block, next_offset { - match_block FunctionBodyItem:match_block - next_offset FunctionBodyItemParseResult:ok - } - case MatchBlockParseResult:err { FunctionBodyItemParseResult:err } - } - } - case TokenType:return_ { - parser offset Parser:parse_return - match { - case ReturnParseResult:ok as return_, next_offset { - return_ FunctionBodyItem:return_ - next_offset FunctionBodyItemParseResult:ok - } - case ReturnParseResult:err { FunctionBodyItemParseResult:err } - } - } - default { FunctionBodyItemParseResult:err } - } - } -} - -fn Parser:parse_function_body args parser as Parser, offset as int return FunctionBodyParseResult { - vec[FunctionBodyItem] - use items { - while true { - parser offset Parser:parse_function_body_item - match { - case FunctionBodyItemParseResult:err { - if items vec:empty { - FunctionBodyParseResult:err - } else { - items offset make_function_body offset FunctionBodyParseResult:ok - } - return - } - case FunctionBodyItemParseResult:ok as item, item_offset { - offset <- { item_offset } - items item vec:push - } - } - } - } -} - -fn Parser:has_token_type args parser as Parser, offset as int, token_type as str return bool { - parser offset Parser:get_token - match { - case OptionalToken:none { false } - case OptionalToken:some as token { - token "type_" ? TokenType:to_str token_type = - } - } -} - -fn Parser:parse_branch args parser as Parser, offset as int return BranchParseResult { - if parser offset "IF" Parser:has_token_type not { - BranchParseResult:err return - } - - offset <- { offset 1 + } - - parser offset Parser:parse_function_body - match { - case FunctionBodyParseResult:ok { nop } - default { BranchParseResult:err return } - } - - use condition, condition_offset { - offset <- { condition_offset } - - if parser offset "BLOCK_START" Parser:has_token_type not { - BranchParseResult:err return - } - - offset <- { offset 1 + } - - parser offset Parser:parse_function_body - match { - case FunctionBodyParseResult:ok { nop } - default { BranchParseResult:err return } - } - - OptionalFunctionBody:none - use if_body, if_body_offset, else_body { - offset <- { if_body_offset } - - if parser offset "BLOCK_END" Parser:has_token_type not { - BranchParseResult:err return - } - - offset <- { offset 1 + } - - if parser offset "ELSE" Parser:has_token_type { - offset <- { offset 1 + } - - if parser offset "BLOCK_START" Parser:has_token_type not { - BranchParseResult:err return - } - - offset <- { offset 1 + } - - parser offset Parser:parse_function_body - - match { - case FunctionBodyParseResult:ok as body, body_offset { - else_body <- { body OptionalFunctionBody:some } - offset <- { body_offset } - } - default { BranchParseResult:err return } - } - - if parser offset "BLOCK_END" Parser:has_token_type not { - BranchParseResult:err return - } - - offset <- { offset 1 + } - } - - condition if_body else_body make_branch - offset - BranchParseResult:ok - } - } -} - -fn Parser:parse_while_loop args parser as Parser, offset as int return WhileLoopParseResult { - if parser offset "WHILE" Parser:has_token_type not { - WhileLoopParseResult:err return - } - - offset <- { offset 1 + } - - parser offset Parser:parse_function_body - match { - case FunctionBodyParseResult:ok { nop } - default { WhileLoopParseResult:err return } - } - - use condition, condition_offset { - offset <- { condition_offset } - - if parser offset "BLOCK_START" Parser:has_token_type not { - WhileLoopParseResult:err return - } - - offset <- { offset 1 + } - - parser offset Parser:parse_function_body - match { - case FunctionBodyParseResult:ok { nop } - default { WhileLoopParseResult:err return } - } - - use body, body_offset { - offset <- { body_offset } - - if parser offset "BLOCK_END" Parser:has_token_type not { - WhileLoopParseResult:err return - } - - offset <- { offset 1 + } - - condition body make_while_loop offset WhileLoopParseResult:ok - } - } -} - -fn Parser:parse_foreach_loop args parser as Parser, offset as int return ForeachLoopParseResult { - if parser offset "FOREACH" Parser:has_token_type not { - ForeachLoopParseResult:err return - } - - offset <- { offset 1 + } - - if parser offset "BLOCK_START" Parser:has_token_type not { - ForeachLoopParseResult:err return - } - - offset <- { offset 1 + } - - parser offset Parser:parse_function_body - match { - case FunctionBodyParseResult:ok { nop } - default { ForeachLoopParseResult:err return } - } - - use body, body_offset { - offset <- { body_offset } - - if parser offset "BLOCK_END" Parser:has_token_type not { - ForeachLoopParseResult:err return - } - - offset <- { offset 1 + } - - body make_foreach_loop offset ForeachLoopParseResult:ok - } -} - -fn Parser:parse_struct_field_query args parser as Parser, offset as int return StructFieldQueryResult { - parser offset Parser:parse_string - match { - case StringParseResult:err { StructFieldQueryResult:err return } - case StringParseResult:ok { nop } - } - - use field_name_literal, field_name_offset { - offset <- { field_name_offset } - - if parser offset "GET_FIELD" Parser:has_token_type not { - StructFieldQueryResult:err return - } - - offset <- { offset 1 + } - - field_name_literal "value" ? - use field_name { - field_name make_struct_field_query offset StructFieldQueryResult:ok - } - } -} - -fn Parser:parse_struct_field_update args parser as Parser, offset as int return StructFieldUpdateResult { - parser offset Parser:parse_string - match { - case StringParseResult:err { StructFieldUpdateResult:err return } - case StringParseResult:ok { nop } - } - - use field_name_literal, field_name_offset { - offset <- { field_name_offset } - - if parser offset "BLOCK_START" Parser:has_token_type not { - StructFieldUpdateResult:err return - } - - offset <- { offset 1 + } - - parser offset Parser:parse_function_body - match { - case FunctionBodyParseResult:err { StructFieldUpdateResult:err return } - case FunctionBodyParseResult:ok { nop } - } - - use body, body_offset { - offset <- { body_offset } - - if parser offset "BLOCK_END" Parser:has_token_type not { - StructFieldUpdateResult:err return - } - - offset <- { offset 1 + } - - if parser offset "SET_FIELD" Parser:has_token_type not { - StructFieldUpdateResult:err return - } - - offset <- { offset 1 + } - - field_name_literal "value" ? - use field_name { - field_name body make_struct_field_update offset StructFieldUpdateResult:ok - } - } - } -} - -fn Parser:parse_case_block args parser as Parser, offset as int { todo } -fn Parser:parse_default_block args parser as Parser, offset as int { todo } -fn Parser:parse_match_block args parser as Parser, offset as int return MatchBlockParseResult { todo } -fn Parser:parse_enum_variant args parser as Parser, offset as int { todo } -fn Parser:parse_enum_variants args parser as Parser, offset as int { todo } -fn Parser:parse_enum_definition args parser as Parser, offset as int { todo } -fn Parser:parse_variables args parser as Parser, offset as int { todo } -fn Parser:parse_use_block args parser as Parser, offset as int return UseBlockParseResult { todo } -fn Parser:parse_assignment args parser as Parser, offset as int return AssignmentParseResult { todo } -fn Parser:parse_call args parser as Parser, offset as int return CallParseResult { todo } -fn Parser:parse_function_name args parser as Parser, offset as int { todo } -fn Parser:parse_function_declaration args parser as Parser, offset as int { todo } -fn Parser:parse_function_definition args parser as Parser, offset as int { todo } - -fn Parser:parse_type_literal args parser as Parser, offset as int { todo } -fn Parser:parse_type_params args parser as Parser, offset as int { todo } -fn Parser:parse_argument args parser as Parser, offset as int { todo } -fn Parser:parse_arguments args parser as Parser, offset as int { todo } -fn Parser:parse_return_types args parser as Parser, offset as int { todo } - -fn Parser:parse_import_item args parser as Parser, offset as int { todo } -fn Parser:parse_import_items args parser as Parser, offset as int { todo } -fn Parser:parse_import_statement args parser as Parser, offset as int { todo } - -fn Parser:parse_struct_definition args parser as Parser, offset as int { todo } -fn Parser:parse_flat_type_params args parser as Parser, offset as int { todo } -fn Parser:parse_flat_type_literal args parser as Parser, offset as int { todo } -fn Parser:parse_type_declaration args parser as Parser, offset as int { todo } -fn Parser:parse_builtins_file_root args parser as Parser, offset as int { todo } -fn Parser:parse_regular_file_root args parser as Parser, offset as int { todo } - -fn main args argv as vec[str] return int { - // TODO move duplicate code out, it's similar to tokenizer main - - if argv vec:len 2 = not { - "Usage: " . - argv 0 vec:get . - " \n" . - 1 return - } - - argv 1 vec:get - - use source_path { - source_path read_file - match { - case FileReadResult:open_error { - "Could not open " . source_path . "\n" . - 1 return - } - case FileReadResult:read_error { - "Could not read " . source_path . "\n" . - 1 return - } - case FileReadResult:ok as content { - source_path content make_tokenizer Tokenizer:run - match { - case TokenizeResult:ok as tokens { - tokens make_parser Parser:run - } - case TokenizeResult:error { - "Tokenization failed.\n" . - 1 return - } - } - } - } - } - - 0 -} diff --git a/examples/selfhosting/syntax_loader.aaa b/examples/selfhosting/syntax_loader.aaa new file mode 100644 index 00000000..1d720bfb --- /dev/null +++ b/examples/selfhosting/syntax_loader.aaa @@ -0,0 +1,26 @@ +from "../json/parser.aaa" import json_from_str, Json, JsonResult, JsonError + +enum SyntaxLoaderResult { + ok as SyntaxLoader, + error as str, +} + +struct SyntaxLoader {} + +fn syntax_loader_new_from_file args filename as str return SyntaxLoaderResult { + // TODO load file and call syntax_loader_new_from_str + todo +} + +fn syntax_loader_new_from_str args text as str return SyntaxLoaderResult { + text json_from_str + match { + case JsonResult:error as error { error JsonError:to_str SyntaxLoaderResult:error return } + case JsonResult:ok as json { json } + } + + todo + // TODO of data validation + + // TODO loading fields of SyntaxLoader +} diff --git a/examples/selfhosting/test_parser.aaa b/examples/selfhosting/test_parser.aaa deleted file mode 100644 index 920b39f5..00000000 --- a/examples/selfhosting/test_parser.aaa +++ /dev/null @@ -1,638 +0,0 @@ -from "tokenizer" import - make_tokenizer, - Tokenizer, - TokenizeResult, - -from "parser" import - BooleanParseResult, - BranchParseResult, - ForeachLoopParseResult, - FunctionBodyItem, - FunctionBodyItemParseResult, - FunctionBodyParseResult, - IdentifierParseResult, - IntegerParseResult, - make_parser, - Parser, - ReturnParseResult, - StringParseResult, - StructFieldQueryResult, - StructFieldUpdateResult, - WhileLoopParseResult, - -fn code_to_parser args code as str return Parser { - "" code make_tokenizer Tokenizer:run - match { - case TokenizeResult:ok { nop } - default { unreachable } - } - - use tokens { - tokens make_parser - } -} - -fn test_parse_boolean_ok_true { - "true" code_to_parser 0 Parser:parse_boolean - match { - case BooleanParseResult:ok { - use boolean, next_offset { - boolean "value" ? assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_boolean_ok_false { - "false" code_to_parser 0 Parser:parse_boolean - match { - case BooleanParseResult:ok { - use boolean, next_offset { - boolean "value" ? not assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_boolean_unreachable_wrong_token_type { - "123" code_to_parser 0 Parser:parse_boolean - match { - case BooleanParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_boolean_unreachable_no_tokens { - "" code_to_parser 0 Parser:parse_boolean - match { - case BooleanParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_integer_ok { - "123" code_to_parser 0 Parser:parse_integer - match { - case IntegerParseResult:ok { - use integer, next_offset { - integer "value" ? 123 = assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_integer_ok_negative { - "-123" code_to_parser 0 Parser:parse_integer - match { - case IntegerParseResult:ok { - use integer, next_offset { - integer "value" ? -123 = assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_integer_unreachable_too_big { - "123456789012345678901234567890" code_to_parser 0 Parser:parse_integer - match { - case IntegerParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_integer_unreachable_wrong_token_type { - "true" code_to_parser 0 Parser:parse_integer - match { - case IntegerParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_integer_unreachable_no_tokens { - "" code_to_parser 0 Parser:parse_integer - match { - case IntegerParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_string_ok_empty { - "\"\"" code_to_parser 0 Parser:parse_string - match { - case StringParseResult:ok { - use string, next_offset { - string "value" ? "" = assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_string_ok_non_empty { - "\"hello world\"" code_to_parser 0 Parser:parse_string - match { - case StringParseResult:ok { - use string, next_offset { - string "value" ? "hello world" = assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_string_ok_escaped_chars { - "\"backslash \\\\ newline \\n carriage return \\r double quote \\\"\"" - code_to_parser 0 Parser:parse_string - match { - case StringParseResult:ok { - use string, next_offset { - string "value" ? - "backslash \\ newline \n carriage return \r double quote \"" - = assert - - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_string_unreachable_wrong_token_type { - "true" code_to_parser 0 Parser:parse_string - match { - case StringParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_string_unreachable_no_tokens { - "" code_to_parser 0 Parser:parse_string - match { - case StringParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_identifier_ok { - "something" code_to_parser 0 Parser:parse_identifier - match { - case IdentifierParseResult:ok { - use identifier, next_offset { - identifier "value" ? "something" = assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_identifier_unreachable_wrong_token_type { - "true" code_to_parser 0 Parser:parse_identifier - match { - case IdentifierParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_identifier_unreachable_no_tokens { - "" code_to_parser 0 Parser:parse_identifier - match { - case IdentifierParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_return_ok { - "return" code_to_parser 0 Parser:parse_return - match { - case ReturnParseResult:ok { - use return_, next_offset { - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_return_unreachable_wrong_token_type { - "true" code_to_parser 0 Parser:parse_return - match { - case ReturnParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_return_unreachable_no_tokens { - "" code_to_parser 0 Parser:parse_return - match { - case ReturnParseResult:err { nop } - default { unreachable } - } -} - -// TODO -// fn test_parse_function_body_item_ok_struct_field_query { -// "\"field\" ?" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:struct_field_query { -// "next_offset" ? 2 = assert -// } -// default { unreachable } -// } -// } - -// TODO -//fn test_parse_function_body_item_ok_struct_field_update { -// "\"field\" { 3 }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:struct_field_update { -// "next_offset" ? 4 = assert -// } -// default { unreachable } -// } -//} - -fn test_parse_function_body_item_ok_string { - "\"something\"" code_to_parser 0 Parser:parse_function_body_item - match { - case FunctionBodyItemParseResult:ok { - use item, next_offset { - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_function_body_item_ok_true { - "true" code_to_parser 0 Parser:parse_function_body_item - match { - case FunctionBodyItemParseResult:ok { - use item, next_offset { - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_function_body_item_ok_false { - "false" code_to_parser 0 Parser:parse_function_body_item - match { - case FunctionBodyItemParseResult:ok { - use item, next_offset { - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_function_body_item_ok_integer { - "69" code_to_parser 0 Parser:parse_function_body_item - match { - case FunctionBodyItemParseResult:ok { - use item, next_offset { - next_offset 1 = assert - } - } - default { unreachable } - } -} - -// TODO -// fn test_parse_function_body_item_ok_assign { -// "foo <- { 3 }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:assignment { -// "next_offset" ? 5 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_assign_with_comma { -// "foo, bar <- { 3 6 }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:assignment { -// "next_offset" ? 8 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_call { -// "foo" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:call { -// "next_offset" ? 1 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_branch { -// "if false { nop }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:branch { -// "next_offset" ? 5 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_while_loop { -// "while false { nop }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:while_loop { -// "next_offset" ? 5 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_foreach_loop { -// "foreach { drop }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:foreach_loop { -// "next_offset" ? 4 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_use_block { -// "use x { drop }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:use_block { -// "next_offset" ? 5 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_match_block { -// "match { default { nop } }" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:match_block { -// "next_offset" ? 7 = assert -// } -// default { unreachable } -// } -// } - -// TODO -// fn test_parse_function_body_item_ok_return { -// "return" code_to_parser 0 Parser:parse_function_body_item -// match { -// case FunctionBodyItemParseResult:return_ { -// "next_offset" ? 1 = assert -// } -// default { unreachable } -// } -// } - -fn test_parse_function_body_item_unreachable { - "123456789012345678901234567890" // too big integer - code_to_parser 0 Parser:parse_function_body_item - match { - case FunctionBodyItemParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_function_body_ok_one_item { - "true" code_to_parser 0 Parser:parse_function_body - match { - case FunctionBodyParseResult:ok { - use items, next_offset { - items "items" ? vec:len 1 = assert - next_offset 1 = assert - } - } - default { unreachable } - } -} - -fn test_parse_function_body_ok_two_items { - "true 69 \"foo\"" code_to_parser 0 Parser:parse_function_body - match { - case FunctionBodyParseResult:ok { - use items, next_offset { - items "items" ? vec:len 3 = assert - next_offset 3 = assert - } - } - default { unreachable } - } -} - -fn test_parse_function_body_unreachable_no_item { - "fn" code_to_parser 0 Parser:parse_function_body - match { - case FunctionBodyParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_function_body_unreachable_no_tokens { - "" code_to_parser 0 Parser:parse_function_body - match { - case FunctionBodyParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_branch_ok_without_else { - "if true { 69 }" code_to_parser 0 Parser:parse_branch - match { - case BranchParseResult:ok { - use branch, next_offset { - next_offset 5 = assert - } - } - default { unreachable } - } -} - -fn test_parse_branch_ok_with_else { - "if true { 69 } else { 420 }" code_to_parser 0 Parser:parse_branch - match { - case BranchParseResult:ok { - use branch, next_offset { - next_offset 9 = assert - } - } - default { unreachable } - } -} - -fn test_parse_branch_ok_with_else_and_more_items { - "if 1 2 3 { 4 5 6 true } else { 7 8 9 \"\" }" code_to_parser 0 Parser:parse_branch - match { - case BranchParseResult:ok { - use branch, next_offset { - next_offset 17 = assert - } - } - default { unreachable } - } -} - -fn check_parse_branch_unreachables args code as str { - code code_to_parser 0 Parser:parse_branch - match { - case BranchParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_branch_unreachable_without_else { - "if true { 69 fn" check_parse_branch_unreachables - "if true { fn" check_parse_branch_unreachables - "if true fn" check_parse_branch_unreachables - "if fn" check_parse_branch_unreachables - "fn" check_parse_branch_unreachables -} - -fn test_parse_branch_unreachable_with_else { - "if true { 69 } else { 420 fn" check_parse_branch_unreachables - "if true { 69 } else { fn" check_parse_branch_unreachables - "if true { 69 } else fn" check_parse_branch_unreachables -} - -fn test_parse_while_loop_ok { - "while true { 69 }" code_to_parser 0 Parser:parse_while_loop - match { - case WhileLoopParseResult:ok { - use loop, next_offset { - next_offset 5 = assert } - } - default { unreachable } - } -} - -fn test_parse_while_loop_ok_more_items { - "while true 1 2 3 \"\" 4 { 69 3 false }" code_to_parser 0 Parser:parse_while_loop - match { - case WhileLoopParseResult:ok { - use loop, next_offset { - next_offset 12 = assert - } - } - default { unreachable } - } -} - -fn check_parse_while_loop_unreachable args code as str { - code code_to_parser 0 Parser:parse_while_loop - match { - case WhileLoopParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_while_loop_unreachable { - "while true { 69 fn" check_parse_while_loop_unreachable - "while true { fn" check_parse_while_loop_unreachable - "while true fn" check_parse_while_loop_unreachable - "while fn" check_parse_while_loop_unreachable - "fn" check_parse_while_loop_unreachable -} - -fn test_parse_foreach_loop_ok { - "foreach { 69 }" code_to_parser 0 Parser:parse_foreach_loop - match { - case ForeachLoopParseResult:ok { - use loop, next_offset { - next_offset 4 = assert - } - } - default { unreachable } - } -} - -fn check_parse_foreach_loop_unreachable args code as str { - code code_to_parser 0 Parser:parse_foreach_loop - match { - case ForeachLoopParseResult:err { nop } - default { unreachable } - } -} - -fn test_parse_foreach_loop_unreachable { - "foreach { 69 fn" check_parse_foreach_loop_unreachable - "foreach { fn" check_parse_foreach_loop_unreachable - "foreach fn" check_parse_foreach_loop_unreachable - "fn" check_parse_foreach_loop_unreachable -} - -fn test_parse_struct_field_query_ok { - "\"foo\" ?" code_to_parser 0 Parser:parse_struct_field_query - match { - case StructFieldQueryResult:ok { - use loop, next_offset { - next_offset 2 = assert - } - } - default { unreachable } - } -} - -fn check_parse_struct_field_query_unreachable args code as str { - code code_to_parser 0 Parser:parse_struct_field_query - match { - case StructFieldQueryResult:err { nop } - default { unreachable } - } -} - -fn test_parse_struct_field_query_unreachable { - "\"foo\" fn" check_parse_struct_field_query_unreachable - "fn" check_parse_struct_field_query_unreachable -} - -fn test_parse_struct_field_update_ok { - "\"foo\" { 69 } !" code_to_parser 0 Parser:parse_struct_field_update - match { - case StructFieldUpdateResult:ok { - use loop, next_offset { - next_offset 5 = assert - } - } - default { unreachable } - } -} - -fn check_parse_struct_field_update_unreachable args code as str { - code code_to_parser 0 Parser:parse_struct_field_update - match { - case StructFieldUpdateResult:err { nop } - default { unreachable } - } -} - -fn test_parse_struct_field_update_unreachable { - "\"foo\" { 69 } fn" check_parse_struct_field_update_unreachable - "\"foo\" { 69 fn" check_parse_struct_field_update_unreachable - "\"foo\" { fn" check_parse_struct_field_update_unreachable - "\"foo\" fn" check_parse_struct_field_update_unreachable - "fn" check_parse_struct_field_update_unreachable -} diff --git a/examples/selfhosting/test_tokenizer.aaa b/examples/selfhosting/test_tokenizer.aaa deleted file mode 100644 index dc765f25..00000000 --- a/examples/selfhosting/test_tokenizer.aaa +++ /dev/null @@ -1,387 +0,0 @@ -from "tokenizer" import - make_tokenizer, - OptionalToken, - Token, - Tokenizer, - TokenType, - - -fn check_tokenize_whitespace_fail args input as str, offset as int { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_whitespace - match { - case OptionalToken:some { unreachable } - case OptionalToken:none { nop } - } -} - -fn test_tokenize_whitespace_fail { - "" 0 check_tokenize_whitespace_fail - "." 0 check_tokenize_whitespace_fail - "a" 0 check_tokenize_whitespace_fail - "\\" 0 check_tokenize_whitespace_fail - "/" 0 check_tokenize_whitespace_fail -} - -fn check_tokenize_whitespace args input as str, offset as int, expected_value as const str { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_whitespace - match { - case OptionalToken:some { - use token_ { - token_ "value" ? - use value { - value expected_value = assert - } - } - } - case OptionalToken:none { unreachable } - } -} - -fn test_tokenize_whitespace_ok { - "a a" 1 " " check_tokenize_whitespace - "a\na" 1 "\n" check_tokenize_whitespace - "a\ra" 1 "\r" check_tokenize_whitespace - "a \n \r\r\n \n\na" 1 " \n \r\r\n \n\n" check_tokenize_whitespace -} - -fn check_tokenize_comment_fail args input as str, offset as int { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_comment - match { - case OptionalToken:some { unreachable } - case OptionalToken:none { nop } - } -} - -fn test_tokenize_comment_fail { - "" 0 check_tokenize_comment_fail - "." 0 check_tokenize_comment_fail - "a" 0 check_tokenize_comment_fail - "/" 0 check_tokenize_comment_fail - "a/" 0 check_tokenize_comment_fail - "/ab" 0 check_tokenize_comment_fail -} - -fn check_tokenize_comment args input as str, offset as int, expected_value as const str { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_comment - match { - case OptionalToken:some { - use token_ { - token_ "value" ? - use value { - value expected_value = assert - } - } - } - case OptionalToken:none { unreachable } - } -} - -fn test_tokenize_comment_ok { - "//" 0 "//" check_tokenize_comment - "// something " 0 "// something " check_tokenize_comment - "a// something " 1 "// something " check_tokenize_comment - "// something \nb" 0 "// something " check_tokenize_comment - "a// something \nb" 1 "// something " check_tokenize_comment - "a// something \n" 1 "// something " check_tokenize_comment -} - -fn check_tokenize_integer_fail args input as str, offset as int { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_integer - match { - case OptionalToken:some { unreachable } - case OptionalToken:none { nop } - } -} - -fn test_tokenize_integer_fail { - "" 0 check_tokenize_integer_fail - "." 0 check_tokenize_integer_fail - "a" 0 check_tokenize_integer_fail - "/" 0 check_tokenize_integer_fail - "\\" 0 check_tokenize_integer_fail - "-" 0 check_tokenize_integer_fail -} - -fn check_tokenize_integer args input as str, offset as int, expected_value as const str { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_integer - match { - case OptionalToken:some { - use token_ { - token_ "value" ? - use value { - value expected_value = assert - } - } - } - case OptionalToken:none { unreachable } - } - "123a" 0 "123" check_tokenize_integer - "a123a" 1 "123" check_tokenize_integer - "a123" 1 "123" check_tokenize_integer - "-123a" 0 "-123" check_tokenize_integer - "a-123a" 1 "-123" check_tokenize_integer - "a-123" 1 "-123" check_tokenize_integer -} - -fn check_tokenize_fixed_size_fail args input as str, offset as int { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_fixed_size - match { - case OptionalToken:some { unreachable } - case OptionalToken:none { nop } - } -} - -fn test_tokenize_fixed_size_fail { - "argsabc" 0 check_tokenize_fixed_size_fail - "asabc" 0 check_tokenize_fixed_size_fail - "caseabc" 0 check_tokenize_fixed_size_fail - "constabc" 0 check_tokenize_fixed_size_fail - "defaultabc" 0 check_tokenize_fixed_size_fail - "elseabc" 0 check_tokenize_fixed_size_fail - "enumabc" 0 check_tokenize_fixed_size_fail - "falseabc" 0 check_tokenize_fixed_size_fail - "fnabc" 0 check_tokenize_fixed_size_fail - "foreachabc" 0 check_tokenize_fixed_size_fail - "fromabc" 0 check_tokenize_fixed_size_fail - "ifabc" 0 check_tokenize_fixed_size_fail - "importabc" 0 check_tokenize_fixed_size_fail - "matchabc" 0 check_tokenize_fixed_size_fail - "neverabc" 0 check_tokenize_fixed_size_fail - "returnabc" 0 check_tokenize_fixed_size_fail - "structabc" 0 check_tokenize_fixed_size_fail - "trueabc" 0 check_tokenize_fixed_size_fail - "typeabc" 0 check_tokenize_fixed_size_fail - "useabc" 0 check_tokenize_fixed_size_fail - "whileabc" 0 check_tokenize_fixed_size_fail -} - -fn check_tokenize_fixed_size args input as str, offset as int, expected_type as TokenType, expected_value as str { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_fixed_size - match { - case OptionalToken:some as token { - token - - Token - dup "type_" { expected_type } ! - dup "value" { expected_value } ! - - = assert - } - case OptionalToken:none { unreachable } - } -} - -fn test_tokenize_fixed_size_ok_bare { - "-" 0 TokenType:identifier "-" check_tokenize_fixed_size - "," 0 TokenType:comma "," check_tokenize_fixed_size - ":" 0 TokenType:colon ":" check_tokenize_fixed_size - "!" 0 TokenType:set_field "!" check_tokenize_fixed_size - "?" 0 TokenType:get_field "?" check_tokenize_fixed_size - "." 0 TokenType:identifier "." check_tokenize_fixed_size - "[" 0 TokenType:sq_start "[" check_tokenize_fixed_size - "]" 0 TokenType:sq_end "]" check_tokenize_fixed_size - "{" 0 TokenType:start "{" check_tokenize_fixed_size - "}" 0 TokenType:end "}" check_tokenize_fixed_size - "*" 0 TokenType:identifier "*" check_tokenize_fixed_size - "/" 0 TokenType:identifier "/" check_tokenize_fixed_size - "%" 0 TokenType:identifier "%" check_tokenize_fixed_size - "+" 0 TokenType:identifier "+" check_tokenize_fixed_size - "<-" 0 TokenType:assign "<-" check_tokenize_fixed_size - "<" 0 TokenType:identifier "<" check_tokenize_fixed_size - "<=" 0 TokenType:identifier "<=" check_tokenize_fixed_size - "=" 0 TokenType:identifier "=" check_tokenize_fixed_size - ">" 0 TokenType:identifier ">" check_tokenize_fixed_size - ">=" 0 TokenType:identifier ">=" check_tokenize_fixed_size - "args" 0 TokenType:args_ "args" check_tokenize_fixed_size - "as" 0 TokenType:as_ "as" check_tokenize_fixed_size - "call" 0 TokenType:call_ "call" check_tokenize_fixed_size - "case" 0 TokenType:case_ "case" check_tokenize_fixed_size - "const" 0 TokenType:const_ "const" check_tokenize_fixed_size - "default" 0 TokenType:default_ "default" check_tokenize_fixed_size - "else" 0 TokenType:else_ "else" check_tokenize_fixed_size - "enum" 0 TokenType:enum_ "enum" check_tokenize_fixed_size - "false" 0 TokenType:false_ "false" check_tokenize_fixed_size - "fn" 0 TokenType:fn_ "fn" check_tokenize_fixed_size - "foreach" 0 TokenType:foreach_ "foreach" check_tokenize_fixed_size - "from" 0 TokenType:from_ "from" check_tokenize_fixed_size - "if" 0 TokenType:if_ "if" check_tokenize_fixed_size - "import" 0 TokenType:import_ "import" check_tokenize_fixed_size - "match" 0 TokenType:match_ "match" check_tokenize_fixed_size - "never" 0 TokenType:never_ "never" check_tokenize_fixed_size - "return" 0 TokenType:return_ "return" check_tokenize_fixed_size - "struct" 0 TokenType:struct_ "struct" check_tokenize_fixed_size - "true" 0 TokenType:true_ "true" check_tokenize_fixed_size - "use" 0 TokenType:use_ "use" check_tokenize_fixed_size - "while" 0 TokenType:while_ "while" check_tokenize_fixed_size -} - -fn test_tokenize_fixed_size_ok_middle { - " - " 1 TokenType:identifier "-" check_tokenize_fixed_size - " , " 1 TokenType:comma "," check_tokenize_fixed_size - " : " 1 TokenType:colon ":" check_tokenize_fixed_size - " ! " 1 TokenType:set_field "!" check_tokenize_fixed_size - " ? " 1 TokenType:get_field "?" check_tokenize_fixed_size - " . " 1 TokenType:identifier "." check_tokenize_fixed_size - " [ " 1 TokenType:sq_start "[" check_tokenize_fixed_size - " ] " 1 TokenType:sq_end "]" check_tokenize_fixed_size - " { " 1 TokenType:start "{" check_tokenize_fixed_size - " } " 1 TokenType:end "}" check_tokenize_fixed_size - " * " 1 TokenType:identifier "*" check_tokenize_fixed_size - " / " 1 TokenType:identifier "/" check_tokenize_fixed_size - " % " 1 TokenType:identifier "%" check_tokenize_fixed_size - " + " 1 TokenType:identifier "+" check_tokenize_fixed_size - " <- " 1 TokenType:assign "<-" check_tokenize_fixed_size - " < " 1 TokenType:identifier "<" check_tokenize_fixed_size - " <= " 1 TokenType:identifier "<=" check_tokenize_fixed_size - " = " 1 TokenType:identifier "=" check_tokenize_fixed_size - " > " 1 TokenType:identifier ">" check_tokenize_fixed_size - " >= " 1 TokenType:identifier ">=" check_tokenize_fixed_size - " args " 1 TokenType:args_ "args" check_tokenize_fixed_size - " as " 1 TokenType:as_ "as" check_tokenize_fixed_size - " call " 1 TokenType:call_ "call" check_tokenize_fixed_size - " case " 1 TokenType:case_ "case" check_tokenize_fixed_size - " const " 1 TokenType:const_ "const" check_tokenize_fixed_size - " default " 1 TokenType:default_ "default" check_tokenize_fixed_size - " else " 1 TokenType:else_ "else" check_tokenize_fixed_size - " enum " 1 TokenType:enum_ "enum" check_tokenize_fixed_size - " false " 1 TokenType:false_ "false" check_tokenize_fixed_size - " fn " 1 TokenType:fn_ "fn" check_tokenize_fixed_size - " foreach " 1 TokenType:foreach_ "foreach" check_tokenize_fixed_size - " from " 1 TokenType:from_ "from" check_tokenize_fixed_size - " if " 1 TokenType:if_ "if" check_tokenize_fixed_size - " import " 1 TokenType:import_ "import" check_tokenize_fixed_size - " match " 1 TokenType:match_ "match" check_tokenize_fixed_size - " never " 1 TokenType:never_ "never" check_tokenize_fixed_size - " return " 1 TokenType:return_ "return" check_tokenize_fixed_size - " struct " 1 TokenType:struct_ "struct" check_tokenize_fixed_size - " true " 1 TokenType:true_ "true" check_tokenize_fixed_size - " use " 1 TokenType:use_ "use" check_tokenize_fixed_size - " while " 1 TokenType:while_ "while" check_tokenize_fixed_size -} - -fn check_tokenize_string_fail args input as str, offset as int { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_string - match { - case OptionalToken:some { unreachable } - case OptionalToken:none { nop } - } -} - -fn test_tokenize_string_fail { - "" 0 check_tokenize_string_fail - "a" 0 check_tokenize_string_fail - "." 0 check_tokenize_string_fail - "." 0 check_tokenize_string_fail - "\"" 0 check_tokenize_string_fail - "a\"" 0 check_tokenize_string_fail - "\"a" 0 check_tokenize_string_fail - "\"\\" 0 check_tokenize_string_fail -} - -fn check_tokenize_string args input as str, offset as int, expected_value as const str { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_string - match { - case OptionalToken:some { - use token_ { - token_ "value" ? - use value { - value expected_value = assert - } - } - } - case OptionalToken:none { unreachable } - } -} - -fn test_tokenize_string_ok { - "\"\"" 0 "\"\"" check_tokenize_string - "\"\" " 0 "\"\"" check_tokenize_string - " \"\"" 1 "\"\"" check_tokenize_string - " \"\" " 1 "\"\"" check_tokenize_string - "\"a\"" 0 "\"a\"" check_tokenize_string - " \"a\"" 1 "\"a\"" check_tokenize_string - " \"a\" " 1 "\"a\"" check_tokenize_string - "\"\n\"" 0 "\"\n\"" check_tokenize_string - " \"\n\"" 1 "\"\n\"" check_tokenize_string - " \"\n\" " 1 "\"\n\"" check_tokenize_string - "\"\\n\"" 0 "\"\\n\"" check_tokenize_string - " \"\\n\"" 1 "\"\\n\"" check_tokenize_string - " \"\\n\" " 1 "\"\\n\"" check_tokenize_string - "\"\\\\\"" 0 "\"\\\\\"" check_tokenize_string - " \"\\\\\"" 1 "\"\\\\\"" check_tokenize_string - " \"\\\\\" " 1 "\"\\\\\"" check_tokenize_string -} - -fn check_tokenize_identifier_fail args input as str, offset as int { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_string - match { - case OptionalToken:some { unreachable } - case OptionalToken:none { nop } - } -} - -fn test_tokenize_identifier_fail { - "" 0 check_tokenize_identifier_fail - "3" 0 check_tokenize_identifier_fail - "." 0 check_tokenize_identifier_fail - "\n" 0 check_tokenize_identifier_fail -} - -fn check_tokenize_identifier args input as str, offset as int, expected_value as const str { - "" input make_tokenizer - dup "offset" { offset } ! - Tokenizer:tokenize_identifier - match { - case OptionalToken:some { - use token_ { - token_ "value" ? - use value { - value expected_value = assert - } - } - } - case OptionalToken:none { unreachable } - } -} - -fn test_tokenize_identifier_ok { - "a" 0 "a" check_tokenize_identifier - "z" 0 "z" check_tokenize_identifier - "A" 0 "A" check_tokenize_identifier - "Z" 0 "Z" check_tokenize_identifier - "_" 0 "_" check_tokenize_identifier - - " a " 1 "a" check_tokenize_identifier - " z " 1 "z" check_tokenize_identifier - " A " 1 "A" check_tokenize_identifier - " Z " 1 "Z" check_tokenize_identifier - " _ " 1 "_" check_tokenize_identifier - - "aaaa" 0 "aaaa" check_tokenize_identifier - "zzzz" 0 "zzzz" check_tokenize_identifier - "AAAA" 0 "AAAA" check_tokenize_identifier - "ZZZZ" 0 "ZZZZ" check_tokenize_identifier - "____" 0 "____" check_tokenize_identifier -} diff --git a/examples/selfhosting/tokenizer.aaa b/examples/selfhosting/tokenizer.aaa deleted file mode 100644 index 2df1088f..00000000 --- a/examples/selfhosting/tokenizer.aaa +++ /dev/null @@ -1,632 +0,0 @@ -struct Range { - next_value as int, - end as int, -} - -fn make_range args start as int, end as int return Range { - Range - dup "next_value" { start } ! - dup "end" { end } ! -} - -fn Range:iter args r as Range return Range { - r -} - -fn Range:next args r as Range return int, bool { - r "next_value" ? - r "end" ? - use next_value, end { - if next_value end >= { - 0 false return - } - - r "next_value" { next_value 1 + } ! - next_value true - } -} - -enum FileReadResult { - ok as str, - open_error, - read_error, -} - -fn read_file args path as const str, return FileReadResult { - path 0 0 open - - use fd, open_ok { - if open_ok not { - FileReadResult:open_error return - } - - "" - use content { - while true { - fd 4096 read - - use buff, read_ok { - if read_ok not { - FileReadResult:read_error return - } - - if buff "" = { - content FileReadResult:ok return - } - - content <- { content buff str:append } - } - } - } - } -} - -enum TokenType { - args_, - as_, - assign, - builtin_, - call_, - case_, - char_, - colon, - comma, - comment, - const_, - default_, - else_, - end, - enum_, - false_, - fn_, - foreach_, - from_, - get_field, - identifier, - if_, - import_, - integer, - match_, - never_, - return_, - set_field, - sq_end, - sq_start, - start, - string, - struct_, - true_, - use_, - while_, - whitespace, -} - -fn TokenType:to_str args token_type as const TokenType return str { - token_type - match { - case TokenType:args_ { "args" } - case TokenType:as_ { "as" } - case TokenType:assign { "assign" } - case TokenType:builtin_ { "builtin" } - case TokenType:call_ { "call" } - case TokenType:case_ { "case" } - case TokenType:char_ { "char" } - case TokenType:colon { "colon" } - case TokenType:comma { "comma" } - case TokenType:comment { "comment" } - case TokenType:const_ { "const" } - case TokenType:default_ { "default" } - case TokenType:else_ { "else" } - case TokenType:end { "end" } - case TokenType:enum_ { "enum" } - case TokenType:false_ { "false" } - case TokenType:fn_ { "fn" } - case TokenType:foreach_ { "foreach" } - case TokenType:from_ { "from" } - case TokenType:get_field { "get_field" } - case TokenType:identifier { "identifier" } - case TokenType:if_ { "if" } - case TokenType:import_ { "import" } - case TokenType:integer { "integer" } - case TokenType:match_ { "match" } - case TokenType:never_ { "never" } - case TokenType:return_ { "return" } - case TokenType:set_field { "set_field" } - case TokenType:sq_end { "sq_end" } - case TokenType:sq_start { "sq_start" } - case TokenType:start { "start" } - case TokenType:string { "string" } - case TokenType:struct_ { "struct" } - case TokenType:true_ { "true" } - case TokenType:use_ { "use" } - case TokenType:while_ { "while" } - case TokenType:whitespace { "whitespace" } - } -} - -struct FixedSizedToken { - value as str, - type_ as TokenType, -} - -fn make_fixed_sized_token args value as str, type_ as TokenType return FixedSizedToken { - FixedSizedToken - dup "type_" { type_ } ! - dup "value" { value } ! -} - -fn get_fixed_sized_tokens return vec[FixedSizedToken] { - vec[FixedSizedToken] - use v { - // NOTE: keep this sorted by longest first, and for same length alphabetical - v "builtin" TokenType:builtin_ make_fixed_sized_token vec:push - v "default" TokenType:default_ make_fixed_sized_token vec:push - v "foreach" TokenType:foreach_ make_fixed_sized_token vec:push - v "import" TokenType:import_ make_fixed_sized_token vec:push - v "return" TokenType:return_ make_fixed_sized_token vec:push - v "struct" TokenType:struct_ make_fixed_sized_token vec:push - v "const" TokenType:const_ make_fixed_sized_token vec:push - v "false" TokenType:false_ make_fixed_sized_token vec:push - v "match" TokenType:match_ make_fixed_sized_token vec:push - v "never" TokenType:never_ make_fixed_sized_token vec:push - v "while" TokenType:while_ make_fixed_sized_token vec:push - v "args" TokenType:args_ make_fixed_sized_token vec:push - v "call" TokenType:call_ make_fixed_sized_token vec:push - v "case" TokenType:case_ make_fixed_sized_token vec:push - v "else" TokenType:else_ make_fixed_sized_token vec:push - v "enum" TokenType:enum_ make_fixed_sized_token vec:push - v "from" TokenType:from_ make_fixed_sized_token vec:push - v "true" TokenType:true_ make_fixed_sized_token vec:push - v "use" TokenType:use_ make_fixed_sized_token vec:push - v "<-" TokenType:assign make_fixed_sized_token vec:push - v "<=" TokenType:identifier make_fixed_sized_token vec:push - v ">=" TokenType:identifier make_fixed_sized_token vec:push - v "as" TokenType:as_ make_fixed_sized_token vec:push - v "fn" TokenType:fn_ make_fixed_sized_token vec:push - v "if" TokenType:if_ make_fixed_sized_token vec:push - v "-" TokenType:identifier make_fixed_sized_token vec:push - v "," TokenType:comma make_fixed_sized_token vec:push - v ":" TokenType:colon make_fixed_sized_token vec:push - v "!" TokenType:set_field make_fixed_sized_token vec:push - v "?" TokenType:get_field make_fixed_sized_token vec:push - v "." TokenType:identifier make_fixed_sized_token vec:push - v "[" TokenType:sq_start make_fixed_sized_token vec:push - v "]" TokenType:sq_end make_fixed_sized_token vec:push - v "{" TokenType:start make_fixed_sized_token vec:push - v "}" TokenType:end make_fixed_sized_token vec:push - v "*" TokenType:identifier make_fixed_sized_token vec:push - v "/" TokenType:identifier make_fixed_sized_token vec:push - v "%" TokenType:identifier make_fixed_sized_token vec:push - v "+" TokenType:identifier make_fixed_sized_token vec:push - v "<" TokenType:identifier make_fixed_sized_token vec:push - v "=" TokenType:identifier make_fixed_sized_token vec:push - v ">" TokenType:identifier make_fixed_sized_token vec:push - - v - } -} - -struct Position { - file as str, - line as int, - column as int, -} - -fn make_position args file as str, line as int, column as int return Position { - Position - dup "file" { file } ! - dup "line" { line } ! - dup "column" { column } ! -} - -fn Position:update args position as Position, token as Token { - token "value" ? - use value { - 0 value str:len make_range - foreach { - use i { - - value i str:at - use char_, ok { - if ok not { - unreachable - } - - if char_ '\n' = { - position "line" { position "line" ? 1 + } ! - position "column" { 1 } ! - } - else { - position "column" { position "column" ? 1 + } ! - } - } - } - } - } -} - -struct Token { - position as Position, - type_ as TokenType, - value as str, -} - -fn Token:= args lhs as const Token, rhs as const Token return bool { - if - lhs "type_" ? TokenType:to_str - rhs "type_" ? TokenType:to_str - = not - { - false return - } - - lhs "value" ? - rhs "value" ? - = -} - -enum OptionalToken { - some as Token, - none, -} - -struct Tokenizer { - filename as str, - input as str, - offset as int, - offset_position as Position, - fixed_sized_tokens as vec[FixedSizedToken], - whitespace_regex as regex, - comment_regex as regex, - integer_regex as regex, - identifier_regex as regex, - string_regex as regex, - character_regex as regex, -} - -fn make_tokenizer args filename as str, input as str return Tokenizer { - Tokenizer - dup "filename" { filename } ! - dup "input" { input } ! - dup "offset_position" { filename 1 1 make_position } ! - dup "fixed_sized_tokens" { get_fixed_sized_tokens } ! - dup "whitespace_regex" { "\\s+" make_regex assert } ! - dup "comment_regex" { "//[^\n]*" make_regex assert } ! - dup "integer_regex" { "(-)?[0-9]+" make_regex assert } ! - dup "identifier_regex" { "[a-zA-Z_]+" make_regex assert } ! - dup "string_regex" { "\"(\\\\.|.|\n)*?\"" make_regex assert } ! - dup "character_regex" { "'(\\\\.|.|\n)'" make_regex assert } ! -} - -fn Tokenizer:tokenize_whitespace args tokenizer as Tokenizer return OptionalToken { - tokenizer "input" ? - tokenizer "offset" ? - tokenizer "whitespace_regex" ? - use input, offset, whitespace_regex { - whitespace_regex input offset regex:find - use matched_str, matched_offset, matched { - if matched not matched_offset offset = not or { - OptionalToken:none - } else { - Token - dup "type_" { TokenType:whitespace } ! - dup "value" { matched_str } ! - OptionalToken:some - } - } - } -} - -fn Tokenizer:tokenize_comment args tokenizer as Tokenizer return OptionalToken { - tokenizer "input" ? - tokenizer "offset" ? - tokenizer "comment_regex" ? - use input, offset, comment_regex { - comment_regex input offset regex:find - use matched_str, matched_offset, matched { - if matched not matched_offset offset = not or { - OptionalToken:none return - } - - Token - dup "type_" { TokenType:comment } ! - dup "value" { matched_str } ! - OptionalToken:some - } - } -} - -fn Tokenizer:tokenize_integer args tokenizer as Tokenizer return OptionalToken { - tokenizer "input" ? - tokenizer "offset" ? - tokenizer "integer_regex" ? - use input, offset, integer_regex { - integer_regex input offset regex:find - use matched_str, matched_offset, matched { - if matched not matched_offset offset = not or { - OptionalToken:none return - } - - Token - dup "type_" { TokenType:integer } ! - dup "value" { matched_str } ! - OptionalToken:some - } - } -} - -fn matches_at args input as const str, search as str, offset as int return bool { - input search offset str:find_after - use found_offset, ok { - ok offset found_offset = and - } -} - -fn Tokenizer:match_fixed_size args tokenizer as Tokenizer, fixed_sized_token as FixedSizedToken return OptionalToken { - tokenizer "input" ? - tokenizer "offset" ? - fixed_sized_token "value" ? - fixed_sized_token "type_" ? - use input, offset, token_value, token_type { - if input token_value offset matches_at not { - OptionalToken:none return - } - - // prevent matching identifier `asdf` as keyword `as` - if - input offset token_value str:len + is_identifier_char - token_value 0 is_identifier_char - and - { - OptionalToken:none return - } - - Token - dup "type_" { token_type } ! - dup "value" { token_value } ! - OptionalToken:some - } -} - -fn Tokenizer:tokenize_fixed_size args tokenizer as Tokenizer return OptionalToken { - tokenizer "fixed_sized_tokens" ? - foreach { - use fixed_sized_token { - tokenizer fixed_sized_token Tokenizer:match_fixed_size - match { - case OptionalToken:some as token { - drop - token OptionalToken:some return - } - default { - nop - } - } - } - } - - OptionalToken:none -} - -fn Tokenizer:tokenize_string args tokenizer as Tokenizer return OptionalToken { - tokenizer "input" ? - tokenizer "offset" ? - tokenizer "string_regex" ? - use input, offset, string_regex { - string_regex input offset regex:find - use matched_str, matched_offset, matched { - if matched matched_offset offset = and not { - OptionalToken:none return - } - - Token - dup "type_" { TokenType:string } ! - dup "value" { matched_str } ! - OptionalToken:some - } - } -} - -fn Tokenizer:tokenize_character args tokenizer as Tokenizer return OptionalToken { - tokenizer "input" ? - tokenizer "offset" ? - tokenizer "character_regex" ? - use input, offset, character_regex { - character_regex input offset regex:find - use matched_str, matched_offset, matched { - if matched matched_offset offset = and not { - OptionalToken:none return - } - - Token - dup "type_" { TokenType:char_ } ! - dup "value" { matched_str } ! - OptionalToken:some - - } - } -} - -fn is_identifier_char args input as const str, offset as int return bool { - input offset str:at - use char_, ok { - if ok not { - false return - } - - char_ char:is_alpha - char_ '_' = - or - } -} - -fn Tokenizer:tokenize_identifier args tokenizer as Tokenizer return OptionalToken { - tokenizer "input" ? - tokenizer "offset" ? - tokenizer "identifier_regex" ? - use input, offset, identifier_regex { - identifier_regex input offset regex:find - use matched_str, matched_offset, matched { - if matched not matched_offset offset = not or { - OptionalToken:none return - } - - Token - dup "type_" { TokenType:identifier } ! - dup "value" { matched_str } ! - OptionalToken:some - } - } -} - -fn Tokenizer:tokenize_at_offset args tokenizer as Tokenizer return OptionalToken { - tokenizer Tokenizer:tokenize_whitespace - match { - case OptionalToken:some { OptionalToken:some return } - default { nop } - } - - tokenizer Tokenizer:tokenize_comment - match { - case OptionalToken:some { OptionalToken:some return } - default { nop } - } - - tokenizer Tokenizer:tokenize_integer - match { - case OptionalToken:some { OptionalToken:some return } - default { nop } - } - - tokenizer Tokenizer:tokenize_fixed_size - match { - case OptionalToken:some { OptionalToken:some return } - default { nop } - } - - tokenizer Tokenizer:tokenize_string - match { - case OptionalToken:some { OptionalToken:some return } - default { nop } - } - - tokenizer Tokenizer:tokenize_identifier - match { - case OptionalToken:some { OptionalToken:some return } - default { nop } - } - - tokenizer Tokenizer:tokenize_character -} - -enum TokenizeResult { - ok as vec[Token], - error, -} - -fn Tokenizer:run args tokenizer as Tokenizer return TokenizeResult { - vec[Token] - tokenizer "input" ? - use tokens, input { - while tokenizer "offset" ? input str:len < { - tokenizer Tokenizer:tokenize_at_offset - match { - case OptionalToken:none { - TokenizeResult:error return - } - case OptionalToken:some as token { - token "position" { tokenizer "offset_position" ? copy swap drop } ! - - token "type_" ? - match { - case TokenType:whitespace { nop } - case TokenType:comment { nop } - default { - tokens token vec:push - } - } - - tokenizer "offset" { - tokenizer "offset" ? - token "value" ? str:len - + - } ! - tokenizer "offset_position" ? token Position:update - } - } - } - tokens TokenizeResult:ok return - } -} - -fn print_tokens args tokens as vec[Token] { - tokens - foreach { - dup drop - use token { - token "type_" ? - match { - case TokenType:whitespace { nop } - case TokenType:comment { nop } - default { - token "position" ? - use position { - position "file" ? . - ":" . - position "line" ? . - ":" . - position "column" ? . - } - " " . - token "type_" ? TokenType:to_str . - " " . - token "value" ? . - "\n" . - } - } - } - } -} - -fn main args argv as vec[str] return int { - if argv vec:len 2 = not { - "Usage: " . - argv 0 vec:get . - " \n" . - 1 return - } - - argv 1 vec:get - - use source_path { - source_path read_file - match { - case FileReadResult:open_error { - "Could not open " . source_path . "\n" . - 1 return - } - case FileReadResult:read_error { - "Could not read " . source_path . "\n" . - 1 return - } - case FileReadResult:ok { - use content { - source_path content make_tokenizer Tokenizer:run - match { - case TokenizeResult:ok { - use tokens { - tokens print_tokens - } - } - case TokenizeResult:error { - "Tokenization failed.\n" . - 1 return - } - } - } - } - } - } - - 0 -} diff --git a/tests/selfhosting/test_tokenizer.py b/tests/selfhosting/test_tokenizer.py index eaf6566c..c0fc0c69 100644 --- a/tests/selfhosting/test_tokenizer.py +++ b/tests/selfhosting/test_tokenizer.py @@ -53,6 +53,7 @@ def aaa_source_files() -> tuple[list[str], list[ParameterSet]]: ) +@pytest.mark.skip() # TODO @pytest.mark.parametrize(*aaa_source_files()) def test_tokenizer_output( tokenizer_excecutable: str, capfd: CaptureFixture[str], source_file: Path From 8e165583b7df9be0471ba83461d2c2f6b82e6519 Mon Sep 17 00:00:00 2001 From: Luuk Verweij Date: Mon, 18 Dec 2023 14:03:41 +0100 Subject: [PATCH 3/5] wip --- .gitignore | 2 + examples/selfhosting/main.aaa | 7 +-- examples/selfhosting/syntax_loader.aaa | 60 ++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index b1dd771e..4e2a954a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ __pycache__ .envrc *.idea *.prof +.ruff_cache/ +.mypy_cache/ aaa-stdlib/target/* aaa-stdlib-user/target/* diff --git a/examples/selfhosting/main.aaa b/examples/selfhosting/main.aaa index f9dc6fb4..e68e8fde 100644 --- a/examples/selfhosting/main.aaa +++ b/examples/selfhosting/main.aaa @@ -1,6 +1,7 @@ -from "syntax_loader" import syntax_loader_new_from_str // TODO remove +from "syntax_loader" import syntax_loader_new_from_file fn main { - nop - // TODO do something useful + "syntax.json" syntax_loader_new_from_file + + drop // TODO do something with SyntaxLoader } diff --git a/examples/selfhosting/syntax_loader.aaa b/examples/selfhosting/syntax_loader.aaa index 1d720bfb..4ef15297 100644 --- a/examples/selfhosting/syntax_loader.aaa +++ b/examples/selfhosting/syntax_loader.aaa @@ -7,9 +7,63 @@ enum SyntaxLoaderResult { struct SyntaxLoader {} -fn syntax_loader_new_from_file args filename as str return SyntaxLoaderResult { - // TODO load file and call syntax_loader_new_from_str - todo +enum FileReadError { + open_error as str, + read_error as str, +} + +fn FileReadError:to_str args error as FileReadError return str { + error + match { + case FileReadError:open_error as path { "Could not open " path str:append } + case FileReadError:read_error as path { "Could not read " path str:append } + } +} + +enum FileReadResult { + ok as str, + error as FileReadError, +} + +fn read_file args path as const str, return FileReadResult { + path 0 0 open + + use fd, open_ok { + if open_ok not { + path + copy swap drop // TODO make ticket for const arguments of enum-ctors + FileReadError:open_error FileReadResult:error return + } + + "" + use content { + while true { + fd 4096 read + + use buff, read_ok { + if read_ok not { + path + copy swap drop + FileReadError:read_error FileReadResult:error return + } + + if buff "" = { + content FileReadResult:ok return + } + + content <- { content buff str:append } + } + } + } + } +} + +fn syntax_loader_new_from_file args path as str return SyntaxLoaderResult { + path read_file + match { + case FileReadResult:ok as text { text syntax_loader_new_from_str } + case FileReadResult:error as error { error FileReadError:to_str SyntaxLoaderResult:error } + } } fn syntax_loader_new_from_str args text as str return SyntaxLoaderResult { From 41921dc188feda67feb27d2df31b6cfe7d7c891c Mon Sep 17 00:00:00 2001 From: Luuk Verweij Date: Mon, 18 Dec 2023 16:27:20 +0100 Subject: [PATCH 4/5] wip --- examples/selfhosting/syntax_loader.aaa | 58 ++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/examples/selfhosting/syntax_loader.aaa b/examples/selfhosting/syntax_loader.aaa index 4ef15297..20973433 100644 --- a/examples/selfhosting/syntax_loader.aaa +++ b/examples/selfhosting/syntax_loader.aaa @@ -66,6 +66,46 @@ fn syntax_loader_new_from_file args path as str return SyntaxLoaderResult { } } +enum LoadTokensResult { + ok as map[str, regex], + error as str, +} + +enum LoadNodesResult { + ok as map[str,str], + error as str, +} + +enum LoadFilteredTokensResult { + ok as vec[str], + error as str, +} + +enum LoadRootNodeResult { + ok as str, + error as str, +} + +fn load_regular_tokens args root_json as map[str, Json] return LoadTokensResult { + todo // TODO +} + +fn load_keyword_tokens args root_json as map[str, Json] return LoadTokensResult { + todo // TODO +} + +fn load_nodes args root_json as map[str, Json] return LoadNodesResult { + todo // TODO +} + +fn load_filter_tokens args root_json as map[str, Json] return LoadFilteredTokensResult { + todo // TODO +} + +fn load_root_node args root_json as map[str, Json] return LoadRootNodeResult { + todo // TODO +} + fn syntax_loader_new_from_str args text as str return SyntaxLoaderResult { text json_from_str match { @@ -73,8 +113,20 @@ fn syntax_loader_new_from_str args text as str return SyntaxLoaderResult { case JsonResult:ok as json { json } } - todo - // TODO of data validation + use json { + json + match { + case Json:object as object { object } + default { "json root should be an object" SyntaxLoaderResult:error return } + } + } + + use root_object { + todo + // TODO call load_* functions + // TODO check for extra values in root_dict + } - // TODO loading fields of SyntaxLoader + // TODO run equivalent of `_check_values()` + // TODO run equivalent of `_load_parsers()` } From fe4348dce3cf725f6af574de079e0ca1d7594cdd Mon Sep 17 00:00:00 2001 From: Luuk Verweij Date: Mon, 18 Dec 2023 20:06:35 +0100 Subject: [PATCH 5/5] wip --- aaa/type_checker/type_checker.py | 8 +++ examples/selfhosting/main.aaa | 14 ++++- examples/selfhosting/syntax_loader.aaa | 86 +++++++++++++++++++++++--- 3 files changed, 98 insertions(+), 10 deletions(-) diff --git a/aaa/type_checker/type_checker.py b/aaa/type_checker/type_checker.py index 4cbb03ba..cc180349 100644 --- a/aaa/type_checker/type_checker.py +++ b/aaa/type_checker/type_checker.py @@ -256,6 +256,10 @@ def _confirm_return_types( if not isinstance(computed_value, VariableType): return False + # A struct is never equal to an enum + if not isinstance(computed_value.type, type(expected_value.type)): + return False + if computed_value.type != expected_value.type: return False @@ -312,6 +316,10 @@ def _match_signature_items( if not isinstance(var_type, VariableType): raise SignatureItemMismatch + # prevent comparing enum with struct + if not isinstance(expected_var_type.type, type(var_type.type)): + raise SignatureItemMismatch + if expected_var_type.type != var_type.type: raise SignatureItemMismatch diff --git a/examples/selfhosting/main.aaa b/examples/selfhosting/main.aaa index e68e8fde..6c917451 100644 --- a/examples/selfhosting/main.aaa +++ b/examples/selfhosting/main.aaa @@ -1,7 +1,19 @@ -from "syntax_loader" import syntax_loader_new_from_file +from "syntax_loader" import + syntax_loader_new_from_file, + SyntaxLoaderResult, fn main { "syntax.json" syntax_loader_new_from_file + match { + case SyntaxLoaderResult:error as error { + error . + "\n" . + 1 exit + } + case SyntaxLoaderResult:ok as syntax_loader { + syntax_loader + } + } drop // TODO do something with SyntaxLoader } diff --git a/examples/selfhosting/syntax_loader.aaa b/examples/selfhosting/syntax_loader.aaa index 20973433..034341b0 100644 --- a/examples/selfhosting/syntax_loader.aaa +++ b/examples/selfhosting/syntax_loader.aaa @@ -5,7 +5,12 @@ enum SyntaxLoaderResult { error as str, } -struct SyntaxLoader {} +struct SyntaxLoader { + tokens as map[str, regex], + nodes as map[str, str], + filtered_tokens as vec[str], + root_node as str, +} enum FileReadError { open_error as str, @@ -72,7 +77,7 @@ enum LoadTokensResult { } enum LoadNodesResult { - ok as map[str,str], + ok as map[str, str], error as str, } @@ -86,23 +91,77 @@ enum LoadRootNodeResult { error as str, } -fn load_regular_tokens args root_json as map[str, Json] return LoadTokensResult { +fn load_regular_tokens args root_object as map[str, Json] return LoadTokensResult { todo // TODO } -fn load_keyword_tokens args root_json as map[str, Json] return LoadTokensResult { +fn load_keyword_tokens args root_object as map[str, Json] return LoadTokensResult { todo // TODO } -fn load_nodes args root_json as map[str, Json] return LoadNodesResult { +enum MergeMapsResult { + ok as map[str, regex], + error as vec[str], +} + +// Merge maps but fail if maps have overlapping keys +fn merge_maps args lhs as map[str, regex], rhs as map[str, regex] return MergeMapsResult { + vec[str] map[str, regex] + use overlap, merged { + lhs foreach { + use key, value { + if rhs key map:has_key { + overlap key vec:push + } + merged key value map:set + } + } + + if overlap vec:empty not { + overlap MergeMapsResult:error return + } + + rhs foreach { + use key, value { + merged key value map:set + } + } + + merged MergeMapsResult:ok + } +} + +fn load_tokens args root_object as map[str, Json] return LoadTokensResult { + root_object load_keyword_tokens + match { + case LoadTokensResult:error as error { error LoadTokensResult:error return } + case LoadTokensResult:ok as keyword_tokens { + root_object load_regular_tokens + match { + case LoadTokensResult:error as error { error LoadTokensResult:error return } + case LoadTokensResult:ok as regular_tokens { keyword_tokens regular_tokens } + } + } + } + + use keyword_tokens, regular_tokens { + keyword_tokens regular_tokens merge_maps + match { + case MergeMapsResult:error { todo } + case MergeMapsResult:ok as tokens { tokens LoadTokensResult:ok } + } + } +} + +fn load_nodes args root_object as map[str, Json] return LoadNodesResult { todo // TODO } -fn load_filter_tokens args root_json as map[str, Json] return LoadFilteredTokensResult { +fn load_filtered_tokens args root_object as map[str, Json] return LoadFilteredTokensResult { todo // TODO } -fn load_root_node args root_json as map[str, Json] return LoadRootNodeResult { +fn load_root_node args root_object as map[str, Json] return LoadRootNodeResult { todo // TODO } @@ -121,9 +180,18 @@ fn syntax_loader_new_from_str args text as str return SyntaxLoaderResult { } } - use root_object { + SyntaxLoader + use root_object, syntax_loader { + root_object load_tokens + match { + case LoadTokensResult:ok as tokens { syntax_loader "tokens" { tokens } ! } + case LoadTokensResult:error as error { error SyntaxLoaderResult:error return } + } + todo - // TODO call load_* functions + // TODO filtered tokens + // TODO load nodes + // TODO root node // TODO check for extra values in root_dict }