From 29ab434cdaee1d8e29b339948ffd70e3d4629082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Backstr=C3=B6m?= Date: Tue, 28 Jan 2025 15:04:17 +0100 Subject: [PATCH 1/5] Add group handling for binaries --- lib/makeup/lexers/erlang_lexer.ex | 8 ++++++++ .../erlang_lexer_tokenizer_test.exs | 17 ++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/makeup/lexers/erlang_lexer.ex b/lib/makeup/lexers/erlang_lexer.ex index d767f94..1cfba55 100644 --- a/lib/makeup/lexers/erlang_lexer.ex +++ b/lib/makeup/lexers/erlang_lexer.ex @@ -409,6 +409,14 @@ defmodule Makeup.Lexers.ErlangLexer do [{:punctuation, %{language: :erlang}, "]"}] ] ], + binary: [ + open: [ + [{:punctuation, %{language: :erlang}, "<<"}] + ], + close: [ + [{:punctuation, %{language: :erlang}, ">>"}] + ] + ], tuple: [ open: [ [{:punctuation, %{language: :erlang}, "{"}] diff --git a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs index ff1344f..3749609 100644 --- a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs +++ b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs @@ -136,22 +136,25 @@ defmodule ErlangLexerTokenizer do describe "binary" do test "<<>> syntax" do - assert lex(~s/<<>>/) == [{:punctuation, %{}, "<<"}, {:punctuation, %{}, ">>"}] + assert lex(~s/<<>>/) == [ + {:punctuation, %{group_id: "group-1"}, "<<"}, + {:punctuation, %{group_id: "group-1"}, ">>"} + ] end test "<<\"\">> syntax" do assert lex(~s/<<"">>/) == [ - {:punctuation, %{}, "<<"}, + {:punctuation, %{group_id: "group-1"}, "<<"}, {:string, %{}, ~s/""/}, - {:punctuation, %{}, ">>"} + {:punctuation, %{group_id: "group-1"}, ">>"} ] end test "<<\"string\">> syntax" do assert lex(~s/<<"string">>/) == [ - {:punctuation, %{}, "<<"}, + {:punctuation, %{group_id: "group-1"}, "<<"}, {:string, %{}, ~s/"string"/}, - {:punctuation, %{}, ">>"} + {:punctuation, %{group_id: "group-1"}, ">>"} ] end end @@ -558,8 +561,8 @@ defmodule ErlangLexerTokenizer do {:generic_prompt, %{selectable: false}, "1> "}, {:name_function, %{}, "list_to_binary"}, {:punctuation, %{group_id: "group-1"}, "("}, - {:punctuation, %{}, "<<"}, - {:punctuation, %{}, ">>"}, + {:punctuation, %{group_id: "group-2"}, "<<"}, + {:punctuation, %{group_id: "group-2"}, ">>"}, {:punctuation, %{group_id: "group-1"}, ")"}, {:punctuation, %{}, "."}, {:whitespace, %{}, "\n"}, From ca403cc8926913850bb07cc49efd1006f6755747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Backstr=C3=B6m?= Date: Tue, 28 Jan 2025 15:11:32 +0100 Subject: [PATCH 2/5] Add tests for list and binary comprehensions --- lib/makeup/lexers/erlang_lexer.ex | 5 +- .../erlang_lexer_tokenizer_test.exs | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/lib/makeup/lexers/erlang_lexer.ex b/lib/makeup/lexers/erlang_lexer.ex index 1cfba55..c3e481e 100644 --- a/lib/makeup/lexers/erlang_lexer.ex +++ b/lib/makeup/lexers/erlang_lexer.ex @@ -219,7 +219,10 @@ defmodule Makeup.Lexers.ErlangLexer do tuple = many_surrounded_by(parsec(:root_element), "{", "}") syntax_operators = - word_from_list(~W[+ - +? ++ = == -- * / < > /= =:= =/= =< >= ==? <- ! ? ?!], :operator) + word_from_list( + ~W[+ - +? ++ = == -- * / < > /= =:= =/= =< >= ==? <- <= ! ? ?!], + :operator + ) record = token(string("#"), :operator) diff --git a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs index 3749609..f8918d6 100644 --- a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs +++ b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs @@ -231,6 +231,57 @@ defmodule ErlangLexerTokenizer do end end + describe "comprehensions" do + test "list" do + assert lex("[A||A<-B]") == [ + {:punctuation, %{group_id: "group-1"}, "["}, + {:name, %{}, "A"}, + {:punctuation, %{}, "||"}, + {:name, %{}, "A"}, + {:operator, %{}, "<-"}, + {:name, %{}, "B"}, + {:punctuation, %{group_id: "group-1"}, "]"} + ] + + assert lex("[A||A<-B,true]") == + [ + {:punctuation, %{group_id: "group-1"}, "["}, + {:name, %{}, "A"}, + {:punctuation, %{}, "||"}, + {:name, %{}, "A"}, + {:operator, %{}, "<-"}, + {:name, %{}, "B"}, + {:punctuation, %{}, ","}, + {:string_symbol, %{}, "true"}, + {:punctuation, %{group_id: "group-1"}, "]"} + ] + end + + test "binary" do + assert lex("[A||A<=B]") == [ + {:punctuation, %{group_id: "group-1"}, "["}, + {:name, %{}, "A"}, + {:punctuation, %{}, "||"}, + {:name, %{}, "A"}, + {:operator, %{}, "<="}, + {:name, %{}, "B"}, + {:punctuation, %{group_id: "group-1"}, "]"} + ] + + assert lex("<>") == [ + {:punctuation, %{group_id: "group-1"}, "<<"}, + {:name, %{}, "A"}, + {:punctuation, %{}, "||"}, + {:name, %{}, "A"}, + {:operator, %{}, "<="}, + {:name, %{}, "B"}, + {:punctuation, %{}, ","}, + {:string_symbol, %{}, "true"}, + {:punctuation, %{group_id: "group-1"}, ">>"} + ] + end + end + describe "atoms" do test "are tokenized as such" do assert lex("atom") == [{:string_symbol, %{}, "atom"}] @@ -312,6 +363,7 @@ defmodule ErlangLexerTokenizer do assert lex("=") == [{:operator, %{}, "="}] assert lex("!") == [{:operator, %{}, "!"}] assert lex("<-") == [{:operator, %{}, "<-"}] + assert lex("<=") == [{:operator, %{}, "<="}] end test "word operators are tokenized as operator" do From f2b40b5968c87de7a4b2606752354c383d95a4b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Backstr=C3=B6m?= Date: Tue, 28 Jan 2025 15:12:06 +0100 Subject: [PATCH 3/5] Add support for parallel comprehensions --- lib/makeup/lexers/erlang_lexer.ex | 2 +- .../erlang_lexer/erlang_lexer_tokenizer_test.exs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/makeup/lexers/erlang_lexer.ex b/lib/makeup/lexers/erlang_lexer.ex index c3e481e..9672272 100644 --- a/lib/makeup/lexers/erlang_lexer.ex +++ b/lib/makeup/lexers/erlang_lexer.ex @@ -212,7 +212,7 @@ defmodule Makeup.Lexers.ErlangLexer do # Combinators that highlight expressions surrounded by a pair of delimiters. punctuation = word_from_list( - [","] ++ ~w[\[ \] : _ @ \" . \#{ { } ( ) | ; => := << >> || -> \#], + [","] ++ ~w[\[ \] : _ @ \" . \#{ { } ( ) | ; => := << >> || -> \# &&], :punctuation ) diff --git a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs index f8918d6..163c4b4 100644 --- a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs +++ b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs @@ -280,6 +280,22 @@ defmodule ErlangLexerTokenizer do {:punctuation, %{group_id: "group-1"}, ">>"} ] end + + test "parallel" do + assert lex("[A||A<-B&&C<-D]") == [ + {:punctuation, %{group_id: "group-1"}, "["}, + {:name, %{}, "A"}, + {:punctuation, %{}, "||"}, + {:name, %{}, "A"}, + {:operator, %{}, "<-"}, + {:name, %{}, "B"}, + {:punctuation, %{}, "&&"}, + {:name, %{}, "C"}, + {:operator, %{}, "<-"}, + {:name, %{}, "D"}, + {:punctuation, %{group_id: "group-1"}, "]"} + ] + end end describe "atoms" do From 4911b79865e24787c49f62051961bbe1d0ab525c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Backstr=C3=B6m?= Date: Tue, 28 Jan 2025 15:13:18 +0100 Subject: [PATCH 4/5] Add support for strict generators --- lib/makeup/lexers/erlang_lexer.ex | 2 +- .../erlang_lexer_tokenizer_test.exs | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/makeup/lexers/erlang_lexer.ex b/lib/makeup/lexers/erlang_lexer.ex index 9672272..fa41daf 100644 --- a/lib/makeup/lexers/erlang_lexer.ex +++ b/lib/makeup/lexers/erlang_lexer.ex @@ -220,7 +220,7 @@ defmodule Makeup.Lexers.ErlangLexer do syntax_operators = word_from_list( - ~W[+ - +? ++ = == -- * / < > /= =:= =/= =< >= ==? <- <= ! ? ?!], + ~W[+ - +? ++ = == -- * / < > /= =:= =/= =< >= ==? <- <:- <= <:= ! ? ?!], :operator ) diff --git a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs index 163c4b4..0f7c5df 100644 --- a/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs +++ b/test/makeup/erlang_lexer/erlang_lexer_tokenizer_test.exs @@ -281,6 +281,28 @@ defmodule ErlangLexerTokenizer do ] end + test "strict" do + assert lex("[A||A<:-B]") == [ + {:punctuation, %{group_id: "group-1"}, "["}, + {:name, %{}, "A"}, + {:punctuation, %{}, "||"}, + {:name, %{}, "A"}, + {:operator, %{}, "<:-"}, + {:name, %{}, "B"}, + {:punctuation, %{group_id: "group-1"}, "]"} + ] + + assert lex("[A||A<:=B]") == [ + {:punctuation, %{group_id: "group-1"}, "["}, + {:name, %{}, "A"}, + {:punctuation, %{}, "||"}, + {:name, %{}, "A"}, + {:operator, %{}, "<:="}, + {:name, %{}, "B"}, + {:punctuation, %{group_id: "group-1"}, "]"} + ] + end + test "parallel" do assert lex("[A||A<-B&&C<-D]") == [ {:punctuation, %{group_id: "group-1"}, "["}, @@ -379,7 +401,9 @@ defmodule ErlangLexerTokenizer do assert lex("=") == [{:operator, %{}, "="}] assert lex("!") == [{:operator, %{}, "!"}] assert lex("<-") == [{:operator, %{}, "<-"}] + assert lex("<:-") == [{:operator, %{}, "<:-"}] assert lex("<=") == [{:operator, %{}, "<="}] + assert lex("<:=") == [{:operator, %{}, "<:="}] end test "word operators are tokenized as operator" do From 6888db038c3f164c8cd3d9c17374a38b2d6b651f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Backstr=C3=B6m?= Date: Tue, 28 Jan 2025 15:18:19 +0100 Subject: [PATCH 5/5] Add Elixir 1.18 and Erlang 27 to CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dab067..3900431 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,9 @@ jobs: - pair: elixir: 1.16 otp: 26 + - pair: + elixir: 1.18 + otp: 27 lint: lint steps: - uses: actions/checkout@v2