From 5b79e1bff1ae3acbc55db2f010335dffd7074001 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:16 -0800 Subject: [PATCH 01/30] WIP: JuliaSyntax integration. --- JuliaSnail.jl | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index ccf832e..9f3433b 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -470,6 +470,120 @@ function replcompletion(identifier, mod) end + ### --- JuliaSyntax wrappers + +module Syntax + +import Base64 +import Base.JuliaSyntax as JS + +function parse(encodedbuf) + try + buf = String(Base64.base64decode(encodedbuf)) + JS.parseall(JS.SyntaxNode, buf) + catch err + println(err) + return nothing + end +end + +function pathat(node::JS.SyntaxNode, offset, path = [(expr=node, start=JS.first_byte(node), stop=JS.last_byte(node))]) + if JS.haschildren(node) + for child in JS.children(node) + if JS.first_byte(child) <= offset <= JS.last_byte(child) + return pathat(child, offset, + [path; [(expr=child, start=JS.first_byte(child), stop=JS.last_byte(child))]]) + end + end + end + return path +end + +function nodename(node::JS.SyntaxNode) + kind = JS.kind(node) + children = JS.children(node) + + if kind == JS.K"module" + return string(children[1]) + + elseif kind == JS.K"function" + first = children[1] + if JS.kind(first) == JS.K"call" + return string(JS.children(first)[1]) + else + return string(first) + end + + elseif kind == JS.K"struct" + return string(children[2]) # First child is usually visibility (mutable/abstract) + + elseif kind == JS.K"primitive" + return string(children[2]) # First child is "type" keyword + + elseif kind == JS.K"abstract" + return string(children[2]) # First child is "type" keyword + + elseif kind == JS.K"macro" + first = children[1] + if JS.kind(first) == JS.K"call" + return string(JS.children(first)[1]) + else + return string(first) + end + end + + return nothing +end + +function moduleat(encodedbuf, byteloc) + tree = parse(encodedbuf) + path = pathat(tree, byteloc) + modules = [] + for node in path + if JS.kind(node.expr) == JS.K"module" + push!(modules, nodename(node.expr)) + end + end + return [:list; modules] +end + +function blockat(encodedbuf, byteloc) + tree = parse(encodedbuf) + path = pathat(tree, byteloc) + modules = [] + description = nothing + start = nothing + stop = nothing + for node in path + if JS.kind(node.expr) == JS.K"module" + description = nothing + push!(modules, nodename(node.expr)) + elseif (isnothing(description) && + (JS.kind(node.expr) ∈ [JS.K"abstract", JS.K"function", + JS.K"struct", JS.K"primitive", + JS.K"macro"])) + description = nodename(node.expr) + start = node.start + stop = node.stop + end + end + # result format equivalent to what Elisp side expects + return isnothing(description) ? + nothing : + [:list; tuple(modules...); start; stop; description] +end + +function codetree(encodedbuf) + # FIXME: Write this. +end + +function includesin(encodedbuf, path="") + # FIXME: Write this. +end + +end + + ### --- CSTParser wrappers module CST From e11aef2f016cb66e87e089d44d5764b3e2b10477 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:16 -0800 Subject: [PATCH 02/30] Port CST.codetree to Syntax.codetree with recursive node traversal --- JuliaSnail.jl | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 9f3433b..ef976b1 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -574,7 +574,44 @@ function blockat(encodedbuf, byteloc) end function codetree(encodedbuf) - # FIXME: Write this. + tree = parse(encodedbuf) + helper = (node, depth = 1) -> begin + res = [] + for child in JS.children(node) + kind = JS.kind(child) + name = nodename(child) + if name !== nothing + if kind == JS.K"module" + push!(res, (:module, name, JS.first_byte(child), helper(child, depth + 1))) + elseif kind == JS.K"function" + # Reconstruct function signature from the call node + if JS.haschildren(child) && JS.kind(JS.children(child)[1]) == JS.K"call" + call_node = JS.children(child)[1] + sig = String(JS.sourcetext(call_node)) + push!(res, (:function, sig, JS.first_byte(child))) + else + push!(res, (:function, name * "()", JS.first_byte(child))) + end + elseif kind ∈ (JS.K"struct", JS.K"mutable") + push!(res, (:struct, name, JS.first_byte(child))) + elseif kind ∈ (JS.K"abstract", JS.K"primitive") + push!(res, (:type, name, JS.first_byte(child))) + elseif kind == JS.K"macro" + push!(res, (:macro, name, JS.first_byte(child))) + end + else + # Flatten results from nested nodes that aren't modules + if JS.haschildren(child) + append!(res, helper(child, depth + 1)) + end + end + end + return res + end + tree_result = helper(tree) + return isempty(tree_result) ? + nothing : + [:list; tree_result] end function includesin(encodedbuf, path="") From 74d7b981c1824ab9729f3526d3a02b470eb4b463 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:16 -0800 Subject: [PATCH 03/30] Implement `includesin` using JuliaSyntax with cleaner traversal and module tracking --- JuliaSnail.jl | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index ef976b1..0f220f2 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -614,8 +614,60 @@ function codetree(encodedbuf) [:list; tree_result] end +""" +For a given buffer, return the files `include()`d in each nested module. + +Result structure: { + filename -> [module names] +} + +Uses JuliaSyntax to parse the code and find include statements within modules. +""" function includesin(encodedbuf, path="") - # FIXME: Write this. + tree = parse(encodedbuf) + results = Dict{String,Vector{Symbol}}() + + helper = (node, modules = Symbol[]) -> begin + for child in JS.children(node) + kind = JS.kind(child) + + # Check for include calls + if kind == JS.K"call" && length(JS.children(child)) >= 2 + call_name = JS.children(child)[1] + if String(JS.sourcetext(call_name)) == "include" + # Get filename from the first argument + filename_node = JS.children(child)[2] + filename = String(JS.sourcetext(filename_node)) + # Remove quotes + filename = replace(filename, r"^\"(.*)\"$" => s"\1") + full_path = joinpath(path, filename) + results[full_path] = copy(modules) + end + # Track module context + elseif kind == JS.K"module" + name = nodename(child) + if name !== nothing + helper(child, [modules; Symbol(name)]) + end + end + + # Recurse into other nodes that may contain includes + if JS.haschildren(child) + helper(child, modules) + end + end + end + + helper(tree) + + # Convert to plist for Emacs + reslist = [] + for (file, modules) in results + push!(reslist, file) + push!(reslist, [:list; modules]) + end + + return isempty(reslist) ? nothing : [:list; reslist] end end From b3004b9fb29131caa6a447d3b91bc37165997c75 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:16 -0800 Subject: [PATCH 04/30] Update and improve test coverage. --- tests/files/codetree.jl | 23 ++++++++++++++++ tests/files/s1.jl | 1 + tests/files/s2.jl | 9 +++++++ tests/files/s3.jl | 4 +++ tests/files/s4.jl | 4 +++ tests/files/s5.jl | 5 ++++ tests/parser.jl | 60 ++++++++++++----------------------------- tests/tests.jl | 14 ++++++++++ 8 files changed, 77 insertions(+), 43 deletions(-) create mode 100644 tests/files/codetree.jl create mode 100644 tests/files/s1.jl create mode 100644 tests/files/s2.jl create mode 100644 tests/files/s3.jl create mode 100644 tests/files/s4.jl create mode 100644 tests/files/s5.jl create mode 100644 tests/tests.jl diff --git a/tests/files/codetree.jl b/tests/files/codetree.jl new file mode 100644 index 0000000..bb0f34d --- /dev/null +++ b/tests/files/codetree.jl @@ -0,0 +1,23 @@ +module Alpha + +module Bravo + +function f1(x) + 2x +end + +module Charlie + +end + +end + +module Delta + +end + +end + +module Echo + +end diff --git a/tests/files/s1.jl b/tests/files/s1.jl new file mode 100644 index 0000000..a594c05 --- /dev/null +++ b/tests/files/s1.jl @@ -0,0 +1 @@ +function f1(); end diff --git a/tests/files/s2.jl b/tests/files/s2.jl new file mode 100644 index 0000000..43bd00e --- /dev/null +++ b/tests/files/s2.jl @@ -0,0 +1,9 @@ +module Alpha +function f1(); end +end +function f2(); end +module Bravo +module Charlie +function f3(); end +end +end diff --git a/tests/files/s3.jl b/tests/files/s3.jl new file mode 100644 index 0000000..34cbbf5 --- /dev/null +++ b/tests/files/s3.jl @@ -0,0 +1,4 @@ +module Geometry +area_circle(radius) = π * r^2 +end +f1() = "f1" diff --git a/tests/files/s4.jl b/tests/files/s4.jl new file mode 100644 index 0000000..e10537b --- /dev/null +++ b/tests/files/s4.jl @@ -0,0 +1,4 @@ +module Alpha +include("a1.jl") +include("a2.jl") +end diff --git a/tests/files/s5.jl b/tests/files/s5.jl new file mode 100644 index 0000000..5b1b4a2 --- /dev/null +++ b/tests/files/s5.jl @@ -0,0 +1,5 @@ +module Alpha +module Bravo +include("a1.jl") +end +end diff --git a/tests/parser.jl b/tests/parser.jl index 4664e9b..2bc4498 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -1,51 +1,13 @@ import Base64 -using Test - - -include("../JuliaSnail.jl") @testset "parser interface" begin - # TODO: Move these samples into supporting files: - - s1 = Base64.base64encode(""" -function f1(); end -""") - - s2 = Base64.base64encode(""" -module Alpha -function f1(); end -end -function f2(); end -module Bravo -module Charlie -function f3(); end -end -end -""") - - s3 = Base64.base64encode(""" -module Geometry -area_circle(radius) = π * r^2 -end -f1() = "f1" -""") - - s4 = Base64.base64encode(""" -module Alpha -include("a1.jl") -include("a2.jl") -end -""") - - s5 = Base64.base64encode(""" -module Alpha -module Bravo -include("a1.jl") -end -end -""") + s1 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s1.jl"), String)) + s2 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s2.jl"), String)) + s3 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s3.jl"), String)) + s4 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s4.jl"), String)) + s5 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s5.jl"), String)) @testset "module detection" begin @test [:list] == JuliaSnail.CST.moduleat(s1, 0) @@ -67,6 +29,18 @@ end @test [:list, tuple("Geometry"), 17, 47, "area_circle"] == JuliaSnail.CST.blockat(s3, 25) end + @testset "code tree" begin + buf = Base64.base64encode(read(joinpath(@__DIR__, "files", "codetree.jl"), String)) + @test Any[:list, + (:module, "Alpha", 1, Any[ + (:module, "Bravo", 15, Any[ + (:function, "f1(x)", 29), + (:module, "Charlie", 54, Any[])]), + (:module, "Delta", 80, Any[])]), + (:module, "Echo", 104, Any[])] == + JuliaSnail.CST.codetree(buf) + end + @testset "include detection" begin # more tests in implicit-modules.el @test [:list, "a1.jl", [:list, "Alpha"], "a2.jl", [:list, "Alpha"]] == JuliaSnail.CST.includesin(s4) diff --git a/tests/tests.jl b/tests/tests.jl new file mode 100644 index 0000000..dcbd4f4 --- /dev/null +++ b/tests/tests.jl @@ -0,0 +1,14 @@ +# running the tests: +# $ julia/tests.jl +# or: +# julia> include("tests/tests.jl") + +module JuliaSnailTests + +using Test + +include("../JuliaSnail.jl") + +include("./parser.jl") + +end From d89d6a1d7859c06676b0b224aee9627f1ef5b22c Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 05/30] Rename Syntax module to JStx for clarity --- JuliaSnail.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 0f220f2..424287c 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -472,7 +472,7 @@ end ### --- JuliaSyntax wrappers -module Syntax +module JStx import Base64 import Base.JuliaSyntax as JS From e435d617cd8a0cac8ef5a2d6fa77fbb8dd891419 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 06/30] Adjust JuliaSyntax blockat to match CST span calculation --- JuliaSnail.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 424287c..f5ec7fb 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -564,7 +564,8 @@ function blockat(encodedbuf, byteloc) JS.K"macro"])) description = nodename(node.expr) start = node.start - stop = node.stop + # Add 1 to stop to match CST behavior which includes the trailing newline + stop = node.stop + 1 end end # result format equivalent to what Elisp side expects From dc152bb8391ba6e2d784c6c54133e206d143a424 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 07/30] Handle function assignments in JuliaSnail.jl blockat method --- JuliaSnail.jl | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index f5ec7fb..4ef5aad 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -558,14 +558,23 @@ function blockat(encodedbuf, byteloc) if JS.kind(node.expr) == JS.K"module" description = nothing push!(modules, nodename(node.expr)) - elseif (isnothing(description) && - (JS.kind(node.expr) ∈ [JS.K"abstract", JS.K"function", - JS.K"struct", JS.K"primitive", - JS.K"macro"])) - description = nodename(node.expr) - start = node.start - # Add 1 to stop to match CST behavior which includes the trailing newline - stop = node.stop + 1 + elseif isnothing(description) + # Handle both regular function definitions and assignments + if JS.kind(node.expr) ∈ [JS.K"abstract", JS.K"function", + JS.K"struct", JS.K"primitive", + JS.K"macro"] + description = nodename(node.expr) + start = node.start + stop = node.stop + 1 + elseif JS.kind(node.expr) == JS.K"=" + # Check for function assignment like f() = ... + children = JS.children(node.expr) + if length(children) >= 1 && JS.kind(children[1]) == JS.K"call" + description = string(JS.children(children[1])[1]) + start = node.start + stop = node.stop + 1 + end + end end end # result format equivalent to what Elisp side expects From 9dd6bed57167202acfbf2f134d723c20ff9e6468 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 08/30] Improve test coverage. --- tests/files/s6.jl | 18 ++++++++++++++++++ tests/parser.jl | 43 +++++++++++++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 tests/files/s6.jl diff --git a/tests/files/s6.jl b/tests/files/s6.jl new file mode 100644 index 0000000..fb5d204 --- /dev/null +++ b/tests/files/s6.jl @@ -0,0 +1,18 @@ +# Nested functions +function outer() + function inner(x) + x + 1 + end + inner(5) +end + +# Type parameters +struct GenericPoint{T} + x::T + y::T +end + +# Multiple definitions +function overloaded() 1 end +function overloaded(x) x end +function overloaded(x,y) x+y end diff --git a/tests/parser.jl b/tests/parser.jl index 2bc4498..ac84401 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -10,15 +10,15 @@ import Base64 s5 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s5.jl"), String)) @testset "module detection" begin - @test [:list] == JuliaSnail.CST.moduleat(s1, 0) - @test [:list] == JuliaSnail.CST.moduleat(s1, 10) - @test [:list] == JuliaSnail.CST.moduleat(s2, 0) - @test [:list, "Alpha"] == JuliaSnail.CST.moduleat(s2, 20) - @test [:list] == JuliaSnail.CST.moduleat(s2, 50) - @test [:list, "Bravo", "Charlie"] == JuliaSnail.CST.moduleat(s2, 90) - @test [:list, "Geometry"] == JuliaSnail.CST.moduleat(s3, 45) - @test [:list, "Geometry"] == JuliaSnail.CST.moduleat(s3, 50) - @test [:list] == JuliaSnail.CST.moduleat(s3, 51) + @test [:list] == JuliaSnail.JStx.moduleat(s1, 0) + @test [:list] == JuliaSnail.JStx.moduleat(s1, 10) + @test [:list] == JuliaSnail.JStx.moduleat(s2, 0) + @test [:list, "Alpha"] == JuliaSnail.JStx.moduleat(s2, 20) + @test [:list] == JuliaSnail.JStx.moduleat(s2, 50) + @test [:list, "Bravo", "Charlie"] == JuliaSnail.JStx.moduleat(s2, 90) + @test [:list, "Geometry"] == JuliaSnail.JStx.moduleat(s3, 45) + @test [:list, "Geometry"] == JuliaSnail.JStx.moduleat(s3, 50) + @test [:list] == JuliaSnail.JStx.moduleat(s3, 51) end @testset "block detection" begin @@ -38,7 +38,7 @@ import Base64 (:module, "Charlie", 54, Any[])]), (:module, "Delta", 80, Any[])]), (:module, "Echo", 104, Any[])] == - JuliaSnail.CST.codetree(buf) + JuliaSnail.JStx.codetree(buf) end @testset "include detection" begin @@ -47,4 +47,27 @@ import Base64 @test [:list, "a1.jl", [:list, "Alpha", "Bravo"]] == JuliaSnail.CST.includesin(s5) end + @testset "JuliaSyntax equivalence to CST" begin + @test [:list, (), 1, 19, "f1"] == JuliaSnail.CST.blockat(s1, 3) == JuliaSnail.JStx.blockat(s1, 3) + @test [:list, tuple("Alpha"), 14, 32, "f1"] == JuliaSnail.CST.blockat(s2, 27) == JuliaSnail.JStx.blockat(s2, 27) + @test [:list, (), 37, 55, "f2"] == JuliaSnail.CST.blockat(s2, 50) == JuliaSnail.JStx.blockat(s2, 50) + @test [:list, ("Bravo", "Charlie"), 84, 102, "f3"] == JuliaSnail.CST.blockat(s2, 97) == JuliaSnail.JStx.blockat(s2, 97) + @test [:list, tuple("Geometry"), 17, 47, "area_circle"] == JuliaSnail.CST.blockat(s3, 25) == JuliaSnail.JStx.blockat(s3, 25) + end + + @testset "Special syntax cases" begin + s6 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s6.jl"), String)) + + # Test nested function (only outer) + @test [:list, (), 20, 97, "outer"] == JuliaSnail.JStx.blockat(s6, 85) + + # Test type with parameters + @test [:list, (), 20, 97, "GenericPoint"] == JuliaSnail.JStx.blockat(s6, 150) + + # Test multiple definitions + @test [:list, (), 161, 184, "overloaded"] == JuliaSnail.JStx.blockat(s6, 170) + @test [:list, (), 185, 210, "overloaded"] == JuliaSnail.JStx.blockat(s6, 195) + @test [:list, (), 211, 241, "overloaded"] == JuliaSnail.JStx.blockat(s6, 220) + end + end From 1fd6dd49276c028fe92f1697874ddca0bca7c290 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 09/30] Improve node name extraction for structs and primitive types --- JuliaSnail.jl | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 4ef5aad..5d51175 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -515,7 +515,13 @@ function nodename(node::JS.SyntaxNode) end elseif kind == JS.K"struct" - return string(children[2]) # First child is usually visibility (mutable/abstract) + # Handle generic type parameters + if JS.kind(children[1]) == JS.K"curly" + curly_children = JS.children(children[1]) + return string(curly_children[1]) # The actual type name + else + return string(children[1]) # No type parameters + end elseif kind == JS.K"primitive" return string(children[2]) # First child is "type" keyword @@ -559,14 +565,15 @@ function blockat(encodedbuf, byteloc) description = nothing push!(modules, nodename(node.expr)) elseif isnothing(description) - # Handle both regular function definitions and assignments - if JS.kind(node.expr) ∈ [JS.K"abstract", JS.K"function", - JS.K"struct", JS.K"primitive", - JS.K"macro"] + if JS.kind(node.expr) ∈ [JS.K"abstract", JS.K"function", JS.K"macro"] + description = nodename(node.expr) + start = node.start + stop = node.stop + 1 + elseif JS.kind(node.expr) ∈ [JS.K"struct", JS.K"primitive"] description = nodename(node.expr) start = node.start stop = node.stop + 1 - elseif JS.kind(node.expr) == JS.K"=" + elseif JS.kind(node.expr) == JS.K"=" # Check for function assignment like f() = ... children = JS.children(node.expr) if length(children) >= 1 && JS.kind(children[1]) == JS.K"call" From 2d344e87c3e5237276a41a7ecbcfb749bf985fcd Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 10/30] Moar test improvements. --- tests/files/s6.jl | 12 ++++++++++++ tests/parser.jl | 27 ++++++++++++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/files/s6.jl b/tests/files/s6.jl index fb5d204..cef68d4 100644 --- a/tests/files/s6.jl +++ b/tests/files/s6.jl @@ -12,6 +12,18 @@ struct GenericPoint{T} y::T end +# Plain struct +struct S10 + x + y +end + +# Abstract type +abstract type AbstractPoint end + +# Primitive type +primitive type Point24 24 end + # Multiple definitions function overloaded() 1 end function overloaded(x) x end diff --git a/tests/parser.jl b/tests/parser.jl index ac84401..83fbbc9 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -8,6 +8,7 @@ import Base64 s3 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s3.jl"), String)) s4 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s4.jl"), String)) s5 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s5.jl"), String)) + s6 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s6.jl"), String)) @testset "module detection" begin @test [:list] == JuliaSnail.JStx.moduleat(s1, 0) @@ -56,18 +57,26 @@ import Base64 end @testset "Special syntax cases" begin - s6 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s6.jl"), String)) - # Test nested function (only outer) + @test [:list, (), 20, 97, "outer"] == JuliaSnail.JStx.blockat(s6, 70) @test [:list, (), 20, 97, "outer"] == JuliaSnail.JStx.blockat(s6, 85) - - # Test type with parameters - @test [:list, (), 20, 97, "GenericPoint"] == JuliaSnail.JStx.blockat(s6, 150) - + + # Test struct with parameters + @test [:list, (), 117, 161, "GenericPoint"] == JuliaSnail.JStx.blockat(s6, 150) + + # Test plain struct + @test [:list, (), 178, 202, "S10"] == JuliaSnail.JStx.blockat(s6, 190) + + # Test abstract type + @test [:list, (), 205, 235, "AbstractPoint"] == JuliaSnail.JStx.blockat(s6, 220) + + # Test primitive type + @test [:list, (), 238, 267, "Point24"] == JuliaSnail.JStx.blockat(s6, 250) + # Test multiple definitions - @test [:list, (), 161, 184, "overloaded"] == JuliaSnail.JStx.blockat(s6, 170) - @test [:list, (), 185, 210, "overloaded"] == JuliaSnail.JStx.blockat(s6, 195) - @test [:list, (), 211, 241, "overloaded"] == JuliaSnail.JStx.blockat(s6, 220) + @test [:list, (), 270, 297, "overloaded"] == JuliaSnail.JStx.blockat(s6, 280) + @test [:list, (), 298, 326, "overloaded"] == JuliaSnail.JStx.blockat(s6, 310) + @test [:list, (), 327, 359, "overloaded"] == JuliaSnail.JStx.blockat(s6, 340) end end From 3c14a01676595ebd26380d3f1689e45e4bfe6377 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 11/30] Handle edge cases in nodename and update test byte positions --- JuliaSnail.jl | 8 ++++++-- tests/parser.jl | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 5d51175..4f2afea 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -524,10 +524,14 @@ function nodename(node::JS.SyntaxNode) end elseif kind == JS.K"primitive" - return string(children[2]) # First child is "type" keyword + if length(children) >= 2 + return string(children[2]) # First child is "type" keyword + end elseif kind == JS.K"abstract" - return string(children[2]) # First child is "type" keyword + if length(children) >= 2 + return string(children[2]) # First child is "type" keyword + end elseif kind == JS.K"macro" first = children[1] diff --git a/tests/parser.jl b/tests/parser.jl index 83fbbc9..a8a6dfa 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -74,9 +74,9 @@ import Base64 @test [:list, (), 238, 267, "Point24"] == JuliaSnail.JStx.blockat(s6, 250) # Test multiple definitions - @test [:list, (), 270, 297, "overloaded"] == JuliaSnail.JStx.blockat(s6, 280) - @test [:list, (), 298, 326, "overloaded"] == JuliaSnail.JStx.blockat(s6, 310) - @test [:list, (), 327, 359, "overloaded"] == JuliaSnail.JStx.blockat(s6, 340) + @test [:list, (), 270, 299, "overloaded"] == JuliaSnail.JStx.blockat(s6, 280) + @test [:list, (), 300, 328, "overloaded"] == JuliaSnail.JStx.blockat(s6, 310) + @test [:list, (), 329, 361, "overloaded"] == JuliaSnail.JStx.blockat(s6, 340) end end From 3050e695028c9c71ff37297de7ab703124471397 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 12/30] Fix tests. --- JuliaSnail.jl | 28 +++++++++++++++++++--------- tests/files/s6.jl | 13 ++++++++++--- tests/parser.jl | 17 +++++++++++------ 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 4f2afea..cd57a93 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -524,13 +524,25 @@ function nodename(node::JS.SyntaxNode) end elseif kind == JS.K"primitive" - if length(children) >= 2 - return string(children[2]) # First child is "type" keyword + # Support both these cases: + # primitive type Point24 24 end + # primitive type Int8 <: Integer 8 end + grandchildren = JS.children(children[1]) + if length(grandchildren) > 0 + return string(grandchildren[1]) + else + return string(children[1]) end elseif kind == JS.K"abstract" - if length(children) >= 2 - return string(children[2]) # First child is "type" keyword + # Support both these cases: + # abstract type AbstractPoint1 end + # abstract type AbstractPoint2 <: Number end + grandchildren = JS.children(children[1]) + if length(grandchildren) > 0 + return string(grandchildren[1]) + else + return string(children[1]) end elseif kind == JS.K"macro" @@ -569,11 +581,9 @@ function blockat(encodedbuf, byteloc) description = nothing push!(modules, nodename(node.expr)) elseif isnothing(description) - if JS.kind(node.expr) ∈ [JS.K"abstract", JS.K"function", JS.K"macro"] - description = nodename(node.expr) - start = node.start - stop = node.stop + 1 - elseif JS.kind(node.expr) ∈ [JS.K"struct", JS.K"primitive"] + if JS.kind(node.expr) ∈ [JS.K"function", JS.K"macro", + JS.K"struct", + JS.K"abstract", JS.K"primitive"] description = nodename(node.expr) start = node.start stop = node.stop + 1 diff --git a/tests/files/s6.jl b/tests/files/s6.jl index cef68d4..90462ff 100644 --- a/tests/files/s6.jl +++ b/tests/files/s6.jl @@ -18,13 +18,20 @@ struct S10 y end -# Abstract type -abstract type AbstractPoint end +# Abstract types +abstract type AbstractPoint1 end +abstract type AbstractPoint2 <: Number end -# Primitive type +# Primitive types primitive type Point24 24 end +primitive type Int8 <: Integer 8 end # Multiple definitions function overloaded() 1 end function overloaded(x) x end function overloaded(x,y) x+y end + +# Macro definition +macro sayhello(name) + return :( println("Hello, ", $name) ) +end diff --git a/tests/parser.jl b/tests/parser.jl index a8a6dfa..861263a 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -68,15 +68,20 @@ import Base64 @test [:list, (), 178, 202, "S10"] == JuliaSnail.JStx.blockat(s6, 190) # Test abstract type - @test [:list, (), 205, 235, "AbstractPoint"] == JuliaSnail.JStx.blockat(s6, 220) + @test [:list, (), 221, 253, "AbstractPoint1"] == JuliaSnail.JStx.blockat(s6, 235) + @test [:list, (), 254, 296, "AbstractPoint2"] == JuliaSnail.JStx.blockat(s6, 268) - # Test primitive type - @test [:list, (), 238, 267, "Point24"] == JuliaSnail.JStx.blockat(s6, 250) + # Test primitive types + @test [:list, (), 316, 345, "Point24"] == JuliaSnail.JStx.blockat(s6, 331) + @test [:list, (), 346, 382, "Int8"] == JuliaSnail.JStx.blockat(s6, 361) # Test multiple definitions - @test [:list, (), 270, 299, "overloaded"] == JuliaSnail.JStx.blockat(s6, 280) - @test [:list, (), 300, 328, "overloaded"] == JuliaSnail.JStx.blockat(s6, 310) - @test [:list, (), 329, 361, "overloaded"] == JuliaSnail.JStx.blockat(s6, 340) + @test [:list, (), 407, 434, "overloaded"] == JuliaSnail.JStx.blockat(s6, 416) + @test [:list, (), 435, 463, "overloaded"] == JuliaSnail.JStx.blockat(s6, 444) + @test [:list, (), 464, 496, "overloaded"] == JuliaSnail.JStx.blockat(s6, 473) + + # Test macro + @test [:list, (), 517, 582, "sayhello"] == JuliaSnail.JStx.blockat(s6, 523) end end From af0fa6d8e49f387903d055aedd64e1a4f4e18fe2 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 13/30] Add docstrings to JuliaSnail.JStx module functions --- JuliaSnail.jl | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index cd57a93..020b9f4 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -477,6 +477,14 @@ module JStx import Base64 import Base.JuliaSyntax as JS +""" +Parse a Base64-encoded buffer containing Julia code using JuliaSyntax. + +Returns a parsed syntax tree or nothing if parsing fails. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code +""" function parse(encodedbuf) try buf = String(Base64.base64decode(encodedbuf)) @@ -487,6 +495,22 @@ function parse(encodedbuf) end end +""" +Return the path from root to the node containing the given byte offset. + +Traverses the syntax tree to find nodes containing the offset position, +building a path of nodes from root to leaf. + +# Arguments +- `node`: Root JuliaSyntax.SyntaxNode to start traversal from +- `offset`: Byte offset to search for in the syntax tree +- `path`: Accumulator for the path being built (internal use) + +Returns an array of named tuples containing: +- expr: The syntax node +- start: Starting byte position +- stop: Ending byte position +""" function pathat(node::JS.SyntaxNode, offset, path = [(expr=node, start=JS.first_byte(node), stop=JS.last_byte(node))]) if JS.haschildren(node) for child in JS.children(node) @@ -499,6 +523,23 @@ function pathat(node::JS.SyntaxNode, offset, path = [(expr=node, start=JS.first_ return path end +""" +Extract the name from a syntax node based on its kind. + +Handles various node types including: +- Modules +- Functions +- Structs (including generic parameters) +- Primitive types +- Abstract types +- Macros + +Returns the extracted name as a string, or nothing if the node type is not supported +or does not contain a name. + +# Arguments +- `node`: JuliaSyntax.SyntaxNode to extract name from +""" function nodename(node::JS.SyntaxNode) kind = JS.kind(node) children = JS.children(node) @@ -557,6 +598,18 @@ function nodename(node::JS.SyntaxNode) return nothing end +""" +Find the module context at a given byte location in code. + +Parses the code and returns a list of module names that enclose the given position, +from outermost to innermost. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code +- `byteloc`: Byte offset to find module context for + +Returns an Elisp-compatible list starting with :list followed by module names. +""" function moduleat(encodedbuf, byteloc) tree = parse(encodedbuf) path = pathat(tree, byteloc) @@ -569,6 +622,25 @@ function moduleat(encodedbuf, byteloc) return [:list; modules] end +""" +Find information about the code block at a given byte location. + +Parses the code and returns details about the enclosing block (function, struct, etc.) +at the specified position. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code +- `byteloc`: Byte offset to find block information for + +Returns an Elisp-compatible list containing: +- :list symbol +- Tuple of enclosing module names +- Starting byte position +- Ending byte position +- Block description (e.g. function name) + +Returns nothing if no block is found at the location. +""" function blockat(encodedbuf, byteloc) tree = parse(encodedbuf) path = pathat(tree, byteloc) @@ -604,6 +676,28 @@ function blockat(encodedbuf, byteloc) [:list; tuple(modules...); start; stop; description] end +""" +Generate a tree representation of code structure. + +Parses the code and builds a tree showing the hierarchical structure of: +- Modules +- Functions (with signatures) +- Structs +- Types (abstract and primitive) +- Macros + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code + +Returns an Elisp-compatible nested list structure starting with :list, +followed by tuples for each code element containing: +- Element type (:module, :function, :struct, :type, :macro) +- Name +- Byte position +- Nested elements (for modules) + +Returns nothing if no structure is found. +""" function codetree(encodedbuf) tree = parse(encodedbuf) helper = (node, depth = 1) -> begin @@ -653,6 +747,19 @@ Result structure: { } Uses JuliaSyntax to parse the code and find include statements within modules. +Find all include() statements and their enclosing module contexts. + +Parses the code and builds a mapping of included files to their module contexts. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code +- `path`: Base path to resolve relative include paths against (default: "") + +Returns an Elisp-compatible plist alternating between: +- Full path to included file +- List of enclosing module names at the include point + +Returns nothing if no includes are found. """ function includesin(encodedbuf, path="") tree = parse(encodedbuf) From 846f038faf4e871591c63054a23ace56b966b309 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 14/30] Remove forcecompile filth. --- JuliaSnail.jl | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 020b9f4..89bec3b 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -1059,16 +1059,6 @@ function includesin(encodedbuf, path="") [:list; reslist] end -# XXX: Dirty hackery to improve perceived startup performance follows. Running -# this function in a separate thread should, in theory, force a bunch of things -# to JIT-compile in the background before the user notices. -# Thank you very much, time-to-first-plot problem! -function forcecompile() - # call these functions before the user does - includesin(Base64.base64encode("module Alpha\ninclude(\"a.jl\")\nend")) - moduleat(Base64.base64encode("module Alpha\nend"), 1) -end - end @@ -1214,17 +1204,6 @@ function start(port=10011; addr="127.0.0.1") println(stderr, "ERROR: Snail will not work correctly.") end end - # XXX: It would be great to do this forcecompile trick in a Thread.@spawn - # block. Unfortunately, as of Julia 1.6.1 this does not work. First, it's - # meaningless unless Julia is started with JULIA_NUM_THREADS or --threads set - # to some number >1 (not the default). Second, and worse, for some reason, - # the spawned thread _still_ blocks the main thread. Non-working code below: - # if VERSION >= v"1.3" && Threads.nthreads() > 1 - # Threads.@spawn begin - # CST.forcecompile() - # end - # end - CST.forcecompile() # main loop: @async begin while running From 4f96ca9f50bdd7a731e6f70cae042a1ad1432033 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 15/30] Remove CSTParser dependency. --- JuliaSnail.jl | 315 +++++++------------------------------------------- Project.toml | 5 - 2 files changed, 41 insertions(+), 279 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 89bec3b..716f02e 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -11,17 +11,52 @@ ## You should have received a copy of the GNU General Public License ## along with this program. If not, see . +module JuliaSnail + +import Markdown import Pkg +import Printf +import REPL +import Sockets +import REPL.REPLCompletions +export start, stop -module JuliaSnail + ### --- package loading support code -# XXX: External dependency hack. Snail's own dependencies need to be listed -# first in LOAD_PATH during initial load, otherwise conflicting versions -# installed in the Julia global environment cause conflicts. Especially -# CSTParser, with its unstable API. However, Snail should not be listed first -# the rest of the time. +""" +XXX: External dependency hack. + +This macro allows Snail, or its extensions, to provide a Project.toml file with +Julia dependencies _without_ forcing these extensions to be shipped to the Julia +package registry. + +The intent is to activate the package directory containing the Project.toml +file, make sure it loads all the dependencies it needs, but put the directory +containing the relevant Project.toml file at the end of the LOAD_PATH. This +allows Snail's dependencies to be loaded, but not disturb any environments the +user wants to activate. + +This is no longer used for the Snail core. Extensions are welcome to use it for +now. + +Snail's own dependencies need to be listed first in LOAD_PATH during initial +load, otherwise conflicting versions installed in the Julia global environment +cause conflicts. However, Snail should not be listed first the rest of the time. + +Historical example, from back when Snail relied on CSTParser: + +``` +@with_pkg_env (@__DIR__) begin + # list all external dependency imports here (from the appropriate Project.toml, either Snail's or an extension's): + import CSTParser + # check for dependency API compatibility + !isdefined(CSTParser, :iscall) && + throw(ArgumentError("CSTParser API not compatible, must install Snail-specific version")) +end +``` +""" macro with_pkg_env(dir, action) :( try @@ -50,23 +85,6 @@ macro with_pkg_env(dir, action) ) end -@with_pkg_env (@__DIR__) begin - # list all external dependency imports here (from the appropriate Project.toml, either Snail's or an extension's): - import CSTParser - # check for dependency API compatibility - !isdefined(CSTParser, :iscall) && - throw(ArgumentError("CSTParser API not compatible, must install Snail-specific version")) -end - - -import Markdown -import Printf -import REPL -import Sockets -import REPL.REPLCompletions - -export start, stop - ### --- configuration @@ -811,257 +829,6 @@ end end - ### --- CSTParser wrappers - -module CST - -import Base64 -import CSTParser - -""" -Helper function: wraps the parser interface. -""" -function parse(encodedbuf) - cst = nothing - try - buf = String(Base64.base64decode(encodedbuf)) - cst = CSTParser.parse(buf, true) - catch err - # probably an IO problem - # TODO: Need better error reporting here. - println(err) - return [] - end - return cst -end - -""" -Return the path of CSTParser.EXPR objects from the root of the CST to the -expression at the offset, while retaining the EXPR objects' full locations. - -The path is represented as an array of named tuples. The first element -represents the root node, and the last element is the expression at the offset. -Each tuple has a start and stop value, showing locations in the original source -where that node begins and ends. - -NB: The locations represent bytes, not characters! - -This is necessary because CSTParser does not include full location data, see -https://github.com/julia-vscode/CSTParser.jl/pull/80. -""" -function pathat(cst, offset, pos = 0, path = [(expr=cst, start=1, stop=cst.span+1)]) - if cst !== nothing && !CSTParser.isnonstdid(cst) - for a in cst - if pos < offset <= (pos + a.span) - return pathat(a, offset, pos, [path; [(expr=a, start=pos+1, stop=pos+a.span+1)]]) - end - # jump forward by fullspan since we need to skip over a's trailing whitespace - pos += a.fullspan - end - elseif (pos < offset <= (pos + cst.fullspan)) - return [path; [(expr=cst, start=pos+1, stop=offset+1)]] - end - return path -end - -""" -Debugging helper: example code for traversing the CST and tracking the location of each node. -""" -function print_cst(cst) - offset = 0 - helper = (node) -> begin - for a in node - if a.args === nothing - val = CSTParser.valof(a) - # Unicode byte fix - if String == typeof(val) - diff = sizeof(val) - length(val) - a.span -= diff - a.fullspan -= diff - end - println(offset, ":", offset+a.span, "\t", val) - offset += a.fullspan - else - helper(a) - end - end - end - helper(cst) - return offset -end - -""" -Return the module active at point as a list of their names. -""" -function moduleat(encodedbuf, byteloc) - cst = parse(encodedbuf) - path = pathat(cst, byteloc) - modules = [] - for node in path - if CSTParser.defines_module(node.expr) - push!(modules, CSTParser.valof(CSTParser.get_name(node.expr))) - end - end - return [:list; modules] -end - -""" -Return information about the block at point. -""" -function blockat(encodedbuf, byteloc) - cst = parse(encodedbuf) - path = pathat(cst, byteloc) - modules = [] - description = nothing - start = nothing - stop = nothing - for node in path - if CSTParser.defines_module(node.expr) - description = nothing - push!(modules, CSTParser.valof(CSTParser.get_name(node.expr))) - elseif (isnothing(description) && - (CSTParser.defines_abstract(node.expr) || - CSTParser.defines_datatype(node.expr) || - CSTParser.defines_function(node.expr) || - CSTParser.defines_macro(node.expr) || - CSTParser.defines_mutable(node.expr) || - CSTParser.defines_primitive(node.expr) || - CSTParser.defines_struct(node.expr))) - description = CSTParser.valof(CSTParser.get_name(node.expr)) - start = node.start - stop = node.stop - end - end - # result format equivalent to what Elisp side expects - return isnothing(description) ? - nothing : - [:list; tuple(modules...); start; stop; description] -end - -""" -Internal: assemble a human-readable function signature from a CSTParser node. -""" -function fnsig_helper(node) - fnsig = "" - reassemble = (n) -> begin - for x in n - if x.args === nothing - candidate = CSTParser.valof(CSTParser.get_name(x)) - if candidate !== nothing && length(candidate) > 0 - if "," == candidate - candidate *= " " - end - fnsig *= candidate - end - else - reassemble(x) - end - end - end - reassemble(node) - return fnsig -end - -""" -For a given buffer, return the overall tree structure of the code. - -Result structure: [ - [type name location extra] -] -where type is :function, :macro, etc.; when type is :module, then extra is a -nested resulting structure. -""" -function codetree(encodedbuf) - cst = parse(encodedbuf) - offset = 1 - helper = (node, depth = 1) -> begin - res = [] - for a in node - if a.args === nothing - val = CSTParser.valof(a) - # XXX: We want bytes here because we convert back to position - # numbers on the Emacs side. So we do not apply the Unicode byte fix - # from print_cst. - offset += a.fullspan - else - curroffset = offset - aname = CSTParser.valof(CSTParser.get_name(a)) - helper_res = helper(a, depth + 1) - if aname !== nothing - if CSTParser.defines_module(a) - push!(res, (:module, aname, curroffset, helper_res)) - elseif CSTParser.defines_function(a) - fnsig = fnsig_helper(a.args[1]) - push!(res, (:function, fnsig, curroffset)) - elseif CSTParser.defines_struct(a) || CSTParser.defines_mutable(a) - push!(res, (:struct, aname, curroffset)) - elseif CSTParser.defines_abstract(a) || CSTParser.defines_datatype(a) || CSTParser.defines_primitive(a) - push!(res, (:type, aname, curroffset)) - elseif CSTParser.defines_macro(a) - push!(res, (:macro, aname, curroffset)) - end - else - # XXX: Flatten on the fly. First, avoid empty entries. Second, - # use append! since only modules should generate nesting. - if length(helper_res) > 0 - append!(res, helper_res) - end - end - end - end - return res - end - tree = helper(cst) - return isempty(tree) ? - nothing : - [:list; tree] -end - -""" -For a given buffer, return the files `include()`d in each nested module. - -Result structure: { - filename -> [module names] -} - -This nasty code relies on internal CSTParser representations of things. -Unfortunately, CSTParser doesn't provide a clean API for this sort of thing: -https://github.com/julia-vscode/CSTParser.jl/issues/56 -""" -function includesin(encodedbuf, path="") - cst = parse(encodedbuf) - results = Dict() - # walk across args, and track the current module - # when a node of type "call" is found, check its args[1] - helper = (node, modules = []) -> begin - for a in node - if (CSTParser.iscall(a) && - "include" == CSTParser.valof(a.args[1])) - # a.args[2] is the file name being included - filename = joinpath(path, CSTParser.valof(a.args[2])) - results[filename] = modules - elseif CSTParser.defines_module(a) - helper(a, [modules; CSTParser.valof(CSTParser.get_name(a))]) - elseif !isnothing(a.args) - helper(a, modules) - end - end - end - helper(cst) - # convert to a plist for returning back to Emacs - reslist = [] - for (file, modules) in results - push!(reslist, file) - push!(reslist, [:list; modules]) - end - return isempty(reslist) ? - nothing : - [:list; reslist] -end - -end - - ### --- multimedia support ### Adapted from a PR by https://github.com/dahtah (https://github.com/gcv/julia-snail/pull/21). diff --git a/Project.toml b/Project.toml index 4e7f05e..e69de29 100644 --- a/Project.toml +++ b/Project.toml @@ -1,5 +0,0 @@ -[deps] -CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" - -[compat] -CSTParser = "~3.4.2" From 9b5c68331fadb4dcfa5759305e0d953af1d76698 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 16/30] Fix JStx.includesin context tracking. --- JuliaSnail.jl | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 716f02e..af88057 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -781,34 +781,38 @@ Returns nothing if no includes are found. """ function includesin(encodedbuf, path="") tree = parse(encodedbuf) - results = Dict{String,Vector{Symbol}}() + results = Dict{String,Vector{String}}() helper = (node, modules = Symbol[]) -> begin - for child in JS.children(node) - kind = JS.kind(child) - - # Check for include calls - if kind == JS.K"call" && length(JS.children(child)) >= 2 - call_name = JS.children(child)[1] - if String(JS.sourcetext(call_name)) == "include" - # Get filename from the first argument - filename_node = JS.children(child)[2] - filename = String(JS.sourcetext(filename_node)) - # Remove quotes - filename = replace(filename, r"^\"(.*)\"$" => s"\1") - full_path = joinpath(path, filename) - results[full_path] = copy(modules) + if JS.haschildren(node) + for child in JS.children(node) + kind = JS.kind(child) + + # Track module context + if kind == JS.K"module" + name = nodename(child) + if name !== nothing + new_modules = [modules; String(name)] + helper(child, new_modules) + end + continue end - # Track module context - elseif kind == JS.K"module" - name = nodename(child) - if name !== nothing - helper(child, [modules; Symbol(name)]) + + # Check for include calls + if kind == JS.K"call" && length(JS.children(child)) >= 2 + call_name = JS.children(child)[1] + if String(JS.sourcetext(call_name)) == "include" + # Get filename from the first argument + filename_node = JS.children(child)[2] + filename = String(JS.sourcetext(filename_node)) + # Remove quotes + filename = replace(filename, r"^\"(.*)\"$" => s"\1") + # Store with current module context + results[filename] = String.(copy(modules)) + end end - end - # Recurse into other nodes that may contain includes - if JS.haschildren(child) + # Recurse into other nodes helper(child, modules) end end From 6d44ac8e9aff171c39549da8a497556d55430736 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 17/30] Remove CST calls from parser tests. --- tests/parser.jl | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/parser.jl b/tests/parser.jl index 861263a..3a9caff 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -23,11 +23,11 @@ import Base64 end @testset "block detection" begin - @test [:list, (), 1, 19, "f1"] == JuliaSnail.CST.blockat(s1, 3) - @test [:list, tuple("Alpha"), 14, 32, "f1"] == JuliaSnail.CST.blockat(s2, 27) - @test [:list, (), 37, 55, "f2"] == JuliaSnail.CST.blockat(s2, 50) - @test [:list, ("Bravo", "Charlie"), 84, 102, "f3"] == JuliaSnail.CST.blockat(s2, 97) - @test [:list, tuple("Geometry"), 17, 47, "area_circle"] == JuliaSnail.CST.blockat(s3, 25) + @test [:list, (), 1, 19, "f1"] == JuliaSnail.JStx.blockat(s1, 3) + @test [:list, tuple("Alpha"), 14, 32, "f1"] == JuliaSnail.JStx.blockat(s2, 27) + @test [:list, (), 37, 55, "f2"] == JuliaSnail.JStx.blockat(s2, 50) + @test [:list, ("Bravo", "Charlie"), 84, 102, "f3"] == JuliaSnail.JStx.blockat(s2, 97) + @test [:list, tuple("Geometry"), 17, 47, "area_circle"] == JuliaSnail.JStx.blockat(s3, 25) end @testset "code tree" begin @@ -44,16 +44,8 @@ import Base64 @testset "include detection" begin # more tests in implicit-modules.el - @test [:list, "a1.jl", [:list, "Alpha"], "a2.jl", [:list, "Alpha"]] == JuliaSnail.CST.includesin(s4) - @test [:list, "a1.jl", [:list, "Alpha", "Bravo"]] == JuliaSnail.CST.includesin(s5) - end - - @testset "JuliaSyntax equivalence to CST" begin - @test [:list, (), 1, 19, "f1"] == JuliaSnail.CST.blockat(s1, 3) == JuliaSnail.JStx.blockat(s1, 3) - @test [:list, tuple("Alpha"), 14, 32, "f1"] == JuliaSnail.CST.blockat(s2, 27) == JuliaSnail.JStx.blockat(s2, 27) - @test [:list, (), 37, 55, "f2"] == JuliaSnail.CST.blockat(s2, 50) == JuliaSnail.JStx.blockat(s2, 50) - @test [:list, ("Bravo", "Charlie"), 84, 102, "f3"] == JuliaSnail.CST.blockat(s2, 97) == JuliaSnail.JStx.blockat(s2, 97) - @test [:list, tuple("Geometry"), 17, 47, "area_circle"] == JuliaSnail.CST.blockat(s3, 25) == JuliaSnail.JStx.blockat(s3, 25) + @test [:list, "a1.jl", [:list, "Alpha"], "a2.jl", [:list, "Alpha"]] == JuliaSnail.JStx.includesin(s4) + @test [:list, "a1.jl", [:list, "Alpha", "Bravo"]] == JuliaSnail.JStx.includesin(s5) end @testset "Special syntax cases" begin From d9128302af46c81e675b3119773e619de0ccde8c Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 18/30] Elisp side of CSTParser removal. --- julia-snail.el | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/julia-snail.el b/julia-snail.el index 67df386..f2cd842 100644 --- a/julia-snail.el +++ b/julia-snail.el @@ -803,8 +803,8 @@ returns \"/home/username/file.jl\"." (setq julia-snail--process netstream) (set-process-filter julia-snail--process #'julia-snail--server-response-filter) ;; TODO: Implement a sanity check on the Julia environment. Not - ;; sure how. But a failed dependency load (like CSTParser) will - ;; leave Snail in a bad state. + ;; sure how. But a failed Julia dependency load will leave Snail + ;; in a bad state. (message "Successfully connected to Snail server in Julia REPL") ;; Query base directory, and cache (puthash process-buf (julia-snail--capture-basedir repl-buf) @@ -1151,36 +1151,36 @@ evaluated in the context of MODULE." (julia-snail--response-base reqid)) - ;;; --- CST parser interface + ;;; --- parser interface -(defun julia-snail--cst-module-at (buf pt) +(defun julia-snail--parser-module-at (buf pt) (let* ((byteloc (position-bytes pt)) (encoded (julia-snail--encode-base64 buf)) (res (julia-snail--send-to-server :Main - (format "JuliaSnail.CST.moduleat(\"%s\", %d)" encoded byteloc) + (format "JuliaSnail.JStx.moduleat(\"%s\", %d)" encoded byteloc) :async nil))) (if (eq res :nothing) nil res))) -(defun julia-snail--cst-block-at (buf pt) +(defun julia-snail--parser-block-at (buf pt) (let* ((byteloc (position-bytes pt)) (encoded (julia-snail--encode-base64 buf)) (res (julia-snail--send-to-server :Main - (format "JuliaSnail.CST.blockat(\"%s\", %d)" encoded byteloc) + (format "JuliaSnail.JStx.blockat(\"%s\", %d)" encoded byteloc) :async nil))) (if (eq res :nothing) nil res))) -(defun julia-snail--cst-includes (buf) +(defun julia-snail--parser-includes (buf) (let* ((encoded (julia-snail--encode-base64 buf)) (pwd (file-name-directory (julia-snail--efn (buffer-file-name buf)))) (res (julia-snail--send-to-server :Main - (format "JuliaSnail.CST.includesin(\"%s\", \"%s\")" encoded pwd) + (format "JuliaSnail.JStx.includesin(\"%s\", \"%s\")" encoded pwd) :async nil)) (includes (make-hash-table :test #'equal))) (unless (eq res :nothing) @@ -1189,11 +1189,11 @@ evaluated in the context of MODULE." ;; TODO: Maybe there's a situation in which returning :error is appropriate? includes)) -(defun julia-snail--cst-code-tree (buf) +(defun julia-snail--parser-code-tree (buf) (let* ((encoded (julia-snail--encode-base64 buf)) (res (julia-snail--send-to-server :Main - (format "JuliaSnail.CST.codetree(\"%s\")" encoded) + (format "JuliaSnail.JStx.codetree(\"%s\")" encoded) :async nil))) res)) @@ -1230,7 +1230,7 @@ evaluated in the context of MODULE." (defun julia-snail--module-at-point (&optional partial-module) "Return the current Julia module at point as an Elisp list, including PARTIAL-MODULE if given." (let ((partial-module (or partial-module - (julia-snail--cst-module-at (current-buffer) (point)))) + (julia-snail--parser-module-at (current-buffer) (point)))) (module-for-file (julia-snail--module-for-file (buffer-file-name (buffer-base-buffer))))) (or (if module-for-file (append module-for-file partial-module) @@ -1434,7 +1434,7 @@ evaluated in the context of MODULE." (cl-return-from julia-snail-imenu (julia-snail--imenu-cache-entry-value julia-snail--imenu-cache)))) ;; cache miss: ask Julia to parse the file and return the imenu index - (let* ((code-tree (julia-snail--cst-code-tree (current-buffer))) + (let* ((code-tree (julia-snail--parser-code-tree (current-buffer))) (imenu-index-raw (julia-snail--imenu-included-module-helper (julia-snail--imenu-helper code-tree (list)))) (imenu-index (if (eq :flat julia-snail-imenu-style) (-flatten imenu-index-raw) @@ -1736,7 +1736,7 @@ activation: This occurs in the context of the current module. Currently only works on blocks terminated with `end'." (interactive) - (let* ((q (julia-snail--cst-block-at (current-buffer) (point))) + (let* ((q (julia-snail--parser-block-at (current-buffer) (point))) (filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) (module (julia-snail--module-at-point (-first-item q))) (block-start (byte-to-position (or (-second-item q) -1))) @@ -1780,7 +1780,7 @@ This will occur in the context of the Main module, just as it would at the REPL. (let* ((jsrb-save julia-snail-repl-buffer) ; save for callback context (filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) (module (or (julia-snail--module-for-file filename) '("Main"))) - (includes (julia-snail--cst-includes (current-buffer)))) + (includes (julia-snail--parser-includes (current-buffer)))) (when (or (not (buffer-modified-p)) (y-or-n-p (format "'%s' is not saved, send to Julia anyway? " filename))) (julia-snail--send-to-server @@ -1793,7 +1793,7 @@ This will occur in the context of the Main module, just as it would at the REPL. ;; julia-snail-repl-buffer will have disappeared (let* ((julia-snail-repl-buffer jsrb-save) (repl-buf (get-buffer julia-snail-repl-buffer))) - ;; NB: At the moment, julia-snail--cst-includes + ;; NB: At the moment, julia-snail--parser-includes ;; does not return :error. However, it might in ;; the future, and this code will then be useful. (if (eq :error includes) @@ -1822,7 +1822,7 @@ This will occur in the context of the Main module, just as it would at the REPL. "Analyze the current buffer's file for include statements" (interactive) (let* ((filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) - (includes (julia-snail--cst-includes (current-buffer)))) + (includes (julia-snail--parser-includes (current-buffer)))) (when (or (not (buffer-modified-p)) (y-or-n-p (format "'%s' is not saved, analyze in Julia anyway? " filename))) (if (eq :error includes) @@ -1916,7 +1916,7 @@ autocompletion aware of the available modules." (interactive) (let* ((filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) (module (or (julia-snail--module-for-file filename) '("Main"))) - (includes (julia-snail--cst-includes (current-buffer)))) + (includes (julia-snail--parser-includes (current-buffer)))) (julia-snail--module-merge-includes filename includes) (message "Caches updated: parent module %s" (julia-snail--construct-module-path module)))) From 17e16b314fe028421d4ffca217fcd0c3c2fd4463 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:17 -0800 Subject: [PATCH 19/30] Remove check for ancient Julia version. --- julia-snail.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/julia-snail.el b/julia-snail.el index f2cd842..26dd26a 100644 --- a/julia-snail.el +++ b/julia-snail.el @@ -493,7 +493,7 @@ Returns nil if the poll timed out, t otherwise." (defun julia-snail--capture-basedir (buf) (julia-snail--send-to-server :Main - "normpath(joinpath(VERSION <= v\"0.7-\" ? JULIA_HOME : Sys.BINDIR, Base.DATAROOTDIR, \"julia\", \"base\"))" + "normpath(joinpath(Sys.BINDIR, Base.DATAROOTDIR, \"julia\", \"base\"))" :repl-buf buf :async nil)) From 4d68a2af85351632f17aee029b7e4fef77678b5e Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:18 -0800 Subject: [PATCH 20/30] fixup --- JuliaSnail.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index af88057..22ca032 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -11,10 +11,11 @@ ## You should have received a copy of the GNU General Public License ## along with this program. If not, see . +import Pkg + module JuliaSnail import Markdown -import Pkg import Printf import REPL import Sockets From 843f37e5e5bd9e24c42554c309714bfa46f16726 Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:18 -0800 Subject: [PATCH 21/30] Fix Emacs tests. --- JuliaSnail.jl | 1 + tests/implicit-modules.el | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 22ca032..6d3300c 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -808,6 +808,7 @@ function includesin(encodedbuf, path="") filename = String(JS.sourcetext(filename_node)) # Remove quotes filename = replace(filename, r"^\"(.*)\"$" => s"\1") + filename = joinpath(path, filename) # Store with current module context results[filename] = String.(copy(modules)) end diff --git a/tests/implicit-modules.el b/tests/implicit-modules.el index 26ad03e..9762870 100644 --- a/tests/implicit-modules.el +++ b/tests/implicit-modules.el @@ -42,7 +42,7 @@ (js-with-julia-session repl-buf (julia-snail) (with-current-buffer source-buf - (let ((includes (julia-snail--cst-includes (get-buffer (current-buffer))))) + (let ((includes (julia-snail--parser-includes (get-buffer (current-buffer))))) (should (equal '("MyModule") From 37633aa7a6fa4e458d1b2b34caea86afe016fc7d Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:17:18 -0800 Subject: [PATCH 22/30] Standardize indentation. --- JuliaSnail.jl | 91 +++++++++++++++++++++++++-------------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 6d3300c..ced68ad 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -781,55 +781,55 @@ Returns an Elisp-compatible plist alternating between: Returns nothing if no includes are found. """ function includesin(encodedbuf, path="") - tree = parse(encodedbuf) - results = Dict{String,Vector{String}}() - - helper = (node, modules = Symbol[]) -> begin - if JS.haschildren(node) - for child in JS.children(node) - kind = JS.kind(child) - - # Track module context - if kind == JS.K"module" - name = nodename(child) - if name !== nothing - new_modules = [modules; String(name)] - helper(child, new_modules) - end - continue - end - - # Check for include calls - if kind == JS.K"call" && length(JS.children(child)) >= 2 - call_name = JS.children(child)[1] - if String(JS.sourcetext(call_name)) == "include" - # Get filename from the first argument - filename_node = JS.children(child)[2] - filename = String(JS.sourcetext(filename_node)) - # Remove quotes - filename = replace(filename, r"^\"(.*)\"$" => s"\1") - filename = joinpath(path, filename) - # Store with current module context - results[filename] = String.(copy(modules)) - end - end - - # Recurse into other nodes - helper(child, modules) + tree = parse(encodedbuf) + results = Dict{String,Vector{String}}() + + helper = (node, modules = Symbol[]) -> begin + if JS.haschildren(node) + for child in JS.children(node) + kind = JS.kind(child) + + # Track module context + if kind == JS.K"module" + name = nodename(child) + if name !== nothing + new_modules = [modules; String(name)] + helper(child, new_modules) + end + continue end - end - end - helper(tree) + # Check for include calls + if kind == JS.K"call" && length(JS.children(child)) >= 2 + call_name = JS.children(child)[1] + if String(JS.sourcetext(call_name)) == "include" + # Get filename from the first argument + filename_node = JS.children(child)[2] + filename = String(JS.sourcetext(filename_node)) + # Remove quotes + filename = replace(filename, r"^\"(.*)\"$" => s"\1") + filename = joinpath(path, filename) + # Store with current module context + results[filename] = String.(copy(modules)) + end + end - # Convert to plist for Emacs - reslist = [] - for (file, modules) in results - push!(reslist, file) - push!(reslist, [:list; modules]) - end + # Recurse into other nodes + helper(child, modules) + end + end + end + + helper(tree) - return isempty(reslist) ? nothing : [:list; reslist] + # Convert to plist for Emacs + reslist = [] + for (file, modules) in results + push!(reslist, file) + push!(reslist, [:list; modules]) + end + + return isempty(reslist) ? nothing : [:list; reslist] end end @@ -1103,5 +1103,4 @@ function send_to_client(expr, client_socket=nothing) println(client_socket, expr) end - end From c1aca39540d3ed9820bc4f8f01e948f7934d9dca Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 13:33:11 -0800 Subject: [PATCH 23/30] Add Julia version check. --- JuliaSnail.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index ced68ad..ccdef2b 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -960,6 +960,13 @@ during evaluation are captured and sent back to the client as Elisp s-expressions. Special queries also write back their responses as s-expressions. """ function start(port=10011; addr="127.0.0.1") + if VERSION < v"1.10" + # JuliaSyntax only ships with 1.10. + # This is an exception-throwing error, and it's placed here so users have + # a chance to see it before the terminal closes. + error("ERROR: Julia Snail now requires Julia 1.10 or higher") + end + global running = false global server_socket = Sockets.listen(Sockets.IPv4(addr), port) let wait_result = timedwait(function(); server_socket.status == Base.StatusActive; end, From d2fec40af48a0fa7d9cc41a11634c78f37aa2b7b Mon Sep 17 00:00:00 2001 From: gcv Date: Wed, 26 Feb 2025 14:18:37 -0800 Subject: [PATCH 24/30] Update documentation. --- CHANGELOG.md | 1 + README.md | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a47f3d0..070aa16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking change:** Switched from `CSTParser` to `JuliaSyntax` ([#149](https://github.com/gcv/julia-snail/issues/149)). Because `JuliaSyntax` ships with Julia versions 1.10 and later, and because I don't want to maintain `Project.toml` files in Snail itself, this means support for older Julia versions is discontinued. - Standardized on using `locate-library` to locate Snail's installation ([#147](https://github.com/gcv/julia-snail/issues/147)). diff --git a/README.md b/README.md index 9a0a92d..615ce41 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Refer to the [changelog](https://github.com/gcv/julia-snail/blob/master/CHANGELO - **Multimedia and plotting:** Snail can display Julia graphics in graphical Emacs instances using packages like [Plots](http://juliaplots.org) and [Gadfly](http://gadflyjl.org). - **Cross-referencing:** Snail is integrated with the built-in Emacs [xref](https://www.gnu.org/software/emacs/manual/html_node/emacs/Xref.html) system. When a Snail session is active, it supports jumping to definitions of functions and macros loaded in the session. - **Completion:** Snail is also integrated with the built-in Emacs [completion-at-point](https://www.gnu.org/software/emacs/manual/html_node/elisp/Completion-in-Buffers.html) facility. Provided it is configured with the `company-capf` backend, [company-mode](http://company-mode.github.io/) completion will also work (this should be the case by default). -- **Parser:** Snail uses [CSTParser](https://github.com/julia-vscode/CSTParser.jl), a full-featured Julia parser, to infer the structure of source files and to enable features which require an understanding of code context, especially the module in which a particular piece of code lives. This enables awareness of the current module for completion and cross-referencing purposes. +- **Parser:** Snail uses [JuliaSyntax](https://github.com/JuliaLang/JuliaSyntax.jl) to infer the structure of source files and to enable features which require an understanding of code context, especially the module in which a particular piece of code lives. This enables awareness of the current module for completion and cross-referencing purposes. ## Demo @@ -62,9 +62,12 @@ https://user-images.githubusercontent.com/10327/128589405-7368bb50-0ef3-4003-b5d ## Installation -Julia versions >1.6.0 should all work with Snail. +Snail now requires Julia version >1.10.0. -Snail’s Julia-side dependencies will automatically be installed when it starts, and will stay out of your way using Julia’s [`LOAD_PATH` mechanism](https://docs.julialang.org/en/v1/base/constants/#Base.LOAD_PATH). +> [!IMPORTANT] +> [Previous versions of Snail](https://github.com/gcv/julia-snail/tree/1.3.2) supported Julia >1.6.0, and should continue to be used in environments that cannot upgrade Julia. Note that older versions can no longer be installed using MELPA or MELPA Stable, and have to be installed either manually or using a Git-enabled Emacs package manager, such as [Elpaca](https://github.com/progfolio/elpaca). + +Core Snail has no Julia-side dependencies. Any Snail extensions that use Julia dependencies will automatically them on startup, and will stay out of your way using Julia’s [`LOAD_PATH` mechanism](https://docs.julialang.org/en/v1/base/constants/#Base.LOAD_PATH). On the Emacs side, you must install one of the supported high-performance terminal emulators to use with the Julia REPL: either `vterm` or [Eat](https://codeberg.org/akib/emacs-eat). From 6f8952dbb4e0e68482117014f96def085d4adaf3 Mon Sep 17 00:00:00 2001 From: MasonProtter Date: Thu, 27 Feb 2025 18:17:53 +0100 Subject: [PATCH 25/30] fix parse calls in JuliaSyntax.jl --- JuliaSnail.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index ccdef2b..cc27aae 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -630,7 +630,7 @@ from outermost to innermost. Returns an Elisp-compatible list starting with :list followed by module names. """ function moduleat(encodedbuf, byteloc) - tree = parse(encodedbuf) + tree = JS.parseall(JS.SyntaxNode, encodedbuf; ignore_errors=true) path = pathat(tree, byteloc) modules = [] for node in path @@ -661,7 +661,7 @@ Returns an Elisp-compatible list containing: Returns nothing if no block is found at the location. """ function blockat(encodedbuf, byteloc) - tree = parse(encodedbuf) + tree = JS.parseall(JS.SyntaxNode, encodedbuf; ignore_errors=true) path = pathat(tree, byteloc) modules = [] description = nothing From b12b2af603399f501b4ba8a815471155aad9b8fd Mon Sep 17 00:00:00 2001 From: gcv Date: Thu, 27 Feb 2025 13:49:27 -0800 Subject: [PATCH 26/30] Unify calls to the JuliaSyntax parser. --- JuliaSnail.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index cc27aae..61e001c 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -507,7 +507,7 @@ Returns a parsed syntax tree or nothing if parsing fails. function parse(encodedbuf) try buf = String(Base64.base64decode(encodedbuf)) - JS.parseall(JS.SyntaxNode, buf) + JS.parseall(JS.SyntaxNode, buf; ignore_errors=true) catch err println(err) return nothing @@ -630,7 +630,7 @@ from outermost to innermost. Returns an Elisp-compatible list starting with :list followed by module names. """ function moduleat(encodedbuf, byteloc) - tree = JS.parseall(JS.SyntaxNode, encodedbuf; ignore_errors=true) + tree = parse(encodedbuf) path = pathat(tree, byteloc) modules = [] for node in path @@ -648,7 +648,7 @@ Parses the code and returns details about the enclosing block (function, struct, at the specified position. # Arguments -- `encodedbuf`: Base64-encoded string containing Julia source code +- `encodedbuf`: Base64-encoded string containing Julia source code - `byteloc`: Byte offset to find block information for Returns an Elisp-compatible list containing: @@ -661,7 +661,7 @@ Returns an Elisp-compatible list containing: Returns nothing if no block is found at the location. """ function blockat(encodedbuf, byteloc) - tree = JS.parseall(JS.SyntaxNode, encodedbuf; ignore_errors=true) + tree = parse(encodedbuf) path = pathat(tree, byteloc) modules = [] description = nothing From 6141d26635760f5cd363fb57cc37e29b382a037c Mon Sep 17 00:00:00 2001 From: gcv Date: Thu, 27 Feb 2025 14:46:07 -0800 Subject: [PATCH 27/30] Fix display of functions with return type tags. --- JuliaSnail.jl | 9 +++++++-- tests/files/s6.jl | 10 ++++++++++ tests/parser.jl | 5 +++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/JuliaSnail.jl b/JuliaSnail.jl index 61e001c..cbaa009 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -569,8 +569,13 @@ function nodename(node::JS.SyntaxNode) elseif kind == JS.K"function" first = children[1] if JS.kind(first) == JS.K"call" + # function load(filepath) end return string(JS.children(first)[1]) + elseif JS.kind(first) == JS.K"::" + # function load(filepath)::DF.DataFrame end + return string(first[1][1]) else + # TODO: This is likely wrong in some cases. return string(first) end @@ -1090,8 +1095,8 @@ function send_to_client(expr, client_socket=nothing) # force the user to choose the client socket options = map( function(cs) - gsn = Sockets.getpeername(cs) - Printf.@sprintf("%s:%d", gsn[1], gsn[2]) + gsn = Sockets.getpeername(cs) + Printf.@sprintf("%s:%d", gsn[1], gsn[2]) end, client_sockets ) diff --git a/tests/files/s6.jl b/tests/files/s6.jl index 90462ff..2c357f4 100644 --- a/tests/files/s6.jl +++ b/tests/files/s6.jl @@ -35,3 +35,13 @@ function overloaded(x,y) x+y end macro sayhello(name) return :( println("Hello, ", $name) ) end + +# Function with type tags +function load(filepath)::DF.DataFrame +end + +function load2(filepath) +end + +function load3(filepath::String)::String +end diff --git a/tests/parser.jl b/tests/parser.jl index 3a9caff..62fe6f3 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -74,6 +74,11 @@ import Base64 # Test macro @test [:list, (), 517, 582, "sayhello"] == JuliaSnail.JStx.blockat(s6, 523) + + # Test function with return type + @test [:list, (), 610, 651, "load"] == JuliaSnail.JStx.blockat(s6, 619) + @test [:list, (), 653, 681, "load2"] == JuliaSnail.JStx.blockat(s6, 662) + @test [:list, (), 683, 727, "load3"] == JuliaSnail.JStx.blockat(s6, 692) end end From 4774334a0a49bbf459525543ee45cf2ffb523493 Mon Sep 17 00:00:00 2001 From: gcv Date: Sun, 2 Mar 2025 07:04:10 -0800 Subject: [PATCH 28/30] Implement module cache to improve completion performance. --- julia-snail.el | 61 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/julia-snail.el b/julia-snail.el index 26dd26a..a539bf5 100644 --- a/julia-snail.el +++ b/julia-snail.el @@ -329,6 +329,12 @@ nil means disable Snail-specific imenu integration (fall back on julia-mode impl (defvar-local julia-snail--imenu-cache nil) +(defvar julia-snail--parser-module-cache (make-hash-table :test #'equal) + "Cache for module-at-point lookups. Keys are (buffer-hash . line-number) pairs.") + +(defvar julia-snail--parser-module-cache-counter 0 + "Counter for cache cleanup. Incremented on each cache miss.") + (defvar julia-snail--compilation-regexp-alist '(;; matches "while loading /tmp/Foo.jl, in expression starting on line 2" (julia-load-error . ("while loading \\([^ ><()\t\n,'\";:]+\\), in expression starting on line \\([0-9]+\\)" 1 2)) @@ -1153,15 +1159,54 @@ evaluated in the context of MODULE." ;;; --- parser interface +(defun julia-snail--clean-module-cache () + "Remove expired entries from the module cache." + (let ((current-time (float-time))) + (cl-loop for key being the hash-keys of julia-snail--parser-module-cache + using (hash-values value) + when (> (- current-time (car value)) 5.0) + do (remhash key julia-snail--parser-module-cache)))) + (defun julia-snail--parser-module-at (buf pt) - (let* ((byteloc (position-bytes pt)) - (encoded (julia-snail--encode-base64 buf)) - (res (julia-snail--send-to-server - :Main - (format "JuliaSnail.JStx.moduleat(\"%s\", %d)" encoded byteloc) - :async nil))) - (if (eq res :nothing) - nil + "Return the Julia module at point PT in buffer BUF. +Uses a cache that expires entries after 5 seconds, and keyed to +BUF and the line number at PT. This cache structure helps this +function withstand heavy use by completion frameworks (Company, +Corfu): they call repeatedly out to the Julia completion system, +which needs to know the current module. The current module will +not have changed as the user is typing on the same line, but the +`module-at` machinery will be called a lot. There's room for +improvement here. For example, instead of expiring the cache 5 +seconds later, it can just expire when the user moves to a +different line." + (let* ((line-num (line-number-at-pos pt buf)) + (cache-key (cons buf line-num)) + (cached-entry (gethash cache-key julia-snail--parser-module-cache)) + (current-time (float-time)) + (byteloc (position-bytes pt)) + res) + ;; use cached entry if it exists and hasn't expired (5 second TTL) + (if (and cached-entry + (< (- current-time (car cached-entry)) 5.0)) + ;; return cached value + (cdr cached-entry) + ;; cache miss or expired entry: perform the lookup + (setq res (let* ((encoded (julia-snail--encode-base64 buf)) + (result (julia-snail--send-to-server + :Main + (format "JuliaSnail.JStx.moduleat(\"%s\", %d)" encoded byteloc) + :async nil))) + (if (eq result :nothing) + nil + result))) + ;; store the result in the cache with current timestamp: + (puthash cache-key (cons current-time res) julia-snail--parser-module-cache) + ;; increment counter and clean expired entries every 50 calls: + (cl-incf julia-snail--parser-module-cache-counter) + (when (>= julia-snail--parser-module-cache-counter 50) + (setq julia-snail--parser-module-cache-counter 0) + (julia-snail--clean-module-cache)) + ;; done res))) (defun julia-snail--parser-block-at (buf pt) From 0fa70553ef198df6ff69e833d4cdc79231a79025 Mon Sep 17 00:00:00 2001 From: gcv Date: Thu, 13 Mar 2025 13:16:16 -0700 Subject: [PATCH 29/30] Rewrite the imenu helper for nested modules. --- julia-snail.el | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/julia-snail.el b/julia-snail.el index a539bf5..999500a 100644 --- a/julia-snail.el +++ b/julia-snail.el @@ -1444,23 +1444,24 @@ different line." (julia-snail--imenu-helper (cdr tree) modules))))) (defun julia-snail--imenu-included-module-helper (normal-tree) - ;; XXX: This fugly kludge transforms imenu trees in a way that injects modules - ;; cached through include()ed files at the root: + ;; This function transforms imenu trees in a way that injects modules cached + ;; through include()ed files at the root: ;; ;; if included-modules is ("Alpha" "Bravo") ;; and normal-tree is ((function "f1" ...)) ;; then this returns (("Alpha" ("Bravo" (function "f1" ...)))) - ;; - ;; There must be a cleaner way to implement this logic. (let ((included-modules (julia-snail--module-for-file (buffer-file-name (buffer-base-buffer))))) - (cl-labels ((some-helper - (incls norms first-time) - (if (null incls) - norms - (let* ((next (some-helper (cdr incls) norms nil)) - (tail (cons (car incls) (if first-time(list next) next)))) - (if first-time (list tail) tail))))) - (some-helper included-modules normal-tree t)))) + (if (null included-modules) + ;; no included modules, return original tree + normal-tree + ;; recursively build the nested tree: + (cl-labels ((build-nested-tree + (modules content) + (if (null modules) + content + (list (cons (car modules) + (build-nested-tree (cdr modules) content)))))) + (build-nested-tree included-modules normal-tree))))) (cl-defun julia-snail-imenu () ;; exit early if Snail's imenu integration is turned off, or no Snail session is running From 1bfa18bc300be54efd83a8f1e3e51d725141067c Mon Sep 17 00:00:00 2001 From: gcv Date: Fri, 28 Mar 2025 12:29:27 -0700 Subject: [PATCH 30/30] Update ob-julia extension to use JuliaSyntax. --- extensions/ob-julia/ob-julia.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ob-julia/ob-julia.el b/extensions/ob-julia/ob-julia.el index 9261068..7846f55 100644 --- a/extensions/ob-julia/ob-julia.el +++ b/extensions/ob-julia/ob-julia.el @@ -155,7 +155,7 @@ your org notebook" (inner-module (with-temp-buffer (let* ((julia-snail-repl-buffer jsrb-save)) (insert contents) - (julia-snail--cst-module-at (current-buffer) pt))))) + (julia-snail--parser-module-at (current-buffer) pt))))) (if inner-module (append src-module inner-module) src-module)))