diff --git a/CHANGELOG.md b/CHANGELOG.md index a47f3d0..070aa16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Breaking change:** Switched from `CSTParser` to `JuliaSyntax` ([#149](https://github.com/gcv/julia-snail/issues/149)). Because `JuliaSyntax` ships with Julia versions 1.10 and later, and because I don't want to maintain `Project.toml` files in Snail itself, this means support for older Julia versions is discontinued. - Standardized on using `locate-library` to locate Snail's installation ([#147](https://github.com/gcv/julia-snail/issues/147)). diff --git a/JuliaSnail.jl b/JuliaSnail.jl index ccf832e..cbaa009 100644 --- a/JuliaSnail.jl +++ b/JuliaSnail.jl @@ -13,15 +13,51 @@ import Pkg - module JuliaSnail +import Markdown +import Printf +import REPL +import Sockets +import REPL.REPLCompletions + +export start, stop + + + ### --- package loading support code + +""" +XXX: External dependency hack. + +This macro allows Snail, or its extensions, to provide a Project.toml file with +Julia dependencies _without_ forcing these extensions to be shipped to the Julia +package registry. + +The intent is to activate the package directory containing the Project.toml +file, make sure it loads all the dependencies it needs, but put the directory +containing the relevant Project.toml file at the end of the LOAD_PATH. This +allows Snail's dependencies to be loaded, but not disturb any environments the +user wants to activate. -# XXX: External dependency hack. Snail's own dependencies need to be listed -# first in LOAD_PATH during initial load, otherwise conflicting versions -# installed in the Julia global environment cause conflicts. Especially -# CSTParser, with its unstable API. However, Snail should not be listed first -# the rest of the time. +This is no longer used for the Snail core. Extensions are welcome to use it for +now. + +Snail's own dependencies need to be listed first in LOAD_PATH during initial +load, otherwise conflicting versions installed in the Julia global environment +cause conflicts. However, Snail should not be listed first the rest of the time. + +Historical example, from back when Snail relied on CSTParser: + +``` +@with_pkg_env (@__DIR__) begin + # list all external dependency imports here (from the appropriate Project.toml, either Snail's or an extension's): + import CSTParser + # check for dependency API compatibility + !isdefined(CSTParser, :iscall) && + throw(ArgumentError("CSTParser API not compatible, must install Snail-specific version")) +end +``` +""" macro with_pkg_env(dir, action) :( try @@ -50,23 +86,6 @@ macro with_pkg_env(dir, action) ) end -@with_pkg_env (@__DIR__) begin - # list all external dependency imports here (from the appropriate Project.toml, either Snail's or an extension's): - import CSTParser - # check for dependency API compatibility - !isdefined(CSTParser, :iscall) && - throw(ArgumentError("CSTParser API not compatible, must install Snail-specific version")) -end - - -import Markdown -import Printf -import REPL -import Sockets -import REPL.REPLCompletions - -export start, stop - ### --- configuration @@ -470,125 +489,209 @@ function replcompletion(identifier, mod) end - ### --- CSTParser wrappers + ### --- JuliaSyntax wrappers -module CST +module JStx import Base64 -import CSTParser +import Base.JuliaSyntax as JS """ -Helper function: wraps the parser interface. +Parse a Base64-encoded buffer containing Julia code using JuliaSyntax. + +Returns a parsed syntax tree or nothing if parsing fails. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code """ function parse(encodedbuf) - cst = nothing try buf = String(Base64.base64decode(encodedbuf)) - cst = CSTParser.parse(buf, true) + JS.parseall(JS.SyntaxNode, buf; ignore_errors=true) catch err - # probably an IO problem - # TODO: Need better error reporting here. println(err) - return [] + return nothing end - return cst end """ -Return the path of CSTParser.EXPR objects from the root of the CST to the -expression at the offset, while retaining the EXPR objects' full locations. +Return the path from root to the node containing the given byte offset. -The path is represented as an array of named tuples. The first element -represents the root node, and the last element is the expression at the offset. -Each tuple has a start and stop value, showing locations in the original source -where that node begins and ends. +Traverses the syntax tree to find nodes containing the offset position, +building a path of nodes from root to leaf. -NB: The locations represent bytes, not characters! +# Arguments +- `node`: Root JuliaSyntax.SyntaxNode to start traversal from +- `offset`: Byte offset to search for in the syntax tree +- `path`: Accumulator for the path being built (internal use) -This is necessary because CSTParser does not include full location data, see -https://github.com/julia-vscode/CSTParser.jl/pull/80. +Returns an array of named tuples containing: +- expr: The syntax node +- start: Starting byte position +- stop: Ending byte position """ -function pathat(cst, offset, pos = 0, path = [(expr=cst, start=1, stop=cst.span+1)]) - if cst !== nothing && !CSTParser.isnonstdid(cst) - for a in cst - if pos < offset <= (pos + a.span) - return pathat(a, offset, pos, [path; [(expr=a, start=pos+1, stop=pos+a.span+1)]]) +function pathat(node::JS.SyntaxNode, offset, path = [(expr=node, start=JS.first_byte(node), stop=JS.last_byte(node))]) + if JS.haschildren(node) + for child in JS.children(node) + if JS.first_byte(child) <= offset <= JS.last_byte(child) + return pathat(child, offset, + [path; [(expr=child, start=JS.first_byte(child), stop=JS.last_byte(child))]]) end - # jump forward by fullspan since we need to skip over a's trailing whitespace - pos += a.fullspan end - elseif (pos < offset <= (pos + cst.fullspan)) - return [path; [(expr=cst, start=pos+1, stop=offset+1)]] end return path end """ -Debugging helper: example code for traversing the CST and tracking the location of each node. +Extract the name from a syntax node based on its kind. + +Handles various node types including: +- Modules +- Functions +- Structs (including generic parameters) +- Primitive types +- Abstract types +- Macros + +Returns the extracted name as a string, or nothing if the node type is not supported +or does not contain a name. + +# Arguments +- `node`: JuliaSyntax.SyntaxNode to extract name from """ -function print_cst(cst) - offset = 0 - helper = (node) -> begin - for a in node - if a.args === nothing - val = CSTParser.valof(a) - # Unicode byte fix - if String == typeof(val) - diff = sizeof(val) - length(val) - a.span -= diff - a.fullspan -= diff - end - println(offset, ":", offset+a.span, "\t", val) - offset += a.fullspan - else - helper(a) - end +function nodename(node::JS.SyntaxNode) + kind = JS.kind(node) + children = JS.children(node) + + if kind == JS.K"module" + return string(children[1]) + + elseif kind == JS.K"function" + first = children[1] + if JS.kind(first) == JS.K"call" + # function load(filepath) end + return string(JS.children(first)[1]) + elseif JS.kind(first) == JS.K"::" + # function load(filepath)::DF.DataFrame end + return string(first[1][1]) + else + # TODO: This is likely wrong in some cases. + return string(first) + end + + elseif kind == JS.K"struct" + # Handle generic type parameters + if JS.kind(children[1]) == JS.K"curly" + curly_children = JS.children(children[1]) + return string(curly_children[1]) # The actual type name + else + return string(children[1]) # No type parameters + end + + elseif kind == JS.K"primitive" + # Support both these cases: + # primitive type Point24 24 end + # primitive type Int8 <: Integer 8 end + grandchildren = JS.children(children[1]) + if length(grandchildren) > 0 + return string(grandchildren[1]) + else + return string(children[1]) + end + + elseif kind == JS.K"abstract" + # Support both these cases: + # abstract type AbstractPoint1 end + # abstract type AbstractPoint2 <: Number end + grandchildren = JS.children(children[1]) + if length(grandchildren) > 0 + return string(grandchildren[1]) + else + return string(children[1]) + end + + elseif kind == JS.K"macro" + first = children[1] + if JS.kind(first) == JS.K"call" + return string(JS.children(first)[1]) + else + return string(first) end end - helper(cst) - return offset + + return nothing end """ -Return the module active at point as a list of their names. +Find the module context at a given byte location in code. + +Parses the code and returns a list of module names that enclose the given position, +from outermost to innermost. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code +- `byteloc`: Byte offset to find module context for + +Returns an Elisp-compatible list starting with :list followed by module names. """ function moduleat(encodedbuf, byteloc) - cst = parse(encodedbuf) - path = pathat(cst, byteloc) + tree = parse(encodedbuf) + path = pathat(tree, byteloc) modules = [] for node in path - if CSTParser.defines_module(node.expr) - push!(modules, CSTParser.valof(CSTParser.get_name(node.expr))) + if JS.kind(node.expr) == JS.K"module" + push!(modules, nodename(node.expr)) end end return [:list; modules] end """ -Return information about the block at point. +Find information about the code block at a given byte location. + +Parses the code and returns details about the enclosing block (function, struct, etc.) +at the specified position. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code +- `byteloc`: Byte offset to find block information for + +Returns an Elisp-compatible list containing: +- :list symbol +- Tuple of enclosing module names +- Starting byte position +- Ending byte position +- Block description (e.g. function name) + +Returns nothing if no block is found at the location. """ function blockat(encodedbuf, byteloc) - cst = parse(encodedbuf) - path = pathat(cst, byteloc) + tree = parse(encodedbuf) + path = pathat(tree, byteloc) modules = [] description = nothing start = nothing stop = nothing for node in path - if CSTParser.defines_module(node.expr) + if JS.kind(node.expr) == JS.K"module" description = nothing - push!(modules, CSTParser.valof(CSTParser.get_name(node.expr))) - elseif (isnothing(description) && - (CSTParser.defines_abstract(node.expr) || - CSTParser.defines_datatype(node.expr) || - CSTParser.defines_function(node.expr) || - CSTParser.defines_macro(node.expr) || - CSTParser.defines_mutable(node.expr) || - CSTParser.defines_primitive(node.expr) || - CSTParser.defines_struct(node.expr))) - description = CSTParser.valof(CSTParser.get_name(node.expr)) - start = node.start - stop = node.stop + push!(modules, nodename(node.expr)) + elseif isnothing(description) + if JS.kind(node.expr) ∈ [JS.K"function", JS.K"macro", + JS.K"struct", + JS.K"abstract", JS.K"primitive"] + description = nodename(node.expr) + start = node.start + stop = node.stop + 1 + elseif JS.kind(node.expr) == JS.K"=" + # Check for function assignment like f() = ... + children = JS.children(node.expr) + if length(children) >= 1 && JS.kind(children[1]) == JS.K"call" + description = string(JS.children(children[1])[1]) + start = node.start + stop = node.stop + 1 + end + end end end # result format equivalent to what Elisp side expects @@ -598,82 +701,66 @@ function blockat(encodedbuf, byteloc) end """ -Internal: assemble a human-readable function signature from a CSTParser node. -""" -function fnsig_helper(node) - fnsig = "" - reassemble = (n) -> begin - for x in n - if x.args === nothing - candidate = CSTParser.valof(CSTParser.get_name(x)) - if candidate !== nothing && length(candidate) > 0 - if "," == candidate - candidate *= " " - end - fnsig *= candidate - end - else - reassemble(x) - end - end - end - reassemble(node) - return fnsig -end - -""" -For a given buffer, return the overall tree structure of the code. - -Result structure: [ - [type name location extra] -] -where type is :function, :macro, etc.; when type is :module, then extra is a -nested resulting structure. +Generate a tree representation of code structure. + +Parses the code and builds a tree showing the hierarchical structure of: +- Modules +- Functions (with signatures) +- Structs +- Types (abstract and primitive) +- Macros + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code + +Returns an Elisp-compatible nested list structure starting with :list, +followed by tuples for each code element containing: +- Element type (:module, :function, :struct, :type, :macro) +- Name +- Byte position +- Nested elements (for modules) + +Returns nothing if no structure is found. """ function codetree(encodedbuf) - cst = parse(encodedbuf) - offset = 1 + tree = parse(encodedbuf) helper = (node, depth = 1) -> begin res = [] - for a in node - if a.args === nothing - val = CSTParser.valof(a) - # XXX: We want bytes here because we convert back to position - # numbers on the Emacs side. So we do not apply the Unicode byte fix - # from print_cst. - offset += a.fullspan - else - curroffset = offset - aname = CSTParser.valof(CSTParser.get_name(a)) - helper_res = helper(a, depth + 1) - if aname !== nothing - if CSTParser.defines_module(a) - push!(res, (:module, aname, curroffset, helper_res)) - elseif CSTParser.defines_function(a) - fnsig = fnsig_helper(a.args[1]) - push!(res, (:function, fnsig, curroffset)) - elseif CSTParser.defines_struct(a) || CSTParser.defines_mutable(a) - push!(res, (:struct, aname, curroffset)) - elseif CSTParser.defines_abstract(a) || CSTParser.defines_datatype(a) || CSTParser.defines_primitive(a) - push!(res, (:type, aname, curroffset)) - elseif CSTParser.defines_macro(a) - push!(res, (:macro, aname, curroffset)) - end - else - # XXX: Flatten on the fly. First, avoid empty entries. Second, - # use append! since only modules should generate nesting. - if length(helper_res) > 0 - append!(res, helper_res) + for child in JS.children(node) + kind = JS.kind(child) + name = nodename(child) + if name !== nothing + if kind == JS.K"module" + push!(res, (:module, name, JS.first_byte(child), helper(child, depth + 1))) + elseif kind == JS.K"function" + # Reconstruct function signature from the call node + if JS.haschildren(child) && JS.kind(JS.children(child)[1]) == JS.K"call" + call_node = JS.children(child)[1] + sig = String(JS.sourcetext(call_node)) + push!(res, (:function, sig, JS.first_byte(child))) + else + push!(res, (:function, name * "()", JS.first_byte(child))) end + elseif kind ∈ (JS.K"struct", JS.K"mutable") + push!(res, (:struct, name, JS.first_byte(child))) + elseif kind ∈ (JS.K"abstract", JS.K"primitive") + push!(res, (:type, name, JS.first_byte(child))) + elseif kind == JS.K"macro" + push!(res, (:macro, name, JS.first_byte(child))) + end + else + # Flatten results from nested nodes that aren't modules + if JS.haschildren(child) + append!(res, helper(child, depth + 1)) end end end return res end - tree = helper(cst) - return isempty(tree) ? + tree_result = helper(tree) + return isempty(tree_result) ? nothing : - [:list; tree] + [:list; tree_result] end """ @@ -683,49 +770,71 @@ Result structure: { filename -> [module names] } -This nasty code relies on internal CSTParser representations of things. -Unfortunately, CSTParser doesn't provide a clean API for this sort of thing: -https://github.com/julia-vscode/CSTParser.jl/issues/56 +Uses JuliaSyntax to parse the code and find include statements within modules. +Find all include() statements and their enclosing module contexts. + +Parses the code and builds a mapping of included files to their module contexts. + +# Arguments +- `encodedbuf`: Base64-encoded string containing Julia source code +- `path`: Base path to resolve relative include paths against (default: "") + +Returns an Elisp-compatible plist alternating between: +- Full path to included file +- List of enclosing module names at the include point + +Returns nothing if no includes are found. """ function includesin(encodedbuf, path="") - cst = parse(encodedbuf) - results = Dict() - # walk across args, and track the current module - # when a node of type "call" is found, check its args[1] - helper = (node, modules = []) -> begin - for a in node - if (CSTParser.iscall(a) && - "include" == CSTParser.valof(a.args[1])) - # a.args[2] is the file name being included - filename = joinpath(path, CSTParser.valof(a.args[2])) - results[filename] = modules - elseif CSTParser.defines_module(a) - helper(a, [modules; CSTParser.valof(CSTParser.get_name(a))]) - elseif !isnothing(a.args) - helper(a, modules) + tree = parse(encodedbuf) + results = Dict{String,Vector{String}}() + + helper = (node, modules = Symbol[]) -> begin + if JS.haschildren(node) + for child in JS.children(node) + kind = JS.kind(child) + + # Track module context + if kind == JS.K"module" + name = nodename(child) + if name !== nothing + new_modules = [modules; String(name)] + helper(child, new_modules) + end + continue + end + + # Check for include calls + if kind == JS.K"call" && length(JS.children(child)) >= 2 + call_name = JS.children(child)[1] + if String(JS.sourcetext(call_name)) == "include" + # Get filename from the first argument + filename_node = JS.children(child)[2] + filename = String(JS.sourcetext(filename_node)) + # Remove quotes + filename = replace(filename, r"^\"(.*)\"$" => s"\1") + filename = joinpath(path, filename) + # Store with current module context + results[filename] = String.(copy(modules)) + end + end + + # Recurse into other nodes + helper(child, modules) end end end - helper(cst) - # convert to a plist for returning back to Emacs + + helper(tree) + + # Convert to plist for Emacs reslist = [] for (file, modules) in results push!(reslist, file) push!(reslist, [:list; modules]) end - return isempty(reslist) ? - nothing : - [:list; reslist] -end -# XXX: Dirty hackery to improve perceived startup performance follows. Running -# this function in a separate thread should, in theory, force a bunch of things -# to JIT-compile in the background before the user notices. -# Thank you very much, time-to-first-plot problem! -function forcecompile() - # call these functions before the user does - includesin(Base64.base64encode("module Alpha\ninclude(\"a.jl\")\nend")) - moduleat(Base64.base64encode("module Alpha\nend"), 1) + return isempty(reslist) ? nothing : [:list; reslist] end end @@ -856,6 +965,13 @@ during evaluation are captured and sent back to the client as Elisp s-expressions. Special queries also write back their responses as s-expressions. """ function start(port=10011; addr="127.0.0.1") + if VERSION < v"1.10" + # JuliaSyntax only ships with 1.10. + # This is an exception-throwing error, and it's placed here so users have + # a chance to see it before the terminal closes. + error("ERROR: Julia Snail now requires Julia 1.10 or higher") + end + global running = false global server_socket = Sockets.listen(Sockets.IPv4(addr), port) let wait_result = timedwait(function(); server_socket.status == Base.StatusActive; end, @@ -873,17 +989,6 @@ function start(port=10011; addr="127.0.0.1") println(stderr, "ERROR: Snail will not work correctly.") end end - # XXX: It would be great to do this forcecompile trick in a Thread.@spawn - # block. Unfortunately, as of Julia 1.6.1 this does not work. First, it's - # meaningless unless Julia is started with JULIA_NUM_THREADS or --threads set - # to some number >1 (not the default). Second, and worse, for some reason, - # the spawned thread _still_ blocks the main thread. Non-working code below: - # if VERSION >= v"1.3" && Threads.nthreads() > 1 - # Threads.@spawn begin - # CST.forcecompile() - # end - # end - CST.forcecompile() # main loop: @async begin while running @@ -990,8 +1095,8 @@ function send_to_client(expr, client_socket=nothing) # force the user to choose the client socket options = map( function(cs) - gsn = Sockets.getpeername(cs) - Printf.@sprintf("%s:%d", gsn[1], gsn[2]) + gsn = Sockets.getpeername(cs) + Printf.@sprintf("%s:%d", gsn[1], gsn[2]) end, client_sockets ) @@ -1010,5 +1115,4 @@ function send_to_client(expr, client_socket=nothing) println(client_socket, expr) end - end diff --git a/Project.toml b/Project.toml index 4e7f05e..e69de29 100644 --- a/Project.toml +++ b/Project.toml @@ -1,5 +0,0 @@ -[deps] -CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f" - -[compat] -CSTParser = "~3.4.2" diff --git a/README.md b/README.md index 9a0a92d..615ce41 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Refer to the [changelog](https://github.com/gcv/julia-snail/blob/master/CHANGELO - **Multimedia and plotting:** Snail can display Julia graphics in graphical Emacs instances using packages like [Plots](http://juliaplots.org) and [Gadfly](http://gadflyjl.org). - **Cross-referencing:** Snail is integrated with the built-in Emacs [xref](https://www.gnu.org/software/emacs/manual/html_node/emacs/Xref.html) system. When a Snail session is active, it supports jumping to definitions of functions and macros loaded in the session. - **Completion:** Snail is also integrated with the built-in Emacs [completion-at-point](https://www.gnu.org/software/emacs/manual/html_node/elisp/Completion-in-Buffers.html) facility. Provided it is configured with the `company-capf` backend, [company-mode](http://company-mode.github.io/) completion will also work (this should be the case by default). -- **Parser:** Snail uses [CSTParser](https://github.com/julia-vscode/CSTParser.jl), a full-featured Julia parser, to infer the structure of source files and to enable features which require an understanding of code context, especially the module in which a particular piece of code lives. This enables awareness of the current module for completion and cross-referencing purposes. +- **Parser:** Snail uses [JuliaSyntax](https://github.com/JuliaLang/JuliaSyntax.jl) to infer the structure of source files and to enable features which require an understanding of code context, especially the module in which a particular piece of code lives. This enables awareness of the current module for completion and cross-referencing purposes. ## Demo @@ -62,9 +62,12 @@ https://user-images.githubusercontent.com/10327/128589405-7368bb50-0ef3-4003-b5d ## Installation -Julia versions >1.6.0 should all work with Snail. +Snail now requires Julia version >1.10.0. -Snail’s Julia-side dependencies will automatically be installed when it starts, and will stay out of your way using Julia’s [`LOAD_PATH` mechanism](https://docs.julialang.org/en/v1/base/constants/#Base.LOAD_PATH). +> [!IMPORTANT] +> [Previous versions of Snail](https://github.com/gcv/julia-snail/tree/1.3.2) supported Julia >1.6.0, and should continue to be used in environments that cannot upgrade Julia. Note that older versions can no longer be installed using MELPA or MELPA Stable, and have to be installed either manually or using a Git-enabled Emacs package manager, such as [Elpaca](https://github.com/progfolio/elpaca). + +Core Snail has no Julia-side dependencies. Any Snail extensions that use Julia dependencies will automatically them on startup, and will stay out of your way using Julia’s [`LOAD_PATH` mechanism](https://docs.julialang.org/en/v1/base/constants/#Base.LOAD_PATH). On the Emacs side, you must install one of the supported high-performance terminal emulators to use with the Julia REPL: either `vterm` or [Eat](https://codeberg.org/akib/emacs-eat). diff --git a/extensions/ob-julia/ob-julia.el b/extensions/ob-julia/ob-julia.el index 9261068..7846f55 100644 --- a/extensions/ob-julia/ob-julia.el +++ b/extensions/ob-julia/ob-julia.el @@ -155,7 +155,7 @@ your org notebook" (inner-module (with-temp-buffer (let* ((julia-snail-repl-buffer jsrb-save)) (insert contents) - (julia-snail--cst-module-at (current-buffer) pt))))) + (julia-snail--parser-module-at (current-buffer) pt))))) (if inner-module (append src-module inner-module) src-module))) diff --git a/julia-snail.el b/julia-snail.el index 67df386..999500a 100644 --- a/julia-snail.el +++ b/julia-snail.el @@ -329,6 +329,12 @@ nil means disable Snail-specific imenu integration (fall back on julia-mode impl (defvar-local julia-snail--imenu-cache nil) +(defvar julia-snail--parser-module-cache (make-hash-table :test #'equal) + "Cache for module-at-point lookups. Keys are (buffer-hash . line-number) pairs.") + +(defvar julia-snail--parser-module-cache-counter 0 + "Counter for cache cleanup. Incremented on each cache miss.") + (defvar julia-snail--compilation-regexp-alist '(;; matches "while loading /tmp/Foo.jl, in expression starting on line 2" (julia-load-error . ("while loading \\([^ ><()\t\n,'\";:]+\\), in expression starting on line \\([0-9]+\\)" 1 2)) @@ -493,7 +499,7 @@ Returns nil if the poll timed out, t otherwise." (defun julia-snail--capture-basedir (buf) (julia-snail--send-to-server :Main - "normpath(joinpath(VERSION <= v\"0.7-\" ? JULIA_HOME : Sys.BINDIR, Base.DATAROOTDIR, \"julia\", \"base\"))" + "normpath(joinpath(Sys.BINDIR, Base.DATAROOTDIR, \"julia\", \"base\"))" :repl-buf buf :async nil)) @@ -803,8 +809,8 @@ returns \"/home/username/file.jl\"." (setq julia-snail--process netstream) (set-process-filter julia-snail--process #'julia-snail--server-response-filter) ;; TODO: Implement a sanity check on the Julia environment. Not - ;; sure how. But a failed dependency load (like CSTParser) will - ;; leave Snail in a bad state. + ;; sure how. But a failed Julia dependency load will leave Snail + ;; in a bad state. (message "Successfully connected to Snail server in Julia REPL") ;; Query base directory, and cache (puthash process-buf (julia-snail--capture-basedir repl-buf) @@ -1151,36 +1157,75 @@ evaluated in the context of MODULE." (julia-snail--response-base reqid)) - ;;; --- CST parser interface - -(defun julia-snail--cst-module-at (buf pt) - (let* ((byteloc (position-bytes pt)) - (encoded (julia-snail--encode-base64 buf)) - (res (julia-snail--send-to-server - :Main - (format "JuliaSnail.CST.moduleat(\"%s\", %d)" encoded byteloc) - :async nil))) - (if (eq res :nothing) - nil + ;;; --- parser interface + +(defun julia-snail--clean-module-cache () + "Remove expired entries from the module cache." + (let ((current-time (float-time))) + (cl-loop for key being the hash-keys of julia-snail--parser-module-cache + using (hash-values value) + when (> (- current-time (car value)) 5.0) + do (remhash key julia-snail--parser-module-cache)))) + +(defun julia-snail--parser-module-at (buf pt) + "Return the Julia module at point PT in buffer BUF. +Uses a cache that expires entries after 5 seconds, and keyed to +BUF and the line number at PT. This cache structure helps this +function withstand heavy use by completion frameworks (Company, +Corfu): they call repeatedly out to the Julia completion system, +which needs to know the current module. The current module will +not have changed as the user is typing on the same line, but the +`module-at` machinery will be called a lot. There's room for +improvement here. For example, instead of expiring the cache 5 +seconds later, it can just expire when the user moves to a +different line." + (let* ((line-num (line-number-at-pos pt buf)) + (cache-key (cons buf line-num)) + (cached-entry (gethash cache-key julia-snail--parser-module-cache)) + (current-time (float-time)) + (byteloc (position-bytes pt)) + res) + ;; use cached entry if it exists and hasn't expired (5 second TTL) + (if (and cached-entry + (< (- current-time (car cached-entry)) 5.0)) + ;; return cached value + (cdr cached-entry) + ;; cache miss or expired entry: perform the lookup + (setq res (let* ((encoded (julia-snail--encode-base64 buf)) + (result (julia-snail--send-to-server + :Main + (format "JuliaSnail.JStx.moduleat(\"%s\", %d)" encoded byteloc) + :async nil))) + (if (eq result :nothing) + nil + result))) + ;; store the result in the cache with current timestamp: + (puthash cache-key (cons current-time res) julia-snail--parser-module-cache) + ;; increment counter and clean expired entries every 50 calls: + (cl-incf julia-snail--parser-module-cache-counter) + (when (>= julia-snail--parser-module-cache-counter 50) + (setq julia-snail--parser-module-cache-counter 0) + (julia-snail--clean-module-cache)) + ;; done res))) -(defun julia-snail--cst-block-at (buf pt) +(defun julia-snail--parser-block-at (buf pt) (let* ((byteloc (position-bytes pt)) (encoded (julia-snail--encode-base64 buf)) (res (julia-snail--send-to-server :Main - (format "JuliaSnail.CST.blockat(\"%s\", %d)" encoded byteloc) + (format "JuliaSnail.JStx.blockat(\"%s\", %d)" encoded byteloc) :async nil))) (if (eq res :nothing) nil res))) -(defun julia-snail--cst-includes (buf) +(defun julia-snail--parser-includes (buf) (let* ((encoded (julia-snail--encode-base64 buf)) (pwd (file-name-directory (julia-snail--efn (buffer-file-name buf)))) (res (julia-snail--send-to-server :Main - (format "JuliaSnail.CST.includesin(\"%s\", \"%s\")" encoded pwd) + (format "JuliaSnail.JStx.includesin(\"%s\", \"%s\")" encoded pwd) :async nil)) (includes (make-hash-table :test #'equal))) (unless (eq res :nothing) @@ -1189,11 +1234,11 @@ evaluated in the context of MODULE." ;; TODO: Maybe there's a situation in which returning :error is appropriate? includes)) -(defun julia-snail--cst-code-tree (buf) +(defun julia-snail--parser-code-tree (buf) (let* ((encoded (julia-snail--encode-base64 buf)) (res (julia-snail--send-to-server :Main - (format "JuliaSnail.CST.codetree(\"%s\")" encoded) + (format "JuliaSnail.JStx.codetree(\"%s\")" encoded) :async nil))) res)) @@ -1230,7 +1275,7 @@ evaluated in the context of MODULE." (defun julia-snail--module-at-point (&optional partial-module) "Return the current Julia module at point as an Elisp list, including PARTIAL-MODULE if given." (let ((partial-module (or partial-module - (julia-snail--cst-module-at (current-buffer) (point)))) + (julia-snail--parser-module-at (current-buffer) (point)))) (module-for-file (julia-snail--module-for-file (buffer-file-name (buffer-base-buffer))))) (or (if module-for-file (append module-for-file partial-module) @@ -1399,23 +1444,24 @@ evaluated in the context of MODULE." (julia-snail--imenu-helper (cdr tree) modules))))) (defun julia-snail--imenu-included-module-helper (normal-tree) - ;; XXX: This fugly kludge transforms imenu trees in a way that injects modules - ;; cached through include()ed files at the root: + ;; This function transforms imenu trees in a way that injects modules cached + ;; through include()ed files at the root: ;; ;; if included-modules is ("Alpha" "Bravo") ;; and normal-tree is ((function "f1" ...)) ;; then this returns (("Alpha" ("Bravo" (function "f1" ...)))) - ;; - ;; There must be a cleaner way to implement this logic. (let ((included-modules (julia-snail--module-for-file (buffer-file-name (buffer-base-buffer))))) - (cl-labels ((some-helper - (incls norms first-time) - (if (null incls) - norms - (let* ((next (some-helper (cdr incls) norms nil)) - (tail (cons (car incls) (if first-time(list next) next)))) - (if first-time (list tail) tail))))) - (some-helper included-modules normal-tree t)))) + (if (null included-modules) + ;; no included modules, return original tree + normal-tree + ;; recursively build the nested tree: + (cl-labels ((build-nested-tree + (modules content) + (if (null modules) + content + (list (cons (car modules) + (build-nested-tree (cdr modules) content)))))) + (build-nested-tree included-modules normal-tree))))) (cl-defun julia-snail-imenu () ;; exit early if Snail's imenu integration is turned off, or no Snail session is running @@ -1434,7 +1480,7 @@ evaluated in the context of MODULE." (cl-return-from julia-snail-imenu (julia-snail--imenu-cache-entry-value julia-snail--imenu-cache)))) ;; cache miss: ask Julia to parse the file and return the imenu index - (let* ((code-tree (julia-snail--cst-code-tree (current-buffer))) + (let* ((code-tree (julia-snail--parser-code-tree (current-buffer))) (imenu-index-raw (julia-snail--imenu-included-module-helper (julia-snail--imenu-helper code-tree (list)))) (imenu-index (if (eq :flat julia-snail-imenu-style) (-flatten imenu-index-raw) @@ -1736,7 +1782,7 @@ activation: This occurs in the context of the current module. Currently only works on blocks terminated with `end'." (interactive) - (let* ((q (julia-snail--cst-block-at (current-buffer) (point))) + (let* ((q (julia-snail--parser-block-at (current-buffer) (point))) (filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) (module (julia-snail--module-at-point (-first-item q))) (block-start (byte-to-position (or (-second-item q) -1))) @@ -1780,7 +1826,7 @@ This will occur in the context of the Main module, just as it would at the REPL. (let* ((jsrb-save julia-snail-repl-buffer) ; save for callback context (filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) (module (or (julia-snail--module-for-file filename) '("Main"))) - (includes (julia-snail--cst-includes (current-buffer)))) + (includes (julia-snail--parser-includes (current-buffer)))) (when (or (not (buffer-modified-p)) (y-or-n-p (format "'%s' is not saved, send to Julia anyway? " filename))) (julia-snail--send-to-server @@ -1793,7 +1839,7 @@ This will occur in the context of the Main module, just as it would at the REPL. ;; julia-snail-repl-buffer will have disappeared (let* ((julia-snail-repl-buffer jsrb-save) (repl-buf (get-buffer julia-snail-repl-buffer))) - ;; NB: At the moment, julia-snail--cst-includes + ;; NB: At the moment, julia-snail--parser-includes ;; does not return :error. However, it might in ;; the future, and this code will then be useful. (if (eq :error includes) @@ -1822,7 +1868,7 @@ This will occur in the context of the Main module, just as it would at the REPL. "Analyze the current buffer's file for include statements" (interactive) (let* ((filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) - (includes (julia-snail--cst-includes (current-buffer)))) + (includes (julia-snail--parser-includes (current-buffer)))) (when (or (not (buffer-modified-p)) (y-or-n-p (format "'%s' is not saved, analyze in Julia anyway? " filename))) (if (eq :error includes) @@ -1916,7 +1962,7 @@ autocompletion aware of the available modules." (interactive) (let* ((filename (julia-snail--efn (buffer-file-name (buffer-base-buffer)))) (module (or (julia-snail--module-for-file filename) '("Main"))) - (includes (julia-snail--cst-includes (current-buffer)))) + (includes (julia-snail--parser-includes (current-buffer)))) (julia-snail--module-merge-includes filename includes) (message "Caches updated: parent module %s" (julia-snail--construct-module-path module)))) diff --git a/tests/files/codetree.jl b/tests/files/codetree.jl new file mode 100644 index 0000000..bb0f34d --- /dev/null +++ b/tests/files/codetree.jl @@ -0,0 +1,23 @@ +module Alpha + +module Bravo + +function f1(x) + 2x +end + +module Charlie + +end + +end + +module Delta + +end + +end + +module Echo + +end diff --git a/tests/files/s1.jl b/tests/files/s1.jl new file mode 100644 index 0000000..a594c05 --- /dev/null +++ b/tests/files/s1.jl @@ -0,0 +1 @@ +function f1(); end diff --git a/tests/files/s2.jl b/tests/files/s2.jl new file mode 100644 index 0000000..43bd00e --- /dev/null +++ b/tests/files/s2.jl @@ -0,0 +1,9 @@ +module Alpha +function f1(); end +end +function f2(); end +module Bravo +module Charlie +function f3(); end +end +end diff --git a/tests/files/s3.jl b/tests/files/s3.jl new file mode 100644 index 0000000..34cbbf5 --- /dev/null +++ b/tests/files/s3.jl @@ -0,0 +1,4 @@ +module Geometry +area_circle(radius) = π * r^2 +end +f1() = "f1" diff --git a/tests/files/s4.jl b/tests/files/s4.jl new file mode 100644 index 0000000..e10537b --- /dev/null +++ b/tests/files/s4.jl @@ -0,0 +1,4 @@ +module Alpha +include("a1.jl") +include("a2.jl") +end diff --git a/tests/files/s5.jl b/tests/files/s5.jl new file mode 100644 index 0000000..5b1b4a2 --- /dev/null +++ b/tests/files/s5.jl @@ -0,0 +1,5 @@ +module Alpha +module Bravo +include("a1.jl") +end +end diff --git a/tests/files/s6.jl b/tests/files/s6.jl new file mode 100644 index 0000000..2c357f4 --- /dev/null +++ b/tests/files/s6.jl @@ -0,0 +1,47 @@ +# Nested functions +function outer() + function inner(x) + x + 1 + end + inner(5) +end + +# Type parameters +struct GenericPoint{T} + x::T + y::T +end + +# Plain struct +struct S10 + x + y +end + +# Abstract types +abstract type AbstractPoint1 end +abstract type AbstractPoint2 <: Number end + +# Primitive types +primitive type Point24 24 end +primitive type Int8 <: Integer 8 end + +# Multiple definitions +function overloaded() 1 end +function overloaded(x) x end +function overloaded(x,y) x+y end + +# Macro definition +macro sayhello(name) + return :( println("Hello, ", $name) ) +end + +# Function with type tags +function load(filepath)::DF.DataFrame +end + +function load2(filepath) +end + +function load3(filepath::String)::String +end diff --git a/tests/implicit-modules.el b/tests/implicit-modules.el index 26ad03e..9762870 100644 --- a/tests/implicit-modules.el +++ b/tests/implicit-modules.el @@ -42,7 +42,7 @@ (js-with-julia-session repl-buf (julia-snail) (with-current-buffer source-buf - (let ((includes (julia-snail--cst-includes (get-buffer (current-buffer))))) + (let ((includes (julia-snail--parser-includes (get-buffer (current-buffer))))) (should (equal '("MyModule") diff --git a/tests/parser.jl b/tests/parser.jl index 4664e9b..62fe6f3 100644 --- a/tests/parser.jl +++ b/tests/parser.jl @@ -1,76 +1,84 @@ import Base64 -using Test - - -include("../JuliaSnail.jl") @testset "parser interface" begin - # TODO: Move these samples into supporting files: - - s1 = Base64.base64encode(""" -function f1(); end -""") - - s2 = Base64.base64encode(""" -module Alpha -function f1(); end -end -function f2(); end -module Bravo -module Charlie -function f3(); end -end -end -""") - - s3 = Base64.base64encode(""" -module Geometry -area_circle(radius) = π * r^2 -end -f1() = "f1" -""") - - s4 = Base64.base64encode(""" -module Alpha -include("a1.jl") -include("a2.jl") -end -""") - - s5 = Base64.base64encode(""" -module Alpha -module Bravo -include("a1.jl") -end -end -""") + s1 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s1.jl"), String)) + s2 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s2.jl"), String)) + s3 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s3.jl"), String)) + s4 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s4.jl"), String)) + s5 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s5.jl"), String)) + s6 = Base64.base64encode(read(joinpath(@__DIR__, "files", "s6.jl"), String)) @testset "module detection" begin - @test [:list] == JuliaSnail.CST.moduleat(s1, 0) - @test [:list] == JuliaSnail.CST.moduleat(s1, 10) - @test [:list] == JuliaSnail.CST.moduleat(s2, 0) - @test [:list, "Alpha"] == JuliaSnail.CST.moduleat(s2, 20) - @test [:list] == JuliaSnail.CST.moduleat(s2, 50) - @test [:list, "Bravo", "Charlie"] == JuliaSnail.CST.moduleat(s2, 90) - @test [:list, "Geometry"] == JuliaSnail.CST.moduleat(s3, 45) - @test [:list, "Geometry"] == JuliaSnail.CST.moduleat(s3, 50) - @test [:list] == JuliaSnail.CST.moduleat(s3, 51) + @test [:list] == JuliaSnail.JStx.moduleat(s1, 0) + @test [:list] == JuliaSnail.JStx.moduleat(s1, 10) + @test [:list] == JuliaSnail.JStx.moduleat(s2, 0) + @test [:list, "Alpha"] == JuliaSnail.JStx.moduleat(s2, 20) + @test [:list] == JuliaSnail.JStx.moduleat(s2, 50) + @test [:list, "Bravo", "Charlie"] == JuliaSnail.JStx.moduleat(s2, 90) + @test [:list, "Geometry"] == JuliaSnail.JStx.moduleat(s3, 45) + @test [:list, "Geometry"] == JuliaSnail.JStx.moduleat(s3, 50) + @test [:list] == JuliaSnail.JStx.moduleat(s3, 51) end @testset "block detection" begin - @test [:list, (), 1, 19, "f1"] == JuliaSnail.CST.blockat(s1, 3) - @test [:list, tuple("Alpha"), 14, 32, "f1"] == JuliaSnail.CST.blockat(s2, 27) - @test [:list, (), 37, 55, "f2"] == JuliaSnail.CST.blockat(s2, 50) - @test [:list, ("Bravo", "Charlie"), 84, 102, "f3"] == JuliaSnail.CST.blockat(s2, 97) - @test [:list, tuple("Geometry"), 17, 47, "area_circle"] == JuliaSnail.CST.blockat(s3, 25) + @test [:list, (), 1, 19, "f1"] == JuliaSnail.JStx.blockat(s1, 3) + @test [:list, tuple("Alpha"), 14, 32, "f1"] == JuliaSnail.JStx.blockat(s2, 27) + @test [:list, (), 37, 55, "f2"] == JuliaSnail.JStx.blockat(s2, 50) + @test [:list, ("Bravo", "Charlie"), 84, 102, "f3"] == JuliaSnail.JStx.blockat(s2, 97) + @test [:list, tuple("Geometry"), 17, 47, "area_circle"] == JuliaSnail.JStx.blockat(s3, 25) + end + + @testset "code tree" begin + buf = Base64.base64encode(read(joinpath(@__DIR__, "files", "codetree.jl"), String)) + @test Any[:list, + (:module, "Alpha", 1, Any[ + (:module, "Bravo", 15, Any[ + (:function, "f1(x)", 29), + (:module, "Charlie", 54, Any[])]), + (:module, "Delta", 80, Any[])]), + (:module, "Echo", 104, Any[])] == + JuliaSnail.JStx.codetree(buf) end @testset "include detection" begin # more tests in implicit-modules.el - @test [:list, "a1.jl", [:list, "Alpha"], "a2.jl", [:list, "Alpha"]] == JuliaSnail.CST.includesin(s4) - @test [:list, "a1.jl", [:list, "Alpha", "Bravo"]] == JuliaSnail.CST.includesin(s5) + @test [:list, "a1.jl", [:list, "Alpha"], "a2.jl", [:list, "Alpha"]] == JuliaSnail.JStx.includesin(s4) + @test [:list, "a1.jl", [:list, "Alpha", "Bravo"]] == JuliaSnail.JStx.includesin(s5) + end + + @testset "Special syntax cases" begin + # Test nested function (only outer) + @test [:list, (), 20, 97, "outer"] == JuliaSnail.JStx.blockat(s6, 70) + @test [:list, (), 20, 97, "outer"] == JuliaSnail.JStx.blockat(s6, 85) + + # Test struct with parameters + @test [:list, (), 117, 161, "GenericPoint"] == JuliaSnail.JStx.blockat(s6, 150) + + # Test plain struct + @test [:list, (), 178, 202, "S10"] == JuliaSnail.JStx.blockat(s6, 190) + + # Test abstract type + @test [:list, (), 221, 253, "AbstractPoint1"] == JuliaSnail.JStx.blockat(s6, 235) + @test [:list, (), 254, 296, "AbstractPoint2"] == JuliaSnail.JStx.blockat(s6, 268) + + # Test primitive types + @test [:list, (), 316, 345, "Point24"] == JuliaSnail.JStx.blockat(s6, 331) + @test [:list, (), 346, 382, "Int8"] == JuliaSnail.JStx.blockat(s6, 361) + + # Test multiple definitions + @test [:list, (), 407, 434, "overloaded"] == JuliaSnail.JStx.blockat(s6, 416) + @test [:list, (), 435, 463, "overloaded"] == JuliaSnail.JStx.blockat(s6, 444) + @test [:list, (), 464, 496, "overloaded"] == JuliaSnail.JStx.blockat(s6, 473) + + # Test macro + @test [:list, (), 517, 582, "sayhello"] == JuliaSnail.JStx.blockat(s6, 523) + + # Test function with return type + @test [:list, (), 610, 651, "load"] == JuliaSnail.JStx.blockat(s6, 619) + @test [:list, (), 653, 681, "load2"] == JuliaSnail.JStx.blockat(s6, 662) + @test [:list, (), 683, 727, "load3"] == JuliaSnail.JStx.blockat(s6, 692) end end diff --git a/tests/tests.jl b/tests/tests.jl new file mode 100644 index 0000000..dcbd4f4 --- /dev/null +++ b/tests/tests.jl @@ -0,0 +1,14 @@ +# running the tests: +# $ julia/tests.jl +# or: +# julia> include("tests/tests.jl") + +module JuliaSnailTests + +using Test + +include("../JuliaSnail.jl") + +include("./parser.jl") + +end