diff --git a/CHANGELOG.md b/CHANGELOG.md index 884c2565e..afa1b720b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,46 @@ * Non-breaking changes: + Supports parsing the arguments' names for `func` and `service` (init args). + + Supports collecting line comments as doc comments in the following cases: + - above services: + ``` + // This is a valid doc comment for the service + service : { + greet : (text) -> (text); + } + ``` + - above actor methods: + ``` + service : { + // This is a valid doc comment for greet + greet : (text) -> (text); + } + ``` + - above type declarations: + ``` + // This is a valid doc comment for type A + type A = record { + my_field : text; + }; + ``` + - above record and variant fields: + ``` + type A = record { + // This is a valid doc comment for my_field + my_field : text; + }; + + type B = record { + // This is a valid doc comment for nat element + nat; + text; + } + + type C = variant { + // This is a valid doc comment for my_variant_field + my_variant_field : nat; + }; + ``` ### candid_derive diff --git a/rust/candid/src/types/syntax.rs b/rust/candid/src/types/syntax.rs index 1f943ea2d..179e17ea9 100644 --- a/rust/candid/src/types/syntax.rs +++ b/rust/candid/src/types/syntax.rs @@ -94,6 +94,7 @@ impl IDLArgType { pub struct TypeField { pub label: Label, pub typ: IDLType, + pub docs: Vec, } #[derive(Debug)] @@ -107,12 +108,19 @@ pub enum Dec { pub struct Binding { pub id: String, pub typ: IDLType, + pub docs: Vec, } -#[derive(Debug, Default)] +#[derive(Debug)] +pub struct IDLActorType { + pub typ: IDLType, + pub docs: Vec, +} + +#[derive(Debug)] pub struct IDLProg { pub decs: Vec, - pub actor: Option, + pub actor: Option, } #[derive(Debug)] diff --git a/rust/candid_parser/src/grammar.lalrpop b/rust/candid_parser/src/grammar.lalrpop index dd22238eb..926cf875c 100644 --- a/rust/candid_parser/src/grammar.lalrpop +++ b/rust/candid_parser/src/grammar.lalrpop @@ -1,12 +1,12 @@ use super::test::{Assert, Input, Test}; -use super::token::{Token, error, error2, LexicalError, Span}; +use super::token::{Token, error, error2, LexicalError, Span, TriviaMap}; use candid::{Principal, types::Label}; -use candid::types::syntax::{IDLType, PrimType, TypeField, FuncType, Binding, Dec, IDLProg, IDLTypes, IDLInitArgs, IDLArgType}; +use candid::types::syntax::{IDLType, PrimType, TypeField, FuncType, Binding, Dec, IDLProg, IDLTypes, IDLInitArgs, IDLArgType, IDLActorType}; use candid::types::value::{IDLField, IDLValue, IDLArgs, VariantValue}; use candid::types::{TypeEnv, FuncMode}; use candid::utils::check_unique; -grammar; +grammar(trivia: Option<&TriviaMap>); extern { type Location = usize; @@ -174,7 +174,7 @@ pub Typ: IDLType = { Label::Unnamed(_) => { id = id + 1; Label::Unnamed(id - 1) }, ref l => { id = l.get_id() + 1; l.clone() }, }; - TypeField { label, typ: f.typ.clone() } + TypeField { label, typ: f.typ.clone(), docs: f.docs.clone() } }).collect(); fs.sort_unstable_by_key(|TypeField { label, .. }| label.get_id()); check_unique(fs.iter().map(|f| &f.label)).map_err(|e| error2(e, span))?; @@ -202,19 +202,19 @@ PrimTyp: IDLType = { } FieldTyp: TypeField = { - ":" =>? Ok(TypeField { label: Label::Id(n), typ: t }), - ":" => TypeField { label: Label::Named(n), typ: t }, + ":" =>? Ok(TypeField { label: Label::Id(id), typ, docs: doc_comment.unwrap_or_default() }), + ":" => TypeField { label: Label::Named(n), typ, docs: doc_comment.unwrap_or_default() }, } RecordFieldTyp: TypeField = { FieldTyp => <>, - Typ => TypeField { label: Label::Unnamed(0), typ: <> }, + => TypeField { label: Label::Unnamed(0), typ, docs: doc_comment.unwrap_or_default() }, } VariantFieldTyp: TypeField = { FieldTyp => <>, - Name => TypeField { label: Label::Named(<>), typ: IDLType::PrimT(PrimType::Null) }, - FieldId =>? Ok(TypeField { label: Label::Id(<>), typ: IDLType::PrimT(PrimType::Null) }), + => TypeField { label: Label::Named(n), typ: IDLType::PrimT(PrimType::Null), docs: doc_comment.unwrap_or_default() }, + =>? Ok(TypeField { label: Label::Id(id), typ: IDLType::PrimT(PrimType::Null), docs: doc_comment.unwrap_or_default() }), } ArgTupTyp: Vec = "(" > ")" =>? { @@ -253,13 +253,13 @@ ActorTyp: Vec = { } MethTyp: Binding = { - ":" => Binding { id: n, typ: IDLType::FuncT(f) }, - ":" => Binding { id: n, typ: IDLType::VarT(id) }, + ":" => Binding { id: n, typ: IDLType::FuncT(f), docs: doc_comment.unwrap_or_default() }, + ":" => Binding { id: n, typ: IDLType::VarT(id), docs: doc_comment.unwrap_or_default() }, } // Type declarations Def: Dec = { - "type" "=" => Dec::TypD(Binding { id: id, typ: t }), + "type" "=" => Dec::TypD(Binding { id: id, typ: t, docs: doc_comment.unwrap_or_default() }), "import" => Dec::ImportType(<>), "import" "service" => Dec::ImportServ(<>), } @@ -269,9 +269,9 @@ Actor: IDLType = { "id" => IDLType::VarT(<>), } -MainActor: IDLType = { - "service" "id"? ":" ";"? => <>, - "service" "id"? ":" "->" ";"? => IDLType::ClassT(args, Box::new(t)), +MainActor: IDLActorType = { + "service" "id"? ":" ";"? => IDLActorType { typ: t, docs: doc_comment.unwrap_or_default() }, + "service" "id"? ":" "->" ";"? => IDLActorType { typ: IDLType::ClassT(args, Box::new(t)), docs: doc_comment.unwrap_or_default() }, } pub IDLProg: IDLProg = { @@ -327,3 +327,9 @@ SepBy: Vec = { #[inline] Sp: (T, Span) = => (t, l..r); + +#[inline] +DocComment: Option> = + => { + trivia.and_then(|t| t.borrow().get(&l).cloned()) + }; diff --git a/rust/candid_parser/src/lib.rs b/rust/candid_parser/src/lib.rs index 4086ac3c8..8aa133eab 100644 --- a/rust/candid_parser/src/lib.rs +++ b/rust/candid_parser/src/lib.rs @@ -143,31 +143,33 @@ pub mod random; pub mod test; pub fn parse_idl_prog(str: &str) -> Result { - let lexer = token::Tokenizer::new(str); - Ok(grammar::IDLProgParser::new().parse(lexer)?) + let trivia = token::TriviaMap::default(); + let lexer = token::Tokenizer::new_with_trivia(str, trivia.clone()); + let res = grammar::IDLProgParser::new().parse(Some(&trivia.clone()), lexer)?; + Ok(res) } pub fn parse_idl_init_args(str: &str) -> Result { let lexer = token::Tokenizer::new(str); - Ok(grammar::IDLInitArgsParser::new().parse(lexer)?) + Ok(grammar::IDLInitArgsParser::new().parse(None, lexer)?) } pub fn parse_idl_type(str: &str) -> Result { let lexer = token::Tokenizer::new(str); - Ok(grammar::TypParser::new().parse(lexer)?) + Ok(grammar::TypParser::new().parse(None, lexer)?) } pub fn parse_idl_types(str: &str) -> Result { let lexer = token::Tokenizer::new(str); - Ok(grammar::TypsParser::new().parse(lexer)?) + Ok(grammar::TypsParser::new().parse(None, lexer)?) } pub fn parse_idl_args(s: &str) -> crate::Result { let lexer = token::Tokenizer::new(s); - Ok(grammar::ArgsParser::new().parse(lexer)?) + Ok(grammar::ArgsParser::new().parse(None, lexer)?) } pub fn parse_idl_value(s: &str) -> crate::Result { let lexer = token::Tokenizer::new(s); - Ok(grammar::ArgParser::new().parse(lexer)?) + Ok(grammar::ArgParser::new().parse(None, lexer)?) } diff --git a/rust/candid_parser/src/test.rs b/rust/candid_parser/src/test.rs index 9f3dbd970..498c7e9f1 100644 --- a/rust/candid_parser/src/test.rs +++ b/rust/candid_parser/src/test.rs @@ -78,7 +78,7 @@ impl std::str::FromStr for Test { type Err = Error; fn from_str(str: &str) -> std::result::Result { let lexer = super::token::Tokenizer::new(str); - Ok(super::grammar::TestParser::new().parse(lexer)?) + Ok(super::grammar::TestParser::new().parse(None, lexer)?) } } diff --git a/rust/candid_parser/src/token.rs b/rust/candid_parser/src/token.rs index db7ae2d61..021ae3f87 100644 --- a/rust/candid_parser/src/token.rs +++ b/rust/candid_parser/src/token.rs @@ -1,10 +1,15 @@ +use std::{cell::RefCell, collections::HashMap, mem, rc::Rc}; + use lalrpop_util::ParseError; use logos::{Lexer, Logos}; +const LINE_COMMENT_PREFIX: &str = "//"; + #[derive(Logos, Debug, Clone, PartialEq, Eq, Ord, PartialOrd)] #[logos(skip r"[ \t\r\n]+")] -#[logos(skip r"//[^\n]*")] // line comment pub enum Token { + #[regex(r"//[^\n]*")] // must start with `LINE_COMMENT_PREFIX` + LineComment, #[token("/*")] StartComment, #[token("=")] @@ -118,22 +123,43 @@ fn parse_number(lex: &mut Lexer) -> String { } } +fn parse_doc_comment(lex: &Lexer) -> String { + lex.slice() + .trim_start_matches(LINE_COMMENT_PREFIX) + .trim() + .to_string() +} + +pub type TriviaMap = Rc>>>; + pub struct Tokenizer<'input> { lex: Lexer<'input, Token>, + comment_buffer: Vec, + trivia: Option, } + impl<'input> Tokenizer<'input> { pub fn new(input: &'input str) -> Self { let lex = Token::lexer(input); - Tokenizer { lex } + Tokenizer { + lex, + comment_buffer: vec![], + trivia: None, + } + } + + pub fn new_with_trivia(input: &'input str, trivia: TriviaMap) -> Self { + let lex = Token::lexer(input); + Tokenizer { + lex, + comment_buffer: vec![], + trivia: Some(trivia), + } } } pub type Span = std::ops::Range; -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Spanned { - pub span: Span, - pub value: T, -} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct LexicalError { pub err: String, @@ -179,6 +205,13 @@ impl Iterator for Tokenizer<'_> { let err = format!("Unknown token {}", self.lex.slice()); Some(Err(LexicalError::new(err, span))) } + Ok(Token::LineComment) => { + let content = parse_doc_comment(&self.lex); + if self.trivia.is_some() { + self.comment_buffer.push(content.to_string()); + } + self.next() + } Ok(Token::StartComment) => { let mut lex = self.lex.to_owned().morph::(); let mut nesting = 1; @@ -278,7 +311,15 @@ impl Iterator for Tokenizer<'_> { self.lex = lex.morph::(); Some(Ok((span.start, Token::Text(result), self.lex.span().end))) } - Ok(token) => Some(Ok((span.start, token, span.end))), + Ok(token) => { + if let Some(trivia) = &mut self.trivia { + if !self.comment_buffer.is_empty() { + let content: Vec = mem::take(&mut self.comment_buffer); + trivia.borrow_mut().insert(span.start, content); + } + } + Some(Ok((span.start, token, span.end))) + } } } } diff --git a/rust/candid_parser/src/typing.rs b/rust/candid_parser/src/typing.rs index a4a7d7daf..f78333583 100644 --- a/rust/candid_parser/src/typing.rs +++ b/rust/candid_parser/src/typing.rs @@ -1,6 +1,8 @@ use crate::{parse_idl_prog, pretty_parse_idl_prog, Error, Result}; use candid::types::{ - syntax::{Binding, Dec, IDLArgType, IDLInitArgs, IDLProg, IDLType, PrimType, TypeField}, + syntax::{ + Binding, Dec, IDLActorType, IDLArgType, IDLInitArgs, IDLProg, IDLType, PrimType, TypeField, + }, ArgType, Field, Function, Type, TypeEnv, TypeInner, }; use candid::utils::check_unique; @@ -141,7 +143,7 @@ fn check_meths(env: &Env, ms: &[Binding]) -> Result> { fn check_defs(env: &mut Env, decs: &[Dec]) -> Result<()> { for dec in decs.iter() { match dec { - Dec::TypD(Binding { id, typ }) => { + Dec::TypD(Binding { id, typ, docs: _ }) => { let t = check_type(env, typ)?; env.te.0.insert(id.to_string(), t); } @@ -176,7 +178,7 @@ fn check_cycle(env: &TypeEnv) -> Result<()> { fn check_decs(env: &mut Env, decs: &[Dec]) -> Result<()> { for dec in decs.iter() { - if let Dec::TypD(Binding { id, typ: _ }) = dec { + if let Dec::TypD(Binding { id, .. }) = dec { let duplicate = env.te.0.insert(id.to_string(), TypeInner::Unknown.into()); if duplicate.is_some() { return Err(Error::msg(format!("duplicate binding for {id}"))); @@ -191,8 +193,8 @@ fn check_decs(env: &mut Env, decs: &[Dec]) -> Result<()> { Ok(()) } -fn check_actor(env: &Env, actor: &Option) -> Result> { - match actor { +fn check_actor(env: &Env, actor: &Option) -> Result> { + match actor.as_ref().map(|a| &a.typ) { None => Ok(None), Some(IDLType::ClassT(ts, t)) => { let mut args = Vec::new(); diff --git a/rust/candid_parser/tests/parse_type.rs b/rust/candid_parser/tests/parse_type.rs index 3129c4afd..2385b9a32 100644 --- a/rust/candid_parser/tests/parse_type.rs +++ b/rust/candid_parser/tests/parse_type.rs @@ -1,4 +1,5 @@ use candid::pretty::candid::compile; +use candid::types::syntax::{Dec, IDLType}; use candid::types::TypeEnv; use candid_parser::bindings::{javascript, motoko, rust, typescript}; use candid_parser::configs::Configs; @@ -12,8 +13,13 @@ use std::path::Path; fn test_parse_idl_prog() { let prog = r#" import "test.did"; +// Doc comment for my_type type my_type = principal; -type List = opt record { head: int; tail: List }; +type List = opt record { + // Doc comment for List.head + head: int; + tail: List +}; type f = func (List, func (int32) -> (int64)) -> (opt List); type broker = service { find : (name: text) -> @@ -21,14 +27,88 @@ type broker = service { }; type nested = record { nat; nat; record { nat; 0x2a:nat; nat8; }; 42:nat; 40:nat; variant{ A; 0x2a; B; C }; }; +// Doc comment for service service server : { + // Doc comment for f f : (test: blob, opt bool) -> () oneway; g : (my_type, List, opt List) -> (int) query; h : (vec opt text, variant { A: nat; B: opt text }, opt List) -> (record { id: nat; 0x2a: record {} }); i : f; } "#; - parse_idl_prog(prog).unwrap(); + let ast = parse_idl_prog(prog).unwrap(); + + // Assert doc comments + let actor = ast.actor.unwrap(); + assert_eq!(actor.docs, vec!["Doc comment for service"]); + + let IDLType::ServT(methods) = &actor.typ else { + panic!("actor is not a service"); + }; + assert_eq!(methods[0].docs, vec!["Doc comment for f"]); + assert!(methods[1].docs.is_empty()); + assert!(methods[2].docs.is_empty()); + assert!(methods[3].docs.is_empty()); + + let my_type = ast + .decs + .iter() + .find_map(|dec| { + if let Dec::TypD(binding) = dec { + if binding.id == "my_type" { + Some(binding) + } else { + None + } + } else { + None + } + }) + .unwrap(); + assert_eq!(my_type.docs, vec!["Doc comment for my_type"]); + + let list = ast + .decs + .iter() + .find_map(|dec| { + if let Dec::TypD(binding) = dec { + if binding.id == "List" { + Some(binding) + } else { + None + } + } else { + None + } + }) + .unwrap(); + match &list.typ { + IDLType::OptT(list_inner) => { + let IDLType::RecordT(fields) = list_inner.as_ref() else { + panic!("inner is not a record"); + }; + assert_eq!(fields[0].docs, vec!["Doc comment for List.head"]); + assert!(fields[1].docs.is_empty()); + } + _ => panic!("list is not an opt"), + } + + let nested = ast + .decs + .iter() + .find_map(|dec| { + if let Dec::TypD(binding) = dec { + if binding.id == "nested" { + Some(binding) + } else { + None + } + } else { + None + } + }) + .unwrap(); + assert!(nested.docs.is_empty()); } #[test_generator::test_resources("rust/candid_parser/tests/assets/*.did")]