feat: support line comments as doc comments

CHANGELOG.md

@@ -46,6 +46,46 @@
 
 * Non-breaking changes:
   + Supports parsing the arguments' names for `func` and `service` (init args).
+  + Supports collecting line comments as doc comments in the following cases:
+    - above services:
+      ```
+      // This is a valid doc comment for the service
+      service : {
+        greet : (text) -> (text);
+      }
+      ```
+    - above actor methods:
+      ```
+      service : {
+        // This is a valid doc comment for greet
+        greet : (text) -> (text);
+      }
+      ```
+    - above type declarations:
+      ```
+      // This is a valid doc comment for type A
+      type A = record {
+        my_field : text;
+      };
+      ```
+    - above record and variant fields:
+      ```
+      type A = record {
+        // This is a valid doc comment for my_field
+        my_field : text;
+      };
+
+      type B = record {
+        // This is a valid doc comment for nat element
+        nat;
+        text;
+      }
+
+      type C = variant {
+        // This is a valid doc comment for my_variant_field
+        my_variant_field : nat;
+      };
+      ```
 
 ### candid_derive
 

rust/candid/src/types/syntax.rs

@@ -94,6 +94,7 @@ impl IDLArgType {
 pub struct TypeField {
     pub label: Label,
     pub typ: IDLType,
+    pub docs: Vec<String>,
 }
 
 #[derive(Debug)]
@@ -107,12 +108,19 @@ pub enum Dec {
 pub struct Binding {
     pub id: String,
     pub typ: IDLType,
+    pub docs: Vec<String>,
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug)]
+pub struct IDLActorType {
+    pub typ: IDLType,
+    pub docs: Vec<String>,
+}
+
+#[derive(Debug)]
 pub struct IDLProg {
     pub decs: Vec<Dec>,
-    pub actor: Option<IDLType>,
+    pub actor: Option<IDLActorType>,
 }
 
 #[derive(Debug)]

rust/candid_parser/src/grammar.lalrpop

@@ -1,12 +1,12 @@
 use super::test::{Assert, Input, Test};
-use super::token::{Token, error, error2, LexicalError, Span};
+use super::token::{Token, error, error2, LexicalError, Span, TriviaMap};
 use candid::{Principal, types::Label};
-use candid::types::syntax::{IDLType, PrimType, TypeField, FuncType, Binding, Dec, IDLProg, IDLTypes, IDLInitArgs, IDLArgType};
+use candid::types::syntax::{IDLType, PrimType, TypeField, FuncType, Binding, Dec, IDLProg, IDLTypes, IDLInitArgs, IDLArgType, IDLActorType};
 use candid::types::value::{IDLField, IDLValue, IDLArgs, VariantValue};
 use candid::types::{TypeEnv, FuncMode};
 use candid::utils::check_unique;
 
-grammar;
+grammar(trivia: Option<&TriviaMap>);
 
 extern {
     type Location = usize;
@@ -174,7 +174,7 @@ pub Typ: IDLType = {
               Label::Unnamed(_) => { id = id + 1; Label::Unnamed(id - 1) },
               ref l => { id = l.get_id() + 1; l.clone() },
           };
-          TypeField { label, typ: f.typ.clone() }
+          TypeField { label, typ: f.typ.clone(), docs: f.docs.clone() }
         }).collect();
         fs.sort_unstable_by_key(|TypeField { label, .. }| label.get_id());
         check_unique(fs.iter().map(|f| &f.label)).map_err(|e| error2(e, span))?;
@@ -202,19 +202,19 @@ PrimTyp: IDLType = {
 }
 
 FieldTyp: TypeField = {
-    <n:FieldId> ":" <t:Typ> =>? Ok(TypeField { label: Label::Id(n), typ: t }),
-    <n:Name> ":" <t:Typ> => TypeField { label: Label::Named(n), typ: t },
+    <doc_comment:DocComment> <id:FieldId> ":" <typ:Typ> =>? Ok(TypeField { label: Label::Id(id), typ, docs: doc_comment.unwrap_or_default() }),
+    <doc_comment:DocComment> <n:Name> ":" <typ:Typ> => TypeField { label: Label::Named(n), typ, docs: doc_comment.unwrap_or_default() },
 }
 
 RecordFieldTyp: TypeField = {
     FieldTyp => <>,
-    Typ => TypeField { label: Label::Unnamed(0), typ: <> },
+    <doc_comment:DocComment> <typ:Typ> => TypeField { label: Label::Unnamed(0), typ, docs: doc_comment.unwrap_or_default() },
 }
 
 VariantFieldTyp: TypeField = {
     FieldTyp => <>,
-    Name => TypeField { label: Label::Named(<>), typ: IDLType::PrimT(PrimType::Null) },
-    FieldId =>? Ok(TypeField { label: Label::Id(<>), typ: IDLType::PrimT(PrimType::Null) }),
+    <doc_comment:DocComment> <n:Name> => TypeField { label: Label::Named(n), typ: IDLType::PrimT(PrimType::Null), docs: doc_comment.unwrap_or_default() },
+    <doc_comment:DocComment> <id:FieldId> =>? Ok(TypeField { label: Label::Id(id), typ: IDLType::PrimT(PrimType::Null), docs: doc_comment.unwrap_or_default() }),
 }
 
 ArgTupTyp: Vec<IDLArgType> = "(" <SepBy<ArgTyp, ",">> ")" =>? {
@@ -253,13 +253,13 @@ ActorTyp: Vec<Binding> = {
 }
 
 MethTyp: Binding = {
-    <n:Name> ":" <f:FuncTyp> => Binding { id: n, typ: IDLType::FuncT(f) },
-    <n:Name> ":" <id:"id"> => Binding { id: n, typ: IDLType::VarT(id) },
+    <doc_comment: DocComment> <n:Name> ":" <f:FuncTyp> => Binding { id: n, typ: IDLType::FuncT(f), docs: doc_comment.unwrap_or_default() },
+    <doc_comment: DocComment> <n:Name> ":" <id:"id"> => Binding { id: n, typ: IDLType::VarT(id), docs: doc_comment.unwrap_or_default() },
 }
 
 // Type declarations
 Def: Dec = {
-    "type" <id:"id"> "=" <t:Typ> => Dec::TypD(Binding { id: id, typ: t }),
+    <doc_comment: DocComment> "type" <id:"id"> "=" <t:Typ> => Dec::TypD(Binding { id: id, typ: t, docs: doc_comment.unwrap_or_default() }),
     "import" <Text> => Dec::ImportType(<>),
     "import" "service" <Text> => Dec::ImportServ(<>),
 }
@@ -269,9 +269,9 @@ Actor: IDLType = {
     "id" => IDLType::VarT(<>),
 }
 
-MainActor: IDLType = {
-    "service" "id"? ":" <t:Actor> ";"? => <>,
-    "service" "id"? ":" <args:ArgTupTyp> "->" <t:Actor> ";"? => IDLType::ClassT(args, Box::new(t)),
+MainActor: IDLActorType = {
+    <doc_comment: DocComment> "service" "id"? ":" <t:Actor> ";"? => IDLActorType { typ: t, docs: doc_comment.unwrap_or_default() },
+    <doc_comment: DocComment> "service" "id"? ":" <args:ArgTupTyp> "->" <t:Actor> ";"? => IDLActorType { typ: IDLType::ClassT(args, Box::new(t)), docs: doc_comment.unwrap_or_default() },
 }
 
 pub IDLProg: IDLProg = {
@@ -327,3 +327,9 @@ SepBy<T, S>: Vec<T> = {
 #[inline]
 Sp<T>: (T, Span) =
     <l: @L> <t: T> <r: @R> => (t, l..r);
+
+#[inline]
+DocComment: Option<Vec<String>> =
+    <l: @L> => {
+      trivia.and_then(|t| t.borrow().get(&l).cloned())
+    };

rust/candid_parser/src/lib.rs

@@ -143,31 +143,33 @@ pub mod random;
 pub mod test;
 
 pub fn parse_idl_prog(str: &str) -> Result<candid::types::syntax::IDLProg> {
-    let lexer = token::Tokenizer::new(str);
-    Ok(grammar::IDLProgParser::new().parse(lexer)?)
+    let trivia = token::TriviaMap::default();
+    let lexer = token::Tokenizer::new_with_trivia(str, trivia.clone());
+    let res = grammar::IDLProgParser::new().parse(Some(&trivia.clone()), lexer)?;
+    Ok(res)
 }
 
 pub fn parse_idl_init_args(str: &str) -> Result<candid::types::syntax::IDLInitArgs> {
     let lexer = token::Tokenizer::new(str);
-    Ok(grammar::IDLInitArgsParser::new().parse(lexer)?)
+    Ok(grammar::IDLInitArgsParser::new().parse(None, lexer)?)
 }
 
 pub fn parse_idl_type(str: &str) -> Result<candid::types::syntax::IDLType> {
     let lexer = token::Tokenizer::new(str);
-    Ok(grammar::TypParser::new().parse(lexer)?)
+    Ok(grammar::TypParser::new().parse(None, lexer)?)
 }
 
 pub fn parse_idl_types(str: &str) -> Result<candid::types::syntax::IDLTypes> {
     let lexer = token::Tokenizer::new(str);
-    Ok(grammar::TypsParser::new().parse(lexer)?)
+    Ok(grammar::TypsParser::new().parse(None, lexer)?)
 }
 
 pub fn parse_idl_args(s: &str) -> crate::Result<candid::IDLArgs> {
     let lexer = token::Tokenizer::new(s);
-    Ok(grammar::ArgsParser::new().parse(lexer)?)
+    Ok(grammar::ArgsParser::new().parse(None, lexer)?)
 }
 
 pub fn parse_idl_value(s: &str) -> crate::Result<candid::IDLValue> {
     let lexer = token::Tokenizer::new(s);
-    Ok(grammar::ArgParser::new().parse(lexer)?)
+    Ok(grammar::ArgParser::new().parse(None, lexer)?)
 }

rust/candid_parser/src/test.rs

@@ -78,7 +78,7 @@ impl std::str::FromStr for Test {
     type Err = Error;
     fn from_str(str: &str) -> std::result::Result<Self, Self::Err> {
         let lexer = super::token::Tokenizer::new(str);
-        Ok(super::grammar::TestParser::new().parse(lexer)?)
+        Ok(super::grammar::TestParser::new().parse(None, lexer)?)
     }
 }
 

rust/candid_parser/src/token.rs

@@ -1,10 +1,15 @@
+use std::{cell::RefCell, collections::HashMap, mem, rc::Rc};
+
 use lalrpop_util::ParseError;
 use logos::{Lexer, Logos};
 
+const LINE_COMMENT_PREFIX: &str = "//";
+
 #[derive(Logos, Debug, Clone, PartialEq, Eq, Ord, PartialOrd)]
 #[logos(skip r"[ \t\r\n]+")]
-#[logos(skip r"//[^\n]*")] // line comment
 pub enum Token {
+    #[regex(r"//[^\n]*")] // must start with `LINE_COMMENT_PREFIX`
+    LineComment,
     #[token("/*")]
     StartComment,
     #[token("=")]
@@ -118,22 +123,43 @@ fn parse_number(lex: &mut Lexer<Token>) -> String {
     }
 }
 
+fn parse_doc_comment(lex: &Lexer<Token>) -> String {
+    lex.slice()
+        .trim_start_matches(LINE_COMMENT_PREFIX)
+        .trim()
+        .to_string()
+}
+
+pub type TriviaMap = Rc<RefCell<HashMap<usize, Vec<String>>>>;
+
 pub struct Tokenizer<'input> {
     lex: Lexer<'input, Token>,
+    comment_buffer: Vec<String>,
+    trivia: Option<TriviaMap>,
 }
+
 impl<'input> Tokenizer<'input> {
     pub fn new(input: &'input str) -> Self {
         let lex = Token::lexer(input);
-        Tokenizer { lex }
+        Tokenizer {
+            lex,
+            comment_buffer: vec![],
+            trivia: None,
+        }
+    }
+
+    pub fn new_with_trivia(input: &'input str, trivia: TriviaMap) -> Self {
+        let lex = Token::lexer(input);
+        Tokenizer {
+            lex,
+            comment_buffer: vec![],
+            trivia: Some(trivia),
+        }
     }
 }
 
 pub type Span = std::ops::Range<usize>;
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct Spanned<T> {
-    pub span: Span,
-    pub value: T,
-}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct LexicalError {
     pub err: String,
@@ -179,6 +205,13 @@ impl Iterator for Tokenizer<'_> {
                 let err = format!("Unknown token {}", self.lex.slice());
                 Some(Err(LexicalError::new(err, span)))
             }
+            Ok(Token::LineComment) => {
+                let content = parse_doc_comment(&self.lex);
+                if self.trivia.is_some() {
+                    self.comment_buffer.push(content.to_string());
+                }
+                self.next()
+            }
             Ok(Token::StartComment) => {
                 let mut lex = self.lex.to_owned().morph::<Comment>();
                 let mut nesting = 1;
@@ -278,7 +311,15 @@ impl Iterator for Tokenizer<'_> {
                 self.lex = lex.morph::<Token>();
                 Some(Ok((span.start, Token::Text(result), self.lex.span().end)))
             }
-            Ok(token) => Some(Ok((span.start, token, span.end))),
+            Ok(token) => {
+                if let Some(trivia) = &mut self.trivia {
+                    if !self.comment_buffer.is_empty() {
+                        let content: Vec<String> = mem::take(&mut self.comment_buffer);
+                        trivia.borrow_mut().insert(span.start, content);
+                    }
+                }
+                Some(Ok((span.start, token, span.end)))
+            }
         }
     }
 }

rust/candid_parser/src/typing.rs

@@ -1,6 +1,8 @@
 use crate::{parse_idl_prog, pretty_parse_idl_prog, Error, Result};
 use candid::types::{
-    syntax::{Binding, Dec, IDLArgType, IDLInitArgs, IDLProg, IDLType, PrimType, TypeField},
+    syntax::{
+        Binding, Dec, IDLActorType, IDLArgType, IDLInitArgs, IDLProg, IDLType, PrimType, TypeField,
+    },
     ArgType, Field, Function, Type, TypeEnv, TypeInner,
 };
 use candid::utils::check_unique;
@@ -141,7 +143,7 @@ fn check_meths(env: &Env, ms: &[Binding]) -> Result<Vec<(String, Type)>> {
 fn check_defs(env: &mut Env, decs: &[Dec]) -> Result<()> {
     for dec in decs.iter() {
         match dec {
-            Dec::TypD(Binding { id, typ }) => {
+            Dec::TypD(Binding { id, typ, docs: _ }) => {
                 let t = check_type(env, typ)?;
                 env.te.0.insert(id.to_string(), t);
             }
@@ -176,7 +178,7 @@ fn check_cycle(env: &TypeEnv) -> Result<()> {
 
 fn check_decs(env: &mut Env, decs: &[Dec]) -> Result<()> {
     for dec in decs.iter() {
-        if let Dec::TypD(Binding { id, typ: _ }) = dec {
+        if let Dec::TypD(Binding { id, .. }) = dec {
             let duplicate = env.te.0.insert(id.to_string(), TypeInner::Unknown.into());
             if duplicate.is_some() {
                 return Err(Error::msg(format!("duplicate binding for {id}")));
@@ -191,8 +193,8 @@ fn check_decs(env: &mut Env, decs: &[Dec]) -> Result<()> {
     Ok(())
 }
 
-fn check_actor(env: &Env, actor: &Option<IDLType>) -> Result<Option<Type>> {
-    match actor {
+fn check_actor(env: &Env, actor: &Option<IDLActorType>) -> Result<Option<Type>> {
+    match actor.as_ref().map(|a| &a.typ) {
         None => Ok(None),
         Some(IDLType::ClassT(ts, t)) => {
             let mut args = Vec::new();