clippy_dev: parsing revamp 1/N

clippy_dev/src/deprecate_lint.rs

@@ -1,4 +1,5 @@
-use crate::update_lints::{DeprecatedLint, Lint, find_lint_decls, generate_lint_files, read_deprecated_lints};
+use crate::parse::{DeprecatedLint, Lint, find_lint_decls, read_deprecated_lints};
+use crate::update_lints::generate_lint_files;
 use crate::utils::{UpdateMode, Version};
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
@@ -14,10 +15,6 @@ use std::{fs, io};
 ///
 /// If a file path could not read from or written to
 pub fn deprecate(clippy_version: Version, name: &str, reason: &str) {
-    if let Some((prefix, _)) = name.split_once("::") {
-        panic!("`{name}` should not contain the `{prefix}` prefix");
-    }
-
     let mut lints = find_lint_decls();
     let (mut deprecated_lints, renamed_lints) = read_deprecated_lints();
 
@@ -135,14 +132,14 @@ fn remove_lint_declaration(name: &str, path: &Path, lints: &mut Vec<Lint>) -> io
             );
 
             assert!(
-                content[lint.declaration_range.clone()].contains(&name.to_uppercase()),
+                content[lint.declaration_range].contains(&name.to_uppercase()),
                 "error: `{}` does not contain lint `{}`'s declaration",
                 path.display(),
                 lint.name
             );
 
             // Remove lint declaration (declare_clippy_lint!)
-            content.replace_range(lint.declaration_range.clone(), "");
+            content.replace_range(lint.declaration_range, "");
 
             // Remove the module declaration (mod xyz;)
             let mod_decl = format!("\nmod {name};");

clippy_dev/src/lib.rs

@@ -1,9 +1,11 @@
 #![feature(
-    rustc_private,
     exit_status_error,
     if_let_guard,
+    new_range,
+    new_range_api,
     os_str_slice,
     os_string_truncate,
+    rustc_private,
     slice_split_once
 )]
 #![warn(
@@ -34,3 +36,5 @@ pub mod update_lints;
 
 mod utils;
 pub use utils::{ClippyInfo, UpdateMode};
+
+mod parse;

clippy_dev/src/main.rs

@@ -7,7 +7,6 @@ use clippy_dev::{
     ClippyInfo, UpdateMode, deprecate_lint, dogfood, fmt, lint, new_lint, release, rename_lint, serve, setup, sync,
     update_lints,
 };
-use std::convert::Infallible;
 use std::env;
 
 fn main() {
@@ -95,6 +94,20 @@ fn main() {
     }
 }
 
+fn lint_name(name: &str) -> Result<String, String> {
+    let name = name.replace('-', "_");
+    if let Some((pre, _)) = name.split_once("::") {
+        Err(format!("lint name should not contain the `{pre}` prefix"))
+    } else if name
+        .bytes()
+        .any(|x| !matches!(x, b'_' | b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z'))
+    {
+        Err("lint name contains invalid characters".to_owned())
+    } else {
+        Ok(name)
+    }
+}
+
 #[derive(Parser)]
 #[command(name = "dev", about)]
 struct Dev {
@@ -150,7 +163,7 @@ enum DevCommand {
         #[arg(
             short,
             long,
-            value_parser = |name: &str| Ok::<_, Infallible>(name.replace('-', "_")),
+            value_parser = lint_name,
         )]
         /// Name of the new lint in snake case, ex: `fn_too_long`
         name: String,
@@ -223,8 +236,12 @@ enum DevCommand {
     /// Rename a lint
     RenameLint {
         /// The name of the lint to rename
+        #[arg(value_parser = lint_name)]
         old_name: String,
-        #[arg(required_unless_present = "uplift")]
+        #[arg(
+            required_unless_present = "uplift",
+            value_parser = lint_name,
+        )]
         /// The new name of the lint
         new_name: Option<String>,
         #[arg(long)]
@@ -234,6 +251,7 @@ enum DevCommand {
     /// Deprecate the given lint
     Deprecate {
         /// The name of the lint to deprecate
+        #[arg(value_parser = lint_name)]
         name: String,
         #[arg(long, short)]
         /// The reason for deprecation

clippy_dev/src/new_lint.rs

@@ -1,4 +1,5 @@
-use crate::utils::{RustSearcher, Token, Version};
+use crate::parse::cursor::{self, Capture, Cursor};
+use crate::utils::Version;
 use clap::ValueEnum;
 use indoc::{formatdoc, writedoc};
 use std::fmt::{self, Write as _};
@@ -516,22 +517,22 @@ fn setup_mod_file(path: &Path, lint: &LintData<'_>) -> io::Result<&'static str>
 // Find both the last lint declaration (declare_clippy_lint!) and the lint pass impl
 fn parse_mod_file(path: &Path, contents: &str) -> (&'static str, usize) {
     #[allow(clippy::enum_glob_use)]
-    use Token::*;
+    use cursor::Pat::*;
 
     let mut context = None;
     let mut decl_end = None;
-    let mut searcher = RustSearcher::new(contents);
-    while let Some(name) = searcher.find_capture_token(CaptureIdent) {
-        match name {
+    let mut cursor = Cursor::new(contents);
+    let mut captures = [Capture::EMPTY];
+    while let Some(name) = cursor.find_any_ident() {
+        match cursor.get_text(name) {
             "declare_clippy_lint" => {
-                if searcher.match_tokens(&[Bang, OpenBrace], &mut []) && searcher.find_token(CloseBrace) {
-                    decl_end = Some(searcher.pos());
+                if cursor.match_all(&[Bang, OpenBrace], &mut []) && cursor.find_pat(CloseBrace) {
+                    decl_end = Some(cursor.pos());
                 }
             },
             "impl" => {
-                let mut capture = "";
-                if searcher.match_tokens(&[Lt, Lifetime, Gt, CaptureIdent], &mut [&mut capture]) {
-                    match capture {
+                if cursor.match_all(&[Lt, Lifetime, Gt, CaptureIdent], &mut captures) {
+                    match cursor.get_text(captures[0]) {
                         "LateLintPass" => context = Some("LateContext"),
                         "EarlyLintPass" => context = Some("EarlyContext"),
                         _ => {},

clippy_dev/src/parse.rs

@@ -0,0 +1,218 @@
+pub mod cursor;
+
+use self::cursor::{Capture, Cursor};
+use crate::utils::{ErrAction, File, expect_action};
+use core::range::Range;
+use std::fs;
+use std::path::{Path, PathBuf};
+use walkdir::{DirEntry, WalkDir};
+
+pub struct Lint {
+    pub name: String,
+    pub group: String,
+    pub module: String,
+    pub path: PathBuf,
+    pub declaration_range: Range<usize>,
+}
+
+pub struct DeprecatedLint {
+    pub name: String,
+    pub reason: String,
+    pub version: String,
+}
+
+pub struct RenamedLint {
+    pub old_name: String,
+    pub new_name: String,
+    pub version: String,
+}
+
+/// Finds all lint declarations (`declare_clippy_lint!`)
+#[must_use]
+pub fn find_lint_decls() -> Vec<Lint> {
+    let mut lints = Vec::with_capacity(1000);
+    let mut contents = String::new();
+    for e in expect_action(fs::read_dir("."), ErrAction::Read, ".") {
+        let e = expect_action(e, ErrAction::Read, ".");
+        if !expect_action(e.file_type(), ErrAction::Read, ".").is_dir() {
+            continue;
+        }
+        let Ok(mut name) = e.file_name().into_string() else {
+            continue;
+        };
+        if name.starts_with("clippy_lints") && name != "clippy_lints_internal" {
+            name.push_str("/src");
+            for (file, module) in read_src_with_module(name.as_ref()) {
+                parse_clippy_lint_decls(
+                    file.path(),
+                    File::open_read_to_cleared_string(file.path(), &mut contents),
+                    &module,
+                    &mut lints,
+                );
+            }
+        }
+    }
+    lints.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name));
+    lints
+}
+
+/// Reads the source files from the given root directory
+fn read_src_with_module(src_root: &Path) -> impl use<'_> + Iterator<Item = (DirEntry, String)> {
+    WalkDir::new(src_root).into_iter().filter_map(move |e| {
+        let e = expect_action(e, ErrAction::Read, src_root);
+        let path = e.path().as_os_str().as_encoded_bytes();
+        if let Some(path) = path.strip_suffix(b".rs")
+            && let Some(path) = path.get(src_root.as_os_str().len() + 1..)
+        {
+            if path == b"lib" {
+                Some((e, String::new()))
+            } else {
+                let path = if let Some(path) = path.strip_suffix(b"mod")
+                    && let Some(path) = path.strip_suffix(b"/").or_else(|| path.strip_suffix(b"\\"))
+                {
+                    path
+                } else {
+                    path
+                };
+                if let Ok(path) = str::from_utf8(path) {
+                    let path = path.replace(['/', '\\'], "::");
+                    Some((e, path))
+                } else {
+                    None
+                }
+            }
+        } else {
+            None
+        }
+    })
+}
+
+/// Parse a source file looking for `declare_clippy_lint` macro invocations.
+fn parse_clippy_lint_decls(path: &Path, contents: &str, module: &str, lints: &mut Vec<Lint>) {
+    #[allow(clippy::enum_glob_use)]
+    use cursor::Pat::*;
+    #[rustfmt::skip]
+    static DECL_TOKENS: &[cursor::Pat<'_>] = &[
+        // !{ /// docs
+        Bang, OpenBrace, AnyComment,
+        // #[clippy::version = "version"]
+        Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, LitStr, CloseBracket,
+        // pub NAME, GROUP,
+        Ident("pub"), CaptureIdent, Comma, AnyComment, CaptureIdent, Comma,
+    ];
+
+    let mut cursor = Cursor::new(contents);
+    let mut captures = [Capture::EMPTY; 2];
+    while let Some(start) = cursor.find_ident("declare_clippy_lint") {
+        if cursor.match_all(DECL_TOKENS, &mut captures) && cursor.find_pat(CloseBrace) {
+            lints.push(Lint {
+                name: cursor.get_text(captures[0]).to_lowercase(),
+                group: cursor.get_text(captures[1]).into(),
+                module: module.into(),
+                path: path.into(),
+                declaration_range: start as usize..cursor.pos() as usize,
+            });
+        }
+    }
+}
+
+#[must_use]
+pub fn read_deprecated_lints() -> (Vec<DeprecatedLint>, Vec<RenamedLint>) {
+    #[allow(clippy::enum_glob_use)]
+    use cursor::Pat::*;
+    #[rustfmt::skip]
+    static DECL_TOKENS: &[cursor::Pat<'_>] = &[
+        // #[clippy::version = "version"]
+        Pound, OpenBracket, Ident("clippy"), DoubleColon, Ident("version"), Eq, CaptureLitStr, CloseBracket,
+        // ("first", "second"),
+        OpenParen, CaptureLitStr, Comma, CaptureLitStr, CloseParen, Comma,
+    ];
+    #[rustfmt::skip]
+    static DEPRECATED_TOKENS: &[cursor::Pat<'_>] = &[
+        // !{ DEPRECATED(DEPRECATED_VERSION) = [
+        Bang, OpenBrace, Ident("DEPRECATED"), OpenParen, Ident("DEPRECATED_VERSION"), CloseParen, Eq, OpenBracket,
+    ];
+    #[rustfmt::skip]
+    static RENAMED_TOKENS: &[cursor::Pat<'_>] = &[
+        // !{ RENAMED(RENAMED_VERSION) = [
+        Bang, OpenBrace, Ident("RENAMED"), OpenParen, Ident("RENAMED_VERSION"), CloseParen, Eq, OpenBracket,
+    ];
+
+    let path = "clippy_lints/src/deprecated_lints.rs";
+    let mut deprecated = Vec::with_capacity(30);
+    let mut renamed = Vec::with_capacity(80);
+    let mut contents = String::new();
+    File::open_read_to_cleared_string(path, &mut contents);
+
+    let mut cursor = Cursor::new(&contents);
+    let mut captures = [Capture::EMPTY; 3];
+
+    // First instance is the macro definition.
+    assert!(
+        cursor.find_ident("declare_with_version").is_some(),
+        "error reading deprecated lints"
+    );
+
+    if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(DEPRECATED_TOKENS, &mut []) {
+        while cursor.match_all(DECL_TOKENS, &mut captures) {
+            deprecated.push(DeprecatedLint {
+                name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])),
+                reason: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])),
+                version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])),
+            });
+        }
+    } else {
+        panic!("error reading deprecated lints");
+    }
+
+    if cursor.find_ident("declare_with_version").is_some() && cursor.match_all(RENAMED_TOKENS, &mut []) {
+        while cursor.match_all(DECL_TOKENS, &mut captures) {
+            renamed.push(RenamedLint {
+                old_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[1])),
+                new_name: parse_str_single_line(path.as_ref(), cursor.get_text(captures[2])),
+                version: parse_str_single_line(path.as_ref(), cursor.get_text(captures[0])),
+            });
+        }
+    } else {
+        panic!("error reading renamed lints");
+    }
+
+    deprecated.sort_by(|lhs, rhs| lhs.name.cmp(&rhs.name));
+    renamed.sort_by(|lhs, rhs| lhs.old_name.cmp(&rhs.old_name));
+    (deprecated, renamed)
+}
+
+/// Removes the line splices and surrounding quotes from a string literal
+fn parse_str_lit(s: &str) -> String {
+    let (s, is_raw) = if let Some(s) = s.strip_prefix("r") {
+        (s.trim_matches('#'), true)
+    } else {
+        (s, false)
+    };
+    let s = s
+        .strip_prefix('"')
+        .and_then(|s| s.strip_suffix('"'))
+        .unwrap_or_else(|| panic!("expected quoted string, found `{s}`"));
+
+    if is_raw {
+        s.into()
+    } else {
+        let mut res = String::with_capacity(s.len());
+        rustc_literal_escaper::unescape_str(s, &mut |_, ch| {
+            if let Ok(ch) = ch {
+                res.push(ch);
+            }
+        });
+        res
+    }
+}
+
+fn parse_str_single_line(path: &Path, s: &str) -> String {
+    let value = parse_str_lit(s);
+    assert!(
+        !value.contains('\n'),
+        "error parsing `{}`: `{s}` should be a single line string",
+        path.display(),
+    );
+    value
+}