Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 7f8a54b

Browse files
VeykrilVeykril
committed
Switch token trees to use Symbols
1 parent a2c46b9 commit 7f8a54b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+593
-399
lines changed

‎src/tools/rust-analyzer/Cargo.lock‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ dependencies = [
149149
"mbe",
150150
"oorandom",
151151
"rustc-hash",
152+
"smol_str",
152153
"syntax",
153154
"tt",
154155
]
@@ -1045,6 +1046,7 @@ version = "0.0.0"
10451046
dependencies = [
10461047
"arrayvec",
10471048
"cov-mark",
1049+
"intern",
10481050
"parser",
10491051
"rustc-hash",
10501052
"smallvec",
@@ -1324,8 +1326,8 @@ version = "0.0.0"
13241326
dependencies = [
13251327
"base-db",
13261328
"indexmap",
1329+
"intern",
13271330
"la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
1328-
"mbe",
13291331
"paths",
13301332
"rustc-hash",
13311333
"serde",
@@ -1343,6 +1345,7 @@ version = "0.0.0"
13431345
dependencies = [
13441346
"base-db",
13451347
"expect-test",
1348+
"intern",
13461349
"libloading",
13471350
"mbe",
13481351
"memmap2",
@@ -1966,6 +1969,7 @@ dependencies = [
19661969
"base-db",
19671970
"cfg",
19681971
"hir-expand",
1972+
"intern",
19691973
"rustc-hash",
19701974
"span",
19711975
"stdx",
@@ -2218,8 +2222,8 @@ name = "tt"
22182222
version = "0.0.0"
22192223
dependencies = [
22202224
"arrayvec",
2225+
"intern",
22212226
"ra-ap-rustc_lexer",
2222-
"smol_str",
22232227
"stdx",
22242228
"text-size",
22252229
]

‎src/tools/rust-analyzer/crates/cfg/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ rustc-hash.workspace = true
1616

1717
# locals deps
1818
tt.workspace = true
19+
smol_str.workspace = true
1920

2021
[dev-dependencies]
2122
expect-test = "1.4.1"

‎src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
55
use std::{fmt, slice::Iter as SliceIter};
66

7-
use tt::SmolStr;
7+
use smol_str::SmolStr;
88

99
/// A simple configuration value passed in from the outside.
1010
#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
@@ -66,7 +66,7 @@ impl CfgExpr {
6666
fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
6767
let name = match it.next() {
6868
None => return None,
69-
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),
69+
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
7070
Some(_) => return Some(CfgExpr::Invalid),
7171
};
7272

@@ -77,10 +77,9 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
7777
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
7878
it.next();
7979
it.next();
80-
// FIXME: escape? raw string?
81-
let value =
82-
SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"'));
83-
CfgAtom::KeyValue { key: name, value }.into()
80+
// FIXME: escape?
81+
let value = literal.symbol.as_str().into();
82+
CfgAtom::KeyValue { key: name.as_str().into(), value }.into()
8483
}
8584
_ => return Some(CfgExpr::Invalid),
8685
}
@@ -96,7 +95,7 @@ fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr>
9695
_ => CfgExpr::Invalid,
9796
}
9897
}
99-
_ => CfgAtom::Flag(name).into(),
98+
_ => CfgAtom::Flag(name.as_str().into()).into(),
10099
};
101100

102101
// Eat comma separator

‎src/tools/rust-analyzer/crates/cfg/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ mod tests;
88
use std::fmt;
99

1010
use rustc_hash::FxHashSet;
11-
use tt::SmolStr;
1211

1312
pub use cfg_expr::{CfgAtom, CfgExpr};
1413
pub use dnf::DnfExpr;
14+
use smol_str::SmolStr;
1515

1616
/// Configuration options used for conditional compilation on items with `cfg` attributes.
1717
/// We have two kind of options in different namespaces: atomic options like `unix`, and

‎src/tools/rust-analyzer/crates/hir-def/src/attr.rs‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -159,14 +159,14 @@ impl Attrs {
159159
pub fn has_doc_hidden(&self) -> bool {
160160
self.by_key("doc").tt_values().any(|tt| {
161161
tt.delimiter.kind == DelimiterKind::Parenthesis &&
162-
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
162+
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::hidden)
163163
})
164164
}
165165

166166
pub fn has_doc_notable_trait(&self) -> bool {
167167
self.by_key("doc").tt_values().any(|tt| {
168168
tt.delimiter.kind == DelimiterKind::Parenthesis &&
169-
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "notable_trait")
169+
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::notable_trait)
170170
})
171171
}
172172

@@ -267,21 +267,24 @@ impl DocExpr {
267267
fn next_doc_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<DocExpr> {
268268
let name = match it.next() {
269269
None => return None,
270-
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),
270+
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
271271
Some(_) => return Some(DocExpr::Invalid),
272272
};
273273

274274
// Peek
275275
let ret = match it.as_slice().first() {
276276
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
277277
match it.as_slice().get(1) {
278-
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
278+
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
279+
symbol: text,
280+
kind: tt::LitKind::Str,
281+
..
282+
}))) => {
279283
it.next();
280284
it.next();
281285
// FIXME: escape? raw string?
282-
let value =
283-
SmolStr::new(literal.text.trim_start_matches('"').trim_end_matches('"'));
284-
DocAtom::KeyValue { key: name, value }.into()
286+
let value = SmolStr::new(text.as_str());
287+
DocAtom::KeyValue { key: name.as_str().into(), value }.into()
285288
}
286289
_ => return Some(DocExpr::Invalid),
287290
}
@@ -294,7 +297,7 @@ fn next_doc_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<DocExpr>
294297
_ => DocExpr::Invalid,
295298
}
296299
}
297-
_ => DocAtom::Flag(name).into(),
300+
_ => DocAtom::Flag(name.as_str().into()).into(),
298301
};
299302

300303
// Eat comma separator
@@ -311,10 +314,11 @@ fn parse_comma_sep<S>(subtree: &tt::Subtree<S>) -> Vec<SmolStr> {
311314
.token_trees
312315
.iter()
313316
.filter_map(|tt| match tt {
314-
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
315-
// FIXME: escape? raw string?
316-
Some(SmolStr::new(lit.text.trim_start_matches('"').trim_end_matches('"')))
317-
}
317+
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
318+
kind: tt::LitKind::Str,
319+
symbol: text,
320+
..
321+
})) => Some(SmolStr::new(text.as_str())),
318322
_ => None,
319323
})
320324
.collect()
@@ -598,14 +602,14 @@ impl<'attr> AttrQuery<'attr> {
598602
/// #[doc(html_root_url = "url")]
599603
/// ^^^^^^^^^^^^^ key
600604
/// ```
601-
pub fn find_string_value_in_tt(self, key: &'attr str) -> Option<&SmolStr> {
605+
pub fn find_string_value_in_tt(self, key: &'attr str) -> Option<&str> {
602606
self.tt_values().find_map(|tt| {
603607
let name = tt.token_trees.iter()
604-
.skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { text, ..} )) if text == key))
608+
.skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { sym, ..} )) if sym.as_str() == key))
605609
.nth(2);
606610

607611
match name {
608-
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text),
612+
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text.as_str()),
609613
_ => None
610614
}
611615
})

‎src/tools/rust-analyzer/crates/hir-def/src/builtin_type.rs‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
use std::fmt;
77

88
use hir_expand::name::{AsName, Name};
9-
use intern::sym;
9+
use intern::{sym, Symbol};
1010
/// Different signed int types.
1111
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1212
pub enum BuiltinInt {
@@ -143,6 +143,18 @@ impl BuiltinInt {
143143
};
144144
Some(res)
145145
}
146+
pub fn from_suffix_sym(suffix: &Symbol) -> Option<BuiltinInt> {
147+
let res = match suffix {
148+
s if *s == sym::isize => Self::Isize,
149+
s if *s == sym::i8 => Self::I8,
150+
s if *s == sym::i16 => Self::I16,
151+
s if *s == sym::i32 => Self::I32,
152+
s if *s == sym::i64 => Self::I64,
153+
s if *s == sym::i128 => Self::I128,
154+
_ => return None,
155+
};
156+
Some(res)
157+
}
146158
}
147159

148160
#[rustfmt::skip]
@@ -160,6 +172,19 @@ impl BuiltinUint {
160172
};
161173
Some(res)
162174
}
175+
pub fn from_suffix_sym(suffix: &Symbol) -> Option<BuiltinUint> {
176+
let res = match suffix {
177+
s if *s == sym::usize => Self::Usize,
178+
s if *s == sym::u8 => Self::U8,
179+
s if *s == sym::u16 => Self::U16,
180+
s if *s == sym::u32 => Self::U32,
181+
s if *s == sym::u64 => Self::U64,
182+
s if *s == sym::u128 => Self::U128,
183+
184+
_ => return None,
185+
};
186+
Some(res)
187+
}
163188
}
164189

165190
#[rustfmt::skip]

‎src/tools/rust-analyzer/crates/hir-def/src/data.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
150150
let mut indices = Vec::new();
151151
for args in tt.token_trees.chunks(2) {
152152
match &args[0] {
153-
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => match lit.text.parse() {
153+
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => match lit.symbol.as_str().parse() {
154154
Ok(index) => indices.push(index),
155155
Err(_) => break,
156156
},

‎src/tools/rust-analyzer/crates/hir-def/src/data/adt.rs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use hir_expand::{
99
name::{AsName, Name},
1010
HirFileId, InFile,
1111
};
12-
use intern::Interned;
12+
use intern::{sym, Interned};
1313
use la_arena::Arena;
1414
use rustc_abi::{Align, Integer, IntegerType, ReprFlags, ReprOptions};
1515
use syntax::ast::{self, HasName, HasVisibility};
@@ -112,12 +112,12 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
112112
let mut tts = tt.token_trees.iter().peekable();
113113
while let Some(tt) = tts.next() {
114114
if let TokenTree::Leaf(Leaf::Ident(ident)) = tt {
115-
flags.insert(match &*ident.text {
116-
"packed" => {
115+
flags.insert(match &ident.sym {
116+
s if *s == sym::packed => {
117117
let pack = if let Some(TokenTree::Subtree(tt)) = tts.peek() {
118118
tts.next();
119119
if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() {
120-
lit.text.parse().unwrap_or_default()
120+
lit.symbol.as_str().parse().unwrap_or_default()
121121
} else {
122122
0
123123
}
@@ -129,25 +129,25 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
129129
Some(if let Some(min_pack) = min_pack { min_pack.min(pack) } else { pack });
130130
ReprFlags::empty()
131131
}
132-
"align" => {
132+
s if *s == sym::align => {
133133
if let Some(TokenTree::Subtree(tt)) = tts.peek() {
134134
tts.next();
135135
if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() {
136-
if let Ok(align) = lit.text.parse() {
136+
if let Ok(align) = lit.symbol.as_str().parse() {
137137
let align = Align::from_bytes(align).ok();
138138
max_align = max_align.max(align);
139139
}
140140
}
141141
}
142142
ReprFlags::empty()
143143
}
144-
"C" => ReprFlags::IS_C,
145-
"transparent" => ReprFlags::IS_TRANSPARENT,
146-
"simd" => ReprFlags::IS_SIMD,
144+
s if *s == sym::C => ReprFlags::IS_C,
145+
s if *s == sym::transparent => ReprFlags::IS_TRANSPARENT,
146+
s if *s == sym::simd => ReprFlags::IS_SIMD,
147147
repr => {
148-
if let Some(builtin) = BuiltinInt::from_suffix(repr)
148+
if let Some(builtin) = BuiltinInt::from_suffix_sym(repr)
149149
.map(Either::Left)
150-
.or_else(|| BuiltinUint::from_suffix(repr).map(Either::Right))
150+
.or_else(|| BuiltinUint::from_suffix_sym(repr).map(Either::Right))
151151
{
152152
int = Some(match builtin {
153153
Either::Left(bi) => match bi {

‎src/tools/rust-analyzer/crates/hir-def/src/db.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ fn crate_supports_no_std(db: &dyn DefDatabase, crate_id: CrateId) -> bool {
278278
tt.split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(p)) if p.char == ','));
279279
for output in segments.skip(1) {
280280
match output {
281-
[tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "no_std" => {
281+
[tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::no_std => {
282282
return true
283283
}
284284
_ => {}

‎src/tools/rust-analyzer/crates/hir-def/src/nameres.rs‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ use base_db::{CrateId, FileId};
6363
use hir_expand::{
6464
name::Name, proc_macro::ProcMacroKind, ErasedAstId, HirFileId, InFile, MacroCallId, MacroDefId,
6565
};
66+
use intern::Symbol;
6667
use itertools::Itertools;
6768
use la_arena::Arena;
6869
use rustc_hash::{FxHashMap, FxHashSet};
@@ -148,11 +149,11 @@ struct DefMapCrateData {
148149
proc_macro_loading_error: Option<Box<str>>,
149150

150151
/// Custom attributes registered with `#![register_attr]`.
151-
registered_attrs: Vec<SmolStr>,
152+
registered_attrs: Vec<Symbol>,
152153
/// Custom tool modules registered with `#![register_tool]`.
153-
registered_tools: Vec<SmolStr>,
154+
registered_tools: Vec<Symbol>,
154155
/// Unstable features of Rust enabled with `#![feature(A, B)]`.
155-
unstable_features: FxHashSet<SmolStr>,
156+
unstable_features: FxHashSet<Symbol>,
156157
/// #[rustc_coherence_is_core]
157158
rustc_coherence_is_core: bool,
158159
no_core: bool,
@@ -170,7 +171,7 @@ impl DefMapCrateData {
170171
fn_proc_macro_mapping: FxHashMap::default(),
171172
proc_macro_loading_error: None,
172173
registered_attrs: Vec::new(),
173-
registered_tools: PREDEFINED_TOOLS.into(),
174+
registered_tools: PREDEFINED_TOOLS.iter().map(|it| Symbol::intern(it)).collect(),
174175
unstable_features: FxHashSet::default(),
175176
rustc_coherence_is_core: false,
176177
no_core: false,
@@ -447,15 +448,15 @@ impl DefMap {
447448
self.derive_helpers_in_scope.get(&id.map(|it| it.upcast())).map(Deref::deref)
448449
}
449450

450-
pub fn registered_tools(&self) -> &[SmolStr] {
451+
pub fn registered_tools(&self) -> &[Symbol] {
451452
&self.data.registered_tools
452453
}
453454

454-
pub fn registered_attrs(&self) -> &[SmolStr] {
455+
pub fn registered_attrs(&self) -> &[Symbol] {
455456
&self.data.registered_attrs
456457
}
457458

458-
pub fn is_unstable_feature_enabled(&self, feature: &str) -> bool {
459+
pub fn is_unstable_feature_enabled(&self, feature: &Symbol) -> bool {
459460
self.data.unstable_features.contains(feature)
460461
}
461462

0 commit comments

Comments
 (0)