Skip to content

Commit e58cfd9

Browse files
borsbors
committed
Auto merge of #70091 - Aaron1011:feature/more-tokens, r=<try>
[WIP] Store tokens alongside more AST expressions See #43081 (comment) This PR calls `collect_tokens` during the parsing of more AST nodes, and stores the captured tokens in the parsed AST structs. These tokens are then used in `nt_to_tokenstream` to avoid needing to stringify AST nodes. Since this implementation completely ignores attributes, it will probably explode when given any kind of complicated input. This PR is intended mainly to estimate the performance impact of collecting and storing more tokens - a correct implementation will most likely have to do more work than this. I've only implemented token collecting for a few types of expressions - the current expression parsing implementation makes it difficult to get the proper tokens for every expression. Nevertheless, this is able to bootstrap libstd, and generate better error messages for a simple proc-macro example: (https://github.com/Aaron1011/for-await-test) ```rust #![feature(stmt_expr_attributes, proc_macro_hygiene)] use futures::stream::Stream; use futures_async_stream::for_await; async fn collect(stream: impl Stream<Item = i32>) -> Vec<i32> { let mut vec = Vec::new(); #[for_await] for value in stream.foo() { vec.push(value); } vec } fn main() { println!("Hello, world!"); } ``` on the latest nightly: ``` error[E0599]: no method named `foo` found for type parameter `impl Stream<Item = i32>` in the current scope --> src/main.rs:7:5 | 7 | #[for_await] | ^^^^^^^^^^^^ method not found in `impl Stream<Item = i32>` error: aborting due to previous error ``` with this PR: ``` error[E0599]: no method named `foo` found for type parameter `impl Stream<Item = i32>` in the current scope --> src/main.rs:8:25 | 8 | for value in stream.foo() { | ^^^ method not found in `impl Stream<Item = i32>` ```
2 parents 6724d58 + e56b697 commit e58cfd9

File tree

22 files changed

+275
-137
lines changed

22 files changed

+275
-137
lines changed

‎src/librustc_ast/ast.rs

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -492,13 +492,15 @@ pub struct Block {
492492
/// Distinguishes between `unsafe { ... }` and `{ ... }`.
493493
pub rules: BlockCheckMode,
494494
pub span: Span,
495+
pub tokens: Option<TokenStream>,
495496
}
496497

497498
#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
498499
pub struct Pat {
499500
pub id: NodeId,
500501
pub kind: PatKind,
501502
pub span: Span,
503+
pub tokens: Option<TokenStream>,
502504
}
503505

504506
impl Pat {
@@ -534,7 +536,7 @@ impl Pat {
534536
_ => return None,
535537
};
536538

537-
Some(P(Ty { kind, id: self.id, span: self.span }))
539+
Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
538540
}
539541

540542
/// Walk top-down and call `it` in each place where a pattern occurs
@@ -875,6 +877,7 @@ pub struct Stmt {
875877
pub id: NodeId,
876878
pub kind: StmtKind,
877879
pub span: Span,
880+
pub tokens: Option<TokenStream>,
878881
}
879882

880883
impl Stmt {
@@ -1011,11 +1014,12 @@ pub struct Expr {
10111014
pub kind: ExprKind,
10121015
pub span: Span,
10131016
pub attrs: AttrVec,
1017+
pub tokens: Option<TokenStream>,
10141018
}
10151019

10161020
// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
10171021
#[cfg(target_arch = "x86_64")]
1018-
rustc_data_structures::static_assert_size!(Expr, 96);
1022+
rustc_data_structures::static_assert_size!(Expr, 104);
10191023

10201024
impl Expr {
10211025
/// Returns `true` if this expression would be valid somewhere that expects a value;
@@ -1091,7 +1095,7 @@ impl Expr {
10911095
_ => return None,
10921096
};
10931097

1094-
Some(P(Ty { kind, id: self.id, span: self.span }))
1098+
Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
10951099
}
10961100

10971101
pub fn precedence(&self) -> ExprPrecedence {
@@ -1829,6 +1833,7 @@ pub struct Ty {
18291833
pub id: NodeId,
18301834
pub kind: TyKind,
18311835
pub span: Span,
1836+
pub tokens: Option<TokenStream>,
18321837
}
18331838

18341839
#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
@@ -2009,13 +2014,14 @@ impl Param {
20092014
/// Builds a `Param` object from `ExplicitSelf`.
20102015
pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param {
20112016
let span = eself.span.to(eself_ident.span);
2012-
let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span });
2017+
let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None });
20132018
let param = |mutbl, ty| Param {
20142019
attrs,
20152020
pat: P(Pat {
20162021
id: DUMMY_NODE_ID,
20172022
kind: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None),
20182023
span,
2024+
tokens: None,
20192025
}),
20202026
span,
20212027
ty,
@@ -2031,6 +2037,7 @@ impl Param {
20312037
id: DUMMY_NODE_ID,
20322038
kind: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl }),
20332039
span,
2040+
tokens: None,
20342041
}),
20352042
),
20362043
}

‎src/librustc_ast/mut_visit.rs

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ pub fn noop_visit_ty_constraint<T: MutVisitor>(
449449
}
450450

451451
pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
452-
let Ty { id, kind, span } = ty.deref_mut();
452+
let Ty { id, kind, span, tokens: _ } = ty.deref_mut();
453453
vis.visit_id(id);
454454
match kind {
455455
TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err | TyKind::Never | TyKind::CVarArgs => {}
@@ -867,7 +867,7 @@ pub fn noop_visit_mt<T: MutVisitor>(MutTy { ty, mutbl: _ }: &mut MutTy, vis: &mu
867867
}
868868

869869
pub fn noop_visit_block<T: MutVisitor>(block: &mut P<Block>, vis: &mut T) {
870-
let Block { id, stmts, rules: _, span } = block.deref_mut();
870+
let Block { id, stmts, rules: _, span, tokens: _ } = block.deref_mut();
871871
vis.visit_id(id);
872872
stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt));
873873
vis.visit_span(span);
@@ -1050,7 +1050,7 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(
10501050
}
10511051

10521052
pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
1053-
let Pat { id, kind, span } = pat.deref_mut();
1053+
let Pat { id, kind, span, tokens: _ } = pat.deref_mut();
10541054
vis.visit_id(id);
10551055
match kind {
10561056
PatKind::Wild | PatKind::Rest => {}
@@ -1092,7 +1092,10 @@ pub fn noop_visit_anon_const<T: MutVisitor>(AnonConst { id, value }: &mut AnonCo
10921092
vis.visit_expr(value);
10931093
}
10941094

1095-
pub fn noop_visit_expr<T: MutVisitor>(Expr { kind, id, span, attrs }: &mut Expr, vis: &mut T) {
1095+
pub fn noop_visit_expr<T: MutVisitor>(
1096+
Expr { kind, id, span, attrs, tokens: _ }: &mut Expr,
1097+
vis: &mut T,
1098+
) {
10961099
match kind {
10971100
ExprKind::Box(expr) => vis.visit_expr(expr),
10981101
ExprKind::Array(exprs) => visit_exprs(exprs, vis),
@@ -1254,12 +1257,15 @@ pub fn noop_filter_map_expr<T: MutVisitor>(mut e: P<Expr>, vis: &mut T) -> Optio
12541257
}
12551258

12561259
pub fn noop_flat_map_stmt<T: MutVisitor>(
1257-
Stmt { kind, mut span, mut id }: Stmt,
1260+
Stmt { kind, mut span, mut id, tokens: _ }: Stmt,
12581261
vis: &mut T,
12591262
) -> SmallVec<[Stmt; 1]> {
12601263
vis.visit_id(&mut id);
12611264
vis.visit_span(&mut span);
1262-
noop_flat_map_stmt_kind(kind, vis).into_iter().map(|kind| Stmt { id, kind, span }).collect()
1265+
noop_flat_map_stmt_kind(kind, vis)
1266+
.into_iter()
1267+
.map(|kind| Stmt { id, kind, span, tokens: None })
1268+
.collect()
12631269
}
12641270

12651271
pub fn noop_flat_map_stmt_kind<T: MutVisitor>(

‎src/librustc_ast/token.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -728,7 +728,7 @@ impl PartialEq<TokenKind> for Token {
728728
}
729729
}
730730

731-
#[derive(Clone, RustcEncodable, RustcDecodable)]
731+
#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
732732
/// For interpolation during macro expansion.
733733
pub enum Nonterminal {
734734
NtItem(P<ast::Item>),
@@ -749,7 +749,7 @@ pub enum Nonterminal {
749749

750750
// `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
751751
#[cfg(target_arch = "x86_64")]
752-
rustc_data_structures::static_assert_size!(Nonterminal, 40);
752+
rustc_data_structures::static_assert_size!(Nonterminal, 48);
753753

754754
impl Nonterminal {
755755
fn span(&self) -> Span {
@@ -786,7 +786,7 @@ impl PartialEq for Nonterminal {
786786
}
787787
}
788788

789-
impl fmt::Debug for Nonterminal {
789+
/*impl fmt::Debug for Nonterminal {
790790
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
791791
match *self {
792792
NtItem(..) => f.pad("NtItem(..)"),
@@ -804,7 +804,7 @@ impl fmt::Debug for Nonterminal {
804804
NtLifetime(..) => f.pad("NtLifetime(..)"),
805805
}
806806
}
807-
}
807+
}*/
808808

809809
impl<CTX> HashStable<CTX> for Nonterminal
810810
where

‎src/librustc_ast_lowering/lib.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1064,6 +1064,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
10641064
id: node_id,
10651065
kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()),
10661066
span: constraint.span,
1067+
tokens: None,
10671068
},
10681069
itctx,
10691070
);
@@ -1128,6 +1129,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
11281129
kind: ExprKind::Path(qself.clone(), path.clone()),
11291130
span: ty.span,
11301131
attrs: AttrVec::new(),
1132+
tokens: None,
11311133
};
11321134

11331135
let ct = self.with_new_scopes(|this| hir::AnonConst {

‎src/librustc_builtin_macros/asm.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ pub fn expand_asm<'cx>(
6161
kind: ast::ExprKind::InlineAsm(P(inline_asm)),
6262
span: cx.with_def_site_ctxt(sp),
6363
attrs: ast::AttrVec::new(),
64+
tokens: None,
6465
}))
6566
}
6667

‎src/librustc_builtin_macros/concat_idents.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ pub fn expand_concat_idents<'cx>(
5252
kind: ast::ExprKind::Path(None, ast::Path::from_ident(self.ident)),
5353
span: self.ident.span,
5454
attrs: ast::AttrVec::new(),
55+
tokens: None,
5556
}))
5657
}
5758

@@ -60,6 +61,7 @@ pub fn expand_concat_idents<'cx>(
6061
id: ast::DUMMY_NODE_ID,
6162
kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)),
6263
span: self.ident.span,
64+
tokens: None,
6365
}))
6466
}
6567
}

‎src/librustc_builtin_macros/deriving/debug.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,5 +133,5 @@ fn stmt_let_undescore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P<ast::Expr>) -> ast
133133
span: sp,
134134
attrs: ast::AttrVec::new(),
135135
});
136-
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
136+
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp, tokens: None }
137137
}

‎src/librustc_builtin_macros/deriving/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ fn call_intrinsic(
7474
id: ast::DUMMY_NODE_ID,
7575
rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated),
7676
span,
77+
tokens: None,
7778
}))
7879
}
7980

‎src/librustc_expand/base.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ macro_rules! make_stmts_default {
392392
id: ast::DUMMY_NODE_ID,
393393
span: e.span,
394394
kind: ast::StmtKind::Expr(e),
395+
tokens: None
395396
}]
396397
})
397398
};
@@ -543,6 +544,7 @@ impl MacResult for MacEager {
543544
id: ast::DUMMY_NODE_ID,
544545
span: e.span,
545546
kind: PatKind::Lit(e),
547+
tokens: None,
546548
}));
547549
}
548550
}
@@ -583,12 +585,13 @@ impl DummyResult {
583585
kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) },
584586
span: sp,
585587
attrs: ast::AttrVec::new(),
588+
tokens: None,
586589
})
587590
}
588591

589592
/// A plain dummy pattern.
590593
pub fn raw_pat(sp: Span) -> ast::Pat {
591-
ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp }
594+
ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None }
592595
}
593596

594597
/// A plain dummy type.
@@ -597,6 +600,7 @@ impl DummyResult {
597600
id: ast::DUMMY_NODE_ID,
598601
kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) },
599602
span: sp,
603+
tokens: None,
600604
})
601605
}
602606
}
@@ -631,6 +635,7 @@ impl MacResult for DummyResult {
631635
id: ast::DUMMY_NODE_ID,
632636
kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)),
633637
span: self.span,
638+
tokens: None
634639
}])
635640
}
636641

‎src/librustc_expand/build.rs

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ impl<'a> ExtCtxt<'a> {
5353
}
5454

5555
pub fn ty(&self, span: Span, kind: ast::TyKind) -> P<ast::Ty> {
56-
P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind })
56+
P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind, tokens: None })
5757
}
5858

5959
pub fn ty_path(&self, path: ast::Path) -> P<ast::Ty> {
@@ -69,7 +69,13 @@ impl<'a> ExtCtxt<'a> {
6969
pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst {
7070
ast::AnonConst {
7171
id: ast::DUMMY_NODE_ID,
72-
value: P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }),
72+
value: P(ast::Expr {
73+
id: ast::DUMMY_NODE_ID,
74+
kind,
75+
span,
76+
attrs: AttrVec::new(),
77+
tokens: None,
78+
}),
7379
}
7480
}
7581

@@ -151,7 +157,12 @@ impl<'a> ExtCtxt<'a> {
151157
}
152158

153159
pub fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt {
154-
ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr) }
160+
ast::Stmt {
161+
id: ast::DUMMY_NODE_ID,
162+
span: expr.span,
163+
kind: ast::StmtKind::Expr(expr),
164+
tokens: None,
165+
}
155166
}
156167

157168
pub fn stmt_let(
@@ -175,7 +186,12 @@ impl<'a> ExtCtxt<'a> {
175186
span: sp,
176187
attrs: AttrVec::new(),
177188
});
178-
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
189+
ast::Stmt {
190+
id: ast::DUMMY_NODE_ID,
191+
kind: ast::StmtKind::Local(local),
192+
span: sp,
193+
tokens: None,
194+
}
179195
}
180196

181197
// Generates `let _: Type;`, which is usually used for type assertions.
@@ -188,11 +204,16 @@ impl<'a> ExtCtxt<'a> {
188204
span,
189205
attrs: AttrVec::new(),
190206
});
191-
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span }
207+
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span, tokens: None }
192208
}
193209

194210
pub fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt {
195-
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(item), span: sp }
211+
ast::Stmt {
212+
id: ast::DUMMY_NODE_ID,
213+
kind: ast::StmtKind::Item(item),
214+
span: sp,
215+
tokens: None,
216+
}
196217
}
197218

198219
pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> {
@@ -202,15 +223,22 @@ impl<'a> ExtCtxt<'a> {
202223
id: ast::DUMMY_NODE_ID,
203224
span: expr.span,
204225
kind: ast::StmtKind::Expr(expr),
226+
tokens: None,
205227
}],
206228
)
207229
}
208230
pub fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> {
209-
P(ast::Block { stmts, id: ast::DUMMY_NODE_ID, rules: BlockCheckMode::Default, span })
231+
P(ast::Block {
232+
stmts,
233+
id: ast::DUMMY_NODE_ID,
234+
rules: BlockCheckMode::Default,
235+
span,
236+
tokens: None,
237+
})
210238
}
211239

212240
pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P<ast::Expr> {
213-
P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() })
241+
P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new(), tokens: None })
214242
}
215243

216244
pub fn expr_path(&self, path: ast::Path) -> P<ast::Expr> {
@@ -396,7 +424,7 @@ impl<'a> ExtCtxt<'a> {
396424
}
397425

398426
pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> {
399-
P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span })
427+
P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span, tokens: None })
400428
}
401429
pub fn pat_wild(&self, span: Span) -> P<ast::Pat> {
402430
self.pat(span, PatKind::Wild)

0 commit comments

Comments
 (0)