Importing rustc-1.59.0
Test: ./build.py --lto=thin
Change-Id: I6dbef2f2e4acd7832f31c209c452b87d4f354704
Bug: 215232614
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index 44b002f..5b8300a 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -227,14 +227,15 @@
}
/// Creates an iterator that produces tokens from the input string.
-pub fn tokenize(mut input: &str) -> impl Iterator<Item = Token> + '_ {
+pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ {
+ let mut cursor = Cursor::new(input);
std::iter::from_fn(move || {
- if input.is_empty() {
- return None;
+ if cursor.is_eof() {
+ None
+ } else {
+ cursor.reset_len_consumed();
+ Some(cursor.advance_token())
}
- let token = first_token(input);
- input = &input[token.len..];
- Some(token)
})
}
@@ -832,11 +833,4 @@
self.eat_while(is_id_continue);
}
-
- /// Eats symbols while predicate returns true or until the end of file is reached.
- fn eat_while(&mut self, mut predicate: impl FnMut(char) -> bool) {
- while predicate(self.first()) && !self.is_eof() {
- self.bump();
- }
- }
}
