Skip to content

Commit

Permalink
perf(parser): skip single space between tokens branchlessly
Browse files Browse the repository at this point in the history
  • Loading branch information
overlookmotel committed Feb 8, 2024
1 parent a3d4ea3 commit 2a906b6
Showing 1 changed file with 18 additions and 2 deletions.
20 changes: 18 additions & 2 deletions crates/oxc_parser/src/lexer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -288,8 +288,24 @@ impl<'a> Lexer<'a> {
/// Whitespace and line terminators are skipped
fn read_next_token(&mut self) -> Kind {
loop {
let offset = self.offset();
self.token.start = offset;
if self.source.is_eof() {
self.token.start = self.offset();
return Kind::Eof;
}

// A single space between tokens is common, so consume one it if it's there.
// This is implemented to be branchless.
// SAFETY: We just checked not at EOF, so safe to peek and consume a byte.
// `advance_by` can only be 0 or 1.
// It's only 1 if next byte is an ASCII space, so then safe to advance past it.
// So either way, we're guaranteed to end up on a UTF-8 character boundary.
unsafe {
let byte = self.source.peek_byte_unchecked();
let advance_by = usize::from(byte == b' ');
self.source.set_position(self.source.position().add(advance_by));
}

self.token.start = self.offset();

let byte = if let Some(byte) = self.source.peek_byte() {
byte
Expand Down

0 comments on commit 2a906b6

Please sign in to comment.