From 4ae3fc021d23e49ba2bf5c4d016fd9e3bb0bec4c Mon Sep 17 00:00:00 2001 From: Ulugbek Abdullaev Date: Thu, 16 Feb 2023 20:07:00 +0100 Subject: [PATCH] rewrite scanner from iterator-like lazy style to eager --- .../platform/contextkey/common/contextkey.ts | 2 +- src/vs/platform/contextkey/common/scanner.ts | 161 +++++++++--------- .../contextkey/test/common/scanner.test.ts | 6 +- 3 files changed, 83 insertions(+), 86 deletions(-) diff --git a/src/vs/platform/contextkey/common/contextkey.ts b/src/vs/platform/contextkey/common/contextkey.ts index e75b653844c..d4fed814cad 100644 --- a/src/vs/platform/contextkey/common/contextkey.ts +++ b/src/vs/platform/contextkey/common/contextkey.ts @@ -338,7 +338,7 @@ export class Parser { this._scanner.reset(input); - this._tokens = [...this._scanner]; + this._tokens = this._scanner.scan(); if (this._scanner.errorTokens.length > 0) { return undefined; diff --git a/src/vs/platform/contextkey/common/scanner.ts b/src/vs/platform/contextkey/common/scanner.ts index fcdf947f6a2..dbc52dde615 100644 --- a/src/vs/platform/contextkey/common/scanner.ts +++ b/src/vs/platform/contextkey/common/scanner.ts @@ -107,6 +107,7 @@ export class Scanner { private _input: string = ''; private _start: number = 0; private _current: number = 0; + private _tokens: Token[] = []; private _errorTokens: Token[] = []; get errorTokens(): Readonly { @@ -118,75 +119,77 @@ export class Scanner { this._start = 0; this._current = 0; + this._tokens = []; this._errorTokens = []; return this; } - next(): Token { - this._eatWhitespace(); + scan() { + while (!this._isAtEnd()) { + this._eatWhitespace(); - if (this._isAtEnd()) { - return { type: TokenType.EOF, offset: this._current }; + this._start = this._current; + + const ch = this._advance(); + switch (ch) { + case '(': this._addToken(TokenType.LParen); break; + case ')': this._addToken(TokenType.RParen); break; + + case '!': + this._addToken(this._match('=') ? TokenType.NotEq : TokenType.Neg); + break; + + case '\'': this._quotedString(); break; + case '/': this._regex(); break; + + case '=': + if (this._match('=')) { // support `==` + this._addToken(TokenType.Eq); + } else if (this._match('~')) { + this._addToken(TokenType.RegexOp); + } else { + this._error(); + } + break; + + case '<': this._addToken(this._match('=') ? TokenType.LtEq : TokenType.Lt); break; + + case '>': this._addToken(this._match('=') ? TokenType.GtEq : TokenType.Gt); break; + + case '&': + if (this._match('&')) { + this._addToken(TokenType.And); + } else { + this._error(); + } + break; + + case '|': + if (this._match('|')) { + this._addToken(TokenType.Or); + } else { + this._error(); + } + break; + + // TODO@ulugbekna: 1) I don't think we need to handle whitespace here, 2) if we do, we should reconsider what characters we consider whitespace, including unicode, nbsp, etc. + case ' ': + case '\r': + case '\t': + case '\n': + case '\u00A0': //   + break; + + default: + this._string(); + } } this._start = this._current; + this._addToken(TokenType.EOF); - const ch = this._advance(); - switch (ch) { - case '(': return this._token(TokenType.LParen); - case ')': return this._token(TokenType.RParen); - - case '!': { - if (this._match('=')) { // support `!=` - return this._token(TokenType.NotEq); - } - return this._token(TokenType.Neg); - } - - case '\'': return this._quotedString(); - case '/': return this._regex(); - - case '=': - if (this._match('=')) { // support `==` - return this._token(TokenType.Eq); - } else if (this._match('~')) { - return this._token(TokenType.RegexOp); - } else { - return this._error(); - } - - case '<': return this._token(this._match('=') ? TokenType.LtEq : TokenType.Lt); - - case '>': return this._token(this._match('=') ? TokenType.GtEq : TokenType.Gt); - - case '&': - if (this._match('&')) { - return this._token(TokenType.And); - } else { - return this._error(); - } - - case '|': - if (this._match('|')) { - return this._token(TokenType.Or); - } else { - return this._error(); - } - - // handle whitespace - case ' ': - case '\r': - case '\t': - case '\n': // TODO@ulugbekna: if we're allowing newlines, we should keep track of line # as well ? - return this.next(); - - case '\u00A0': //   - return this.next(); - - default: - return this._string(); - } + return Array.from(this._tokens); } private _match(expected: string): boolean { @@ -216,22 +219,22 @@ export class Scanner { } } - private _token(type: TokenType, captureLexeme: boolean = false): Token { + private _addToken(type: TokenType, captureLexeme: boolean = false) { if (captureLexeme) { const lexeme = this._input.substring(this._start, this._current); - return { type, lexeme, offset: this._start }; + this._tokens.push({ type, lexeme, offset: this._start }); } else { - return { type, offset: this._start }; + this._tokens.push({ type, offset: this._start }); } } - private _error(): Token { + private _error() { const errToken = { type: TokenType.Error, offset: this._start, lexeme: this._input.substring(this._start, this._current) }; this._errorTokens.push(errToken); if (!this._isAtEnd()) { ++this._current; } - return errToken; + this._tokens.push(errToken); } private _string() { @@ -247,9 +250,11 @@ export class Scanner { const keyword = Scanner._keywords.get(lexeme); if (keyword) { - return this._token(keyword); + this._addToken(keyword); } else { - return this._token(TokenType.Str, true); + if (lexeme.length > 0) { + this._tokens.push({ type: TokenType.Str, lexeme, offset: this._start }); + } } } @@ -304,19 +309,20 @@ export class Scanner { } // captures the lexeme without the leading and trailing ' - private _quotedString(): Token { + private _quotedString() { while (this._peek() !== `'` && !this._isAtEnd()) { // TODO@ulugbekna: add support for escaping ' ? this._advance(); } if (this._isAtEnd()) { - return this._error(); + this._error(); + return; } // consume the closing ' this._advance(); - return { type: TokenType.QuotedStr, lexeme: this._input.substring(this._start + 1, this._current - 1), offset: this._start + 1 }; + this._tokens.push({ type: TokenType.QuotedStr, lexeme: this._input.substring(this._start + 1, this._current - 1), offset: this._start + 1 }); } /* @@ -325,7 +331,7 @@ export class Scanner { * * Note that we want slashes within a regex to be escaped, e.g., /file:\\/\\/\\// should match `file:///` */ - private _regex(): Token { + private _regex() { let p = this._current; let inEscape = false; @@ -333,7 +339,8 @@ export class Scanner { while (true) { if (p >= this._input.length) { this._current = p; - return this._error(); + this._error(); + return; } const ch = this._input.charCodeAt(p); @@ -360,7 +367,7 @@ export class Scanner { this._current = p; - return this._token(TokenType.RegexStr, true); + this._addToken(TokenType.RegexStr, true); } // invariant: this must not fail if at end of `this._value` @@ -375,14 +382,4 @@ export class Scanner { private _isAtEnd() { return this._current >= this._input.length; } - - *[Symbol.iterator](): Iterator { - while (true) { - const token = this.next(); - yield token; - if (token?.type === TokenType.EOF) { - break; - } - } - } } diff --git a/src/vs/platform/contextkey/test/common/scanner.test.ts b/src/vs/platform/contextkey/test/common/scanner.test.ts index 6cf91a2020f..a4e994398f1 100644 --- a/src/vs/platform/contextkey/test/common/scanner.test.ts +++ b/src/vs/platform/contextkey/test/common/scanner.test.ts @@ -7,7 +7,7 @@ import { Scanner, TokenType } from 'vs/platform/contextkey/common/scanner'; suite('Context Key Scanner', () => { function scan(input: string) { - return [...((new Scanner()).reset(input))]; + return (new Scanner()).reset(input).scan(); } suite('scanning a single token', () => { @@ -181,8 +181,8 @@ suite('Context Key Scanner', () => { test(`foo === bar'`, () => { const input = `foo === bar'`; - const scanner = new Scanner().reset(input); - const r = [...scanner].filter(t => t.type === TokenType.Error).map(Scanner.reportError); + const tokens = new Scanner().reset(input).scan(); + const r = tokens.filter(t => t.type === TokenType.Error).map(Scanner.reportError); assert.deepStrictEqual(r, (["Unexpected token '=' at offset 6. Did you mean '==' or '=~'?", "Unexpected token ''' at offset 11"])); });