From ddf0df9cbb92bca14861b683535a51258291fa4c Mon Sep 17 00:00:00 2001 From: Anders Hejlsberg Date: Sun, 12 Nov 2017 09:11:09 -0800 Subject: [PATCH] Introduce TokenFlags enum --- src/compiler/binder.ts | 4 +- src/compiler/checker.ts | 2 +- src/compiler/parser.ts | 2 +- src/compiler/scanner.ts | 69 ++++++++++++----------------- src/compiler/transformers/es2015.ts | 2 +- src/compiler/types.ts | 19 +++++--- 6 files changed, 46 insertions(+), 52 deletions(-) diff --git a/src/compiler/binder.ts b/src/compiler/binder.ts index a0146740678..eeeeab1d81b 100644 --- a/src/compiler/binder.ts +++ b/src/compiler/binder.ts @@ -1835,7 +1835,7 @@ namespace ts { } function checkStrictModeNumericLiteral(node: NumericLiteral) { - if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) { + if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) { file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode)); } } @@ -3319,7 +3319,7 @@ namespace ts { break; case SyntaxKind.NumericLiteral: - if ((node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) { + if ((node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) { transformFlags |= TransformFlags.AssertES2015; } break; diff --git a/src/compiler/checker.ts b/src/compiler/checker.ts index 981380c70de..7ed7580b483 100644 --- a/src/compiler/checker.ts +++ b/src/compiler/checker.ts @@ -25779,7 +25779,7 @@ namespace ts { function checkGrammarNumericLiteral(node: NumericLiteral): boolean { // Grammar checking - if (node.numericLiteralFlags & NumericLiteralFlags.Octal) { + if (node.numericLiteralFlags & TokenFlags.Octal) { let diagnosticMessage: DiagnosticMessage | undefined; if (languageVersion >= ScriptTarget.ES5) { diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0; diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 79b5eb30c49..db13e843529 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -2111,7 +2111,7 @@ namespace ts { // We also do not need to check for negatives because any prefix operator would be part of a // parent unary expression. if (node.kind === SyntaxKind.NumericLiteral) { - (node).numericLiteralFlags = scanner.getNumericLiteralFlags(); + (node).numericLiteralFlags = scanner.getTokenFlags() & TokenFlags.NumericLiteralFlags; } nextToken(); diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 9fddece11d0..ddc89a60e2a 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -27,7 +27,7 @@ namespace ts { isReservedWord(): boolean; isUnterminated(): boolean; /* @internal */ - getNumericLiteralFlags(): NumericLiteralFlags; + getTokenFlags(): TokenFlags; reScanGreaterToken(): SyntaxKind; reScanSlashToken(): SyntaxKind; reScanTemplateToken(): SyntaxKind; @@ -814,10 +814,7 @@ namespace ts { let token: SyntaxKind; let tokenValue: string; - let precedingLineBreak: boolean; - let hasExtendedUnicodeEscape: boolean; - let tokenIsUnterminated: boolean; - let numericLiteralFlags: NumericLiteralFlags; + let tokenFlags: TokenFlags; setText(text, start, length); @@ -828,12 +825,12 @@ namespace ts { getTokenPos: () => tokenPos, getTokenText: () => text.substring(tokenPos, pos), getTokenValue: () => tokenValue, - hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape, - hasPrecedingLineBreak: () => precedingLineBreak, + hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0, + hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0, isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord, isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord, - isUnterminated: () => tokenIsUnterminated, - getNumericLiteralFlags: () => numericLiteralFlags, + isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0, + getTokenFlags: () => tokenFlags, reScanGreaterToken, reScanSlashToken, reScanTemplateToken, @@ -870,7 +867,7 @@ namespace ts { let end = pos; if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) { pos++; - numericLiteralFlags = NumericLiteralFlags.Scientific; + tokenFlags |= TokenFlags.Scientific; if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++; if (isDigit(text.charCodeAt(pos))) { pos++; @@ -942,7 +939,7 @@ namespace ts { while (true) { if (pos >= end) { result += text.substring(start, pos); - tokenIsUnterminated = true; + tokenFlags |= TokenFlags.Unterminated; error(Diagnostics.Unterminated_string_literal); break; } @@ -960,7 +957,7 @@ namespace ts { } if (isLineBreak(ch)) { result += text.substring(start, pos); - tokenIsUnterminated = true; + tokenFlags |= TokenFlags.Unterminated; error(Diagnostics.Unterminated_string_literal); break; } @@ -984,7 +981,7 @@ namespace ts { while (true) { if (pos >= end) { contents += text.substring(start, pos); - tokenIsUnterminated = true; + tokenFlags |= TokenFlags.Unterminated; error(Diagnostics.Unterminated_template_literal); resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail; break; @@ -1070,7 +1067,7 @@ namespace ts { case CharacterCodes.u: // '\u{DDDDDDDD}' if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) { - hasExtendedUnicodeEscape = true; + tokenFlags |= TokenFlags.ExtendedUnicodeEscape; pos++; return scanExtendedUnicodeEscape(); } @@ -1239,10 +1236,7 @@ namespace ts { function scan(): SyntaxKind { startPos = pos; - hasExtendedUnicodeEscape = false; - precedingLineBreak = false; - tokenIsUnterminated = false; - numericLiteralFlags = 0; + tokenFlags = 0; while (true) { tokenPos = pos; if (pos >= end) { @@ -1264,7 +1258,7 @@ namespace ts { switch (ch) { case CharacterCodes.lineFeed: case CharacterCodes.carriageReturn: - precedingLineBreak = true; + tokenFlags |= TokenFlags.PrecedingLineBreak; if (skipTrivia) { pos++; continue; @@ -1407,7 +1401,7 @@ namespace ts { } if (isLineBreak(ch)) { - precedingLineBreak = true; + tokenFlags |= TokenFlags.PrecedingLineBreak; } pos++; } @@ -1420,7 +1414,9 @@ namespace ts { continue; } else { - tokenIsUnterminated = !commentClosed; + if (!commentClosed) { + tokenFlags |= TokenFlags.Unterminated; + } return token = SyntaxKind.MultiLineCommentTrivia; } } @@ -1441,7 +1437,7 @@ namespace ts { value = 0; } tokenValue = "" + value; - numericLiteralFlags = NumericLiteralFlags.HexSpecifier; + tokenFlags |= TokenFlags.HexSpecifier; return token = SyntaxKind.NumericLiteral; } else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) { @@ -1452,7 +1448,7 @@ namespace ts { value = 0; } tokenValue = "" + value; - numericLiteralFlags = NumericLiteralFlags.BinarySpecifier; + tokenFlags |= TokenFlags.BinarySpecifier; return token = SyntaxKind.NumericLiteral; } else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) { @@ -1463,13 +1459,13 @@ namespace ts { value = 0; } tokenValue = "" + value; - numericLiteralFlags = NumericLiteralFlags.OctalSpecifier; + tokenFlags |= TokenFlags.OctalSpecifier; return token = SyntaxKind.NumericLiteral; } // Try to parse as an octal if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) { tokenValue = "" + scanOctalDigits(); - numericLiteralFlags = NumericLiteralFlags.Octal; + tokenFlags |= TokenFlags.Octal; return token = SyntaxKind.NumericLiteral; } // This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero @@ -1626,7 +1622,7 @@ namespace ts { continue; } else if (isLineBreak(ch)) { - precedingLineBreak = true; + tokenFlags |= TokenFlags.PrecedingLineBreak; pos++; continue; } @@ -1669,14 +1665,14 @@ namespace ts { // If we reach the end of a file, or hit a newline, then this is an unterminated // regex. Report error and return what we have so far. if (p >= end) { - tokenIsUnterminated = true; + tokenFlags |= TokenFlags.Unterminated; error(Diagnostics.Unterminated_regular_expression_literal); break; } const ch = text.charCodeAt(p); if (isLineBreak(ch)) { - tokenIsUnterminated = true; + tokenFlags |= TokenFlags.Unterminated; error(Diagnostics.Unterminated_regular_expression_literal); break; } @@ -1894,7 +1890,7 @@ namespace ts { const saveTokenPos = tokenPos; const saveToken = token; const saveTokenValue = tokenValue; - const savePrecedingLineBreak = precedingLineBreak; + const saveTokenFlags = tokenFlags; const result = callback(); // If our callback returned something 'falsy' or we're just looking ahead, @@ -1905,7 +1901,7 @@ namespace ts { tokenPos = saveTokenPos; token = saveToken; tokenValue = saveTokenValue; - precedingLineBreak = savePrecedingLineBreak; + tokenFlags = saveTokenFlags; } return result; } @@ -1916,10 +1912,8 @@ namespace ts { const saveStartPos = startPos; const saveTokenPos = tokenPos; const saveToken = token; - const savePrecedingLineBreak = precedingLineBreak; const saveTokenValue = tokenValue; - const saveHasExtendedUnicodeEscape = hasExtendedUnicodeEscape; - const saveTokenIsUnterminated = tokenIsUnterminated; + const saveTokenFlags = tokenFlags; setText(text, start, length); const result = callback(); @@ -1929,10 +1923,8 @@ namespace ts { startPos = saveStartPos; tokenPos = saveTokenPos; token = saveToken; - precedingLineBreak = savePrecedingLineBreak; tokenValue = saveTokenValue; - hasExtendedUnicodeEscape = saveHasExtendedUnicodeEscape; - tokenIsUnterminated = saveTokenIsUnterminated; + tokenFlags = saveTokenFlags; return result; } @@ -1973,11 +1965,8 @@ namespace ts { startPos = textPos; tokenPos = textPos; token = SyntaxKind.Unknown; - precedingLineBreak = false; - tokenValue = undefined; - hasExtendedUnicodeEscape = false; - tokenIsUnterminated = false; + tokenFlags = 0; } } } diff --git a/src/compiler/transformers/es2015.ts b/src/compiler/transformers/es2015.ts index 989b0827570..0c5c3cfbd3a 100644 --- a/src/compiler/transformers/es2015.ts +++ b/src/compiler/transformers/es2015.ts @@ -3633,7 +3633,7 @@ namespace ts { * @param node A string literal. */ function visitNumericLiteral(node: NumericLiteral) { - if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) { + if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) { return setTextRange(createNumericLiteral(node.text), node); } return node; diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 3141e3f22d9..7b0145ea134 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -1451,20 +1451,25 @@ namespace ts { } /* @internal */ - export const enum NumericLiteralFlags { + export const enum TokenFlags { None = 0, - Scientific = 1 << 1, // e.g. `10e2` - Octal = 1 << 2, // e.g. `0777` - HexSpecifier = 1 << 3, // e.g. `0x00000000` - BinarySpecifier = 1 << 4, // e.g. `0b0110010000000000` - OctalSpecifier = 1 << 5, // e.g. `0o777` + PrecedingLineBreak = 1 << 0, + PrecedingComment = 1 << 1, + Unterminated = 1 << 2, + ExtendedUnicodeEscape = 1 << 3, + Scientific = 1 << 4, // e.g. `10e2` + Octal = 1 << 5, // e.g. `0777` + HexSpecifier = 1 << 6, // e.g. `0x00000000` + BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000` + OctalSpecifier = 1 << 8, // e.g. `0o777` BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier, + NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier } export interface NumericLiteral extends LiteralExpression { kind: SyntaxKind.NumericLiteral; /* @internal */ - numericLiteralFlags?: NumericLiteralFlags; + numericLiteralFlags?: TokenFlags; } export interface TemplateHead extends LiteralLikeNode {