microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-05-20 22:51:17 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Introduce TokenFlags enum

Browse Source
This commit is contained in:
Anders Hejlsberg
2017-11-12 09:11:09 -08:00
parent b8fbf884d0
commit ddf0df9cbb
6 changed files with 46 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/compiler/binder.ts
View File

@@ -1835,7 +1835,7 @@ namespace ts {
}
function checkStrictModeNumericLiteral(node: NumericLiteral) {
if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {
if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {
file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));
}
}
@@ -3319,7 +3319,7 @@ namespace ts {
break;
case SyntaxKind.NumericLiteral:
if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
transformFlags |= TransformFlags.AssertES2015;
}
break;

2
src/compiler/checker.ts
View File

@@ -25779,7 +25779,7 @@ namespace ts {
function checkGrammarNumericLiteral(node: NumericLiteral): boolean {
// Grammar checking
if (node.numericLiteralFlags & NumericLiteralFlags.Octal) {
if (node.numericLiteralFlags & TokenFlags.Octal) {
let diagnosticMessage: DiagnosticMessage | undefined;
if (languageVersion >= ScriptTarget.ES5) {
diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0;

2
src/compiler/parser.ts
View File

@@ -2111,7 +2111,7 @@ namespace ts {
// We also do not need to check for negatives because any prefix operator would be part of a
// parent unary expression.
if (node.kind === SyntaxKind.NumericLiteral) {
(<NumericLiteral>node).numericLiteralFlags = scanner.getNumericLiteralFlags();
(<NumericLiteral>node).numericLiteralFlags = scanner.getTokenFlags() & TokenFlags.NumericLiteralFlags;
}
nextToken();

69
src/compiler/scanner.ts
View File

@@ -27,7 +27,7 @@ namespace ts {
isReservedWord(): boolean;
isUnterminated(): boolean;
/* @internal */
getNumericLiteralFlags(): NumericLiteralFlags;
getTokenFlags(): TokenFlags;
reScanGreaterToken(): SyntaxKind;
reScanSlashToken(): SyntaxKind;
reScanTemplateToken(): SyntaxKind;
@@ -814,10 +814,7 @@ namespace ts {
let token: SyntaxKind;
let tokenValue: string;
let precedingLineBreak: boolean;
let hasExtendedUnicodeEscape: boolean;
let tokenIsUnterminated: boolean;
let numericLiteralFlags: NumericLiteralFlags;
let tokenFlags: TokenFlags;
setText(text, start, length);
@@ -828,12 +825,12 @@ namespace ts {
getTokenPos: () => tokenPos,
getTokenText: () => text.substring(tokenPos, pos),
getTokenValue: () => tokenValue,
hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
hasPrecedingLineBreak: () => precedingLineBreak,
hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0,
isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
isUnterminated: () => tokenIsUnterminated,
getNumericLiteralFlags: () => numericLiteralFlags,
isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0,
getTokenFlags: () => tokenFlags,
reScanGreaterToken,
reScanSlashToken,
reScanTemplateToken,
@@ -870,7 +867,7 @@ namespace ts {
let end = pos;
if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
pos++;
numericLiteralFlags = NumericLiteralFlags.Scientific;
tokenFlags |= TokenFlags.Scientific;
if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
if (isDigit(text.charCodeAt(pos))) {
pos++;
@@ -942,7 +939,7 @@ namespace ts {
while (true) {
if (pos >= end) {
result += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_string_literal);
break;
}
@@ -960,7 +957,7 @@ namespace ts {
}
if (isLineBreak(ch)) {
result += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_string_literal);
break;
}
@@ -984,7 +981,7 @@ namespace ts {
while (true) {
if (pos >= end) {
contents += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_template_literal);
resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail;
break;
@@ -1070,7 +1067,7 @@ namespace ts {
case CharacterCodes.u:
// '\u{DDDDDDDD}'
if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
hasExtendedUnicodeEscape = true;
tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
pos++;
return scanExtendedUnicodeEscape();
}
@@ -1239,10 +1236,7 @@ namespace ts {
function scan(): SyntaxKind {
startPos = pos;
hasExtendedUnicodeEscape = false;
precedingLineBreak = false;
tokenIsUnterminated = false;
numericLiteralFlags = 0;
tokenFlags = 0;
while (true) {
tokenPos = pos;
if (pos >= end) {
@@ -1264,7 +1258,7 @@ namespace ts {
switch (ch) {
case CharacterCodes.lineFeed:
case CharacterCodes.carriageReturn:
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
if (skipTrivia) {
pos++;
continue;
@@ -1407,7 +1401,7 @@ namespace ts {
}
if (isLineBreak(ch)) {
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
}
pos++;
}
@@ -1420,7 +1414,9 @@ namespace ts {
continue;
}
else {
tokenIsUnterminated = !commentClosed;
if (!commentClosed) {
tokenFlags |= TokenFlags.Unterminated;
}
return token = SyntaxKind.MultiLineCommentTrivia;
}
}
@@ -1441,7 +1437,7 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.HexSpecifier;
tokenFlags |= TokenFlags.HexSpecifier;
return token = SyntaxKind.NumericLiteral;
}
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
@@ -1452,7 +1448,7 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.BinarySpecifier;
tokenFlags |= TokenFlags.BinarySpecifier;
return token = SyntaxKind.NumericLiteral;
}
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
@@ -1463,13 +1459,13 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.OctalSpecifier;
tokenFlags |= TokenFlags.OctalSpecifier;
return token = SyntaxKind.NumericLiteral;
}
// Try to parse as an octal
if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
tokenValue = "" + scanOctalDigits();
numericLiteralFlags = NumericLiteralFlags.Octal;
tokenFlags |= TokenFlags.Octal;
return token = SyntaxKind.NumericLiteral;
}
// This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero
@@ -1626,7 +1622,7 @@ namespace ts {
continue;
}
else if (isLineBreak(ch)) {
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
pos++;
continue;
}
@@ -1669,14 +1665,14 @@ namespace ts {
// If we reach the end of a file, or hit a newline, then this is an unterminated
// regex. Report error and return what we have so far.
if (p >= end) {
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_regular_expression_literal);
break;
}
const ch = text.charCodeAt(p);
if (isLineBreak(ch)) {
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_regular_expression_literal);
break;
}
@@ -1894,7 +1890,7 @@ namespace ts {
const saveTokenPos = tokenPos;
const saveToken = token;
const saveTokenValue = tokenValue;
const savePrecedingLineBreak = precedingLineBreak;
const saveTokenFlags = tokenFlags;
const result = callback();
// If our callback returned something 'falsy' or we're just looking ahead,
@@ -1905,7 +1901,7 @@ namespace ts {
tokenPos = saveTokenPos;
token = saveToken;
tokenValue = saveTokenValue;
precedingLineBreak = savePrecedingLineBreak;
tokenFlags = saveTokenFlags;
}
return result;
}
@@ -1916,10 +1912,8 @@ namespace ts {
const saveStartPos = startPos;
const saveTokenPos = tokenPos;
const saveToken = token;
const savePrecedingLineBreak = precedingLineBreak;
const saveTokenValue = tokenValue;
const saveHasExtendedUnicodeEscape = hasExtendedUnicodeEscape;
const saveTokenIsUnterminated = tokenIsUnterminated;
const saveTokenFlags = tokenFlags;
setText(text, start, length);
const result = callback();
@@ -1929,10 +1923,8 @@ namespace ts {
startPos = saveStartPos;
tokenPos = saveTokenPos;
token = saveToken;
precedingLineBreak = savePrecedingLineBreak;
tokenValue = saveTokenValue;
hasExtendedUnicodeEscape = saveHasExtendedUnicodeEscape;
tokenIsUnterminated = saveTokenIsUnterminated;
tokenFlags = saveTokenFlags;
return result;
}
@@ -1973,11 +1965,8 @@ namespace ts {
startPos = textPos;
tokenPos = textPos;
token = SyntaxKind.Unknown;
precedingLineBreak = false;
tokenValue = undefined;
hasExtendedUnicodeEscape = false;
tokenIsUnterminated = false;
tokenFlags = 0;
}
}
}

2
src/compiler/transformers/es2015.ts
View File

@@ -3633,7 +3633,7 @@ namespace ts {
* @param node A string literal.
*/
function visitNumericLiteral(node: NumericLiteral) {
if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
return setTextRange(createNumericLiteral(node.text), node);
}
return node;

19
src/compiler/types.ts
View File

@@ -1451,20 +1451,25 @@ namespace ts {
}
/* @internal */
export const enum NumericLiteralFlags {
export const enum TokenFlags {
None = 0,
Scientific = 1 << 1, // e.g. `10e2`
Octal = 1 << 2, // e.g. `0777`
HexSpecifier = 1 << 3, // e.g. `0x00000000`
BinarySpecifier = 1 << 4, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 5, // e.g. `0o777`
PrecedingLineBreak = 1 << 0,
PrecedingComment = 1 << 1,
Unterminated = 1 << 2,
ExtendedUnicodeEscape = 1 << 3,
Scientific = 1 << 4, // e.g. `10e2`
Octal = 1 << 5, // e.g. `0777`
HexSpecifier = 1 << 6, // e.g. `0x00000000`
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 8, // e.g. `0o777`
BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
}
export interface NumericLiteral extends LiteralExpression {
kind: SyntaxKind.NumericLiteral;
/* @internal */
numericLiteralFlags?: NumericLiteralFlags;
numericLiteralFlags?: TokenFlags;
}
export interface TemplateHead extends LiteralLikeNode {