microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-02-11 01:34:55 -06:00
Code Issues Packages Projects Releases 212 Wiki Activity

Merge pull request #19976 from Microsoft/optimizeParser

Browse Source 
Optimize parsing
This commit is contained in:
Anders Hejlsberg 2017-11-16 10:19:10 -08:00 committed by GitHub
commit b6f96052d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 310 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/compiler/binder.ts
View File

@ -1835,7 +1835,7 @@ namespace ts {
}
function checkStrictModeNumericLiteral(node: NumericLiteral) {
if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {
if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {
file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));
}
}
@ -3319,7 +3319,7 @@ namespace ts {
break;
case SyntaxKind.NumericLiteral:
if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
transformFlags |= TransformFlags.AssertES2015;
}
break;

2
src/compiler/checker.ts
View File

@ -26274,7 +26274,7 @@ namespace ts {
function checkGrammarNumericLiteral(node: NumericLiteral): boolean {
// Grammar checking
if (node.numericLiteralFlags & NumericLiteralFlags.Octal) {
if (node.numericLiteralFlags & TokenFlags.Octal) {
let diagnosticMessage: DiagnosticMessage | undefined;
if (languageVersion >= ScriptTarget.ES5) {
diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0;

543
src/compiler/parser.ts
View File

File diff suppressed because it is too large Load Diff

72
src/compiler/scanner.ts
View File

@ -27,7 +27,7 @@ namespace ts {
isReservedWord(): boolean;
isUnterminated(): boolean;
/* @internal */
getNumericLiteralFlags(): NumericLiteralFlags;
getTokenFlags(): TokenFlags;
reScanGreaterToken(): SyntaxKind;
reScanSlashToken(): SyntaxKind;
reScanTemplateToken(): SyntaxKind;
@ -815,10 +815,7 @@ namespace ts {
let token: SyntaxKind;
let tokenValue: string;
let precedingLineBreak: boolean;
let hasExtendedUnicodeEscape: boolean;
let tokenIsUnterminated: boolean;
let numericLiteralFlags: NumericLiteralFlags;
let tokenFlags: TokenFlags;
setText(text, start, length);
@ -829,12 +826,12 @@ namespace ts {
getTokenPos: () => tokenPos,
getTokenText: () => text.substring(tokenPos, pos),
getTokenValue: () => tokenValue,
hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
hasPrecedingLineBreak: () => precedingLineBreak,
hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0,
isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
isUnterminated: () => tokenIsUnterminated,
getNumericLiteralFlags: () => numericLiteralFlags,
isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0,
getTokenFlags: () => tokenFlags,
reScanGreaterToken,
reScanSlashToken,
reScanTemplateToken,
@ -871,7 +868,7 @@ namespace ts {
let end = pos;
if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
pos++;
numericLiteralFlags = NumericLiteralFlags.Scientific;
tokenFlags |= TokenFlags.Scientific;
if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
if (isDigit(text.charCodeAt(pos))) {
pos++;
@ -943,7 +940,7 @@ namespace ts {
while (true) {
if (pos >= end) {
result += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_string_literal);
break;
}
@ -961,7 +958,7 @@ namespace ts {
}
if (isLineBreak(ch)) {
result += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_string_literal);
break;
}
@ -985,7 +982,7 @@ namespace ts {
while (true) {
if (pos >= end) {
contents += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_template_literal);
resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail;
break;
@ -1071,7 +1068,7 @@ namespace ts {
case CharacterCodes.u:
// '\u{DDDDDDDD}'
if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
hasExtendedUnicodeEscape = true;
tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
pos++;
return scanExtendedUnicodeEscape();
}
@ -1240,10 +1237,7 @@ namespace ts {
function scan(): SyntaxKind {
startPos = pos;
hasExtendedUnicodeEscape = false;
precedingLineBreak = false;
tokenIsUnterminated = false;
numericLiteralFlags = 0;
tokenFlags = 0;
while (true) {
tokenPos = pos;
if (pos >= end) {
@ -1265,7 +1259,7 @@ namespace ts {
switch (ch) {
case CharacterCodes.lineFeed:
case CharacterCodes.carriageReturn:
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
if (skipTrivia) {
pos++;
continue;
@ -1396,6 +1390,9 @@ namespace ts {
// Multi-line comment
if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) {
pos += 2;
if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) {
tokenFlags |= TokenFlags.PrecedingJSDocComment;
}
let commentClosed = false;
while (pos < end) {
@ -1408,7 +1405,7 @@ namespace ts {
}
if (isLineBreak(ch)) {
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
}
pos++;
}
@ -1421,7 +1418,9 @@ namespace ts {
continue;
}
else {
tokenIsUnterminated = !commentClosed;
if (!commentClosed) {
tokenFlags |= TokenFlags.Unterminated;
}
return token = SyntaxKind.MultiLineCommentTrivia;
}
}
@ -1442,7 +1441,7 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.HexSpecifier;
tokenFlags |= TokenFlags.HexSpecifier;
return token = SyntaxKind.NumericLiteral;
}
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
@ -1453,7 +1452,7 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.BinarySpecifier;
tokenFlags |= TokenFlags.BinarySpecifier;
return token = SyntaxKind.NumericLiteral;
}
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
@ -1464,13 +1463,13 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.OctalSpecifier;
tokenFlags |= TokenFlags.OctalSpecifier;
return token = SyntaxKind.NumericLiteral;
}
// Try to parse as an octal
if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
tokenValue = "" + scanOctalDigits();
numericLiteralFlags = NumericLiteralFlags.Octal;
tokenFlags |= TokenFlags.Octal;
return token = SyntaxKind.NumericLiteral;
}
// This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero
@ -1627,7 +1626,7 @@ namespace ts {
continue;
}
else if (isLineBreak(ch)) {
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
pos++;
continue;
}
@ -1670,14 +1669,14 @@ namespace ts {
// If we reach the end of a file, or hit a newline, then this is an unterminated
// regex. Report error and return what we have so far.
if (p >= end) {
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_regular_expression_literal);
break;
}
const ch = text.charCodeAt(p);
if (isLineBreak(ch)) {
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_regular_expression_literal);
break;
}
@ -1895,7 +1894,7 @@ namespace ts {
const saveTokenPos = tokenPos;
const saveToken = token;
const saveTokenValue = tokenValue;
const savePrecedingLineBreak = precedingLineBreak;
const saveTokenFlags = tokenFlags;
const result = callback();
// If our callback returned something 'falsy' or we're just looking ahead,
@ -1906,7 +1905,7 @@ namespace ts {
tokenPos = saveTokenPos;
token = saveToken;
tokenValue = saveTokenValue;
precedingLineBreak = savePrecedingLineBreak;
tokenFlags = saveTokenFlags;
}
return result;
}
@ -1917,10 +1916,8 @@ namespace ts {
const saveStartPos = startPos;
const saveTokenPos = tokenPos;
const saveToken = token;
const savePrecedingLineBreak = precedingLineBreak;
const saveTokenValue = tokenValue;
const saveHasExtendedUnicodeEscape = hasExtendedUnicodeEscape;
const saveTokenIsUnterminated = tokenIsUnterminated;
const saveTokenFlags = tokenFlags;
setText(text, start, length);
const result = callback();
@ -1930,10 +1927,8 @@ namespace ts {
startPos = saveStartPos;
tokenPos = saveTokenPos;
token = saveToken;
precedingLineBreak = savePrecedingLineBreak;
tokenValue = saveTokenValue;
hasExtendedUnicodeEscape = saveHasExtendedUnicodeEscape;
tokenIsUnterminated = saveTokenIsUnterminated;
tokenFlags = saveTokenFlags;
return result;
}
@ -1974,11 +1969,8 @@ namespace ts {
startPos = textPos;
tokenPos = textPos;
token = SyntaxKind.Unknown;
precedingLineBreak = false;
tokenValue = undefined;
hasExtendedUnicodeEscape = false;
tokenIsUnterminated = false;
tokenFlags = 0;
}
}
}

2
src/compiler/transformers/es2015.ts
View File

@ -3633,7 +3633,7 @@ namespace ts {
* @param node A string literal.
*/
function visitNumericLiteral(node: NumericLiteral) {
if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
return setTextRange(createNumericLiteral(node.text), node);
}
return node;

19
src/compiler/types.ts
View File

@ -1474,20 +1474,25 @@ namespace ts {
}
/* @internal */
export const enum NumericLiteralFlags {
export const enum TokenFlags {
None = 0,
Scientific = 1 << 1, // e.g. `10e2`
Octal = 1 << 2, // e.g. `0777`
HexSpecifier = 1 << 3, // e.g. `0x00000000`
BinarySpecifier = 1 << 4, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 5, // e.g. `0o777`
PrecedingLineBreak = 1 << 0,
PrecedingJSDocComment = 1 << 1,
Unterminated = 1 << 2,
ExtendedUnicodeEscape = 1 << 3,
Scientific = 1 << 4, // e.g. `10e2`
Octal = 1 << 5, // e.g. `0777`
HexSpecifier = 1 << 6, // e.g. `0x00000000`
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 8, // e.g. `0o777`
BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
}
export interface NumericLiteral extends LiteralExpression {
kind: SyntaxKind.NumericLiteral;
/* @internal */
numericLiteralFlags?: NumericLiteralFlags;
numericLiteralFlags?: TokenFlags;
}
export interface TemplateHead extends LiteralLikeNode {

2
tests/baselines/reference/duplicateIdentifierDifferentSpelling.js
View File

@ -10,7 +10,7 @@ var X = { 0b11: '', 3: '' };
//// [duplicateIdentifierDifferentSpelling.js]
var A = /** @class */ (function () {
function A() {
this[0b11] = '';
this[3] = '';
this[3] = '';
}
return A;

37
tests/baselines/reference/parserArrowFunctionExpression6.errors.txt
View File

@ -1,37 +0,0 @@
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,23): error TS2300: Duplicate identifier '(Missing)'.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,24): error TS1005: ',' expected.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,29): error TS1138: Parameter declaration expected.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,30): error TS2300: Duplicate identifier '(Missing)'.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,31): error TS1003: Identifier expected.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,40): error TS2355: A function whose declared type is neither 'void' nor 'any' must return a value.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,41): error TS2371: A parameter initializer is only allowed in a function or constructor implementation.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,55): error TS1138: Parameter declaration expected.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,66): error TS1005: '=>' expected.
tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts(2,69): error TS1005: ':' expected.
==== tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts (10 errors) ====
function foo(q: string, b: number) {
return true ? (q ? true : false) : (b = q.length, function() { });
!!! error TS2300: Duplicate identifier '(Missing)'.
~~~~
!!! error TS1005: ',' expected.
~
!!! error TS1138: Parameter declaration expected.
!!! error TS2300: Duplicate identifier '(Missing)'.
~~~~~
!!! error TS1003: Identifier expected.
~~~~~~~~~~~~~~~~~~~~~~~~~
!!! error TS2355: A function whose declared type is neither 'void' nor 'any' must return a value.
~~~~~~~~~~~~
!!! error TS2371: A parameter initializer is only allowed in a function or constructor implementation.
~~~~~~~~
!!! error TS1138: Parameter declaration expected.
~
!!! error TS1005: '=>' expected.
~
!!! error TS1005: ':' expected.
};

3
tests/baselines/reference/parserArrowFunctionExpression6.js
View File

@ -6,7 +6,6 @@ function foo(q: string, b: number) {
//// [parserArrowFunctionExpression6.js]
function foo(q, b) {
return true ? function (q, , ) { } : ;
;
return true ? (q ? true : false) : (b = q.length, function () { });
}
;

10
tests/baselines/reference/parserArrowFunctionExpression6.symbols
View File

@ -5,11 +5,11 @@ function foo(q: string, b: number) {
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))
return true ? (q ? true : false) : (b = q.length, function() { });
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 22))
> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 29))
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 1, 40))
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))
>q.length : Symbol(String.length, Decl(lib.d.ts, --, --))
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
>length : Symbol(String.length, Decl(lib.d.ts, --, --))
};

26
tests/baselines/reference/parserArrowFunctionExpression6.types
View File

@ -1,21 +1,25 @@
=== tests/cases/conformance/parser/ecmascript5/ArrowFunctionExpressions/parserArrowFunctionExpression6.ts ===
function foo(q: string, b: number) {
>foo : (q: string, b: number) => any
>foo : (q: string, b: number) => boolean | (() => void)
>q : string
>b : number
return true ? (q ? true : false) : (b = q.length, function() { });
>true ? (q ? true : false) : (b = q.length, function() { } : any
>true ? (q ? true : false) : (b = q.length, function() { }) : boolean | (() => void)
>true : true
>(q ? true : false) : (b = q.length, function() { } : (q?: any, : any, : any) => (b?: any) => () => any
>q : any
> : any
> : any
>b : any
>q.length : any
>q : any
>length : any
> : any
>(q ? true : false) : boolean
>q ? true : false : boolean
>q : string
>true : true
>false : false
>(b = q.length, function() { }) : () => void
>b = q.length, function() { } : () => void
>b = q.length : number
>b : number
>q.length : number
>q : string
>length : number
>function() { } : () => void
};