microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-06-10 17:05:58 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Speed up incremental parsing by caching child tokens instead of walking the tree for them.

Browse Source
This commit is contained in:
Cyrus Najmabadi
2014-11-02 17:42:47 -08:00
parent f15d87bec7
commit d89d9282ff
9 changed files with 186 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/services/syntax/SyntaxGenerator.js
View File

@@ -1041,7 +1041,7 @@ var definitions = [
name: 'VariableDeclaratorSyntax',
baseType: 'ISyntaxNode',
children: [
{ name: 'propertyName', isToken: true, tokenKinds: ['IdentifierName', 'StringLiteral', 'NumericLiteral'] },
{ name: 'propertyName', isToken: true },
{ name: 'typeAnnotation', type: 'TypeAnnotationSyntax', isOptional: true, isTypeScriptSpecific: true },
{ name: 'equalsValueClause', type: 'EqualsValueClauseSyntax', isOptional: true }
]
@@ -2838,14 +2838,22 @@ function max(array, func) {
return max;
}
function generateScannerUtilities() {
var result = "///<reference path='references.ts' />\r\n" + "\r\n" + "module TypeScript {\r\n" + " export class ScannerUtilities {\r\n";
var result = "///<reference path='references.ts' />\r\n" + "\r\n" + "module TypeScript {\r\n" + " export module ScannerUtilities {\r\n";
result += " export function fixedWidthTokenLength(kind: SyntaxKind) {\r\n";
result += " switch (kind) {\r\n";
for (var k = TypeScript.SyntaxKind.FirstFixedWidth; k <= TypeScript.SyntaxKind.LastFixedWidth; k++) {
result += " case SyntaxKind." + syntaxKindName(k) + ": return " + TypeScript.SyntaxFacts.getText(k).length + ";\r\n";
}
result += " default: throw new Error();\r\n";
result += " }\r\n";
result += " }\r\n\r\n";
var i;
var keywords = [];
for (i = TypeScript.SyntaxKind.FirstKeyword; i <= TypeScript.SyntaxKind.LastKeyword; i++) {
keywords.push({ kind: i, text: TypeScript.SyntaxFacts.getText(i) });
}
keywords.sort(function (a, b) { return a.text.localeCompare(b.text); });
result += " public static identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n";
result += " export function identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n";
var minTokenLength = min(keywords, function (k) { return k.text.length; });
var maxTokenLength = max(keywords, function (k) { return k.text.length; });
result += " switch (length) {\r\n";

40
src/services/syntax/incrementalParser.ts
View File

@@ -247,12 +247,37 @@ module TypeScript.IncrementalParser {
var tokenWasMoved = isPastChangeRange() && fullStart(nodeOrToken) !== position;
if (tokenWasMoved) {
setTokenFullStartWalker.position = position;
if (isToken(nodeOrToken)) {
(<ISyntaxToken>nodeOrToken).setFullStart(position);
}
else {
var tokens = getTokens(<ISyntaxNode>nodeOrToken);
visitNodeOrToken(setTokenFullStartWalker, nodeOrToken);
for (var i = 0, n = tokens.length; i < n; i++) {
var token = tokens[i];
token.setFullStart(position);
position += token.fullWidth();
}
}
}
}
function getTokens(node: ISyntaxNode): ISyntaxToken[] {
var tokens = node.__cachedTokens;
if (!tokens) {
tokens = [];
tokenCollectorWalker.tokens = tokens;
visitNodeOrToken(tokenCollectorWalker, node);
node.__cachedTokens = tokens;
tokenCollectorWalker.tokens = undefined;
}
return tokens;
}
function currentNode(): ISyntaxNode {
if (canReadFromOldSourceUnit()) {
// Try to read a node. If we can't then our caller will call back in and just try
@@ -841,18 +866,15 @@ module TypeScript.IncrementalParser {
// A simple walker we use to hit all the tokens of a node and update their positions when they
// are reused in a different location because of an incremental parse.
class SetTokenFullStartWalker extends SyntaxWalker {
public position: number;
class TokenCollectorWalker extends SyntaxWalker {
public tokens: ISyntaxToken[] = [];
public visitToken(token: ISyntaxToken): void {
var position = this.position;
token.setFullStart(position);
this.position = position + token.fullWidth();
this.tokens.push(token);
}
}
var setTokenFullStartWalker = new SetTokenFullStartWalker();
var tokenCollectorWalker = new TokenCollectorWalker();
export function parse(oldSyntaxTree: SyntaxTree, textChangeRange: TextChangeRange, newText: ISimpleText): SyntaxTree {
if (textChangeRange.isUnchanged()) {

8
src/services/syntax/parser.ts
View File

@@ -554,11 +554,13 @@ module TypeScript.Parser {
while (true) {
// Parent must be a list or a node. All of those have a 'data' element.
Debug.assert(isNode(parent) || isList(parent));
var dataElement = <{ data: number }><any>parent;
if (dataElement.data) {
dataElement.data &= SyntaxConstants.NodeParsedInStrictModeMask
var dataElement = <ISyntaxNode>parent;
if (dataElement.__data) {
dataElement.__data &= SyntaxConstants.NodeParsedInStrictModeMask
}
dataElement.__cachedTokens = undefined;
if (parent === node) {
break;
}

113
src/services/syntax/scanner.ts
View File

@@ -275,7 +275,7 @@ module TypeScript.Scanner {
public trailingTriviaWidth(): number { return 0; }
public kind(): SyntaxKind { return this._packedData & ScannerConstants.KindMask; }
public fullWidth(): number { return this.fullText().length; }
public fullWidth(): number { return fixedWidthTokenLength(this._packedData & ScannerConstants.KindMask); }
public fullStart(): number { return fixedWidthTokenUnpackFullStart(this._packedData); }
public hasLeadingTrivia(): boolean { return false; }
public hasTrailingTrivia(): boolean { return false; }
@@ -1745,4 +1745,115 @@ module TypeScript.Scanner {
resetToPosition: resetToPosition,
};
}
function fixedWidthTokenLength(kind: SyntaxKind) {
switch (kind) {
case SyntaxKind.BreakKeyword: return 5;
case SyntaxKind.CaseKeyword: return 4;
case SyntaxKind.CatchKeyword: return 5;
case SyntaxKind.ContinueKeyword: return 8;
case SyntaxKind.DebuggerKeyword: return 8;
case SyntaxKind.DefaultKeyword: return 7;
case SyntaxKind.DeleteKeyword: return 6;
case SyntaxKind.DoKeyword: return 2;
case SyntaxKind.ElseKeyword: return 4;
case SyntaxKind.FalseKeyword: return 5;
case SyntaxKind.FinallyKeyword: return 7;
case SyntaxKind.ForKeyword: return 3;
case SyntaxKind.FunctionKeyword: return 8;
case SyntaxKind.IfKeyword: return 2;
case SyntaxKind.InKeyword: return 2;
case SyntaxKind.InstanceOfKeyword: return 10;
case SyntaxKind.NewKeyword: return 3;
case SyntaxKind.NullKeyword: return 4;
case SyntaxKind.ReturnKeyword: return 6;
case SyntaxKind.SwitchKeyword: return 6;
case SyntaxKind.ThisKeyword: return 4;
case SyntaxKind.ThrowKeyword: return 5;
case SyntaxKind.TrueKeyword: return 4;
case SyntaxKind.TryKeyword: return 3;
case SyntaxKind.TypeOfKeyword: return 6;
case SyntaxKind.VarKeyword: return 3;
case SyntaxKind.VoidKeyword: return 4;
case SyntaxKind.WhileKeyword: return 5;
case SyntaxKind.WithKeyword: return 4;
case SyntaxKind.ClassKeyword: return 5;
case SyntaxKind.ConstKeyword: return 5;
case SyntaxKind.EnumKeyword: return 4;
case SyntaxKind.ExportKeyword: return 6;
case SyntaxKind.ExtendsKeyword: return 7;
case SyntaxKind.ImportKeyword: return 6;
case SyntaxKind.SuperKeyword: return 5;
case SyntaxKind.ImplementsKeyword: return 10;
case SyntaxKind.InterfaceKeyword: return 9;
case SyntaxKind.LetKeyword: return 3;
case SyntaxKind.PackageKeyword: return 7;
case SyntaxKind.PrivateKeyword: return 7;
case SyntaxKind.ProtectedKeyword: return 9;
case SyntaxKind.PublicKeyword: return 6;
case SyntaxKind.StaticKeyword: return 6;
case SyntaxKind.YieldKeyword: return 5;
case SyntaxKind.AnyKeyword: return 3;
case SyntaxKind.BooleanKeyword: return 7;
case SyntaxKind.ConstructorKeyword: return 11;
case SyntaxKind.DeclareKeyword: return 7;
case SyntaxKind.GetKeyword: return 3;
case SyntaxKind.ModuleKeyword: return 6;
case SyntaxKind.RequireKeyword: return 7;
case SyntaxKind.NumberKeyword: return 6;
case SyntaxKind.SetKeyword: return 3;
case SyntaxKind.StringKeyword: return 6;
case SyntaxKind.OpenBraceToken: return 1;
case SyntaxKind.CloseBraceToken: return 1;
case SyntaxKind.OpenParenToken: return 1;
case SyntaxKind.CloseParenToken: return 1;
case SyntaxKind.OpenBracketToken: return 1;
case SyntaxKind.CloseBracketToken: return 1;
case SyntaxKind.DotToken: return 1;
case SyntaxKind.DotDotDotToken: return 3;
case SyntaxKind.SemicolonToken: return 1;
case SyntaxKind.CommaToken: return 1;
case SyntaxKind.LessThanToken: return 1;
case SyntaxKind.GreaterThanToken: return 1;
case SyntaxKind.LessThanEqualsToken: return 2;
case SyntaxKind.GreaterThanEqualsToken: return 2;
case SyntaxKind.EqualsEqualsToken: return 2;
case SyntaxKind.EqualsGreaterThanToken: return 2;
case SyntaxKind.ExclamationEqualsToken: return 2;
case SyntaxKind.EqualsEqualsEqualsToken: return 3;
case SyntaxKind.ExclamationEqualsEqualsToken: return 3;
case SyntaxKind.PlusToken: return 1;
case SyntaxKind.MinusToken: return 1;
case SyntaxKind.AsteriskToken: return 1;
case SyntaxKind.PercentToken: return 1;
case SyntaxKind.PlusPlusToken: return 2;
case SyntaxKind.MinusMinusToken: return 2;
case SyntaxKind.LessThanLessThanToken: return 2;
case SyntaxKind.GreaterThanGreaterThanToken: return 2;
case SyntaxKind.GreaterThanGreaterThanGreaterThanToken: return 3;
case SyntaxKind.AmpersandToken: return 1;
case SyntaxKind.BarToken: return 1;
case SyntaxKind.CaretToken: return 1;
case SyntaxKind.ExclamationToken: return 1;
case SyntaxKind.TildeToken: return 1;
case SyntaxKind.AmpersandAmpersandToken: return 2;
case SyntaxKind.BarBarToken: return 2;
case SyntaxKind.QuestionToken: return 1;
case SyntaxKind.ColonToken: return 1;
case SyntaxKind.EqualsToken: return 1;
case SyntaxKind.PlusEqualsToken: return 2;
case SyntaxKind.MinusEqualsToken: return 2;
case SyntaxKind.AsteriskEqualsToken: return 2;
case SyntaxKind.PercentEqualsToken: return 2;
case SyntaxKind.LessThanLessThanEqualsToken: return 3;
case SyntaxKind.GreaterThanGreaterThanEqualsToken: return 3;
case SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: return 4;
case SyntaxKind.AmpersandEqualsToken: return 2;
case SyntaxKind.BarEqualsToken: return 2;
case SyntaxKind.CaretEqualsToken: return 2;
case SyntaxKind.SlashToken: return 1;
case SyntaxKind.SlashEqualsToken: return 2;
default: throw new Error();
}
}
}

4
src/services/syntax/scannerUtilities.generated.ts
View File

@@ -1,8 +1,8 @@
///<reference path='references.ts' />
module TypeScript {
export class ScannerUtilities {
public static identifierKind(str: string, start: number, length: number): SyntaxKind {
export module ScannerUtilities {
export function identifierKind(str: string, start: number, length: number): SyntaxKind {
switch (length) {
case 2: // do, if, in
switch(str.charCodeAt(start)) {

15
src/services/syntax/syntaxElement.ts
View File

@@ -28,7 +28,7 @@ module TypeScript {
}
export function parsedInStrictMode(node: ISyntaxNode): boolean {
var info = node.data;
var info = node.__data;
if (info === undefined) {
return false;
}
@@ -73,7 +73,7 @@ module TypeScript {
* Note: findToken will always return a non-missing token with width greater than or equal to
* 1 (except for EOF). Empty tokens synthesized by the parser are never returned.
*/
export function findToken(element: ISyntaxElement, position: number, includeSkippedTokens: boolean = false): ISyntaxToken {
export function findToken(element: ISyntaxElement, position: number, includeSkippedTokens?: boolean): ISyntaxToken {
var endOfFileToken = tryGetEndOfFileAt(element, position);
if (endOfFileToken) {
return endOfFileToken;
@@ -277,7 +277,7 @@ module TypeScript {
var kind = element.kind();
if (isTokenKind(kind)) {
return fullWidth(element) > 0 || element.kind() === SyntaxKind.EndOfFileToken ? <ISyntaxToken>element : undefined;
return (<ISyntaxToken>element).fullWidth() > 0 || kind === SyntaxKind.EndOfFileToken ? <ISyntaxToken>element : undefined;
}
for (var i = 0, n = element.childCount(); i < n; i++) {
@@ -349,16 +349,16 @@ module TypeScript {
Debug.assert(isNode(element) || isList(element));
// Lists and nodes all have a 'data' element.
var dataElement = <{ data: number }><any>element;
var dataElement = <ISyntaxNode>element;
var info = dataElement.data;
var info = dataElement.__data;
if (info === undefined) {
info = 0;
}
if ((info & SyntaxConstants.NodeDataComputed) === 0) {
info |= computeData(element);
dataElement.data = info;
dataElement.__data = info;
}
return info;
@@ -429,7 +429,8 @@ module TypeScript {
}
export interface ISyntaxNode extends ISyntaxNodeOrToken {
data: number;
__data: number;
__cachedTokens: ISyntaxToken[];
}
export interface IModuleReferenceSyntax extends ISyntaxNode {

14
src/services/syntax/syntaxGenerator.ts
View File

@@ -2407,7 +2407,17 @@ function generateScannerUtilities(): string {
var result = "///<reference path='references.ts' />\r\n" +
"\r\n" +
"module TypeScript {\r\n" +
" export class ScannerUtilities {\r\n";
" export module ScannerUtilities {\r\n";
result += " export function fixedWidthTokenLength(kind: SyntaxKind) {\r\n";
result += " switch (kind) {\r\n";
for (var k = TypeScript.SyntaxKind.FirstFixedWidth; k <= TypeScript.SyntaxKind.LastFixedWidth; k++) {
result += " case SyntaxKind." + syntaxKindName(k) + ": return " + TypeScript.SyntaxFacts.getText(k).length + ";\r\n";
}
result += " default: throw new Error();\r\n";
result += " }\r\n";
result += " }\r\n\r\n";
var i: number;
var keywords: { text: string; kind: TypeScript.SyntaxKind; }[] = [];
@@ -2418,7 +2428,7 @@ function generateScannerUtilities(): string {
keywords.sort((a, b) => a.text.localeCompare(b.text));
result += " public static identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n";
result += " export function identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n";
var minTokenLength = min(keywords, k => k.text.length);
var maxTokenLength = max(keywords, k => k.text.length);

2
src/services/syntax/syntaxList.ts
View File

@@ -1,7 +1,7 @@
///<reference path='references.ts' />
interface Array<T> {
data: number;
__data: number;
kind(): TypeScript.SyntaxKind;
parent: TypeScript.ISyntaxElement;

7
src/services/syntax/syntaxNode.ts
View File

@@ -1,14 +1,15 @@
///<reference path='references.ts' />
module TypeScript {
export class SyntaxNode implements ISyntaxNodeOrToken {
export class SyntaxNode implements ISyntaxNode {
// private __kind: SyntaxKind;
public data: number;
public __data: number;
public __cachedTokens: ISyntaxToken[];
public parent: ISyntaxElement;
constructor(data: number) {
if (data) {
this.data = data;
this.__data = data;
}
}