diff --git a/src/services/syntax/SyntaxGenerator.js b/src/services/syntax/SyntaxGenerator.js index 0c3dd9e5b38..3d30ba1531c 100644 --- a/src/services/syntax/SyntaxGenerator.js +++ b/src/services/syntax/SyntaxGenerator.js @@ -1041,7 +1041,7 @@ var definitions = [ name: 'VariableDeclaratorSyntax', baseType: 'ISyntaxNode', children: [ - { name: 'propertyName', isToken: true, tokenKinds: ['IdentifierName', 'StringLiteral', 'NumericLiteral'] }, + { name: 'propertyName', isToken: true }, { name: 'typeAnnotation', type: 'TypeAnnotationSyntax', isOptional: true, isTypeScriptSpecific: true }, { name: 'equalsValueClause', type: 'EqualsValueClauseSyntax', isOptional: true } ] @@ -2838,14 +2838,22 @@ function max(array, func) { return max; } function generateScannerUtilities() { - var result = "///\r\n" + "\r\n" + "module TypeScript {\r\n" + " export class ScannerUtilities {\r\n"; + var result = "///\r\n" + "\r\n" + "module TypeScript {\r\n" + " export module ScannerUtilities {\r\n"; + result += " export function fixedWidthTokenLength(kind: SyntaxKind) {\r\n"; + result += " switch (kind) {\r\n"; + for (var k = TypeScript.SyntaxKind.FirstFixedWidth; k <= TypeScript.SyntaxKind.LastFixedWidth; k++) { + result += " case SyntaxKind." + syntaxKindName(k) + ": return " + TypeScript.SyntaxFacts.getText(k).length + ";\r\n"; + } + result += " default: throw new Error();\r\n"; + result += " }\r\n"; + result += " }\r\n\r\n"; var i; var keywords = []; for (i = TypeScript.SyntaxKind.FirstKeyword; i <= TypeScript.SyntaxKind.LastKeyword; i++) { keywords.push({ kind: i, text: TypeScript.SyntaxFacts.getText(i) }); } keywords.sort(function (a, b) { return a.text.localeCompare(b.text); }); - result += " public static identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n"; + result += " export function identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n"; var minTokenLength = min(keywords, function (k) { return k.text.length; }); var maxTokenLength = max(keywords, function (k) { return k.text.length; }); result += " switch (length) {\r\n"; diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index c30672e5042..3c9938a56de 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -247,12 +247,37 @@ module TypeScript.IncrementalParser { var tokenWasMoved = isPastChangeRange() && fullStart(nodeOrToken) !== position; if (tokenWasMoved) { - setTokenFullStartWalker.position = position; + if (isToken(nodeOrToken)) { + (nodeOrToken).setFullStart(position); + } + else { + var tokens = getTokens(nodeOrToken); - visitNodeOrToken(setTokenFullStartWalker, nodeOrToken); + for (var i = 0, n = tokens.length; i < n; i++) { + var token = tokens[i]; + token.setFullStart(position); + + position += token.fullWidth(); + } + } } } + function getTokens(node: ISyntaxNode): ISyntaxToken[] { + var tokens = node.__cachedTokens; + if (!tokens) { + tokens = []; + tokenCollectorWalker.tokens = tokens; + + visitNodeOrToken(tokenCollectorWalker, node); + + node.__cachedTokens = tokens; + tokenCollectorWalker.tokens = undefined; + } + + return tokens; + } + function currentNode(): ISyntaxNode { if (canReadFromOldSourceUnit()) { // Try to read a node. If we can't then our caller will call back in and just try @@ -841,18 +866,15 @@ module TypeScript.IncrementalParser { // A simple walker we use to hit all the tokens of a node and update their positions when they // are reused in a different location because of an incremental parse. - class SetTokenFullStartWalker extends SyntaxWalker { - public position: number; + class TokenCollectorWalker extends SyntaxWalker { + public tokens: ISyntaxToken[] = []; public visitToken(token: ISyntaxToken): void { - var position = this.position; - token.setFullStart(position); - - this.position = position + token.fullWidth(); + this.tokens.push(token); } } - var setTokenFullStartWalker = new SetTokenFullStartWalker(); + var tokenCollectorWalker = new TokenCollectorWalker(); export function parse(oldSyntaxTree: SyntaxTree, textChangeRange: TextChangeRange, newText: ISimpleText): SyntaxTree { if (textChangeRange.isUnchanged()) { diff --git a/src/services/syntax/parser.ts b/src/services/syntax/parser.ts index d0e556f8d6c..71c88b42982 100644 --- a/src/services/syntax/parser.ts +++ b/src/services/syntax/parser.ts @@ -554,11 +554,13 @@ module TypeScript.Parser { while (true) { // Parent must be a list or a node. All of those have a 'data' element. Debug.assert(isNode(parent) || isList(parent)); - var dataElement = <{ data: number }>parent; - if (dataElement.data) { - dataElement.data &= SyntaxConstants.NodeParsedInStrictModeMask + var dataElement = parent; + if (dataElement.__data) { + dataElement.__data &= SyntaxConstants.NodeParsedInStrictModeMask } + dataElement.__cachedTokens = undefined; + if (parent === node) { break; } diff --git a/src/services/syntax/scanner.ts b/src/services/syntax/scanner.ts index a1be20adddb..b42b4d16986 100644 --- a/src/services/syntax/scanner.ts +++ b/src/services/syntax/scanner.ts @@ -275,7 +275,7 @@ module TypeScript.Scanner { public trailingTriviaWidth(): number { return 0; } public kind(): SyntaxKind { return this._packedData & ScannerConstants.KindMask; } - public fullWidth(): number { return this.fullText().length; } + public fullWidth(): number { return fixedWidthTokenLength(this._packedData & ScannerConstants.KindMask); } public fullStart(): number { return fixedWidthTokenUnpackFullStart(this._packedData); } public hasLeadingTrivia(): boolean { return false; } public hasTrailingTrivia(): boolean { return false; } @@ -1745,4 +1745,115 @@ module TypeScript.Scanner { resetToPosition: resetToPosition, }; } + + function fixedWidthTokenLength(kind: SyntaxKind) { + switch (kind) { + case SyntaxKind.BreakKeyword: return 5; + case SyntaxKind.CaseKeyword: return 4; + case SyntaxKind.CatchKeyword: return 5; + case SyntaxKind.ContinueKeyword: return 8; + case SyntaxKind.DebuggerKeyword: return 8; + case SyntaxKind.DefaultKeyword: return 7; + case SyntaxKind.DeleteKeyword: return 6; + case SyntaxKind.DoKeyword: return 2; + case SyntaxKind.ElseKeyword: return 4; + case SyntaxKind.FalseKeyword: return 5; + case SyntaxKind.FinallyKeyword: return 7; + case SyntaxKind.ForKeyword: return 3; + case SyntaxKind.FunctionKeyword: return 8; + case SyntaxKind.IfKeyword: return 2; + case SyntaxKind.InKeyword: return 2; + case SyntaxKind.InstanceOfKeyword: return 10; + case SyntaxKind.NewKeyword: return 3; + case SyntaxKind.NullKeyword: return 4; + case SyntaxKind.ReturnKeyword: return 6; + case SyntaxKind.SwitchKeyword: return 6; + case SyntaxKind.ThisKeyword: return 4; + case SyntaxKind.ThrowKeyword: return 5; + case SyntaxKind.TrueKeyword: return 4; + case SyntaxKind.TryKeyword: return 3; + case SyntaxKind.TypeOfKeyword: return 6; + case SyntaxKind.VarKeyword: return 3; + case SyntaxKind.VoidKeyword: return 4; + case SyntaxKind.WhileKeyword: return 5; + case SyntaxKind.WithKeyword: return 4; + case SyntaxKind.ClassKeyword: return 5; + case SyntaxKind.ConstKeyword: return 5; + case SyntaxKind.EnumKeyword: return 4; + case SyntaxKind.ExportKeyword: return 6; + case SyntaxKind.ExtendsKeyword: return 7; + case SyntaxKind.ImportKeyword: return 6; + case SyntaxKind.SuperKeyword: return 5; + case SyntaxKind.ImplementsKeyword: return 10; + case SyntaxKind.InterfaceKeyword: return 9; + case SyntaxKind.LetKeyword: return 3; + case SyntaxKind.PackageKeyword: return 7; + case SyntaxKind.PrivateKeyword: return 7; + case SyntaxKind.ProtectedKeyword: return 9; + case SyntaxKind.PublicKeyword: return 6; + case SyntaxKind.StaticKeyword: return 6; + case SyntaxKind.YieldKeyword: return 5; + case SyntaxKind.AnyKeyword: return 3; + case SyntaxKind.BooleanKeyword: return 7; + case SyntaxKind.ConstructorKeyword: return 11; + case SyntaxKind.DeclareKeyword: return 7; + case SyntaxKind.GetKeyword: return 3; + case SyntaxKind.ModuleKeyword: return 6; + case SyntaxKind.RequireKeyword: return 7; + case SyntaxKind.NumberKeyword: return 6; + case SyntaxKind.SetKeyword: return 3; + case SyntaxKind.StringKeyword: return 6; + case SyntaxKind.OpenBraceToken: return 1; + case SyntaxKind.CloseBraceToken: return 1; + case SyntaxKind.OpenParenToken: return 1; + case SyntaxKind.CloseParenToken: return 1; + case SyntaxKind.OpenBracketToken: return 1; + case SyntaxKind.CloseBracketToken: return 1; + case SyntaxKind.DotToken: return 1; + case SyntaxKind.DotDotDotToken: return 3; + case SyntaxKind.SemicolonToken: return 1; + case SyntaxKind.CommaToken: return 1; + case SyntaxKind.LessThanToken: return 1; + case SyntaxKind.GreaterThanToken: return 1; + case SyntaxKind.LessThanEqualsToken: return 2; + case SyntaxKind.GreaterThanEqualsToken: return 2; + case SyntaxKind.EqualsEqualsToken: return 2; + case SyntaxKind.EqualsGreaterThanToken: return 2; + case SyntaxKind.ExclamationEqualsToken: return 2; + case SyntaxKind.EqualsEqualsEqualsToken: return 3; + case SyntaxKind.ExclamationEqualsEqualsToken: return 3; + case SyntaxKind.PlusToken: return 1; + case SyntaxKind.MinusToken: return 1; + case SyntaxKind.AsteriskToken: return 1; + case SyntaxKind.PercentToken: return 1; + case SyntaxKind.PlusPlusToken: return 2; + case SyntaxKind.MinusMinusToken: return 2; + case SyntaxKind.LessThanLessThanToken: return 2; + case SyntaxKind.GreaterThanGreaterThanToken: return 2; + case SyntaxKind.GreaterThanGreaterThanGreaterThanToken: return 3; + case SyntaxKind.AmpersandToken: return 1; + case SyntaxKind.BarToken: return 1; + case SyntaxKind.CaretToken: return 1; + case SyntaxKind.ExclamationToken: return 1; + case SyntaxKind.TildeToken: return 1; + case SyntaxKind.AmpersandAmpersandToken: return 2; + case SyntaxKind.BarBarToken: return 2; + case SyntaxKind.QuestionToken: return 1; + case SyntaxKind.ColonToken: return 1; + case SyntaxKind.EqualsToken: return 1; + case SyntaxKind.PlusEqualsToken: return 2; + case SyntaxKind.MinusEqualsToken: return 2; + case SyntaxKind.AsteriskEqualsToken: return 2; + case SyntaxKind.PercentEqualsToken: return 2; + case SyntaxKind.LessThanLessThanEqualsToken: return 3; + case SyntaxKind.GreaterThanGreaterThanEqualsToken: return 3; + case SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: return 4; + case SyntaxKind.AmpersandEqualsToken: return 2; + case SyntaxKind.BarEqualsToken: return 2; + case SyntaxKind.CaretEqualsToken: return 2; + case SyntaxKind.SlashToken: return 1; + case SyntaxKind.SlashEqualsToken: return 2; + default: throw new Error(); + } + } } \ No newline at end of file diff --git a/src/services/syntax/scannerUtilities.generated.ts b/src/services/syntax/scannerUtilities.generated.ts index fbf1b1cb952..97f0093a672 100644 --- a/src/services/syntax/scannerUtilities.generated.ts +++ b/src/services/syntax/scannerUtilities.generated.ts @@ -1,8 +1,8 @@ /// module TypeScript { - export class ScannerUtilities { - public static identifierKind(str: string, start: number, length: number): SyntaxKind { + export module ScannerUtilities { + export function identifierKind(str: string, start: number, length: number): SyntaxKind { switch (length) { case 2: // do, if, in switch(str.charCodeAt(start)) { diff --git a/src/services/syntax/syntaxElement.ts b/src/services/syntax/syntaxElement.ts index 7b5498d6550..53f3a63cce4 100644 --- a/src/services/syntax/syntaxElement.ts +++ b/src/services/syntax/syntaxElement.ts @@ -28,7 +28,7 @@ module TypeScript { } export function parsedInStrictMode(node: ISyntaxNode): boolean { - var info = node.data; + var info = node.__data; if (info === undefined) { return false; } @@ -73,7 +73,7 @@ module TypeScript { * Note: findToken will always return a non-missing token with width greater than or equal to * 1 (except for EOF). Empty tokens synthesized by the parser are never returned. */ - export function findToken(element: ISyntaxElement, position: number, includeSkippedTokens: boolean = false): ISyntaxToken { + export function findToken(element: ISyntaxElement, position: number, includeSkippedTokens?: boolean): ISyntaxToken { var endOfFileToken = tryGetEndOfFileAt(element, position); if (endOfFileToken) { return endOfFileToken; @@ -277,7 +277,7 @@ module TypeScript { var kind = element.kind(); if (isTokenKind(kind)) { - return fullWidth(element) > 0 || element.kind() === SyntaxKind.EndOfFileToken ? element : undefined; + return (element).fullWidth() > 0 || kind === SyntaxKind.EndOfFileToken ? element : undefined; } for (var i = 0, n = element.childCount(); i < n; i++) { @@ -349,16 +349,16 @@ module TypeScript { Debug.assert(isNode(element) || isList(element)); // Lists and nodes all have a 'data' element. - var dataElement = <{ data: number }>element; + var dataElement = element; - var info = dataElement.data; + var info = dataElement.__data; if (info === undefined) { info = 0; } if ((info & SyntaxConstants.NodeDataComputed) === 0) { info |= computeData(element); - dataElement.data = info; + dataElement.__data = info; } return info; @@ -429,7 +429,8 @@ module TypeScript { } export interface ISyntaxNode extends ISyntaxNodeOrToken { - data: number; + __data: number; + __cachedTokens: ISyntaxToken[]; } export interface IModuleReferenceSyntax extends ISyntaxNode { diff --git a/src/services/syntax/syntaxGenerator.ts b/src/services/syntax/syntaxGenerator.ts index 7ab1dfc289a..94932d701df 100644 --- a/src/services/syntax/syntaxGenerator.ts +++ b/src/services/syntax/syntaxGenerator.ts @@ -2407,7 +2407,17 @@ function generateScannerUtilities(): string { var result = "///\r\n" + "\r\n" + "module TypeScript {\r\n" + - " export class ScannerUtilities {\r\n"; + " export module ScannerUtilities {\r\n"; + + result += " export function fixedWidthTokenLength(kind: SyntaxKind) {\r\n"; + result += " switch (kind) {\r\n"; + + for (var k = TypeScript.SyntaxKind.FirstFixedWidth; k <= TypeScript.SyntaxKind.LastFixedWidth; k++) { + result += " case SyntaxKind." + syntaxKindName(k) + ": return " + TypeScript.SyntaxFacts.getText(k).length + ";\r\n"; + } + result += " default: throw new Error();\r\n"; + result += " }\r\n"; + result += " }\r\n\r\n"; var i: number; var keywords: { text: string; kind: TypeScript.SyntaxKind; }[] = []; @@ -2418,7 +2428,7 @@ function generateScannerUtilities(): string { keywords.sort((a, b) => a.text.localeCompare(b.text)); - result += " public static identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n"; + result += " export function identifierKind(str: string, start: number, length: number): SyntaxKind {\r\n"; var minTokenLength = min(keywords, k => k.text.length); var maxTokenLength = max(keywords, k => k.text.length); diff --git a/src/services/syntax/syntaxList.ts b/src/services/syntax/syntaxList.ts index 1be7eb25ddf..653bec8a393 100644 --- a/src/services/syntax/syntaxList.ts +++ b/src/services/syntax/syntaxList.ts @@ -1,7 +1,7 @@ /// interface Array { - data: number; + __data: number; kind(): TypeScript.SyntaxKind; parent: TypeScript.ISyntaxElement; diff --git a/src/services/syntax/syntaxNode.ts b/src/services/syntax/syntaxNode.ts index ea1cd17db5b..0cd0177b3fb 100644 --- a/src/services/syntax/syntaxNode.ts +++ b/src/services/syntax/syntaxNode.ts @@ -1,14 +1,15 @@ /// module TypeScript { - export class SyntaxNode implements ISyntaxNodeOrToken { + export class SyntaxNode implements ISyntaxNode { // private __kind: SyntaxKind; - public data: number; + public __data: number; + public __cachedTokens: ISyntaxToken[]; public parent: ISyntaxElement; constructor(data: number) { if (data) { - this.data = data; + this.__data = data; } }