From a0464435c9d112bcfd4abb1527a89010878a5f4a Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 6 Nov 2014 09:30:49 -0800 Subject: [PATCH] Attach skipped tokens to the following token, not the preceding one. This makes handling skipped tokens much simpler, and helps put us in a position where tokens only have leading trivia and never trailing trivia. --- src/services/syntax/incrementalParser.ts | 2 + src/services/syntax/parser.ts | 549 ++++++++--------------- src/services/syntax/scanner.ts | 2 + 3 files changed, 185 insertions(+), 368 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index c0d1d4aa0ed..b234b77e03f 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -529,6 +529,8 @@ module TypeScript.IncrementalParser { } function consumeToken(currentToken: ISyntaxToken): void { + // Debug.assert(currentToken.fullWidth() > 0 || currentToken.kind === SyntaxKind.EndOfFileToken); + // This token may have come from the old source unit, or from the new text. Handle // both accordingly. diff --git a/src/services/syntax/parser.ts b/src/services/syntax/parser.ts index efa73c8164a..c495b71cbc8 100644 --- a/src/services/syntax/parser.ts +++ b/src/services/syntax/parser.ts @@ -116,27 +116,6 @@ module TypeScript.Parser { export interface IRewindPoint { } - var arrayPool: any[][] = []; - var arrayPoolCount: number = 0; - - function getArray(): any[] { - if (arrayPoolCount === 0) { - return []; - } - - arrayPoolCount--; - var result = arrayPool[arrayPoolCount]; - arrayPool[arrayPoolCount] = undefined; - - return result; - } - - function returnArray(array: any[]) { - array.length = 0; - arrayPool[arrayPoolCount] = array; - arrayPoolCount++; - } - interface IParserRewindPoint extends IRewindPoint { // As we speculatively parse, we may build up diagnostics. When we rewind we want to // 'forget' that information.In order to do that we store the count of diagnostics and @@ -144,6 +123,12 @@ module TypeScript.Parser { // speculative parse does not affect any further results. diagnosticsCount: number; + // As we speculatively parse we may end up adding additional skipped tokens to the + // _skippedTokens array in the parser. When we rewind we don't want those items in the + // array. We may also, during speculative parsing, attach our skipped tokens to some + // new token. When we rewind we need to restore whatever skipped tokens we started with. + skippedTokens: ISyntaxToken[]; + // isInStrictMode and listParsingState should not have to be tracked by a rewind point. // Because they are naturally mutated and restored based on the normal stack movement of // the parser, they should automatically return to whatever value they had to begin with @@ -190,6 +175,8 @@ module TypeScript.Parser { var parseNodeData: number = 0; + var _skippedTokens: ISyntaxToken[] = undefined; + function parseSyntaxTree(_source: IParserSource, isDeclaration: boolean): SyntaxTree { // First, set up our state. fileName = _source.fileName; @@ -222,7 +209,10 @@ module TypeScript.Parser { function getRewindPoint(): IParserRewindPoint { var rewindPoint = source.getRewindPoint(); + // See the comments in IParserRewindPoint for the explanation on why we need to store + // this data, and what it is used for. rewindPoint.diagnosticsCount = diagnostics.length; + rewindPoint.skippedTokens = _skippedTokens ? _skippedTokens.slice(0) : undefined; // Values we keep around for debug asserting purposes. rewindPoint.isInStrictMode = isInStrictMode; @@ -235,6 +225,7 @@ module TypeScript.Parser { source.rewind(rewindPoint); diagnostics.length = rewindPoint.diagnosticsCount; + _skippedTokens = rewindPoint.skippedTokens; } function releaseRewindPoint(rewindPoint: IParserRewindPoint): void { @@ -245,6 +236,14 @@ module TypeScript.Parser { } function currentNode(): ISyntaxNode { + // If we have any outstanding tokens, then don't reuse a node. + // TODO(cyrusn): This may be too conservative. Perhaps we could reuse hte node and + // attach the skipped tokens in front? For now though, being conservative is nice and + // safe, and likely won't ever affect perf. + if (_skippedTokens) { + return null; + } + var node = source.currentNode(); // We can only reuse a node if it was parsed under the same strict mode that we're @@ -277,12 +276,82 @@ module TypeScript.Parser { return source.peekToken(n); } + function skipToken(token: ISyntaxToken): void { + _skippedTokens = _skippedTokens || []; + _skippedTokens.push(token); + + // directly tell the source to just consume the token we're skipping. i.e. do not + // call 'consumeToken'. Doing so would attempt to add any previous skipped tokens + // to this token we're skipping. We don't want to do that. Instead, we want to add + // all the skipped tokens when we finally eat the next good token. + source.consumeToken(token) + } + function consumeToken(token: ISyntaxToken): ISyntaxToken { + // Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); + + // First, tell our source that the token has been consumed. source.consumeToken(token); + + // Now, if we had any skipped tokens, we want to add them to the start of this token + // we're consuming. + if (_skippedTokens) { + token = addSkippedTokensBeforeToken(token, _skippedTokens); + _skippedTokens = undefined; + } + return token; } + function addSkippedTokensBeforeToken(token: ISyntaxToken, skippedTokens: ISyntaxToken[]): ISyntaxToken { + //Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); + //Debug.assert(skippedTokens.length > 0); + + var leadingTrivia: ISyntaxTrivia[] = []; + for (var i = 0, n = skippedTokens.length; i < n; i++) { + var skippedToken = skippedTokens[i]; + addSkippedTokenToTriviaArray(leadingTrivia, skippedToken); + } + + addTriviaTo(token.leadingTrivia(source.text), leadingTrivia); + + var updatedToken = Syntax.withLeadingTrivia(token, Syntax.triviaList(leadingTrivia), source.text); + + // We've prepending this token with new leading trivia. This means the full start of + // the token is not where the scanner originally thought it was, but is instead at the + // start of the first skipped token. + updatedToken.setFullStart(skippedTokens[0].fullStart()); + + return updatedToken; + } + + function addSkippedTokenToTriviaArray(array: ISyntaxTrivia[], skippedToken: ISyntaxToken): void { + // Debug.assert(skippedToken.text().length > 0); + + // first, add the leading trivia of the skipped token to the array + addTriviaTo(skippedToken.leadingTrivia(source.text), array); + + // now, add the text of the token as skipped text to the trivia array. + var trimmedToken = Syntax.withTrailingTrivia(Syntax.withLeadingTrivia(skippedToken, Syntax.emptyTriviaList, source.text), Syntax.emptyTriviaList, source.text); + + // Because we removed the leading trivia from the skipped token, the full start of the + // trimmed token is the start of the skipped token. + trimmedToken.setFullStart(start(skippedToken, source.text)); + + array.push(Syntax.skippedTokenTrivia(trimmedToken, source.text)); + + // Finally, add the trailing trivia of the skipped token to the trivia array. + addTriviaTo(skippedToken.trailingTrivia(source.text), array); + } + + function addTriviaTo(list: ISyntaxTriviaList, array: ISyntaxTrivia[]): void { + for (var i = 0, n = list.count(); i < n; i++) { + array.push(list.syntaxTriviaAt(i)); + } + } + function consumeNode(node: ISyntaxNode): void { + Debug.assert(_skippedTokens === undefined); source.consumeNode(node); } @@ -366,13 +435,11 @@ module TypeScript.Parser { function eatIdentifierToken(diagnosticCode?: string): ISyntaxToken { var token = currentToken(); if (isIdentifier(token)) { - consumeToken(token); - if (token.kind === SyntaxKind.IdentifierName) { - return token; + return consumeToken(token); } - return TypeScript.Syntax.convertKeywordToIdentifier(token); + return TypeScript.Syntax.convertKeywordToIdentifier(consumeToken(token)); } return createMissingToken(SyntaxKind.IdentifierName, token, diagnosticCode); @@ -523,182 +590,24 @@ module TypeScript.Parser { throw Errors.invalidOperation(); } - function addSkippedTokenAfterNodeOrToken(nodeOrToken: ISyntaxNodeOrToken, skippedToken: ISyntaxToken): ISyntaxNodeOrToken { - if (isToken(nodeOrToken)) { - return addSkippedTokenAfterToken(nodeOrToken, skippedToken); - } - else if (isNode(nodeOrToken)) { - return addSkippedTokenAfterNode(nodeOrToken, skippedToken); - } - else { - throw Errors.invalidOperation(); - } - } - - function replaceTokenInParent(node: ISyntaxNode, oldToken: ISyntaxToken, newToken: ISyntaxToken): void { - // oldToken may be parented by a node or a list. - replaceTokenInParentWorker(oldToken, newToken); - - var parent = oldToken.parent; - newToken.parent = parent; - - // Walk upwards to our outermost node, clearing hte cached 'data' in it. This will - // make sure that the fullWidths and incrementally unusable bits are computed correctly - // when next requested. - while (true) { - // Parent must be a list or a node. All of those have a 'data' element. - Debug.assert(isNode(parent) || isList(parent)); - var dataElement = parent; - if (dataElement.__data) { - dataElement.__data &= SyntaxConstants.NodeParsedInStrictModeMask - } - - dataElement.__cachedTokens = undefined; - - if (parent === node) { - break; - } - - parent = parent.parent; - } - } - - function replaceTokenInParentWorker(oldToken: ISyntaxToken, newToken: ISyntaxToken): void { - var parent = oldToken.parent; - - if (isNode(parent)) { - var node = parent; - for (var key in node) { - if (node[key] === oldToken) { - node[key] = newToken; - return; - } - } - } - else if (isList(parent)) { - var list1 = parent; - for (var i = 0, n = list1.length; i < n; i++) { - if (list1[i] === oldToken) { - list1[i] = newToken; - return; - } - } - } - - throw Errors.invalidOperation(); - } - - function addSkippedTokenAfterNode(node: ISyntaxNode, skippedToken: ISyntaxToken): ISyntaxNode { - var oldToken = lastToken(node); - var newToken = addSkippedTokenAfterToken(oldToken, skippedToken); - - replaceTokenInParent(node, oldToken, newToken); - return node; - } - - function addSkippedTokensBeforeNode(node: ISyntaxNode, skippedTokens: ISyntaxToken[]): ISyntaxNode { - if (skippedTokens.length > 0) { - var oldToken = firstToken(node); - var newToken = addSkippedTokensBeforeToken(oldToken, skippedTokens); - - replaceTokenInParent(node, oldToken, newToken); - } - - return node; - } - - function addSkippedTokensBeforeToken(token: ISyntaxToken, skippedTokens: ISyntaxToken[]): ISyntaxToken { - // Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); - // Debug.assert(skippedTokens.length > 0); - - var leadingTrivia: ISyntaxTrivia[] = []; - for (var i = 0, n = skippedTokens.length; i < n; i++) { - var skippedToken = skippedTokens[i]; - addSkippedTokenToTriviaArray(leadingTrivia, skippedToken); - } - - addTriviaTo(token.leadingTrivia(source.text), leadingTrivia); - - var updatedToken = Syntax.withLeadingTrivia(token, Syntax.triviaList(leadingTrivia), source.text); - - // We've prepending this token with new leading trivia. This means the full start of - // the token is not where the scanner originally thought it was, but is instead at the - // start of the first skipped token. - updatedToken.setFullStart(skippedTokens[0].fullStart()); - - // Don't need this array anymore. Give it back so we can reuse it. - returnArray(skippedTokens); - - return updatedToken; - } - - function addSkippedTokensAfterToken(token: ISyntaxToken, skippedTokens: ISyntaxToken[]): ISyntaxToken { - // Debug.assert(token.fullWidth() > 0); - if (skippedTokens.length === 0) { - returnArray(skippedTokens); - return token; - } - - var trailingTrivia = token.trailingTrivia(source.text).toArray(); - - for (var i = 0, n = skippedTokens.length; i < n; i++) { - addSkippedTokenToTriviaArray(trailingTrivia, skippedTokens[i]); - } - - // Don't need this array anymore. Give it back so we can reuse it. - returnArray(skippedTokens); - return Syntax.withTrailingTrivia(token, Syntax.triviaList(trailingTrivia), source.text); - } - - function addSkippedTokenAfterToken(token: ISyntaxToken, skippedToken: ISyntaxToken): ISyntaxToken { - // Debug.assert(token.fullWidth() > 0); - var trailingTrivia = token.trailingTrivia(source.text).toArray(); - addSkippedTokenToTriviaArray(trailingTrivia, skippedToken); - - return Syntax.withTrailingTrivia(token, Syntax.triviaList(trailingTrivia), source.text); - } - - function addSkippedTokenToTriviaArray(array: ISyntaxTrivia[], skippedToken: ISyntaxToken): void { - // Debug.assert(skippedToken.text().length > 0); - - // first, add the leading trivia of the skipped token to the array - addTriviaTo(skippedToken.leadingTrivia(source.text), array); - - // now, add the text of the token as skipped text to the trivia array. - var trimmedToken = Syntax.withTrailingTrivia(Syntax.withLeadingTrivia(skippedToken, Syntax.emptyTriviaList, source.text), Syntax.emptyTriviaList, source.text); - - // Because we removed the leading trivia from the skipped token, the full start of the - // trimmed token is the start of the skipped token. - trimmedToken.setFullStart(start(skippedToken, source.text)); - - array.push(Syntax.skippedTokenTrivia(trimmedToken, source.text)); - - // Finally, add the trailing trivia of the skipped token to the trivia array. - addTriviaTo(skippedToken.trailingTrivia(source.text), array); - } - - function addTriviaTo(list: ISyntaxTriviaList, array: ISyntaxTrivia[]): void { - for (var i = 0, n = list.count(); i < n; i++) { - array.push(list.syntaxTriviaAt(i)); - } - } - function setStrictMode(_isInStrictMode: boolean) { isInStrictMode = _isInStrictMode; parseNodeData = _isInStrictMode ? SyntaxConstants.NodeParsedInStrictModeMask : 0; } function parseSourceUnit(): SourceUnitSyntax { + // Note: saving and restoring the 'isInStrictMode' state is not really necessary here + // (as it will never be read afterwards). However, for symmetry with the rest of the + // parsing code, we do the same here. var savedIsInStrictMode = isInStrictMode - var skippedTokens: ISyntaxToken[] = getArray(); - var moduleElements = parseSyntaxList(ListParsingState.SourceUnit_ModuleElements, skippedTokens, updateStrictModeState); - + // Note: any skipped tokens produced after the end of all the module elements will be + // added as skipped trivia to the start of the EOF token. + var moduleElements = parseSyntaxList(ListParsingState.SourceUnit_ModuleElements, updateStrictModeState); + setStrictMode(savedIsInStrictMode); - var sourceUnit = new SourceUnitSyntax(parseNodeData, moduleElements, currentToken()); - - sourceUnit = addSkippedTokensBeforeNode(sourceUnit, skippedTokens); + var sourceUnit = new SourceUnitSyntax(parseNodeData, moduleElements, consumeToken(currentToken())); if (Debug.shouldAssert(AssertionLevel.Aggressive)) { Debug.assert(fullWidth(sourceUnit) === source.text.length()); @@ -849,13 +758,10 @@ module TypeScript.Parser { if (!inExpression) { // if we're not in an expression, this must be a type argument list. Just parse // it out as such. - var lessThanToken = consumeToken(_currentToken); - - var skippedTokens: ISyntaxToken[] = getArray(); - var typeArguments = parseSeparatedSyntaxList(ListParsingState.TypeArgumentList_Types, skippedTokens); - lessThanToken = addSkippedTokensAfterToken(lessThanToken, skippedTokens); - - return new TypeArgumentListSyntax(parseNodeData, lessThanToken, typeArguments, eatToken(SyntaxKind.GreaterThanToken)); + return new TypeArgumentListSyntax(parseNodeData, + consumeToken(_currentToken), + parseSeparatedSyntaxList(ListParsingState.TypeArgumentList_Types), + eatToken(SyntaxKind.GreaterThanToken)); } // If we're in an expression, then we only want to consume this as a type argument list @@ -865,11 +771,7 @@ module TypeScript.Parser { // We've seen a '<'. Try to parse it out as a type argument list. var lessThanToken = consumeToken(_currentToken); - - var skippedTokens: ISyntaxToken[] = getArray(); - var typeArguments = parseSeparatedSyntaxList(ListParsingState.TypeArgumentList_Types, skippedTokens); - var lessThanToken = addSkippedTokensAfterToken(lessThanToken, skippedTokens); - + var typeArguments = parseSeparatedSyntaxList(ListParsingState.TypeArgumentList_Types); var greaterThanToken = eatToken(SyntaxKind.GreaterThanToken); // We're in a context where '<' could be the start of a type argument list, or part @@ -991,9 +893,7 @@ module TypeScript.Parser { var enumElements: ISeparatedSyntaxList; if (openBraceToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - enumElements = parseSeparatedSyntaxList(ListParsingState.EnumDeclaration_EnumElements, skippedTokens); - openBraceToken = addSkippedTokensAfterToken(openBraceToken, skippedTokens); + enumElements = parseSeparatedSyntaxList(ListParsingState.EnumDeclaration_EnumElements); } return new EnumDeclarationSyntax(parseNodeData, modifiers, enumKeyword, identifier, openBraceToken, enumElements || [], eatToken(SyntaxKind.CloseBraceToken)); @@ -1073,7 +973,7 @@ module TypeScript.Parser { } function parseModifiers(): ISyntaxToken[] { - var tokens: ISyntaxToken[] = getArray(); + var tokens: ISyntaxToken[] = []; while (true) { var token = currentToken(); @@ -1115,9 +1015,7 @@ module TypeScript.Parser { var classElements: IClassElementSyntax[]; if (openBraceToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - classElements = parseSyntaxList(ListParsingState.ClassDeclaration_ClassElements, skippedTokens); - openBraceToken = addSkippedTokensAfterToken(openBraceToken, skippedTokens); + classElements = parseSyntaxList(ListParsingState.ClassDeclaration_ClassElements); }; return new ClassDeclarationSyntax(parseNodeData, @@ -1333,9 +1231,8 @@ module TypeScript.Parser { start(token0, source.text), width(token0), DiagnosticCode.Unexpected_token_0_expected, [SyntaxFacts.getText(SyntaxKind.OpenBraceToken)]); addDiagnostic(diagnostic); - consumeToken(token0); - - addSkippedTokenAfterNode(callSignature, token0); + // Skip over the => It will get attached to whatever comes next. + skipToken(token0); return true; } } @@ -1390,9 +1287,7 @@ module TypeScript.Parser { var moduleElements: IModuleElementSyntax[]; if (openBraceToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - moduleElements = parseSyntaxList(ListParsingState.ModuleDeclaration_ModuleElements, skippedTokens); - openBraceToken = addSkippedTokensAfterToken(openBraceToken, skippedTokens); + moduleElements = parseSyntaxList(ListParsingState.ModuleDeclaration_ModuleElements); } return new ModuleDeclarationSyntax(parseNodeData, @@ -1410,25 +1305,17 @@ module TypeScript.Parser { var typeMembers: ISeparatedSyntaxList; if (openBraceToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - typeMembers = parseSeparatedSyntaxList(ListParsingState.ObjectType_TypeMembers, skippedTokens); - openBraceToken = addSkippedTokensAfterToken(openBraceToken, skippedTokens); + typeMembers = parseSeparatedSyntaxList(ListParsingState.ObjectType_TypeMembers); } return new ObjectTypeSyntax(parseNodeData, openBraceToken, typeMembers || [], eatToken(SyntaxKind.CloseBraceToken)); } function parseTupleType(currentToken: ISyntaxToken): TupleTypeSyntax { - var openBracket = consumeToken(currentToken); - - var types: ISeparatedSyntaxList; - if (openBracket.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - types = parseSeparatedSyntaxList(ListParsingState.TupleType_Types, skippedTokens); - openBracket = addSkippedTokensAfterToken(openBracket, skippedTokens); - } - - return new TupleTypeSyntax(parseNodeData, openBracket, types || [], eatToken(SyntaxKind.CloseBracketToken)); + return new TupleTypeSyntax(parseNodeData, + consumeToken(currentToken), + parseSeparatedSyntaxList(ListParsingState.TupleType_Types), + eatToken(SyntaxKind.CloseBracketToken)); } function isTypeMember(inErrorRecovery: boolean): boolean { @@ -1508,14 +1395,10 @@ module TypeScript.Parser { } function parseIndexSignature(): IndexSignatureSyntax { - var openBracketToken = eatToken(SyntaxKind.OpenBracketToken); - - var skippedTokens: ISyntaxToken[] = getArray(); - var parameters = parseSeparatedSyntaxList(ListParsingState.IndexSignature_Parameters, skippedTokens); - openBracketToken = addSkippedTokensAfterToken(openBracketToken, skippedTokens); - return new IndexSignatureSyntax(parseNodeData, - openBracketToken, parameters, eatToken(SyntaxKind.CloseBracketToken), parseOptionalTypeAnnotation(/*allowStringLiteral:*/ false)); + eatToken(SyntaxKind.OpenBracketToken), + parseSeparatedSyntaxList(ListParsingState.IndexSignature_Parameters), + eatToken(SyntaxKind.CloseBracketToken), parseOptionalTypeAnnotation(/*allowStringLiteral:*/ false)); } function parseMethodSignature(propertyName: IPropertyNameSyntax, questionToken: ISyntaxToken): MethodSignatureSyntax { @@ -1605,13 +1488,9 @@ module TypeScript.Parser { return undefined; } - consumeToken(extendsOrImplementsKeyword); - - var skippedTokens: ISyntaxToken[] = getArray(); - var typeNames = parseSeparatedSyntaxList(ListParsingState.HeritageClause_TypeNameList, skippedTokens); - extendsOrImplementsKeyword = addSkippedTokensAfterToken(extendsOrImplementsKeyword, skippedTokens); - - return new HeritageClauseSyntax(parseNodeData, extendsOrImplementsKeyword, typeNames); + return new HeritageClauseSyntax(parseNodeData, + consumeToken(extendsOrImplementsKeyword), + parseSeparatedSyntaxList(ListParsingState.HeritageClause_TypeNameList)); } function isInterfaceEnumClassModuleImportOrExport(modifierCount: number, _currentToken?: ISyntaxToken): boolean { @@ -1886,7 +1765,7 @@ module TypeScript.Parser { function parseForOrForInStatement(forKeyword: ISyntaxToken): IStatementSyntax { // Debug.assert(isForOrForInStatement()); - consumeToken(forKeyword); + forKeyword = consumeToken(forKeyword); var openParenToken = eatToken(SyntaxKind.OpenParenToken); var _currentToken = currentToken(); @@ -2007,7 +1886,7 @@ module TypeScript.Parser { function parseSwitchStatement(switchKeyword: ISyntaxToken) { // Debug.assert(isSwitchStatement()); - consumeToken(switchKeyword); + switchKeyword = consumeToken(switchKeyword); var openParenToken = eatToken(SyntaxKind.OpenParenToken); var expression: IExpressionSyntax; @@ -2025,9 +1904,7 @@ module TypeScript.Parser { var switchClauses: ISwitchClauseSyntax[]; if (openBraceToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - switchClauses = parseSyntaxList(ListParsingState.SwitchStatement_SwitchClauses, skippedTokens); - openBraceToken = addSkippedTokensAfterToken(openBraceToken, skippedTokens); + switchClauses = parseSyntaxList(ListParsingState.SwitchStatement_SwitchClauses); } return new SwitchStatementSyntax(parseNodeData, switchKeyword, openParenToken, expression, closeParenToken, openBraceToken, switchClauses || [], eatToken(SyntaxKind.CloseBraceToken)); @@ -2066,38 +1943,20 @@ module TypeScript.Parser { function parseCaseSwitchClause(caseKeyword: ISyntaxToken): CaseSwitchClauseSyntax { // Debug.assert(isCaseSwitchClause()); - consumeToken(caseKeyword); - var expression = parseExpression(/*allowIn:*/ true); - var colonToken = eatToken(SyntaxKind.ColonToken); - var statements: IStatementSyntax[]; - - // TODO: allow parsing of the list evne if there's no colon. However, we have to make - // sure we add any skipped tokens to the right previous node or token. - if (colonToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - statements = parseSyntaxList(ListParsingState.SwitchClause_Statements, skippedTokens); - colonToken = addSkippedTokensAfterToken(colonToken, skippedTokens); - } - - return new CaseSwitchClauseSyntax(parseNodeData, caseKeyword, expression, colonToken, statements || []); + return new CaseSwitchClauseSyntax(parseNodeData, + consumeToken(caseKeyword), + parseExpression(/*allowIn:*/ true), + eatToken(SyntaxKind.ColonToken), + parseSyntaxList(ListParsingState.SwitchClause_Statements)); } function parseDefaultSwitchClause(defaultKeyword: ISyntaxToken): DefaultSwitchClauseSyntax { // Debug.assert(isDefaultSwitchClause()); - consumeToken(defaultKeyword); - var colonToken = eatToken(SyntaxKind.ColonToken); - var statements: IStatementSyntax[]; - - // TODO: Allow parsing without a colon here. However, ensure that we attach any skipped - // tokens to the defaultKeyword. - if (colonToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - statements = parseSyntaxList(ListParsingState.SwitchClause_Statements, skippedTokens); - colonToken = addSkippedTokensAfterToken(colonToken, skippedTokens); - } - - return new DefaultSwitchClauseSyntax(parseNodeData, defaultKeyword, colonToken, statements || []); + return new DefaultSwitchClauseSyntax(parseNodeData, + consumeToken(defaultKeyword), + eatToken(SyntaxKind.ColonToken), + parseSyntaxList(ListParsingState.SwitchClause_Statements)); } function parseThrowStatementExpression(): IExpressionSyntax { @@ -2245,18 +2104,13 @@ module TypeScript.Parser { function parseVariableDeclaration(allowIn: boolean): VariableDeclarationSyntax { // Debug.assert(currentToken().kind === SyntaxKind.VarKeyword); - var varKeyword = eatToken(SyntaxKind.VarKeyword); - // Debug.assert(varKeyword.fullWidth() > 0); - var listParsingState = allowIn ? ListParsingState.VariableDeclaration_VariableDeclarators_AllowIn : ListParsingState.VariableDeclaration_VariableDeclarators_DisallowIn; - var skippedTokens: ISyntaxToken[] = getArray(); - var variableDeclarators = parseSeparatedSyntaxList(listParsingState, skippedTokens); - varKeyword = addSkippedTokensAfterToken(varKeyword, skippedTokens); - - return new VariableDeclarationSyntax(parseNodeData, varKeyword, variableDeclarators); + return new VariableDeclarationSyntax(parseNodeData, + eatToken(SyntaxKind.VarKeyword), + parseSeparatedSyntaxList(listParsingState)); } function isVariableDeclarator(): boolean { @@ -2633,7 +2487,7 @@ module TypeScript.Parser { switch (currentTokenKind) { case SyntaxKind.OpenParenToken: - expression = new InvocationExpressionSyntax(parseNodeData, expression, parseArgumentList(/*typeArgumentList:*/ undefined)); + expression = new InvocationExpressionSyntax(parseNodeData, expression, parseArgumentList(/*typeArgumentList:*/ undefined, _currentToken)); continue; case SyntaxKind.LessThanToken: @@ -2740,7 +2594,7 @@ module TypeScript.Parser { } function parseSuperExpression(superToken: ISyntaxToken): ILeftHandSideExpressionSyntax { - var expression: ILeftHandSideExpressionSyntax = consumeToken(superToken); + var expression = consumeToken(superToken); // If we have seen "super" it must be followed by '(' or '.'. // If it wasn't then just try to parse out a '.' and report an error. @@ -2797,6 +2651,8 @@ module TypeScript.Parser { return undefined; } else { + Debug.assert(typeArgumentList && isOpenParenOrDot); + releaseRewindPoint(rewindPoint); // It's not uncommon for a user to type: "Foo." // @@ -2814,37 +2670,34 @@ module TypeScript.Parser { Syntax.emptyToken(SyntaxKind.OpenParenToken), [], Syntax.emptyToken(SyntaxKind.CloseParenToken)); } else { - return parseArgumentList(typeArgumentList); + Debug.assert(token0.kind === SyntaxKind.OpenParenToken); + return parseArgumentList(typeArgumentList, token0); } } } function tryParseArgumentList(): ArgumentListSyntax { - var tokenKind = currentToken().kind; + var _currentToken = currentToken(); + var tokenKind = _currentToken.kind; if (tokenKind === SyntaxKind.LessThanToken) { return tryParseGenericArgumentList(); } if (tokenKind === SyntaxKind.OpenParenToken) { - return parseArgumentList(undefined); + return parseArgumentList(/*typeArgumentList:*/ undefined, /*openParenToken:*/ _currentToken); } return undefined; } - function parseArgumentList(typeArgumentList: TypeArgumentListSyntax): ArgumentListSyntax { - var openParenToken = eatToken(SyntaxKind.OpenParenToken); + function parseArgumentList(typeArgumentList: TypeArgumentListSyntax, openParenToken: ISyntaxToken): ArgumentListSyntax { + Debug.assert(openParenToken.kind === SyntaxKind.OpenParenToken && openParenToken.fullWidth() > 0); - // Don't use the name 'arguments' it prevents V8 from optimizing this method. - var _arguments: ISeparatedSyntaxList; - - if (openParenToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - _arguments = parseSeparatedSyntaxList(ListParsingState.ArgumentList_AssignmentExpressions, skippedTokens); - openParenToken = addSkippedTokensAfterToken(openParenToken, skippedTokens); - } - - return new ArgumentListSyntax(parseNodeData, typeArgumentList, openParenToken, _arguments || [], eatToken(SyntaxKind.CloseParenToken)); + return new ArgumentListSyntax(parseNodeData, + typeArgumentList, + consumeToken(openParenToken), + parseSeparatedSyntaxList(ListParsingState.ArgumentList_AssignmentExpressions), + eatToken(SyntaxKind.CloseParenToken)); } function tryParseArgumentListExpression(): IExpressionSyntax { @@ -2994,13 +2847,13 @@ module TypeScript.Parser { } function parseTemplateExpression(startToken: ISyntaxToken): IPrimaryExpressionSyntax { - consumeToken(startToken); + startToken = consumeToken(startToken); if (startToken.kind === SyntaxKind.NoSubstitutionTemplateToken) { return startToken; } - var templateClauses: TemplateClauseSyntax[] = getArray(); + var templateClauses: TemplateClauseSyntax[] = []; do { // Keep consuming template spans as long as the last one we keep getting template @@ -3019,7 +2872,7 @@ module TypeScript.Parser { if (token.kind === SyntaxKind.CloseBraceToken) { token = currentContextualToken(); Debug.assert(token.kind === SyntaxKind.TemplateMiddleToken || token.kind === SyntaxKind.TemplateEndToken); - consumeToken(token); + token = consumeToken(token); } else { var diagnostic = getExpectedTokenDiagnostic(SyntaxKind.CloseBraceToken); @@ -3321,15 +3174,10 @@ module TypeScript.Parser { function parseObjectLiteralExpression(openBraceToken: ISyntaxToken): ObjectLiteralExpressionSyntax { // Debug.assert(currentToken().kind === SyntaxKind.OpenBraceToken); - - consumeToken(openBraceToken); - // Debug.assert(openBraceToken.fullWidth() > 0); - - var skippedTokens: ISyntaxToken[] = getArray(); - var propertyAssignments = parseSeparatedSyntaxList(ListParsingState.ObjectLiteralExpression_PropertyAssignments, skippedTokens); - openBraceToken = addSkippedTokensAfterToken(openBraceToken, skippedTokens); - - return new ObjectLiteralExpressionSyntax(parseNodeData, openBraceToken, propertyAssignments, eatToken(SyntaxKind.CloseBraceToken)); + return new ObjectLiteralExpressionSyntax(parseNodeData, + consumeToken(openBraceToken), + parseSeparatedSyntaxList(ListParsingState.ObjectLiteralExpression_PropertyAssignments), + eatToken(SyntaxKind.CloseBraceToken)); } function tryParsePropertyAssignment(inErrorRecovery: boolean): IPropertyAssignmentSyntax { @@ -3461,14 +3309,10 @@ module TypeScript.Parser { function parseArrayLiteralExpression(openBracketToken: ISyntaxToken): ArrayLiteralExpressionSyntax { // Debug.assert(currentToken().kind === SyntaxKind.OpenBracketToken); - consumeToken(openBracketToken); - // Debug.assert(openBracketToken.fullWidth() > 0); - - var skippedTokens: ISyntaxToken[] = getArray(); - var expressions = parseSeparatedSyntaxList(ListParsingState.ArrayLiteralExpression_AssignmentExpressions, skippedTokens); - openBracketToken = addSkippedTokensAfterToken(openBracketToken, skippedTokens); - - return new ArrayLiteralExpressionSyntax(parseNodeData, openBracketToken, expressions, eatToken(SyntaxKind.CloseBracketToken)); + return new ArrayLiteralExpressionSyntax(parseNodeData, + consumeToken(openBracketToken), + parseSeparatedSyntaxList(ListParsingState.ArrayLiteralExpression_AssignmentExpressions), + eatToken(SyntaxKind.CloseBracketToken)); } function parseBlock(parseBlockEvenWithNoOpenBrace: boolean, checkForStrictMode: boolean): BlockSyntax { @@ -3479,9 +3323,7 @@ module TypeScript.Parser { var savedIsInStrictMode = isInStrictMode; var processItems = checkForStrictMode ? updateStrictModeState : undefined; - var skippedTokens: ISyntaxToken[] = getArray(); - var statements = parseSyntaxList(ListParsingState.Block_Statements, skippedTokens, processItems); - openBraceToken = addSkippedTokensAfterToken(openBraceToken, skippedTokens); + var statements = parseSyntaxList(ListParsingState.Block_Statements, processItems); setStrictMode(savedIsInStrictMode); } @@ -3503,10 +3345,7 @@ module TypeScript.Parser { var rewindPoint = getRewindPoint(); var lessThanToken = consumeToken(_currentToken); - - var skippedTokens: ISyntaxToken[] = getArray(); - var typeParameters = parseSeparatedSyntaxList(ListParsingState.TypeParameterList_TypeParameters, skippedTokens); - lessThanToken = addSkippedTokensAfterToken(lessThanToken, skippedTokens); + var typeParameters = parseSeparatedSyntaxList(ListParsingState.TypeParameterList_TypeParameters); var greaterThanToken = eatToken(SyntaxKind.GreaterThanToken); @@ -3560,9 +3399,7 @@ module TypeScript.Parser { var parameters: ISeparatedSyntaxList; if (openParenToken.fullWidth() > 0) { - var skippedTokens: ISyntaxToken[] = getArray(); - parameters = parseSeparatedSyntaxList(ListParsingState.ParameterList_Parameters, skippedTokens); - openParenToken = addSkippedTokensAfterToken(openParenToken, skippedTokens); + parameters = parseSeparatedSyntaxList(ListParsingState.ParameterList_Parameters); } return new ParameterListSyntax(parseNodeData, openParenToken, parameters || [], eatToken(SyntaxKind.CloseParenToken)); @@ -3651,10 +3488,7 @@ module TypeScript.Parser { if (type) { var barToken: ISyntaxToken; while ((barToken = currentToken()).kind === SyntaxKind.BarToken) { - consumeToken(barToken); - var right = parsePrimaryType(); - - type = new UnionTypeSyntax(parseNodeData, type, barToken, right); + type = new UnionTypeSyntax(parseNodeData, type, consumeToken(barToken), parsePrimaryType()); } } @@ -3875,23 +3709,22 @@ module TypeScript.Parser { return new ParameterSyntax(parseNodeData, dotDotDotToken, modifiers, identifier, questionToken, typeAnnotation, equalsValueClause); } - function parseSyntaxList( - currentListType: ListParsingState, skippedTokens: ISyntaxToken[], processItems?: (items: any[]) => void): T[] { + function parseSyntaxList(currentListType: ListParsingState, processItems?: (items: any[]) => void): T[] { var savedListParsingState = listParsingState; listParsingState |= (1 << currentListType); - var result = parseSyntaxListWorker(currentListType, skippedTokens, processItems); + var result = parseSyntaxListWorker(currentListType, processItems); listParsingState = savedListParsingState; return result; } - function parseSeparatedSyntaxList(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): ISeparatedSyntaxList { + function parseSeparatedSyntaxList(currentListType: ListParsingState): ISeparatedSyntaxList { var savedListParsingState = listParsingState; listParsingState |= (1 << currentListType); - var result = parseSeparatedSyntaxListWorker(currentListType, skippedTokens); + var result = parseSeparatedSyntaxListWorker(currentListType); listParsingState = savedListParsingState; @@ -3900,7 +3733,9 @@ module TypeScript.Parser { // Returns true if we should abort parsing. function abortParsingListOrMoveToNextToken( - currentListType: ListParsingState, nodeAndSeparators: ISyntaxNodeOrToken[], skippedTokens: ISyntaxToken[]): boolean { + currentListType: ListParsingState, + nodeAndSeparators: ISyntaxNodeOrToken[]): boolean { + // Ok. We're at a token that is not a terminator for the list and wasn't the start of // an item in the list. Definitely report an error for this token. reportUnexpectedTokenDiagnostic(currentListType); @@ -3920,34 +3755,12 @@ module TypeScript.Parser { // Otherwise, if none of the lists we're in can capture this token, then we need to // unilaterally skip it. Note: we've already reported an error above. - addSkippedTokenToList(nodeAndSeparators, skippedTokens, consumeToken(currentToken())); - + skipToken(currentToken()); + // Continue parsing this list. Attach this token to whatever we've seen already. return false; } - function addSkippedTokenToList( - nodesAndSeparators: ISyntaxNodeOrToken[], skippedTokens: ISyntaxToken[], skippedToken: ISyntaxToken): void { - // Now, add this skipped token to the last item we successfully parsed in the list. Or - // add it to the list of skipped tokens if we haven't parsed anything. Our caller will - // have to deal with them. - // - var length = nodesAndSeparators.length; - - for (var i = length - 1; i >= 0; i--) { - var item = nodesAndSeparators[i]; - var _lastToken = lastToken(item); - if (_lastToken && _lastToken.fullWidth() > 0) { - nodesAndSeparators[i] = addSkippedTokenAfterNodeOrToken(item, skippedToken); - return; - } - } - - // Didn't have anything in the list we could add to. Add to the skipped items array - // for our caller to handle. - skippedTokens.push(skippedToken); - } - function tryParseExpectedListItem( currentListType: ListParsingState, inErrorRecovery: boolean, items: ISyntaxNodeOrToken[], processItems: (items: ISyntaxNodeOrToken[]) => void): boolean { var item = tryParseExpectedListItemWorker(currentListType, inErrorRecovery); @@ -3971,7 +3784,7 @@ module TypeScript.Parser { currentToken().kind === SyntaxKind.EndOfFileToken; } - function parseSyntaxListWorker(currentListType: ListParsingState, skippedTokens: ISyntaxToken[], processItems: (items: ISyntaxNodeOrToken[]) => void ): T[] { + function parseSyntaxListWorker(currentListType: ListParsingState, processItems: (items: ISyntaxNodeOrToken[]) => void ): T[] { var items: T[] = []; while (true) { @@ -3990,7 +3803,7 @@ module TypeScript.Parser { // List wasn't complete and we didn't get an item. Figure out if we should bail out // or skip a token and continue. - var abort = abortParsingListOrMoveToNextToken(currentListType, items, skippedTokens); + var abort = abortParsingListOrMoveToNextToken(currentListType, items); if (abort) { break; } @@ -4003,7 +3816,7 @@ module TypeScript.Parser { return Syntax.list(items); } - function parseSeparatedSyntaxListWorker(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): ISeparatedSyntaxList { + function parseSeparatedSyntaxListWorker(currentListType: ListParsingState): ISeparatedSyntaxList { var nodesAndSeparators: ISyntaxNodeOrToken[] = []; // Debug.assert(nodes.length === 0); @@ -4036,7 +3849,7 @@ module TypeScript.Parser { // List wasn't complete and we didn't get an item. Figure out if we should bail out // or skip a token and continue. - var abort = abortParsingListOrMoveToNextToken(currentListType, nodesAndSeparators, skippedTokens); + var abort = abortParsingListOrMoveToNextToken(currentListType, nodesAndSeparators); if (abort) { break; } diff --git a/src/services/syntax/scanner.ts b/src/services/syntax/scanner.ts index a56dbcdc474..3d2b61e6dd5 100644 --- a/src/services/syntax/scanner.ts +++ b/src/services/syntax/scanner.ts @@ -1622,6 +1622,8 @@ module TypeScript.Scanner { } function consumeToken(token: ISyntaxToken): void { + // Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); + // Debug.assert(currentToken() === token); _absolutePosition += token.fullWidth();