From 48bef4698b566680de826ac080c81ca5c38dfb9d Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 18 Dec 2014 19:18:13 -0800 Subject: [PATCH] Provide better error recovery when we encounter merge markers in the source. Previously we would just treat each merge marker as trivia and then continue scanning and parsing like normal. This worked well in some scenarios, but fell down in others like: ``` class C { public foo() { <<<<<<< HEAD this.bar(); } ======= this.baz(); } >>>>>>> Branch public bar() { } } ``` The problem stems from the previous approach trying to incorporate both branches of the merge into the final tree. In a case like this, that approach breaks down entirely. The the parser ends up seeing the close curly in both included sections, and it considers the class finished. Then, it starts erroring when it encounters "public bar()". The fix is to only incorporate one of these sections into the tree. Specifically, we only include the first section. The second sectoin is treated like trivia and does not affect the parse at all. To make the experience more pleasant we do *lexically* classify the second section. That way it does not appear as just plain black text in the editor. Instead, it will have appropriate lexicla classifications for keywords, literals, comments, operators, punctuation, etc. However, any syntactic or semantic feature will not work in the second block due to this being trivia as far as any feature is concerned. This experience is still much better than what we had originally (where merge markers would absolutely) destroy the parse tree. And it is better than what we checked in last week, which could easily create a borked tree for many types of merges. Now, almost all merges should still leave the tree in good shape. All LS features will work in the first section, and lexical classification will work in the second. --- src/compiler/scanner.ts | 50 ++-- src/services/services.ts | 224 ++++++++++++------ .../conflictMarkerTrivia1.errors.txt | 8 +- .../conflictMarkerTrivia2.errors.txt | 17 +- ...yntacticClassificationsConflictMarkers1.ts | 19 ++ ...yntacticClassificationsConflictMarkers2.ts | 15 ++ .../cases/unittests/services/colorization.ts | 100 ++++---- 7 files changed, 270 insertions(+), 163 deletions(-) create mode 100644 tests/cases/fourslash/syntacticClassificationsConflictMarkers1.ts create mode 100644 tests/cases/fourslash/syntacticClassificationsConflictMarkers2.ts diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 67814c99462..18a9d72b17d 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -7,10 +7,6 @@ module ts { (message: DiagnosticMessage, length: number): void; } - export interface CommentCallback { - (pos: number, end: number): void; - } - export interface Scanner { getStartPos(): number; getToken(): SyntaxKind; @@ -396,8 +392,10 @@ module ts { var mergeConflictMarkerLength = "<<<<<<<".length; function isConflictMarkerTrivia(text: string, pos: number) { + Debug.assert(pos >= 0); + // Conflict markers must be at the start of a line. - if (pos > 0 && isLineBreak(text.charCodeAt(pos - 1))) { + if (pos === 0 || isLineBreak(text.charCodeAt(pos - 1))) { var ch = text.charCodeAt(pos); if ((pos + mergeConflictMarkerLength) < text.length) { @@ -415,10 +413,31 @@ module ts { return false; } - function scanConflictMarkerTrivia(text: string, pos: number) { - var len = text.length; - while (pos < len && !isLineBreak(text.charCodeAt(pos))) { - pos++; + function scanConflictMarkerTrivia(text: string, pos: number, error?: ErrorCallback) { + if (error) { + error(Diagnostics.Merge_conflict_marker_encountered, mergeConflictMarkerLength); + } + + var ch = text.charCodeAt(pos); + if (ch === CharacterCodes.lessThan || ch === CharacterCodes.greaterThan) { + var len = text.length; + while (pos < len && !isLineBreak(text.charCodeAt(pos))) { + pos++; + } + } + else { + Debug.assert(ch === CharacterCodes.equals); + // Consume everything from the start of the mid-conlict marker to the start of the next + // end-conflict marker. + var len = text.length; + while (pos < len) { + var ch = text.charCodeAt(pos); + if (ch === CharacterCodes.greaterThan && isConflictMarkerTrivia(text, pos)) { + break; + } + + pos++; + } } return pos; @@ -1057,8 +1076,7 @@ module ts { return pos++, token = SyntaxKind.SemicolonToken; case CharacterCodes.lessThan: if (isConflictMarkerTrivia(text, pos)) { - mergeConflictError(); - pos = scanConflictMarkerTrivia(text, pos); + pos = scanConflictMarkerTrivia(text, pos, error); if (skipTrivia) { continue; } @@ -1079,8 +1097,7 @@ module ts { return pos++, token = SyntaxKind.LessThanToken; case CharacterCodes.equals: if (isConflictMarkerTrivia(text, pos)) { - mergeConflictError(); - pos = scanConflictMarkerTrivia(text, pos); + pos = scanConflictMarkerTrivia(text, pos, error); if (skipTrivia) { continue; } @@ -1101,8 +1118,7 @@ module ts { return pos++, token = SyntaxKind.EqualsToken; case CharacterCodes.greaterThan: if (isConflictMarkerTrivia(text, pos)) { - mergeConflictError(); - pos = scanConflictMarkerTrivia(text, pos); + pos = scanConflictMarkerTrivia(text, pos, error); if (skipTrivia) { continue; } @@ -1171,10 +1187,6 @@ module ts { } } - function mergeConflictError() { - error(Diagnostics.Merge_conflict_marker_encountered, mergeConflictMarkerLength); - } - function reScanGreaterToken(): SyntaxKind { if (token === SyntaxKind.GreaterThanToken) { if (text.charCodeAt(pos) === CharacterCodes.greaterThan) { diff --git a/src/services/services.ts b/src/services/services.ts index 4951ff258f6..83a00d2dea0 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -2155,7 +2155,7 @@ module ts { // invalid identifier name. We need to check if whatever was inside the quotes is actually a valid identifier name. displayName = displayName.substring(1, displayName.length - 1); } - + var isValid = isIdentifierStart(displayName.charCodeAt(0), target); for (var i = 1, n = displayName.length; isValid && i < n; i++) { isValid = isIdentifierPart(displayName.charCodeAt(i), target); @@ -2206,7 +2206,7 @@ module ts { // Completion not allowed inside comments, bail out if this is the case var insideComment = isInsideComment(sourceFile, currentToken, position); host.log("getCompletionsAtPosition: Is inside comment: " + (new Date().getTime() - start)); - + if (insideComment) { host.log("Returning an empty list because completion was inside a comment."); return undefined; @@ -2593,7 +2593,7 @@ module ts { if (flags & SymbolFlags.TypeAlias) return ScriptElementKind.typeElement; if (flags & SymbolFlags.Interface) return ScriptElementKind.interfaceElement; if (flags & SymbolFlags.TypeParameter) return ScriptElementKind.typeParameterElement; - + var result = getSymbolKindOfConstructorPropertyMethodAccessorFunctionOrVar(symbol, flags, typeResolver, location); if (result === ScriptElementKind.unknown) { if (flags & SymbolFlags.TypeParameter) return ScriptElementKind.typeParameterElement; @@ -2656,7 +2656,7 @@ module ts { return ScriptElementKind.unknown; } - + function getTypeKind(type: Type): string { var flags = type.getFlags(); @@ -2730,7 +2730,7 @@ module ts { if (location.parent && location.parent.kind === SyntaxKind.PropertyAccessExpression) { var right = (location.parent).name; // Either the location is on the right of a property access, or on the left and the right is missing - if (right === location || (right && right.getFullWidth() === 0)){ + if (right === location || (right && right.getFullWidth() === 0)) { location = location.parent; } } @@ -2974,7 +2974,7 @@ module ts { symbolFlags & SymbolFlags.Method || symbolFlags & SymbolFlags.Constructor || symbolFlags & SymbolFlags.Signature || - symbolFlags & SymbolFlags.Accessor || + symbolFlags & SymbolFlags.Accessor || symbolKind === ScriptElementKind.memberFunctionElement) { var allSignatures = type.getCallSignatures(); addSignatureDisplayParts(allSignatures[0], allSignatures); @@ -3333,7 +3333,7 @@ module ts { if (!hasKind(ifStatement.elseStatement, SyntaxKind.IfStatement)) { break } - + ifStatement = ifStatement.elseStatement; } @@ -3355,7 +3355,7 @@ module ts { break; } } - + if (shouldHighlightNextKeyword) { result.push({ fileName: filename, @@ -3394,7 +3394,7 @@ module ts { return map(keywords, getReferenceEntryFromNode); } - + function getThrowOccurrences(throwStatement: ThrowStatement) { var owner = getThrowStatementOwner(throwStatement); @@ -3403,7 +3403,7 @@ module ts { } var keywords: Node[] = []; - + forEach(aggregateOwnedThrowStatements(owner), throwStatement => { pushKeywordIf(keywords, throwStatement.getFirstToken(), SyntaxKind.ThrowKeyword); }); @@ -3415,7 +3415,7 @@ module ts { pushKeywordIf(keywords, returnStatement.getFirstToken(), SyntaxKind.ReturnKeyword); }); } - + return map(keywords, getReferenceEntryFromNode); } @@ -3551,7 +3551,7 @@ module ts { return map(keywords, getReferenceEntryFromNode); } - function getBreakOrContinueStatementOccurences(breakOrContinueStatement: BreakOrContinueStatement): ReferenceEntry[]{ + function getBreakOrContinueStatementOccurences(breakOrContinueStatement: BreakOrContinueStatement): ReferenceEntry[] { var owner = getBreakOrContinueOwner(breakOrContinueStatement); if (owner) { @@ -3599,7 +3599,7 @@ module ts { if (statement.kind === SyntaxKind.ContinueStatement) { continue; } - // Fall through. + // Fall through. case SyntaxKind.ForStatement: case SyntaxKind.ForInStatement: case SyntaxKind.WhileStatement: @@ -4024,13 +4024,13 @@ module ts { * searchLocation: a node where the search value */ function getReferencesInNode(container: Node, - searchSymbol: Symbol, - searchText: string, - searchLocation: Node, - searchMeaning: SemanticMeaning, - findInStrings: boolean, - findInComments: boolean, - result: ReferenceEntry[]): void { + searchSymbol: Symbol, + searchText: string, + searchLocation: Node, + searchMeaning: SemanticMeaning, + findInStrings: boolean, + findInComments: boolean, + result: ReferenceEntry[]): void { var sourceFile = container.getSourceFile(); var tripleSlashDirectivePrefixRegex = /^\/\/\/\s*searchSpaceNode)) { return undefined; } - // Fall through + // Fall through case SyntaxKind.FunctionDeclaration: case SyntaxKind.FunctionExpression: break; @@ -4861,26 +4861,105 @@ module ts { fileName = normalizeSlashes(fileName); var sourceFile = getCurrentSourceFile(fileName); + // Make a scanner we can get trivia from. + var triviaScanner = createScanner(ScriptTarget.Latest, /*skipTrivia:*/ false, sourceFile.text); + var mergeConflictScanner = createScanner(ScriptTarget.Latest, /*skipTrivia:*/ false, sourceFile.text); + var result: ClassifiedSpan[] = []; processElement(sourceFile); return result; - function classifyComment(comment: CommentRange) { - var width = comment.end - comment.pos; - if (textSpanIntersectsWith(span, comment.pos, width)) { + function classifyLeadingTrivia(token: Node): void { + var tokenStart = skipTrivia(sourceFile.text, token.pos, /*stopAfterLineBreak:*/ false); + if (tokenStart === token.pos) { + return; + } + + // token has trivia. Classify them appropriately. + triviaScanner.setTextPos(token.pos); + while (true) { + var start = triviaScanner.getTextPos(); + var kind = triviaScanner.scan(); + var end = triviaScanner.getTextPos(); + var width = end - start; + + if (textSpanIntersectsWith(span, start, width)) { + if (!isTrivia(kind)) { + return; + } + + if (isComment(kind)) { + // Simple comment. Just add as is. + result.push({ + textSpan: createTextSpan(start, width), + classificationType: ClassificationTypeNames.comment + }) + continue; + } + + if (kind === SyntaxKind.ConflictMarkerTrivia) { + var text = sourceFile.text; + var ch = text.charCodeAt(start); + + // for the <<<<<<< and >>>>>>> markers, we just add them as in as + // comments in the classification stream. + if (ch === CharacterCodes.lessThan || ch === CharacterCodes.greaterThan) { + result.push({ + textSpan: createTextSpan(start, width), + classificationType: ClassificationTypeNames.comment + }); + continue; + } + + // for the ======== add a comment for the first line, and then lex all + // subsequent lines up until the end of the conflict marker. + Debug.assert(ch === CharacterCodes.equals); + classifyDisabledCode(text, start, end); + } + } + } + } + + function classifyDisabledCode(text: string, start: number, end: number) { + // Classify the line that the ======= marker is on as a comment. Then just lex + // all further tokens and add them to the result. + for (var i = start; i < end; i++) { + if (isLineBreak(text.charCodeAt(i))) { + break; + } + } + result.push({ + textSpan: createTextSpanFromBounds(start, i), + classificationType: ClassificationTypeNames.comment + }); + + mergeConflictScanner.setTextPos(i); + + while (mergeConflictScanner.getTextPos() < end) { + classifyDisabledCodeToken(); + } + } + + function classifyDisabledCodeToken() { + var start = mergeConflictScanner.getTextPos(); + var tokenKind = mergeConflictScanner.scan(); + var end = mergeConflictScanner.getTextPos(); + + var type = classifyTokenType(tokenKind); + if (type) { result.push({ - textSpan: createTextSpan(comment.pos, width), - classificationType: ClassificationTypeNames.comment + textSpan: createTextSpanFromBounds(start, end), + classificationType: type }); } } function classifyToken(token: Node): void { - forEach(getLeadingCommentRanges(sourceFile.text, token.getFullStart()), classifyComment); + classifyLeadingTrivia(token); if (token.getWidth() > 0) { - var type = classifyTokenType(token); + var type = classifyTokenType(token.kind, token); if (type) { result.push({ textSpan: createTextSpan(token.getStart(), token.getWidth()), @@ -4888,12 +4967,9 @@ module ts { }); } } - - forEach(getTrailingCommentRanges(sourceFile.text, token.getEnd()), classifyComment); } - function classifyTokenType(token: Node): string { - var tokenKind = token.kind; + function classifyTokenType(tokenKind: SyntaxKind, token?: Node): string { if (isKeyword(tokenKind)) { return ClassificationTypeNames.keyword; } @@ -4903,23 +4979,24 @@ module ts { if (tokenKind === SyntaxKind.LessThanToken || tokenKind === SyntaxKind.GreaterThanToken) { // If the node owning the token has a type argument list or type parameter list, then // we can effectively assume that a '<' and '>' belong to those lists. - if (getTypeArgumentOrTypeParameterList(token.parent)) { + if (token && getTypeArgumentOrTypeParameterList(token.parent)) { return ClassificationTypeNames.punctuation; } } - if (isPunctuation(token.kind)) { + if (isPunctuation(tokenKind)) { // the '=' in a variable declaration is special cased here. - if (token.parent.kind === SyntaxKind.BinaryExpression || - token.parent.kind === SyntaxKind.VariableDeclaration || - token.parent.kind === SyntaxKind.PrefixUnaryExpression || - token.parent.kind === SyntaxKind.PostfixUnaryExpression || - token.parent.kind === SyntaxKind.ConditionalExpression) { - return ClassificationTypeNames.operator; - } - else { - return ClassificationTypeNames.punctuation; + if (token) { + if (token.parent.kind === SyntaxKind.BinaryExpression || + token.parent.kind === SyntaxKind.VariableDeclaration || + token.parent.kind === SyntaxKind.PrefixUnaryExpression || + token.parent.kind === SyntaxKind.PostfixUnaryExpression || + token.parent.kind === SyntaxKind.ConditionalExpression) { + return ClassificationTypeNames.operator; + } } + + return ClassificationTypeNames.punctuation; } else if (tokenKind === SyntaxKind.NumericLiteral) { return ClassificationTypeNames.numericLiteral; @@ -4936,35 +5013,37 @@ module ts { return ClassificationTypeNames.stringLiteral; } else if (tokenKind === SyntaxKind.Identifier) { - switch (token.parent.kind) { - case SyntaxKind.ClassDeclaration: - if ((token.parent).name === token) { - return ClassificationTypeNames.className; - } - return; - case SyntaxKind.TypeParameter: - if ((token.parent).name === token) { - return ClassificationTypeNames.typeParameterName; - } - return; - case SyntaxKind.InterfaceDeclaration: - if ((token.parent).name === token) { - return ClassificationTypeNames.interfaceName; - } - return; - case SyntaxKind.EnumDeclaration: - if ((token.parent).name === token) { - return ClassificationTypeNames.enumName; - } - return; - case SyntaxKind.ModuleDeclaration: - if ((token.parent).name === token) { - return ClassificationTypeNames.moduleName; - } - return; - default: - return ClassificationTypeNames.text; + if (token) { + switch (token.parent.kind) { + case SyntaxKind.ClassDeclaration: + if ((token.parent).name === token) { + return ClassificationTypeNames.className; + } + return; + case SyntaxKind.TypeParameter: + if ((token.parent).name === token) { + return ClassificationTypeNames.typeParameterName; + } + return; + case SyntaxKind.InterfaceDeclaration: + if ((token.parent).name === token) { + return ClassificationTypeNames.interfaceName; + } + return; + case SyntaxKind.EnumDeclaration: + if ((token.parent).name === token) { + return ClassificationTypeNames.enumName; + } + return; + case SyntaxKind.ModuleDeclaration: + if ((token.parent).name === token) { + return ClassificationTypeNames.moduleName; + } + return; + } } + + return ClassificationTypeNames.text; } } @@ -5493,7 +5572,6 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - // add the token addResult(end - start, classFromKind(token)); if (end >= text.length) { diff --git a/tests/baselines/reference/conflictMarkerTrivia1.errors.txt b/tests/baselines/reference/conflictMarkerTrivia1.errors.txt index 1448894a0ee..9f93ab52758 100644 --- a/tests/baselines/reference/conflictMarkerTrivia1.errors.txt +++ b/tests/baselines/reference/conflictMarkerTrivia1.errors.txt @@ -1,24 +1,18 @@ tests/cases/compiler/conflictMarkerTrivia1.ts(2,1): error TS1185: Merge conflict marker encountered. -tests/cases/compiler/conflictMarkerTrivia1.ts(3,5): error TS2300: Duplicate identifier 'v'. tests/cases/compiler/conflictMarkerTrivia1.ts(4,1): error TS1185: Merge conflict marker encountered. -tests/cases/compiler/conflictMarkerTrivia1.ts(5,5): error TS2300: Duplicate identifier 'v'. tests/cases/compiler/conflictMarkerTrivia1.ts(6,1): error TS1185: Merge conflict marker encountered. -==== tests/cases/compiler/conflictMarkerTrivia1.ts (5 errors) ==== +==== tests/cases/compiler/conflictMarkerTrivia1.ts (3 errors) ==== class C { <<<<<<< HEAD ~~~~~~~ !!! error TS1185: Merge conflict marker encountered. v = 1; - ~ -!!! error TS2300: Duplicate identifier 'v'. ======= ~~~~~~~ !!! error TS1185: Merge conflict marker encountered. v = 2; - ~ -!!! error TS2300: Duplicate identifier 'v'. >>>>>>> Branch-a ~~~~~~~ !!! error TS1185: Merge conflict marker encountered. diff --git a/tests/baselines/reference/conflictMarkerTrivia2.errors.txt b/tests/baselines/reference/conflictMarkerTrivia2.errors.txt index 7dec00327f2..150dbbec69b 100644 --- a/tests/baselines/reference/conflictMarkerTrivia2.errors.txt +++ b/tests/baselines/reference/conflictMarkerTrivia2.errors.txt @@ -1,15 +1,10 @@ tests/cases/compiler/conflictMarkerTrivia2.ts(3,1): error TS1185: Merge conflict marker encountered. tests/cases/compiler/conflictMarkerTrivia2.ts(4,6): error TS2304: Cannot find name 'a'. tests/cases/compiler/conflictMarkerTrivia2.ts(6,1): error TS1185: Merge conflict marker encountered. -tests/cases/compiler/conflictMarkerTrivia2.ts(7,6): error TS2391: Function implementation is missing or not immediately following the declaration. tests/cases/compiler/conflictMarkerTrivia2.ts(9,1): error TS1185: Merge conflict marker encountered. -tests/cases/compiler/conflictMarkerTrivia2.ts(11,3): error TS1128: Declaration or statement expected. -tests/cases/compiler/conflictMarkerTrivia2.ts(11,10): error TS2304: Cannot find name 'bar'. -tests/cases/compiler/conflictMarkerTrivia2.ts(11,16): error TS1005: ';' expected. -tests/cases/compiler/conflictMarkerTrivia2.ts(12,1): error TS1128: Declaration or statement expected. -==== tests/cases/compiler/conflictMarkerTrivia2.ts (9 errors) ==== +==== tests/cases/compiler/conflictMarkerTrivia2.ts (4 errors) ==== class C { foo() { <<<<<<< B @@ -23,21 +18,11 @@ tests/cases/compiler/conflictMarkerTrivia2.ts(12,1): error TS1128: Declaration o ~~~~~~~ !!! error TS1185: Merge conflict marker encountered. b(); - ~ -!!! error TS2391: Function implementation is missing or not immediately following the declaration. } >>>>>>> A ~~~~~~~ !!! error TS1185: Merge conflict marker encountered. public bar() { } - ~~~~~~ -!!! error TS1128: Declaration or statement expected. - ~~~ -!!! error TS2304: Cannot find name 'bar'. - ~ -!!! error TS1005: ';' expected. } - ~ -!!! error TS1128: Declaration or statement expected. \ No newline at end of file diff --git a/tests/cases/fourslash/syntacticClassificationsConflictMarkers1.ts b/tests/cases/fourslash/syntacticClassificationsConflictMarkers1.ts new file mode 100644 index 00000000000..15309d1431e --- /dev/null +++ b/tests/cases/fourslash/syntacticClassificationsConflictMarkers1.ts @@ -0,0 +1,19 @@ +/// + +////class C { +////<<<<<<< HEAD +//// v = 1; +////======= +//// v = 2; +////>>>>>>> Branch - a +////} +debugger; +var c = classification; +verify.syntacticClassificationsAre( + c.keyword("class"), c.className("C"), c.punctuation("{"), + c.comment("<<<<<<< HEAD"), + c.text("v"), c.punctuation("="), c.numericLiteral("1"), c.punctuation(";"), + c.comment("======="), + c.text("v"), c.punctuation("="), c.numericLiteral("2"), c.punctuation(";"), + c.comment(">>>>>>> Branch - a"), + c.punctuation("}")); \ No newline at end of file diff --git a/tests/cases/fourslash/syntacticClassificationsConflictMarkers2.ts b/tests/cases/fourslash/syntacticClassificationsConflictMarkers2.ts new file mode 100644 index 00000000000..92ede0d104b --- /dev/null +++ b/tests/cases/fourslash/syntacticClassificationsConflictMarkers2.ts @@ -0,0 +1,15 @@ +/// + +////<<<<<<< HEAD +////class C { } +////======= +////class D { } +////>>>>>>> Branch - a + +var c = classification; +verify.syntacticClassificationsAre( + c.comment("<<<<<<< HEAD"), + c.keyword("class"), c.className("C"), c.punctuation("{"), c.punctuation("}"), + c.comment("======="), + c.keyword("class"), c.text("D"), c.punctuation("{"), c.punctuation("}"), + c.comment(">>>>>>> Branch - a")); \ No newline at end of file diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 44804e4f9f5..20038800a3e 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -21,7 +21,7 @@ describe('Colorization', function () { var mytypescriptLS = new Harness.LanguageService.TypeScriptLS(); var myclassifier = mytypescriptLS.getClassifier(); - function getClassifications(code: string, initialEndOfLineState: ts.EndOfLineState = ts.EndOfLineState.Start): ClassiferResult { + function getLexicalClassifications(code: string, initialEndOfLineState: ts.EndOfLineState = ts.EndOfLineState.Start): ClassiferResult { var classResult = myclassifier.getClassificationsForLine(code, initialEndOfLineState).split('\n'); var tuples: Classification[] = []; var i = 0; @@ -71,8 +71,8 @@ describe('Colorization', function () { function regExpLiteral(text: string) { return { value: text, class: ts.TokenClass.RegExpLiteral }; } function finalEndOfLineState(value: number) { return { value: value, class: undefined }; } - function test(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { - var result = getClassifications(text, initialEndOfLineState); + function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { + var result = getLexicalClassifications(text, initialEndOfLineState); for (var i = 0, n = expectedEntries.length; i < n; i++) { var expectedEntry = expectedEntries[i]; @@ -95,7 +95,7 @@ describe('Colorization', function () { describe("test getClassifications", function () { it("Returns correct token classes", function () { - test("var x: string = \"foo\"; //Hello", + testLexicalClassification("var x: string = \"foo\"; //Hello", ts.EndOfLineState.Start, keyword("var"), whitespace(" "), @@ -109,7 +109,7 @@ describe('Colorization', function () { }); it("correctly classifies a comment after a divide operator", function () { - test("1 / 2 // comment", + testLexicalClassification("1 / 2 // comment", ts.EndOfLineState.Start, numberLiteral("1"), whitespace(" "), @@ -119,7 +119,7 @@ describe('Colorization', function () { }); it("correctly classifies a literal after a divide operator", function () { - test("1 / 2, 3 / 4", + testLexicalClassification("1 / 2, 3 / 4", ts.EndOfLineState.Start, numberLiteral("1"), whitespace(" "), @@ -131,131 +131,131 @@ describe('Colorization', function () { }); it("correctly classifies a multi-line string with one backslash", function () { - test("'line1\\", + testLexicalClassification("'line1\\", ts.EndOfLineState.Start, stringLiteral("'line1\\"), finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); }); it("correctly classifies a multi-line string with three backslashes", function () { - test("'line1\\\\\\", + testLexicalClassification("'line1\\\\\\", ts.EndOfLineState.Start, stringLiteral("'line1\\\\\\"), finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); }); it("correctly classifies an unterminated single-line string with no backslashes", function () { - test("'line1", + testLexicalClassification("'line1", ts.EndOfLineState.Start, stringLiteral("'line1"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies an unterminated single-line string with two backslashes", function () { - test("'line1\\\\", + testLexicalClassification("'line1\\\\", ts.EndOfLineState.Start, stringLiteral("'line1\\\\"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies an unterminated single-line string with four backslashes", function () { - test("'line1\\\\\\\\", + testLexicalClassification("'line1\\\\\\\\", ts.EndOfLineState.Start, stringLiteral("'line1\\\\\\\\"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies the continuing line of a multi-line string ending in one backslash", function () { - test("\\", + testLexicalClassification("\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\"), finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); }); it("correctly classifies the continuing line of a multi-line string ending in three backslashes", function () { - test("\\", + testLexicalClassification("\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\"), finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); }); it("correctly classifies the last line of an unterminated multi-line string ending in no backslashes", function () { - test(" ", + testLexicalClassification(" ", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral(" "), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies the last line of an unterminated multi-line string ending in two backslashes", function () { - test("\\\\", + testLexicalClassification("\\\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\\\"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies the last line of an unterminated multi-line string ending in four backslashes", function () { - test("\\\\\\\\", + testLexicalClassification("\\\\\\\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\\\\\\\"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies the last line of a multi-line string", function () { - test("'", + testLexicalClassification("'", ts.EndOfLineState.InSingleQuoteStringLiteral, stringLiteral("'"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies an unterminated multiline comment", function () { - test("/*", + testLexicalClassification("/*", ts.EndOfLineState.Start, comment("/*"), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); it("correctly classifies the termination of a multiline comment", function () { - test(" */ ", + testLexicalClassification(" */ ", ts.EndOfLineState.InMultiLineCommentTrivia, comment(" */"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("correctly classifies the continuation of a multiline comment", function () { - test("LOREM IPSUM DOLOR ", + testLexicalClassification("LOREM IPSUM DOLOR ", ts.EndOfLineState.InMultiLineCommentTrivia, comment("LOREM IPSUM DOLOR "), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); it("correctly classifies an unterminated multiline comment on a line ending in '/*/'", function () { - test(" /*/", + testLexicalClassification(" /*/", ts.EndOfLineState.Start, comment("/*/"), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); it("correctly classifies an unterminated multiline comment with trailing space", function () { - test("/* ", + testLexicalClassification("/* ", ts.EndOfLineState.Start, comment("/* "), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); it("correctly classifies a keyword after a dot", function () { - test("a.var", + testLexicalClassification("a.var", ts.EndOfLineState.Start, identifier("var")); }); it("correctly classifies a string literal after a dot", function () { - test("a.\"var\"", + testLexicalClassification("a.\"var\"", ts.EndOfLineState.Start, stringLiteral("\"var\"")); }); it("correctly classifies a keyword after a dot separated by comment trivia", function () { - test("a./*hello world*/ var", + testLexicalClassification("a./*hello world*/ var", ts.EndOfLineState.Start, identifier("a"), punctuation("."), @@ -264,27 +264,27 @@ describe('Colorization', function () { }); it("classifies a property access with whitespace around the dot", function () { - test(" x .\tfoo ()", + testLexicalClassification(" x .\tfoo ()", ts.EndOfLineState.Start, identifier("x"), identifier("foo")); }); it("classifies a keyword after a dot on previous line", function () { - test("var", + testLexicalClassification("var", ts.EndOfLineState.Start, keyword("var"), finalEndOfLineState(ts.EndOfLineState.Start)); }); it("classifies multiple keywords properly", function () { - test("public static", + testLexicalClassification("public static", ts.EndOfLineState.Start, keyword("public"), keyword("static"), finalEndOfLineState(ts.EndOfLineState.Start)); - test("public var", + testLexicalClassification("public var", ts.EndOfLineState.Start, keyword("public"), identifier("var"), @@ -292,7 +292,7 @@ describe('Colorization', function () { }); it("classifies partially written generics correctly.", function () { - test("Foo { - // no longer in something that looks generic. - test("Foo number", - ts.EndOfLineState.Start, - identifier("Foo"), - operator("<"), - identifier("Foo"), - operator(">"), - keyword("number"), - finalEndOfLineState(ts.EndOfLineState.Start)); + it("LexicallyClassifiesConflictTokens", () => { + debugger; // Test conflict markers. - test( + testLexicalClassification( "class C {\r\n\ <<<<<<< HEAD\r\n\ v = 1;\r\n\ @@ -348,14 +340,26 @@ describe('Colorization', function () { operator("="), numberLiteral("1"), punctuation(";"), - comment("======="), - identifier("v"), - operator("="), - numberLiteral("2"), - punctuation(";"), + comment("=======\r\n v = 2;\r\n"), comment(">>>>>>> Branch - a"), punctuation("}"), finalEndOfLineState(ts.EndOfLineState.Start)); + + testLexicalClassification( +"<<<<<<< HEAD\r\n\ +class C { }\r\n\ +=======\r\n\ +class D { }\r\n\ +>>>>>>> Branch - a\r\n", + ts.EndOfLineState.Start, + comment("<<<<<<< HEAD"), + keyword("class"), + identifier("C"), + punctuation("{"), + punctuation("}"), + comment("=======\r\nclass D { }\r\n"), + comment(">>>>>>> Branch - a"), + finalEndOfLineState(ts.EndOfLineState.Start)); }); }); }); \ No newline at end of file