Merge pull request #672 from Microsoft/getSyntacticClassifications

Add support for syntactic classification.

src/compiler/types.ts

@@ -222,7 +222,7 @@ module ts {
         FirstTypeNode = TypeReference,
         LastTypeNode = TupleType,
         FirstPunctuation = OpenBraceToken,
-        LastPunctuation = CaretEqualsToken
+        LastPunctuation = CaretEqualsToken,
     }
 
     export enum NodeFlags {

src/harness/fourslash.ts

@@ -1428,6 +1428,46 @@ module FourSlash {
             Harness.IO.log(this.getNameOrDottedNameSpan(pos));
         }
 
+        private verifyClassifications(expected: { classificationType: string; text: string }[], actual: ts.ClassifiedSpan[]) {
+            if (actual.length !== expected.length) {
+                throw new Error('verifySyntacticClassification failed - expected total classifications to be ' + expected.length + ', but was ' + actual.length);
+            }
+
+            for (var i = 0; i < expected.length; i++) {
+                var expectedClassification = expected[i];
+                var actualClassification = actual[i];
+
+                var expectedType: string = (<any>ts.ClassificationTypeNames)[expectedClassification.classificationType];
+                if (expectedType !== actualClassification.classificationType) {
+                    throw new Error('verifySyntacticClassification failed - expected classifications type to be ' +
+                        expectedType + ', but was ' +
+                        actualClassification.classificationType);
+                }
+
+                var actualSpan = actualClassification.textSpan;
+                var actualText = this.activeFile.content.substr(actualSpan.start(), actualSpan.length());
+                if (expectedClassification.text !== actualText) {
+                    throw new Error('verifySyntacticClassification failed - expected classificatied text to be ' +
+                        expectedClassification.text + ', but was ' +
+                        actualText);
+                }
+            }
+        }
+
+        public verifySemanticClassifications(expected: { classificationType: string; text: string }[]) {
+            var actual = this.languageService.getSemanticClassifications(this.activeFile.fileName,
+                new TypeScript.TextSpan(0, this.activeFile.content.length));
+
+            this.verifyClassifications(expected, actual);
+        }
+
+        public verifySyntacticClassifications(expected: { classificationType: string; text: string }[]) {
+            var actual = this.languageService.getSyntacticClassifications(this.activeFile.fileName, 
+                new TypeScript.TextSpan(0, this.activeFile.content.length));
+
+            this.verifyClassifications(expected, actual);
+        }
+
         public verifyOutliningSpans(spans: TextSpan[]) {
             this.taoInvalidReason = 'verifyOutliningSpans NYI';
 

src/services/services.ts

@@ -75,7 +75,7 @@ module ts {
 
     var scanner: Scanner = createScanner(ScriptTarget.ES5);
 
-    var emptyArray: any [] = [];
+    var emptyArray: any[] = [];
 
     function createNode(kind: SyntaxKind, pos: number, end: number, flags: NodeFlags, parent?: Node): NodeObject {
         var node = <NodeObject> new (getNodeConstructor(kind))();
@@ -259,7 +259,7 @@ module ts {
         getProperty(propertyName: string): Symbol {
             return this.checker.getPropertyOfType(this, propertyName);
         }
-        getApparentProperties(): Symbol[]{
+        getApparentProperties(): Symbol[] {
             return this.checker.getAugmentedPropertiesOfApparentType(this);
         }
         getCallSignatures(): Signature[] {
@@ -302,7 +302,7 @@ module ts {
         }
     }
 
-     var incrementalParse: IncrementalParse = TypeScript.IncrementalParser.parse;
+    var incrementalParse: IncrementalParse = TypeScript.IncrementalParser.parse;
 
     class SourceFileObject extends NodeObject implements SourceFile {
         public filename: string;
@@ -430,6 +430,9 @@ module ts {
         getSemanticDiagnostics(fileName: string): Diagnostic[];
         getCompilerOptionsDiagnostics(): Diagnostic[];
 
+        getSyntacticClassifications(fileName: string, span: TypeScript.TextSpan): ClassifiedSpan[];
+        getSemanticClassifications(fileName: string, span: TypeScript.TextSpan): ClassifiedSpan[];
+
         getCompletionsAtPosition(fileName: string, position: number, isMemberCompletion: boolean): CompletionInfo;
         getCompletionEntryDetails(fileName: string, position: number, entryName: string): CompletionEntryDetails;
 
@@ -467,6 +470,32 @@ module ts {
         dispose(): void;
     }
 
+    export class ClassificationTypeNames {
+        public static comment = "comment";
+        public static identifier = "identifier";
+        public static keyword = "keyword";
+        public static numericLiteral = "number";
+        public static operator = "operator";
+        public static stringLiteral = "string";
+        public static whiteSpace = "whitespace";
+        public static text = "text";
+
+        public static punctuation = "punctuation";
+
+        public static className = "class name";
+        public static enumName = "enum name";
+        public static interfaceName = "interface name";
+        public static moduleName = "module name";
+        public static typeParameterName = "type parameter name";
+    }
+
+    export class ClassifiedSpan {
+        constructor(public textSpan: TypeScript.TextSpan,
+                    public classificationType: string) {
+
+        }
+    }
+
     export class NavigationBarItem {
         constructor(public text: string,
                     public kind: string,
@@ -3214,6 +3243,198 @@ module ts {
             return new TypeScript.Services.NavigationBarItemGetter().getItems(syntaxTree.sourceUnit());
         }
 
+        function getSemanticClassifications(fileName: string, span: TypeScript.TextSpan): ClassifiedSpan[] {
+            synchronizeHostData();
+            fileName = TypeScript.switchToForwardSlashes(fileName);
+
+            var sourceFile = getSourceFile(fileName);
+
+            var result: ClassifiedSpan[] = [];
+            processNode(sourceFile);
+
+            return result;
+
+            function classifySymbol(symbol: Symbol) {
+                var flags = symbol.getFlags();
+
+                if (flags & SymbolFlags.Class) {
+                    return ClassificationTypeNames.className;
+                }
+                else if (flags & SymbolFlags.Enum) {
+                    return ClassificationTypeNames.enumName;
+                }
+                else if (flags & SymbolFlags.Interface) {
+                    return ClassificationTypeNames.interfaceName;
+                }
+                else if (flags & SymbolFlags.Module) {
+                    return ClassificationTypeNames.moduleName;
+                }
+                else if (flags & SymbolFlags.TypeParameter) {
+                    return ClassificationTypeNames.typeParameterName;
+                }
+            }
+
+            function processNode(node: Node) {
+                // Only walk into nodes that intersect the requested span.
+                if (node && span.intersectsWith(node.getStart(), node.getWidth())) {
+                    if (node.kind === SyntaxKind.Identifier && node.getWidth() > 0) {
+                        var symbol = typeInfoResolver.getSymbolInfo(node);
+                        if (symbol) {
+                            var type = classifySymbol(symbol);
+                            if (type) {
+                                result.push(new ClassifiedSpan(
+                                    new TypeScript.TextSpan(node.getStart(), node.getWidth()),
+                                    type));
+                            }
+                        }
+                    }
+
+                    forEachChild(node, processNode);
+                }
+            }
+        }
+
+        function getSyntacticClassifications(fileName: string, span: TypeScript.TextSpan): ClassifiedSpan[] {
+            // doesn't use compiler - no need to synchronize with host
+            fileName = TypeScript.switchToForwardSlashes(fileName);
+            var sourceFile = getCurrentSourceFile(fileName);
+
+            var result: ClassifiedSpan[] = [];
+            processElement(sourceFile.getSourceUnit());
+
+            return result;
+
+            function classifyTrivia(trivia: TypeScript.ISyntaxTrivia) {
+                if (trivia.isComment() && span.intersectsWith(trivia.fullStart(), trivia.fullWidth())) {
+                    result.push(new ClassifiedSpan(
+                        new TypeScript.TextSpan(trivia.fullStart(), trivia.fullWidth()),
+                        ClassificationTypeNames.comment));
+                }
+            }
+
+            function classifyTriviaList(trivia: TypeScript.ISyntaxTriviaList) {
+                for (var i = 0, n = trivia.count(); i < n; i++) {
+                    classifyTrivia(trivia.syntaxTriviaAt(i));
+                }
+            }
+
+            function classifyToken(token: TypeScript.ISyntaxToken) {
+                if (token.hasLeadingComment()) {
+                    classifyTriviaList(token.leadingTrivia());
+                }
+
+                if (TypeScript.width(token) > 0) {
+                    var type = classifyTokenType(token);
+                    if (type) {
+                        result.push(new ClassifiedSpan(
+                            new TypeScript.TextSpan(TypeScript.start(token), TypeScript.width(token)),
+                            type));
+                    }
+                }
+
+                if (token.hasTrailingComment()) {
+                    classifyTriviaList(token.trailingTrivia());
+                }
+            }
+
+            function classifyTokenType(token: TypeScript.ISyntaxToken): string {
+                var tokenKind = token.kind();
+                if (TypeScript.SyntaxFacts.isAnyKeyword(token.kind())) {
+                    return ClassificationTypeNames.keyword;
+                }
+
+                // Special case < and >  If they appear in a generic context they are punctation,
+                // not operators.
+                if (tokenKind === TypeScript.SyntaxKind.LessThanToken || tokenKind === TypeScript.SyntaxKind.GreaterThanToken) {
+                    var tokenParentKind = token.parent.kind();
+                    if (tokenParentKind === TypeScript.SyntaxKind.TypeArgumentList ||
+                        tokenParentKind === TypeScript.SyntaxKind.TypeParameterList) {
+
+                        return ClassificationTypeNames.punctuation;
+                    }
+                }
+
+                if (TypeScript.SyntaxFacts.isBinaryExpressionOperatorToken(tokenKind) ||
+                    TypeScript.SyntaxFacts.isPrefixUnaryExpressionOperatorToken(tokenKind)) {
+                    return ClassificationTypeNames.operator;
+                }
+                else if (TypeScript.SyntaxFacts.isAnyPunctuation(tokenKind)) {
+                    return ClassificationTypeNames.punctuation;
+                }
+                else if (tokenKind === TypeScript.SyntaxKind.NumericLiteral) {
+                    return ClassificationTypeNames.numericLiteral;
+                }
+                else if (tokenKind === TypeScript.SyntaxKind.StringLiteral) {
+                    return ClassificationTypeNames.stringLiteral;
+                }
+                else if (tokenKind === TypeScript.SyntaxKind.RegularExpressionLiteral) {
+                    // TODO: we shoudl get another classification type for these literals.
+                    return ClassificationTypeNames.stringLiteral;
+                }
+                else if (tokenKind === TypeScript.SyntaxKind.IdentifierName) {
+                    var current: TypeScript.ISyntaxNodeOrToken = token;
+                    var parent = token.parent;
+                    while (parent.kind() === TypeScript.SyntaxKind.QualifiedName) {
+                        current = parent;
+                        parent = parent.parent;
+                    }
+
+                    switch (parent.kind()) {
+                        case TypeScript.SyntaxKind.SimplePropertyAssignment:
+                            if ((<TypeScript.SimplePropertyAssignmentSyntax>parent).propertyName === token) {
+                                return ClassificationTypeNames.identifier;
+                            }
+                            return;
+                        case TypeScript.SyntaxKind.ClassDeclaration:
+                            if ((<TypeScript.ClassDeclarationSyntax>parent).identifier === token) {
+                                return ClassificationTypeNames.className;
+                            }
+                            return;
+                        case TypeScript.SyntaxKind.TypeParameter:
+                            if ((<TypeScript.TypeParameterSyntax>parent).identifier === token) {
+                                return ClassificationTypeNames.typeParameterName;
+                            }
+                            return;
+                        case TypeScript.SyntaxKind.InterfaceDeclaration:
+                            if ((<TypeScript.InterfaceDeclarationSyntax>parent).identifier === token) {
+                                return ClassificationTypeNames.interfaceName;
+                            }
+                            return;
+                        case TypeScript.SyntaxKind.EnumDeclaration:
+                            if ((<TypeScript.EnumDeclarationSyntax>parent).identifier === token) {
+                                return ClassificationTypeNames.enumName;
+                            }
+                            return;
+                        case TypeScript.SyntaxKind.ModuleDeclaration:
+                            if ((<TypeScript.ModuleDeclarationSyntax>parent).name === current) {
+                                return ClassificationTypeNames.moduleName;
+                            }
+                            return;
+                        default:
+                            return ClassificationTypeNames.text;
+                    }
+                }
+            }
+
+            function processElement(element: TypeScript.ISyntaxElement) {
+                // Ignore nodes that don't intersect the original span to classify.
+                if (!TypeScript.isShared(element) && span.intersectsWith(TypeScript.fullStart(element), TypeScript.fullWidth(element))) {
+                    for (var i = 0, n = TypeScript.childCount(element); i < n; i++) {
+                        var child = TypeScript.childAt(element, i);
+                        if (child) {
+                            if (TypeScript.isToken(child)) {
+                                classifyToken(<TypeScript.ISyntaxToken>child);
+                            }
+                            else {
+                                // Recurse into our child nodes.
+                                processElement(child);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         function getOutliningSpans(filename: string): OutliningSpan[] {
             // doesn't use compiler - no need to synchronize with host
             filename = TypeScript.switchToForwardSlashes(filename);
@@ -3461,6 +3682,8 @@ module ts {
             getSyntacticDiagnostics: getSyntacticDiagnostics,
             getSemanticDiagnostics: getSemanticDiagnostics,
             getCompilerOptionsDiagnostics: getCompilerOptionsDiagnostics,
+            getSyntacticClassifications: getSyntacticClassifications,
+            getSemanticClassifications: getSemanticClassifications,
             getCompletionsAtPosition: getCompletionsAtPosition,
             getCompletionEntryDetails: getCompletionEntryDetails,
             getTypeAtPosition: getTypeAtPosition,

src/services/shims.ts

@@ -80,6 +80,8 @@ module ts {
         getSemanticDiagnostics(fileName: string): string;
         getCompilerOptionsDiagnostics(): string;
 
+        getSyntacticClassifications(fileName: string, start: number, length: number): string;
+
         getCompletionsAtPosition(fileName: string, position: number, isMemberCompletion: boolean): string;
         getCompletionEntryDetails(fileName: string, position: number, entryName: string): string;
 
@@ -477,6 +479,24 @@ module ts {
             };
         }
 
+        public getSyntacticClassifications(fileName: string, start: number, length: number): string {
+            return this.forwardJSONCall(
+                "getSyntacticClassifications('" + fileName + "', " + start + ", " + length + ")",
+                () => {
+                    var classifications = this.languageService.getSyntacticClassifications(fileName, new TypeScript.TextSpan(start, length));
+                    return classifications;
+                });
+        }
+
+        public getSemanticClassifications(fileName: string, start: number, length: number): string {
+            return this.forwardJSONCall(
+                "getSemanticClassifications('" + fileName + "', " + start + ", " + length + ")",
+                () => {
+                    var classifications = this.languageService.getSemanticClassifications(fileName, new TypeScript.TextSpan(start, length));
+                    return classifications;
+                });
+        }
+
         public getSyntacticDiagnostics(fileName: string): string {
             return this.forwardJSONCall(
                 "getSyntacticDiagnostics('" + fileName + "')",

src/services/text/textSpan.ts

@@ -1,6 +1,7 @@
 ///<reference path='references.ts' />
 
 module TypeScript {
+
     export interface ISpan {
         start(): number;
         end(): number;

tests/cases/fourslash/fourslash.ts

@@ -382,6 +382,14 @@ module FourSlashInterface {
         public completionEntryDetailIs(entryName: string, type: string, docComment?: string, fullSymbolName?: string, kind?: string) {
             FourSlash.currentTestState.verifyCompletionEntryDetails(entryName, type, docComment, fullSymbolName, kind);
         }
+
+        public syntacticClassificationsAre(...classifications: { classificationType: string; text: string }[]) {
+            FourSlash.currentTestState.verifySyntacticClassifications(classifications);
+        }
+
+        public semanticClassificationsAre(...classifications: { classificationType: string; text: string }[]) {
+            FourSlash.currentTestState.verifySemanticClassifications(classifications);
+        }
     }
 
     export class edit {
@@ -524,6 +532,64 @@ module FourSlashInterface {
             FourSlash.currentTestState.cancellationToken.setCancelled(numberOfCalls);
         }
     }
+
+    export class classification {
+        public static comment(text: string): { classificationType: string; text: string } {
+            return { classificationType: "comment", text: text };
+        }
+
+        public static identifier(text: string): { classificationType: string; text: string } {
+            return { classificationType: "identifier", text: text };
+        }
+
+        public static keyword(text: string): { classificationType: string; text: string } {
+            return { classificationType: "keyword", text: text };
+        }
+
+        public static numericLiteral(text: string): { classificationType: string; text: string } {
+            return { classificationType: "numericLiteral", text: text };
+        }
+
+        public static operator(text: string): { classificationType: string; text: string } {
+            return { classificationType: "operator", text: text };
+        }
+
+        public static stringLiteral(text: string): { classificationType: string; text: string } {
+            return { classificationType: "stringLiteral", text: text };
+        }
+
+        public static whiteSpace(text: string): { classificationType: string; text: string } {
+            return { classificationType: "whiteSpace", text: text };
+        }
+
+        public static text(text: string): { classificationType: string; text: string } {
+            return { classificationType: "text", text: text };
+        }
+
+        public static punctuation(text: string): { classificationType: string; text: string } {
+            return { classificationType: "punctuation", text: text };
+        }
+
+        public static className(text: string): { classificationType: string; text: string } {
+            return { classificationType: "className", text: text };
+        }
+
+        public static enumName(text: string): { classificationType: string; text: string } {
+            return { classificationType: "enumName", text: text };
+        }
+
+        public static interfaceName(text: string): { classificationType: string; text: string } {
+            return { classificationType: "interfaceName", text: text };
+        }
+
+        public static moduleName(text: string): { classificationType: string; text: string } {
+            return { classificationType: "moduleName", text: text };
+        }
+
+        public static typeParameterName(text: string): { classificationType: string; text: string } {
+            return { classificationType: "typeParameterName", text: text };
+        }
+    }
 }
 
 module fs {
@@ -547,3 +613,4 @@ var debug = new FourSlashInterface.debug();
 var format = new FourSlashInterface.format();
 var diagnostics = new FourSlashInterface.diagnostics();
 var cancellation = new FourSlashInterface.cancellation();
+var classification = FourSlashInterface.classification;

tests/cases/fourslash/semanticClassification1.ts

@@ -0,0 +1,12 @@
+/// <reference path="fourslash.ts"/>
+
+//// module M {
+////     export interface I {
+////     }
+//// }
+//// interface X extends M.I { }
+
+debugger;
+var c = classification;
+verify.semanticClassificationsAre(
+    c.moduleName("M"), c.interfaceName("I"), c.interfaceName("X"), c.moduleName("M"), c.interfaceName("I"));

tests/cases/fourslash/syntacticClassifications1.ts

@@ -0,0 +1,36 @@
+/// <reference path="fourslash.ts"/>
+
+//// // comment
+//// module M {
+////     var v = 0 + 1;
+////     var s = "string";
+////
+////     class C<T> {
+////     }
+////
+////     enum E {
+////     }
+////
+////     interface I {
+////     }
+////
+////     module M1.M2 {
+////     }
+//// }
+
+debugger;
+var c = classification;
+verify.syntacticClassificationsAre(
+    c.comment("// comment"),
+    c.keyword("module"), c.moduleName("M"), c.punctuation("{"),
+        c.keyword("var"), c.text("v"), c.operator("="), c.numericLiteral("0"), c.operator("+"), c.numericLiteral("1"), c.punctuation(";"),
+        c.keyword("var"), c.text("s"), c.operator("="), c.stringLiteral('"string"'), c.punctuation(";"),
+        c.keyword("class"), c.className("C"), c.punctuation("<"), c.typeParameterName("T"), c.punctuation(">"), c.punctuation("{"),
+        c.punctuation("}"),
+        c.keyword("enum"), c.enumName("E"), c.punctuation("{"),
+        c.punctuation("}"),
+        c.keyword("interface"), c.interfaceName("I"), c.punctuation("{"),
+        c.punctuation("}"),
+        c.keyword("module"), c.moduleName("M1"), c.punctuation("."), c.moduleName("M2"), c.punctuation("{"),
+        c.punctuation("}"),
+    c.punctuation("}"));