Import the semantic highlighter from typescript-vscode-sh-plugin (#39119)

* Initial import of the vscode semantic highlight code * Adds the ability to test modern semantic classification via strings instead of numbers * Adds existing tests * Port over the semantic classification tests * Update baselines * Update src/harness/fourslashImpl.ts Co-authored-by: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> * Handle feedback from #39119 * Consistent formatting in the 2020 classifier * Update baselines * Apply suggestions from code review Co-authored-by: Daniel Rosenwasser <DanielRosenwasser@users.noreply.github.com> * Update src/harness/fourslashImpl.ts Co-authored-by: Daniel Rosenwasser <DanielRosenwasser@users.noreply.github.com> * Reafactor after comments * Use 2020 everywhere * Handle feedback * WIP - don't provide a breaking change * Fix all build errors * Update baselines * Update src/services/classifier2020.ts Co-authored-by: Sheetal Nandi <shkamat@microsoft.com> * Addresses Ron's feedback Co-authored-by: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Co-authored-by: Daniel Rosenwasser <DanielRosenwasser@users.noreply.github.com> Co-authored-by: Sheetal Nandi <shkamat@microsoft.com>
2026-06-11 10:46:28 -05:00 · 2020-09-11 15:10:55 -04:00
parent a36f17c1f8
commit db5368dc1d
81 changed files with 1169 additions and 194 deletions
--- a/src/services/classifier2020.ts
+++ b/src/services/classifier2020.ts
@@ -0,0 +1,249 @@
+/** @internal */
+namespace ts.classifier.v2020 {
+
+    export const enum TokenEncodingConsts {
+        typeOffset = 8,
+        modifierMask = (1 << typeOffset) - 1
+    }
+
+    export const enum TokenType {
+        class, enum, interface, namespace, typeParameter, type, parameter, variable, enumMember, property, function, member
+    }
+
+    export const enum TokenModifier {
+        declaration, static, async, readonly, defaultLibrary, local
+    }
+
+    /** This is mainly used internally for testing */
+    export function getSemanticClassifications(program: Program, cancellationToken: CancellationToken, sourceFile: SourceFile, span: TextSpan): ClassifiedSpan2020[] {
+        const classifications = getEncodedSemanticClassifications(program, cancellationToken, sourceFile, span);
+
+        Debug.assert(classifications.spans.length % 3 === 0);
+        const dense = classifications.spans;
+        const result: ClassifiedSpan2020[] = [];
+        for (let i = 0; i < dense.length; i += 3) {
+            result.push({
+                textSpan: createTextSpan(dense[i], dense[i + 1]),
+                classificationType: dense[i + 2]
+            });
+        }
+
+        return result;
+    }
+
+    export function getEncodedSemanticClassifications(program: Program, cancellationToken: CancellationToken, sourceFile: SourceFile, span: TextSpan): Classifications {
+        return {
+            spans: getSemanticTokens(program, sourceFile, span, cancellationToken),
+            endOfLineState: EndOfLineState.None
+        };
+    }
+
+    function getSemanticTokens(program: Program, sourceFile: SourceFile, span: TextSpan, cancellationToken: CancellationToken): number[] {
+        const resultTokens: number[] = [];
+
+        const collector = (node: Node, typeIdx: number, modifierSet: number) => {
+            resultTokens.push(node.getStart(sourceFile), node.getWidth(sourceFile), ((typeIdx + 1) << TokenEncodingConsts.typeOffset) + modifierSet);
+        };
+
+        if (program && sourceFile) {
+            collectTokens(program, sourceFile, span, collector, cancellationToken);
+        }
+        return resultTokens;
+    }
+
+    function collectTokens(program: Program, sourceFile: SourceFile, span: TextSpan, collector: (node: Node, tokenType: number, tokenModifier: number) => void, cancellationToken: CancellationToken) {
+        const typeChecker = program.getTypeChecker();
+
+        let inJSXElement = false;
+
+        function visit(node: Node) {
+            switch(node.kind) {
+                case SyntaxKind.ModuleDeclaration:
+                case SyntaxKind.ClassDeclaration:
+                case SyntaxKind.InterfaceDeclaration:
+                case SyntaxKind.FunctionDeclaration:
+                case SyntaxKind.ClassExpression:
+                case SyntaxKind.FunctionExpression:
+                case SyntaxKind.ArrowFunction:
+                    cancellationToken.throwIfCancellationRequested();
+            }
+
+            if (!node || !textSpanIntersectsWith(span, node.pos, node.getFullWidth()) || node.getFullWidth() === 0) {
+                return;
+            }
+            const prevInJSXElement = inJSXElement;
+            if (isJsxElement(node) || isJsxSelfClosingElement(node)) {
+                inJSXElement = true;
+            }
+            if (isJsxExpression(node)) {
+                inJSXElement = false;
+            }
+
+            if (isIdentifier(node) && !inJSXElement && !inImportClause(node)) {
+                let symbol = typeChecker.getSymbolAtLocation(node);
+                if (symbol) {
+                    if (symbol.flags & SymbolFlags.Alias) {
+                        symbol = typeChecker.getAliasedSymbol(symbol);
+                    }
+                    let typeIdx = classifySymbol(symbol, getMeaningFromLocation(node));
+                    if (typeIdx !== undefined) {
+                        let modifierSet = 0;
+                        if (node.parent) {
+                            const parentIsDeclaration = (isBindingElement(node.parent) || tokenFromDeclarationMapping.get(node.parent.kind) === typeIdx);
+                            if (parentIsDeclaration && (<NamedDeclaration>node.parent).name === node) {
+                                modifierSet = 1 << TokenModifier.declaration;
+                            }
+                        }
+
+                        // property declaration in constructor
+                        if (typeIdx === TokenType.parameter && isRightSideOfQualifiedNameOrPropertyAccess(node)) {
+                            typeIdx = TokenType.property;
+                        }
+
+                        typeIdx = reclassifyByType(typeChecker, node, typeIdx);
+
+                        const decl = symbol.valueDeclaration;
+                        if (decl) {
+                            const modifiers = getCombinedModifierFlags(decl);
+                            const nodeFlags = getCombinedNodeFlags(decl);
+                            if (modifiers & ModifierFlags.Static) {
+                                modifierSet |= 1 << TokenModifier.static;
+                            }
+                            if (modifiers & ModifierFlags.Async) {
+                                modifierSet |= 1 << TokenModifier.async;
+                            }
+                            if (typeIdx !== TokenType.class && typeIdx !== TokenType.interface) {
+                                if ((modifiers & ModifierFlags.Readonly) || (nodeFlags & NodeFlags.Const) || (symbol.getFlags() & SymbolFlags.EnumMember)) {
+                                    modifierSet |= 1 << TokenModifier.readonly;
+                                }
+                            }
+                            if ((typeIdx === TokenType.variable || typeIdx === TokenType.function) && isLocalDeclaration(decl, sourceFile)) {
+                                modifierSet |= 1 << TokenModifier.local;
+                            }
+                            if (program.isSourceFileDefaultLibrary(decl.getSourceFile())) {
+                                modifierSet |= 1 << TokenModifier.defaultLibrary;
+                            }
+                        }
+                        else if (symbol.declarations && symbol.declarations.some(d => program.isSourceFileDefaultLibrary(d.getSourceFile()))) {
+                            modifierSet |= 1 << TokenModifier.defaultLibrary;
+                        }
+
+                        collector(node, typeIdx, modifierSet);
+
+                    }
+                }
+            }
+            forEachChild(node, visit);
+
+            inJSXElement = prevInJSXElement;
+        }
+        visit(sourceFile);
+    }
+
+    function classifySymbol(symbol: Symbol, meaning: SemanticMeaning): TokenType | undefined {
+        const flags = symbol.getFlags();
+        if (flags & SymbolFlags.Class) {
+            return TokenType.class;
+        }
+        else if (flags & SymbolFlags.Enum) {
+            return TokenType.enum;
+        }
+         else if (flags & SymbolFlags.TypeAlias) {
+            return TokenType.type;
+        }
+        else if (flags & SymbolFlags.Interface) {
+            if (meaning & SemanticMeaning.Type) {
+                return TokenType.interface;
+            }
+        }
+        else if (flags & SymbolFlags.TypeParameter) {
+            return TokenType.typeParameter;
+        }
+        let decl = symbol.valueDeclaration || symbol.declarations && symbol.declarations[0];
+        if (decl && isBindingElement(decl)) {
+            decl = getDeclarationForBindingElement(decl);
+        }
+        return decl && tokenFromDeclarationMapping.get(decl.kind);
+    }
+
+    function reclassifyByType(typeChecker: TypeChecker, node: Node, typeIdx: TokenType): TokenType {
+        // type based classifications
+        if (typeIdx === TokenType.variable || typeIdx === TokenType.property || typeIdx === TokenType.parameter) {
+            const type = typeChecker.getTypeAtLocation(node);
+            if (type) {
+                const test = (condition: (type: Type) => boolean) => {
+                    return condition(type) || type.isUnion() && type.types.some(condition);
+                };
+                if (typeIdx !== TokenType.parameter && test(t => t.getConstructSignatures().length > 0)) {
+                    return TokenType.class;
+                }
+                if (test(t => t.getCallSignatures().length > 0) && !test(t => t.getProperties().length > 0) || isExpressionInCallExpression(node)) {
+                    return typeIdx === TokenType.property ? TokenType.member : TokenType.function;
+                }
+            }
+        }
+        return typeIdx;
+    }
+
+    function isLocalDeclaration(decl: Declaration, sourceFile: SourceFile): boolean {
+        if (isBindingElement(decl)) {
+            decl = getDeclarationForBindingElement(decl);
+        }
+        if (isVariableDeclaration(decl)) {
+            return (!isSourceFile(decl.parent.parent.parent) || isCatchClause(decl.parent)) && decl.getSourceFile() === sourceFile;
+        }
+        else if (isFunctionDeclaration(decl)) {
+            return !isSourceFile(decl.parent) && decl.getSourceFile() === sourceFile;
+        }
+        return false;
+    }
+
+    function getDeclarationForBindingElement(element: BindingElement): VariableDeclaration | ParameterDeclaration {
+        while (true) {
+            if (isBindingElement(element.parent.parent)) {
+                element = element.parent.parent;
+            }
+            else {
+                return element.parent.parent;
+            }
+        }
+    }
+
+    function inImportClause(node: Node): boolean {
+        const parent = node.parent;
+        return parent && (isImportClause(parent) || isImportSpecifier(parent) || isNamespaceImport(parent));
+    }
+
+    function isExpressionInCallExpression(node: Node): boolean {
+        while (isRightSideOfQualifiedNameOrPropertyAccess(node)) {
+            node = node.parent;
+        }
+        return isCallExpression(node.parent) && node.parent.expression === node;
+    }
+
+    function isRightSideOfQualifiedNameOrPropertyAccess(node: Node): boolean {
+        return (isQualifiedName(node.parent) && node.parent.right === node) || (isPropertyAccessExpression(node.parent) && node.parent.name === node);
+    }
+
+    const tokenFromDeclarationMapping = new Map<SyntaxKind, TokenType>([
+        [SyntaxKind.VariableDeclaration, TokenType.variable],
+        [SyntaxKind.Parameter, TokenType.parameter],
+        [SyntaxKind.PropertyDeclaration, TokenType.property],
+        [SyntaxKind.ModuleDeclaration, TokenType.namespace],
+        [SyntaxKind.EnumDeclaration, TokenType.enum],
+        [SyntaxKind.EnumMember, TokenType.enumMember],
+        [SyntaxKind.ClassDeclaration, TokenType.class],
+        [SyntaxKind.MethodDeclaration, TokenType.member],
+        [SyntaxKind.FunctionDeclaration, TokenType.function],
+        [SyntaxKind.FunctionExpression, TokenType.function],
+        [SyntaxKind.MethodSignature, TokenType.member],
+        [SyntaxKind.GetAccessor, TokenType.property],
+        [SyntaxKind.SetAccessor, TokenType.property],
+        [SyntaxKind.PropertySignature, TokenType.property],
+        [SyntaxKind.InterfaceDeclaration, TokenType.interface],
+        [SyntaxKind.TypeAliasDeclaration, TokenType.type],
+        [SyntaxKind.TypeParameter, TokenType.typeParameter],
+        [SyntaxKind.PropertyAssignment, TokenType.property],
+        [SyntaxKind.ShorthandPropertyAssignment, TokenType.property]
+    ]);
+}
--- a/src/services/services.ts
+++ b/src/services/services.ts
@@ -1827,22 +1827,37 @@ namespace ts {
            return kind === ScriptKind.TS || kind === ScriptKind.TSX;
        }

-        function getSemanticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[] {
+        function getSemanticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[];
+        function getSemanticClassifications(fileName: string, span: TextSpan, format?: SemanticClassificationFormat): ClassifiedSpan[] | ClassifiedSpan2020[] {
            if (!isTsOrTsxFile(fileName)) {
                // do not run semantic classification on non-ts-or-tsx files
                return [];
            }
            synchronizeHostData();
-            return ts.getSemanticClassifications(program.getTypeChecker(), cancellationToken, getValidSourceFile(fileName), program.getClassifiableNames(), span);
+
+            const responseFormat = format || SemanticClassificationFormat.Original;
+            if (responseFormat === SemanticClassificationFormat.TwentyTwenty) {
+                return classifier.v2020.getSemanticClassifications(program, cancellationToken, getValidSourceFile(fileName), span);
+            }
+            else {
+                return ts.getSemanticClassifications(program.getTypeChecker(), cancellationToken, getValidSourceFile(fileName), program.getClassifiableNames(), span);
+            }
        }

-        function getEncodedSemanticClassifications(fileName: string, span: TextSpan): Classifications {
+        function getEncodedSemanticClassifications(fileName: string, span: TextSpan, format?: SemanticClassificationFormat): Classifications {
            if (!isTsOrTsxFile(fileName)) {
                // do not run semantic classification on non-ts-or-tsx files
                return { spans: [], endOfLineState: EndOfLineState.None };
            }
            synchronizeHostData();
-            return ts.getEncodedSemanticClassifications(program.getTypeChecker(), cancellationToken, getValidSourceFile(fileName), program.getClassifiableNames(), span);
+
+            const responseFormat = format || SemanticClassificationFormat.Original;
+            if (responseFormat === SemanticClassificationFormat.Original) {
+                return ts.getEncodedSemanticClassifications(program.getTypeChecker(), cancellationToken, getValidSourceFile(fileName), program.getClassifiableNames(), span);
+            }
+            else {
+                return classifier.v2020.getEncodedSemanticClassifications(program, cancellationToken, getValidSourceFile(fileName), span);
+            }
        }

        function getSyntacticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[] {
--- a/src/services/shims.ts
+++ b/src/services/shims.ts
@@ -145,9 +145,9 @@ namespace ts {
        getCompilerOptionsDiagnostics(): string;

        getSyntacticClassifications(fileName: string, start: number, length: number): string;
-        getSemanticClassifications(fileName: string, start: number, length: number): string;
+        getSemanticClassifications(fileName: string, start: number, length: number, format?: SemanticClassificationFormat): string;
        getEncodedSyntacticClassifications(fileName: string, start: number, length: number): string;
-        getEncodedSemanticClassifications(fileName: string, start: number, length: number): string;
+        getEncodedSemanticClassifications(fileName: string, start: number, length: number, format?: SemanticClassificationFormat): string;

        getCompletionsAtPosition(fileName: string, position: number, preferences: UserPreferences | undefined): string;
        getCompletionEntryDetails(fileName: string, position: number, entryName: string, formatOptions: string/*Services.FormatCodeOptions*/ | undefined, source: string | undefined, preferences: UserPreferences | undefined): string;
--- a/src/services/tsconfig.json
+++ b/src/services/tsconfig.json
@@ -12,6 +12,7 @@
        "types.ts",
        "utilities.ts",
        "classifier.ts",
+        "classifier2020.ts",
        "stringCompletions.ts",
        "completions.ts",
        "documentHighlights.ts",
--- a/src/services/types.ts
+++ b/src/services/types.ts
@@ -319,6 +319,11 @@ namespace ts {

    export type WithMetadata<T> = T & { metadata?: unknown; };

+    export const enum SemanticClassificationFormat {
+        Original = "original",
+        TwentyTwenty = "2020"
+    }
+
    //
    // Public services of a language service instance associated
    // with a language service host instance
@@ -382,13 +387,25 @@ namespace ts {

        /** @deprecated Use getEncodedSyntacticClassifications instead. */
        getSyntacticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[];
+        getSyntacticClassifications(fileName: string, span: TextSpan, format: SemanticClassificationFormat): ClassifiedSpan[] | ClassifiedSpan2020[];

        /** @deprecated Use getEncodedSemanticClassifications instead. */
        getSemanticClassifications(fileName: string, span: TextSpan): ClassifiedSpan[];
+        getSemanticClassifications(fileName: string, span: TextSpan, format: SemanticClassificationFormat): ClassifiedSpan[] | ClassifiedSpan2020[];

-        // Encoded as triples of [start, length, ClassificationType].
+        /** Encoded as triples of [start, length, ClassificationType]. */
        getEncodedSyntacticClassifications(fileName: string, span: TextSpan): Classifications;
-        getEncodedSemanticClassifications(fileName: string, span: TextSpan): Classifications;
+
+        /**
+         * Gets semantic highlights information for a particular file. Has two formats, an older
+         * version used by VS and a format used by VS Code.
+         *
+         * @param fileName The path to the file
+         * @param position A text span to return results within
+         * @param format Which format to use, defaults to "original"
+         * @returns a number array encoded as triples of [start, length, ClassificationType, ...].
+         */
+        getEncodedSemanticClassifications(fileName: string, span: TextSpan, format?: SemanticClassificationFormat): Classifications;

        /**
         * Gets completion entries at a particular position in a file.
@@ -603,6 +620,11 @@ namespace ts {
        classificationType: ClassificationTypeNames;
    }

+    export interface ClassifiedSpan2020 {
+        textSpan: TextSpan;
+        classificationType: number;
+    }
+
    /**
     * Navigation bar interface designed for visual studio's dual-column layout.
     * This does not form a proper tree.