vscode/extensions/copilot/test/simulation/debugTools.stest.ts

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
import { LanguageToolsProvider } from '../../src/extension/onboardDebug/node/languageToolsProvider';
import { TestingServiceCollection } from '../../src/platform/test/node/services';
import { CancellationToken } from '../../src/util/vs/base/common/cancellation';
import { IInstantiationService } from '../../src/util/vs/platform/instantiation/common/instantiation';
import { ISimulationTestRuntime, ssuite, stest } from '../base/stest';

// Use to print the model's suggested tool list for all languages.
// Set to true and run with: npm run simulate -- --grep "print all languages" -n=1
const PRINT_LANGUAGE_TOOLS = false;

ssuite({ title: 'Debug tools list', location: 'context' }, () => {

	async function score(testingServiceCollection: TestingServiceCollection, languages: string[], expected: string[]) {
		const accessor = testingServiceCollection.createTestingAccessor();
		const tools = accessor.get(IInstantiationService).createInstance(LanguageToolsProvider);
		const result = await tools.getToolsForLanguages(languages, CancellationToken.None);
		if (!result.ok) {
			throw new Error('Expected tools to be found');
		}

		let found = 0;
		for (const tool of expected) {
			if (result.commands.includes(tool)) {
				found++;
			}
		}

		accessor.get(ISimulationTestRuntime).setExplicitScore(found / expected.length);
	}

	stest({ description: 'javascript' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['javascript'], ['npm', 'node', 'npx', 'mocha']);
	});

	stest({ description: 'c' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['c'], ['gcc', 'clang', 'make', 'cmake', 'gdb']);
	});

	stest({ description: 'python' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['python'], ['python', 'pip', 'pytest', 'tox']);
	});

	stest({ description: 'typescript' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['javascript'], ['npm', 'node', 'npx', 'mocha']);
	});

	stest({ description: 'ruby' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['ruby'], ['ruby', 'cucumber', 'rake', 'irb']);
	});

	stest({ description: 'csharp' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['csharp'], ['dotnet', 'msbuild', 'xunit', 'vstest']);
	});

	stest({ description: 'elixir' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['elixir'], ['mix', 'iex']);
	});

	stest({ description: 'lua' }, async (testingServiceCollection) => {
		await score(testingServiceCollection, ['lua'], ['lua', 'busted']);
	});

	stest({ description: 'go' }, async (testingServiceCollection) => {
		// it's a short list, because everything in Go is invoked with `go`.
		// Amusingly the model sometimes just lists "go" 10 times.
		await score(testingServiceCollection, ['go'], ['go']);
	});

	if (PRINT_LANGUAGE_TOOLS) {
		stest({ description: 'print all languages' }, async (testingServiceCollection) => {
			const accessor = testingServiceCollection.createTestingAccessor();
			const tools = accessor.get(IInstantiationService).createInstance(LanguageToolsProvider);
			const allTools = new Set<string>();
			const allLanguageIds = new Set([
				...baseLanguageIds,
				...additionalLanguageIds,
				...omittedLanguages,
			]);

			for (const language of allLanguageIds) {
				if (omittedLanguages.includes(language)) {
					continue;
				}

				console.log('Getting tools for', language);
				const result = await tools.getToolsForLanguages([language], CancellationToken.None);
				if (!result.ok) {
					throw new Error('Expected tools to be found');
				}
				for (const tool of result.commands) {
					allTools.add(tool);
				}
			}

			console.log(`const KNOWN_DEBUGGABLE_LANGUAGES = ${JSON.stringify([...allLanguageIds].sort())};`);
			console.log(`const KNOWN_DEBUGGABLE_COMMANDS = ${JSON.stringify([...allTools].sort())};`);
		});
	}
});

// Some additional languages popular in the 2024 SO developer survey
const additionalLanguageIds = [
	'dart',
	'zig',
	'kotlin',
	'matlab',
];

// Languages we don't want to bother getting tools for. These are text
// languages, markup languages that aren't specific to any one set of tools,
// or duplicates (e.g. vue and vue-html).
const omittedLanguages = [
	'bat',
	'bibtex',
	'code-refactoring',
	'coffeescript',
	'css',
	'diff',
	'dockercompose',
	'dockerfile',
	'git-commit',
	'git-rebase',
	'github-issues',
	'graphql',
	'haml',
	'handlebars',
	'html',
	'ini',
	'jade',
	'json',
	'jsonc',
	'less',
	'log',
	'pip-requirements',
	'plaintext',
	'pug',
	'razor',
	'scss',
	'shellscript',
	'slim',
	'snippets',
	'stylus',
	'tex',
	'text',
	'toml',
	'vue-html',
	'xml',
	'xsl',
	'yaml',
];

// Base language list seeded from https://code.visualstudio.com/docs/languages/identifiers#_known-language-identifiers
const baseLanguageIds = [
	'abap',
	'bat',
	'bibtex',
	'clojure',
	'coffeescript',
	'c',
	'cpp',
	'csharp',
	'dockercompose',
	'css',
	'cuda-cpp',
	'd',
	'pascal',
	'diff',
	'dockerfile',
	'erlang',
	'fsharp',
	'git-commit',
	'git-rebase',
	'go',
	'groovy',
	'handlebars',
	'haml',
	'haskell',
	'html',
	'ini',
	'java',
	'javascript',
	'javascriptreact',
	'json',
	'jsonc',
	'julia',
	'latex',
	'less',
	'lua',
	'makefile',
	'markdown',
	'objective-c',
	'objective-cpp',
	'ocaml',
	'pascal',
	'perl',
	'perl6',
	'php',
	'plaintext',
	'powershell',
	'jade',
	'pug',
	'python',
	'r',
	'razor',
	'ruby',
	'rust',
	'scss',
	'sass',
	'shaderlab',
	'shellscript',
	'slim',
	'sql',
	'stylus',
	'svelte',
	'swift',
	'typescript',
	'typescriptreact',
	'tex',
	'vb',
	'vue',
	'vue-html',
	'xml',
	'xsl',
	'yaml'
];