/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; import * as path from 'path'; import type * as vscode from 'vscode'; import { Intent } from '../../src/extension/common/constants'; import { CopilotInteractiveEditorResponse, InteractionOutcome, InteractionOutcomeComputer } from '../../src/extension/inlineChat/node/promptCraftingTypes'; import { ICopilotChatResult } from '../../src/extension/prompt/common/conversation'; import { ChatParticipantRequestHandler, IChatAgentArgs } from '../../src/extension/prompt/node/chatParticipantRequestHandler'; import { guessFileIndentInfo } from '../../src/extension/prompt/node/indentationGuesser'; import { IntentDetector } from '../../src/extension/prompt/node/intentDetector'; import { IIntent } from '../../src/extension/prompt/node/intents'; import { WorkingCopyOriginalDocument } from '../../src/extension/prompts/node/inline/workingCopies'; import { IToolsService } from '../../src/extension/tools/common/toolsService'; import { TestEditFileTool } from '../../src/extension/tools/node/test/testTools'; import { TestToolsService } from '../../src/extension/tools/node/test/testToolsService'; import { editorAgentName, getChatParticipantIdFromName } from '../../src/platform/chat/common/chatAgents'; import { IChatMLFetcher } from '../../src/platform/chat/common/chatMLFetcher'; import { ILanguageDiagnosticsService } from '../../src/platform/languages/common/languageDiagnosticsService'; import { ILanguageFeaturesService } from '../../src/platform/languages/common/languageFeaturesService'; import { ITabsAndEditorsService } from '../../src/platform/tabs/common/tabsAndEditorsService'; import { isInExtensionHost } from '../../src/platform/test/node/isInExtensionHost'; import { IDeserializedWorkspaceState } from '../../src/platform/test/node/promptContextModel'; import { ITestingServicesAccessor, TestingServiceCollection } from '../../src/platform/test/node/services'; import { IFile, isNotebook, SimulationWorkspace } from '../../src/platform/test/node/simulationWorkspace'; import { ChatResponseStreamImpl } from '../../src/util/common/chatResponseStreamImpl'; import { getLanguage, getLanguageForResource } from '../../src/util/common/languages'; import { ChatRequestTurn, ChatResponseTurn } from '../../src/util/common/test/shims/chatTypes'; import { ExtHostNotebookDocumentData } from '../../src/util/common/test/shims/notebookDocument'; import { createTextDocumentData, IExtHostDocumentData } from '../../src/util/common/test/shims/textDocument'; import { CancellationToken } from '../../src/util/vs/base/common/cancellation'; import { ResourceMap } from '../../src/util/vs/base/common/map'; import { isEqual } from '../../src/util/vs/base/common/resources'; import { commonPrefixLength, commonSuffixLength } from '../../src/util/vs/base/common/strings'; import { URI } from '../../src/util/vs/base/common/uri'; import { SyncDescriptor } from '../../src/util/vs/platform/instantiation/common/descriptors'; import { IInstantiationService } from '../../src/util/vs/platform/instantiation/common/instantiation'; import { ChatLocation, ChatReferenceDiagnostic, ChatRequest, ChatRequestEditorData, ChatResponseMarkdownPart, ChatResponseNotebookEditPart, ChatResponseTextEditPart, Diagnostic, DiagnosticRelatedInformation, LanguageModelToolResult, Location, NotebookRange, Range, Selection, TextEdit, Uri, WorkspaceEdit } from '../../src/vscodeTypes'; import { SimulationExtHostToolsService } from '../base/extHostContext/simulationExtHostToolsService'; import { SimulationWorkspaceExtHost } from '../base/extHostContext/simulationWorkspaceExtHost'; import { SpyingChatMLFetcher } from '../base/spyingChatMLFetcher'; import { ISimulationTestRuntime, NonExtensionConfiguration } from '../base/stest'; import { createWorkingSetFileVariable, parseQueryForTest } from '../e2e/testHelper'; import { readBuiltinIntents } from '../intent/intentTest'; import { getDiagnostics } from './diagnosticProviders'; import { convertTestToVSCodeDiagnostics } from './diagnosticProviders/utils'; import { SimulationLanguageFeaturesService } from './language/simulationLanguageFeatureService'; import { IDiagnostic, IDiagnosticComparison, INLINE_CHANGED_DOC_TAG, INLINE_INITIAL_DOC_TAG, INLINE_STATE_TAG, IRange, IWorkspaceState, IWorkspaceStateFile } from './shared/sharedTypes'; import { DiagnosticProviderId, EditTestStrategy, IDeserializedWorkspaceStateBasedScenario, IInlineEdit, IOutcome, IScenario, IScenarioDiagnostic, IScenarioQuery, OutcomeAnnotation } from './types'; export type SimulationWorkspaceInput = { files: IFile[]; workspaceFolders?: Uri[] } | { workspaceState: IDeserializedWorkspaceState }; export function setupSimulationWorkspace(testingServiceCollection: TestingServiceCollection, input: SimulationWorkspaceInput): SimulationWorkspace { const workspace = isInExtensionHost ? new SimulationWorkspaceExtHost() : new SimulationWorkspace(); if ('workspaceState' in input) { workspace.resetFromDeserializedWorkspaceState(input.workspaceState); } else { workspace.resetFromFiles(input.files, input.workspaceFolders); } workspace.setupServices(testingServiceCollection); testingServiceCollection.define(ILanguageFeaturesService, new SyncDescriptor(SimulationLanguageFeaturesService, [workspace])); return workspace; } export async function teardownSimulationWorkspace(accessor: ITestingServicesAccessor, workbench: SimulationWorkspace): Promise { const ls = accessor.get(ILanguageFeaturesService); if (ls instanceof SimulationLanguageFeaturesService) { await ls.teardown(); } workbench.dispose(); } function isDeserializedWorkspaceStateBasedScenario(scenario: IScenario): scenario is IDeserializedWorkspaceStateBasedScenario { return 'workspaceState' in scenario; } export function simulateInlineChatWithStrategy(strategy: EditTestStrategy, testingServiceCollection: TestingServiceCollection, scenario: IScenario) { if (strategy === EditTestStrategy.InlineChatIntent) { return simulateInlineChatIntent(testingServiceCollection, scenario); } else { return simulateInlineChat(testingServiceCollection, scenario); } } export async function simulateInlineChat( testingServiceCollection: TestingServiceCollection, scenario: IScenario ): Promise { const host: EditingSimulationHost = { prepareChatRequestLocation: (accessor: ITestingServicesAccessor, wholeRange?: Range) => { const editor = accessor.get(ITabsAndEditorsService).activeTextEditor; if (!editor) { throw new Error(`No active editor`); } return { location: ChatLocation.Editor, location2: new ChatRequestEditorData(editor, editor.document, editor.selection, wholeRange ?? editor.selection), }; } }; return simulateEditingScenario(testingServiceCollection, scenario, host); } class ChatReferenceDiagnostic2 extends ChatReferenceDiagnostic { constructor(uri: Uri, d: Diagnostic) { super([[uri, [d]]]); } } export async function simulateInlineChatIntent( testingServiceCollection: TestingServiceCollection, scenario: IScenario ): Promise { const overrideCommand = `/${Intent.InlineChat}`; const ensureSlashEdit = (query: string) => { return query.startsWith(overrideCommand) ? query : `${overrideCommand} ${query}`; }; const prependEditToUserQueries = (queries: IScenarioQuery[]) => { return queries.map(scenarioQuery => { return { ...scenarioQuery, query: ensureSlashEdit(scenarioQuery.query), }; }); }; const massagedScenario = { ...scenario, queries: prependEditToUserQueries(scenario.queries) }; const host: EditingSimulationHost = { prepareChatRequestLocation: (accessor: ITestingServicesAccessor, wholeRange?: Range) => { const editor = accessor.get(ITabsAndEditorsService).activeTextEditor; if (!editor) { throw new Error(`No active editor`); } return { location: ChatLocation.Editor, location2: new ChatRequestEditorData(editor, editor.document, editor.selection, wholeRange ?? editor.selection), }; }, contributeAdditionalReferences(accessor, existingReferences) { const diagnosticService = accessor.get(ILanguageDiagnosticsService); const editor = accessor.get(ITabsAndEditorsService).activeTextEditor; if (!editor) { return existingReferences.slice(); } const result = existingReferences.slice(); const diagnostics = diagnosticService.getDiagnostics(editor.document.uri); for (const d of diagnostics) { if (d.range.intersection(editor.selection)) { result.push({ id: `diagnostic/${editor.document.uri}/${JSON.stringify(d)}`, name: d.message, value: new ChatReferenceDiagnostic2(editor.document.uri, d) }); } } return result; }, }; return simulateEditingScenario(testingServiceCollection, massagedScenario, host); } export type EditingSimulationHostResponseProcessor = { spyOnStream(stream: vscode.ChatResponseStream): vscode.ChatResponseStream; postProcess(accessor: ITestingServicesAccessor, workspace: SimulationWorkspace, stream: vscode.ChatResponseStream, result?: ICopilotChatResult): Promise; }; export interface EditingSimulationHost { agentArgs?: IChatAgentArgs; prepareChatRequestLocation(accessor: ITestingServicesAccessor, range?: Range): { location: vscode.ChatLocation; location2: vscode.ChatRequestEditorData | undefined; }; contributeAdditionalReferences?: (accessor: ITestingServicesAccessor, existingReferences: readonly vscode.ChatPromptReference[]) => vscode.ChatPromptReference[]; provideResponseProcessor?: (query: IScenarioQuery) => EditingSimulationHostResponseProcessor; } export async function simulateEditingScenario( testingServiceCollection: TestingServiceCollection, scenario: IScenario, host: EditingSimulationHost ): Promise { assert(scenario.queries.length > 0, `Cannot simulate scenario with no queries`); assert(isDeserializedWorkspaceStateBasedScenario(scenario) || scenario.files.length > 0, `Cannot simulate scenario with no files`); const workspace = setupSimulationWorkspace(testingServiceCollection, scenario); await scenario.extraWorkspaceSetup?.(workspace); const accessor = testingServiceCollection.createTestingAccessor(); await scenario.onBeforeStart?.(accessor); const instaService = accessor.get(IInstantiationService); const testRuntime = accessor.get(ISimulationTestRuntime); const states: IWorkspaceState[] = []; let range: Range | undefined; let isFirst = true; const history: (ChatRequestTurn | ChatResponseTurn)[] = []; /** * A map from doc to relative path with initial contents which is populated right before modifying a document. */ const changedDocsInitialStates = new Map | null>(); // run each query for the scenario try { const seenFiles: vscode.ChatPromptReference[] = []; for (const query of scenario.queries) { if (query.file) { if (isNotebook(query.file)) { const notebook = workspace.getNotebook(query.file); if (!notebook) { throw new Error(`Missing notebook file ${query.file}`); } const cell = notebook.cellAt(query.activeCell ?? 0); if (!cell) { throw new Error(`Missing cell ${query.activeCell} in notebook file ${query.file}`); } workspace.addNotebookDocument(notebook); workspace.setCurrentNotebookDocument(notebook); workspace.setCurrentDocument(cell.document.uri); } else if (typeof query.file !== 'string') { workspace.setCurrentDocument(query.file); } else { workspace.setCurrentDocument( workspace.getDocument(query.file).document.uri); } } if (query.selection) { const selection = toSelection(query.selection); workspace.setCurrentSelection(selection); } if (query.visibleRanges) { workspace.setCurrentVisibleRanges(query.visibleRanges.map((range) => toRange(range))); } if (query.activeCell) { const cellSelection = new NotebookRange(query.activeCell, query.activeCell + 1); workspace.setCurrentNotebookSelection([cellSelection]); } const queryWholeRange = query.wholeRange ? toSelection(query.wholeRange) : undefined; const activeEditor = accessor.get(ITabsAndEditorsService).activeTextEditor; if (query.file && !activeEditor) { throw new Error(`query.file is defined but no editor is active`); } let initialDiagnostics: ResourceMap | undefined; if (typeof query.diagnostics === 'string') { // diagnostics are computed try { initialDiagnostics = await fetchDiagnostics(accessor, workspace, query.diagnostics); workspace.setDiagnostics(initialDiagnostics); } catch (error) { throw new Error(`Error obtained while fetching the diagnostics: ${error}`); } } else if (Array.isArray(query.diagnostics)) { if (!activeEditor) { throw new Error(`diagnostics can only be an array if there's an active editor (is 'file' specified?)`); } // diagnostics are set explicitly const diagnostics = new ResourceMap(); diagnostics.set(activeEditor.document.uri, convertToDiagnostics(workspace, query.diagnostics)); workspace.setDiagnostics(diagnostics); } if (query.fileIndentInfo) { workspace.setCurrentDocumentIndentInfo(query.fileIndentInfo); } else if (activeEditor) { workspace.setCurrentDocumentIndentInfo(guessFileIndentInfo(activeEditor.document)); } if (isFirst && activeEditor) { isFirst = false; range = activeEditor.selection; const documentUri = activeEditor.document.uri; const workspacePath = workspace.getFilePath(documentUri); let relativeDiskPath: string | undefined; if (isNotebook(documentUri)) { const notebookDocument = workspace.getNotebook(documentUri); if (!notebookDocument) { throw new Error(`Missing notebook document ${documentUri}`); } relativeDiskPath = await testRuntime.writeFile(workspacePath + '.txt', notebookDocument.getText(), INLINE_INITIAL_DOC_TAG); // using .txt instead of real file extension to avoid breaking automation scripts } else { relativeDiskPath = await testRuntime.writeFile(workspacePath + '.txt', activeEditor.document.getText(), INLINE_INITIAL_DOC_TAG); // using .txt instead of real file extension to avoid breaking automation scripts } changedDocsInitialStates.set(activeEditor.document, null); // just mark that it doesn't get written twice if (!relativeDiskPath) { throw new Error(`Failed to write initial document to disk`); } states.push({ kind: 'initial', file: { workspacePath, relativeDiskPath, languageId: activeEditor.document.languageId }, additionalFiles: [], languageId: getLanguage(activeEditor.document).languageId, selection: toIRange(activeEditor.selection), range: toIRange(range), diagnostics: workspace.activeFileDiagnostics.map(toIDiagnostic), }); } else { range = queryWholeRange ?? range; states.push({ kind: 'initial', additionalFiles: [], diagnostics: workspace.activeFileDiagnostics.map(toIDiagnostic), }); } let command: string | undefined; let prompt = query.query; if (prompt.startsWith('/')) { const groups = /\/(?\w+)(?\s.*)?/s.exec(query.query)?.groups; command = groups?.intentId ?? undefined; prompt = groups?.restOfQuery?.trim() ?? ''; } const changedDocs: vscode.TextDocument[] = []; const references: vscode.ChatPromptReference[] = [...seenFiles]; const toolReferences: vscode.ChatLanguageModelToolReference[] = []; try { const parsedQuery = parseQueryForTest(accessor, prompt, workspace); for (const variable of parsedQuery.variables) { if (!URI.isUri(variable.value)) { references.push(variable); continue; } const uri = variable.value; if (!seenFiles.find(ref => URI.isUri(ref.value) && isEqual(ref.value, uri))) { seenFiles.push(variable); references.push(variable); } } toolReferences.push(...parsedQuery.toolReferences); } catch (error) { // No problem! } references.push(...(host.contributeAdditionalReferences?.(accessor, references) ?? [])); const { location, location2 } = host.prepareChatRequestLocation(accessor, range); let request: vscode.ChatRequest = { location, location2, command, prompt, references, attempt: 0, isParticipantDetected: false, enableCommandDetection: true, // TODO@ulugbekna: add support for disabling intent detection? toolReferences, toolInvocationToken: (isInExtensionHost ? undefined : {}) as never, model: null!, // https://github.com/microsoft/vscode-copilot/issues/9475 tools: new Map(), id: '1', sessionId: '1', sessionResource: Uri.parse('chat:/1'), hasHooksEnabled: false, }; // Run intent detection if (!request.command) { const intentDetector = instaService.createInstance(IntentDetector); const participants = readBuiltinIntents(location); const detectedParticipant = await intentDetector.provideParticipantDetection(request, { history, yieldRequested: false }, { participants, location: ChatLocation.Editor }, CancellationToken.None); if (detectedParticipant?.command) { request = { ...request, command: detectedParticipant.command }; } } const markdownChunks: string[] = []; const changedDocuments = new ResourceMap(); let hasActualEdits = false; let stream: vscode.ChatResponseStream = new ChatResponseStreamImpl((value) => { if (value instanceof ChatResponseTextEditPart && value.edits.length > 0) { const { uri, edits } = value; let doc: IExtHostDocumentData; if (!workspace.hasDocument(uri)) { // this is a new file const language = getLanguageForResource(uri); doc = createTextDocumentData(uri, '', language.languageId); workspace.addDocument(doc); } else { doc = workspace.getDocument(uri); if (!changedDocsInitialStates.has(doc.document)) { const workspacePath = workspace.getFilePath(doc.document.uri); const workspaceStateFilePromise = testRuntime.writeFile(workspacePath, doc.document.getText(), INLINE_CHANGED_DOC_TAG).then((relativeDiskPath) => { return { workspacePath, relativeDiskPath, languageId: doc.document.languageId }; }); changedDocsInitialStates.set(doc.document, workspaceStateFilePromise); } } let workingCopyDocument = changedDocuments.get(uri); if (!workingCopyDocument) { workingCopyDocument = new WorkingCopyOriginalDocument(doc.document.getText()); changedDocuments.set(uri, workingCopyDocument); } const offsetEdits = workingCopyDocument.transformer.toOffsetEdit(edits); if (!workingCopyDocument.isNoop(offsetEdits)) { hasActualEdits = true; workingCopyDocument.applyOffsetEdits(offsetEdits); changedDocs.push(doc.document); if (activeEditor && isEqual(doc.document.uri, activeEditor.document.uri)) { // edit in the same document, adjust the range range = applyEditsAndExpandRange(workspace, activeEditor.document, edits, range); } else { workspace.applyEdits(doc.document.uri, edits); } } } else if (value instanceof ChatResponseNotebookEditPart) { const { uri, edits } = value; const validEdits = edits.filter(edit => typeof edit !== 'boolean'); let notebookDoc: ExtHostNotebookDocumentData; if (!workspace.hasNotebookDocument(uri)) { notebookDoc = ExtHostNotebookDocumentData.createJupyterNotebook(uri, `{ "cells": [] }`); workspace.addNotebookDocument(notebookDoc); } else { notebookDoc = workspace.getNotebook(uri); } let workingCopyDocument = changedDocuments.get(uri); if (!workingCopyDocument) { workingCopyDocument = new WorkingCopyOriginalDocument(notebookDoc.getText()); changedDocuments.set(uri, workingCopyDocument); } if (validEdits.length > 0) { hasActualEdits = true; workspace.applyNotebookEdits(notebookDoc.uri, validEdits); workingCopyDocument = new WorkingCopyOriginalDocument(notebookDoc.getText()); changedDocuments.set(uri, workingCopyDocument); } } else if (value instanceof ChatResponseMarkdownPart) { markdownChunks.push(value.value.value); } }, () => { }, undefined, undefined, undefined, () => Promise.resolve(undefined)); const interactionOutcomeComputer = new InteractionOutcomeComputer(activeEditor?.document.uri); stream = interactionOutcomeComputer.spyOnStream(stream); const responseProcessor = host.provideResponseProcessor?.(query); if (responseProcessor) { stream = responseProcessor.spyOnStream(stream); } const documentStateBeforeInvocation = activeEditor?.document.getText(); setupTools(stream, request, accessor); const agentArgs = host.agentArgs ?? { agentId: getChatParticipantIdFromName(editorAgentName), agentName: editorAgentName, intentId: request.command }; const requestHandler = instaService.createInstance(ChatParticipantRequestHandler, history, request, stream, CancellationToken.None, agentArgs, () => false, undefined); const result = await requestHandler.getResult(); history.push(new ChatRequestTurn(request.prompt, request.command, [...request.references], '', [])); history.push(new ChatResponseTurn([new ChatResponseMarkdownPart(markdownChunks.join(''))], result, '')); let annotations = await responseProcessor?.postProcess(accessor, workspace, stream, result) ?? []; let interactionOutcomeKind = interactionOutcomeComputer.interactionOutcome.kind; if (interactionOutcomeKind === 'inlineEdit' || interactionOutcomeKind === 'workspaceEdit') { // sometimes we push noop edits which can trick the outcome computer if (!hasActualEdits) { interactionOutcomeKind = 'noopEdit'; } } let intent: IIntent | undefined; { // TODO@Alex: extract to host object const response = requestHandler.conversation.getLatestTurn()?.getMetadata(CopilotInteractiveEditorResponse); intent = (response ? response.promptQuery.intent : undefined); } annotations = annotations.concat(requestHandler.conversation.getLatestTurn()?.getMetadata(InteractionOutcome)?.annotations ?? []); let outcome: IOutcome; if (interactionOutcomeKind === 'none') { outcome = { type: 'none', annotations, chatResponseMarkdown: markdownChunks.join('') }; } else if (result.errorDetails) { outcome = { type: 'error', errorDetails: result.errorDetails, annotations }; } else if (interactionOutcomeKind === 'noopEdit') { outcome = { type: 'none', annotations, chatResponseMarkdown: markdownChunks.join('') }; } else if (interactionOutcomeKind === 'inlineEdit' || interactionOutcomeKind === 'workspaceEdit') { const outcomeFiles: IFile[] = []; const workspaceEdit = new WorkspaceEdit(); const outcomeEdits: IInlineEdit[] = []; for (const [uri, workingCopyDoc] of changedDocuments.entries()) { if (uri.scheme === 'file') { outcomeFiles.push({ kind: 'relativeFile', fileName: path.basename(uri.fsPath), fileContents: workspace.tryGetNotebook(uri)?.getText() ?? workspace.getDocument(uri).getText() }); } else { outcomeFiles.push({ kind: 'qualifiedFile', uri: uri, fileContents: workspace.tryGetNotebook(uri)?.getText() ?? workspace.getDocument(uri).getText() }); } const offsetEdits = workingCopyDoc.appliedEdits; const textEdits = workingCopyDoc.transformer.toTextEdits(offsetEdits); if (activeEditor && isEqual(uri, activeEditor.document.uri)) { // edit in the same document for (let i = 0; i < offsetEdits.replacements.length; i++) { const offsetEdit = offsetEdits.replacements[i]; const textEdit = textEdits[i]; outcomeEdits.push({ offset: offsetEdit.replaceRange.start, length: offsetEdit.replaceRange.length, range: textEdit.range, newText: textEdit.newText, }); } } workspaceEdit.set(uri, textEdits); } if (interactionOutcomeKind === 'inlineEdit') { if (!activeEditor) { throw new Error(`inlineEdit should always have an open editor`); } outcome = { type: 'inlineEdit', initialDiagnostics, appliedEdits: outcomeEdits, originalFileContents: documentStateBeforeInvocation ?? '', fileContents: activeEditor.document.getText(), chatResponseMarkdown: markdownChunks.join(''), annotations }; } else { outcome = { type: 'workspaceEdit', files: outcomeFiles, annotations, edits: workspaceEdit, chatResponseMarkdown: markdownChunks.join('') }; } } else { outcome = { type: 'conversational', chatResponseMarkdown: markdownChunks.join(''), annotations }; } const changedFilePaths: IWorkspaceStateFile[] = []; if (changedDocs.length > 0) { const seenDoc = new Set(); for (const changedDoc of changedDocs) { const workspacePath = workspace.getFilePath(changedDoc.uri); if (seenDoc.has(workspacePath)) { continue; } seenDoc.add(workspacePath); if (location !== ChatLocation.Editor && !seenFiles.find((v) => URI.isUri(v.value) && isEqual(v.value, changedDoc.uri))) { seenFiles.push(createWorkingSetFileVariable(changedDoc.uri)); } if (isNotebook(changedDoc.uri)) { const notebook = workspace.getNotebook(changedDoc.uri); changedFilePaths.push({ workspacePath, relativeDiskPath: await testRuntime.writeFile(workspacePath, notebook.getText(), INLINE_CHANGED_DOC_TAG), languageId: changedDoc.languageId }); } else { changedFilePaths.push({ workspacePath, relativeDiskPath: await testRuntime.writeFile(workspacePath, changedDoc.getText(), INLINE_CHANGED_DOC_TAG), languageId: changedDoc.languageId }); } } // We managed to edit some files! testRuntime.setOutcome({ kind: 'edit', files: changedFilePaths.map(f => ({ srcUri: f.workspacePath, post: f.relativeDiskPath })), annotations: outcome.annotations }); } else { if (activeEditor) { const workspacePath = workspace.getFilePath(activeEditor.document.uri); changedFilePaths.push({ workspacePath, relativeDiskPath: await testRuntime.writeFile(workspacePath, activeEditor.document.getText(), INLINE_CHANGED_DOC_TAG), languageId: activeEditor.document.languageId }); } if (markdownChunks.length > 0) { testRuntime.setOutcome({ kind: 'answer', content: markdownChunks.join(''), annotations: outcome.annotations }); } else { const chatMLFetcher = accessor.get(IChatMLFetcher); let contentFilterCount = 0; if (chatMLFetcher instanceof SpyingChatMLFetcher) { contentFilterCount = chatMLFetcher.contentFilterCount; } testRuntime.setOutcome({ kind: 'failed', hitContentFilter: contentFilterCount > 0, error: 'No contents.', annotations: outcome.annotations, critical: false, }); } } let requestCount = 0; const fetcher = accessor.get(IChatMLFetcher); if (fetcher instanceof SpyingChatMLFetcher) { requestCount = fetcher.interceptedRequests.length; } let diagnostics: { [workspacePath: string]: IDiagnosticComparison } | undefined = undefined; if (typeof query.diagnostics === 'string') { const diagnosticsAfter = await fetchDiagnostics(accessor, workspace, query.diagnostics); diagnostics = {}; for (const changedFilePath of changedFilePaths) { const uri = workspace.getUriFromFilePath(changedFilePath.workspacePath); const before = (initialDiagnostics?.get(uri) ?? []).map(toIDiagnostic); const after = (diagnosticsAfter.get(uri) ?? []).map(toIDiagnostic); diagnostics[changedFilePath.workspacePath] = { before, after }; } } states.push({ kind: 'interaction', changedFiles: changedFilePaths, annotations: outcome.annotations, fileName: activeEditor ? workspace.getFilePath(activeEditor.document.uri) : undefined, languageId: activeEditor?.document.languageId, diagnostics, selection: activeEditor ? toIRange(activeEditor.selection) : undefined, range: activeEditor ? toIRange(range ?? activeEditor.selection) : undefined, interaction: { query: query.query, actualIntent: query.expectedIntent, detectedIntent: intent?.id, }, requestCount, }); await Promise.resolve(query.validate(outcome, workspace, accessor)); } for (const [_, workspaceStateFilePromise] of changedDocsInitialStates) { if (workspaceStateFilePromise === null) { continue; } const workspaceStateFile = await workspaceStateFilePromise; if (states.length > 0 && states[0].kind === 'initial') { states[0].additionalFiles?.push(workspaceStateFile); } } } finally { await teardownSimulationWorkspace(accessor, workspace); await testRuntime.writeFile('inline-simulator.txt', JSON.stringify(states, undefined, 2), INLINE_STATE_TAG); // TODO@test: using .txt instead of .json to avoid breaking test scripts } } function setupTools(stream: vscode.ChatResponseStream, request: ChatRequest, accessor: ITestingServicesAccessor) { const toolsService = accessor.get(IToolsService) as TestToolsService | SimulationExtHostToolsService; const instaService = accessor.get(IInstantiationService); const editTool = instaService.createInstance(TestEditFileTool, stream); toolsService.addTestToolOverride( editTool.info, editTool); toolsService.addTestToolOverride( { name: 'inline_chat_exit', description: 'Moves the inline chat session to the richer panel chat which supports edits across files, creating new files, and multi-turn conversations between the user and the assistant.', inputSchema: {}, source: undefined, tags: [], }, { invoke() { return new LanguageModelToolResult([]); } } ); } function computeMoreMinimalEdit(document: vscode.TextDocument, edit: vscode.TextEdit): vscode.TextEdit { edit = reduceCommonPrefix(document, edit); edit = reduceCommonSuffix(document, edit); return edit; function reduceCommonPrefix(document: vscode.TextDocument, edit: vscode.TextEdit): vscode.TextEdit { const start = document.offsetAt(edit.range.start); const end = document.offsetAt(edit.range.end); const oldText = document.getText().substring(start, end); const newText = edit.newText; const commonPrefixLen = commonPrefixLength(oldText, newText); return new TextEdit( new Range( document.positionAt(start + commonPrefixLen), edit.range.end ), edit.newText.substring(commonPrefixLen) ); } function reduceCommonSuffix(document: vscode.TextDocument, edit: vscode.TextEdit): vscode.TextEdit { const start = document.offsetAt(edit.range.start); const end = document.offsetAt(edit.range.end); const oldText = document.getText().substring(start, end); const newText = edit.newText; const commonSuffixLen = commonSuffixLength(oldText, newText); return new TextEdit( new Range( edit.range.start, document.positionAt(end - commonSuffixLen) ), edit.newText.substring(0, newText.length - commonSuffixLen) ); } } function applyEditsAndExpandRange(workspace: SimulationWorkspace, document: vscode.TextDocument, edits: vscode.TextEdit[], range: vscode.Range): vscode.Range; function applyEditsAndExpandRange(workspace: SimulationWorkspace, document: vscode.TextDocument, edits: vscode.TextEdit[], range: vscode.Range | undefined): vscode.Range | undefined; function applyEditsAndExpandRange(workspace: SimulationWorkspace, document: vscode.TextDocument, edits: vscode.TextEdit[], range: vscode.Range | undefined): vscode.Range | undefined { if (typeof range === 'undefined') { workspace.applyEdits(document.uri, edits, range); return undefined; } edits = edits.map(edit => computeMoreMinimalEdit(document, edit)); const touchedRanges = new Set<[number, number]>(); let deltaOffset = 0; for (const edit of edits) { const startOffset = deltaOffset + document.offsetAt(edit.range.start); const endOffset = deltaOffset + document.offsetAt(edit.range.end); const textLen = edit.newText.length; deltaOffset += textLen - (endOffset - startOffset); touchedRanges.add([startOffset, textLen]); } range = workspace.applyEdits(document.uri, edits, range); for (const touchedRange of touchedRanges) { const [startOffset, textLen] = touchedRange; const start = document.positionAt(startOffset); const end = document.positionAt(startOffset + textLen); range = range?.union(new Range(start, end)); } return range; } function convertToDiagnostics(workspace: SimulationWorkspace, diagnostics: IScenarioDiagnostic[] | undefined): vscode.Diagnostic[] { return (diagnostics ?? []).map((d) => { const diagnostic = new Diagnostic(new Range(d.startLine, d.startCharacter, d.endLine, d.endCharacter), d.message); diagnostic.relatedInformation = d.relatedInformation?.map(r => { const range = new Range(r.location.startLine, r.location.startCharacter, r.location.endLine, r.location.endCharacter); const relatedDocument = workspace.getDocument(r.location.path); const relatedLocation = new Location(relatedDocument.document.uri, range); return new DiagnosticRelatedInformation(relatedLocation, r.message); }); return diagnostic; }); } async function fetchDiagnostics(accessor: ITestingServicesAccessor, workspace: SimulationWorkspace, providerId: DiagnosticProviderId) { const files = workspace.documents.map(doc => ({ fileName: workspace.getFilePath(doc.document.uri), fileContents: doc.document.getText() })); const diagnostics = await getDiagnostics(accessor, files, providerId); return convertTestToVSCodeDiagnostics(diagnostics, path => workspace.getUriFromFilePath(path)); } function toIDiagnostic(diagnostic: vscode.Diagnostic): IDiagnostic { return { range: toIRange(diagnostic.range), message: diagnostic.message }; } export function toIRange(range: vscode.Range): IRange { return { start: { line: range.start.line, character: range.start.character }, end: { line: range.end.line, character: range.end.character }, }; } export function toSelection(selection: [number, number] | [number, number, number, number]): vscode.Selection { if (selection.length === 2) { return new Selection(selection[0], selection[1], selection[0], selection[1]); } else { return new Selection(selection[0], selection[1], selection[2], selection[3]); } } export function toRange(range: [number, number] | [number, number, number, number]): vscode.Range { if (range.length === 2) { return new Range(range[0], 0, range[1], 0); } else { return new Range(range[0], range[1], range[2], range[3]); } } export function forInlineAndInlineChatIntent(callback: (strategy: EditTestStrategy, configurations: NonExtensionConfiguration[] | undefined, suffix: string) => void): void { callback(EditTestStrategy.Inline, undefined, ''); callback(EditTestStrategy.InlineChatIntent, [['inlineChat.enableV2', true], ['chat.agent.autoFix', false]], '-InlineChatIntent'); } export function forInline(callback: (strategy: EditTestStrategy, configurations: NonExtensionConfiguration[] | undefined, suffix: string) => void): void { callback(EditTestStrategy.Inline, undefined, ''); }