mirror of
https://github.com/microsoft/vscode.git
synced 2026-04-24 03:57:41 -05:00
Explicitly dispose of the SimulationWorkspace (#929)
Looking into why this is being retained as well during testing but hopefully this reduces the memory usage
This commit is contained in:
Matt Bierner
@@ -14,6 +14,7 @@ import { createTextDocumentData, IExtHostDocumentData, setDocText } from '../../
|
||||
import { ExtHostTextEditor } from '../../../util/common/test/shims/textEditor';
|
||||
import { isUri } from '../../../util/common/types';
|
||||
import { Emitter } from '../../../util/vs/base/common/event';
|
||||
import { Disposable } from '../../../util/vs/base/common/lifecycle';
|
||||
import { ResourceMap } from '../../../util/vs/base/common/map';
|
||||
import { Schemas } from '../../../util/vs/base/common/network';
|
||||
import * as path from '../../../util/vs/base/common/path';
|
||||
@@ -113,9 +114,9 @@ export function isNotebook(file: string | vscode.Uri | vscode.TextDocument) {
|
||||
return file.uri.scheme === Schemas.vscodeNotebookCell || file.uri.fsPath.endsWith('.ipynb');
|
||||
}
|
||||
|
||||
export class SimulationWorkspace {
|
||||
export class SimulationWorkspace extends Disposable {
|
||||
|
||||
private readonly _onDidChangeDiagnostics = new Emitter<vscode.DiagnosticChangeEvent>();
|
||||
private readonly _onDidChangeDiagnostics = this._register(new Emitter<vscode.DiagnosticChangeEvent>());
|
||||
public readonly onDidChangeDiagnostics = this._onDidChangeDiagnostics.event;
|
||||
|
||||
private _workspaceState: IDeserializedWorkspaceState | undefined;
|
||||
@@ -163,6 +164,12 @@ export class SimulationWorkspace {
|
||||
}
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this._clear();
|
||||
}
|
||||
|
||||
public override dispose(): void {
|
||||
super.dispose();
|
||||
this._clear();
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import { SpyChatResponseStream } from '../../src/util/common/test/mockChatRespon
|
||||
import { ChatRequestTurn, ChatResponseTurn } from '../../src/util/common/test/shims/chatTypes';
|
||||
import { CancellationToken } from '../../src/util/vs/base/common/cancellation';
|
||||
import { Event } from '../../src/util/vs/base/common/event';
|
||||
import { DisposableStore } from '../../src/util/vs/base/common/lifecycle';
|
||||
import { IInstantiationService } from '../../src/util/vs/platform/instantiation/common/instantiation';
|
||||
import { ChatLocation, ChatRequest, ChatResponseAnchorPart, ChatResponseMarkdownPart } from '../../src/vscodeTypes';
|
||||
import { SimulationWorkspaceExtHost } from '../base/extHostContext/simulationWorkspaceExtHost';
|
||||
@@ -53,188 +54,193 @@ export function fetchConversationOptions() {
|
||||
|
||||
export function generateScenarioTestRunner(scenario: Scenario, evaluator: ScenarioEvaluator): SimulationTestFunction {
|
||||
return async function (testingServiceCollection) {
|
||||
testingServiceCollection.define(IConversationOptions, fetchConversationOptions());
|
||||
const simulationWorkspace = isInExtensionHost ? new SimulationWorkspaceExtHost() : new SimulationWorkspace();
|
||||
simulationWorkspace.setupServices(testingServiceCollection);
|
||||
const accessor = testingServiceCollection.createTestingAccessor();
|
||||
const disposables = new DisposableStore();
|
||||
try {
|
||||
testingServiceCollection.define(IConversationOptions, fetchConversationOptions());
|
||||
const simulationWorkspace = disposables.add(isInExtensionHost ? new SimulationWorkspaceExtHost() : new SimulationWorkspace());
|
||||
simulationWorkspace.setupServices(testingServiceCollection);
|
||||
const accessor = testingServiceCollection.createTestingAccessor();
|
||||
|
||||
const testContext = accessor.get(ISimulationTestRuntime);
|
||||
const log = (message: string, err?: any) => testContext.log(message, err);
|
||||
const testContext = accessor.get(ISimulationTestRuntime);
|
||||
const log = (message: string, err?: any) => testContext.log(message, err);
|
||||
|
||||
const history: (ChatRequestTurn | ChatResponseTurn)[] = [];
|
||||
for (let i = 0; i < scenario.length; i++) {
|
||||
const testCase = scenario[i];
|
||||
simulationWorkspace.resetFromDeserializedWorkspaceState(testCase.getState?.());
|
||||
await testCase.setupCase?.(accessor, simulationWorkspace);
|
||||
const mockProgressReporter = new SpyChatResponseStream();
|
||||
log(`> Query "${testCase.question}"\n`);
|
||||
const history: (ChatRequestTurn | ChatResponseTurn)[] = [];
|
||||
for (let i = 0; i < scenario.length; i++) {
|
||||
const testCase = scenario[i];
|
||||
simulationWorkspace.resetFromDeserializedWorkspaceState(testCase.getState?.());
|
||||
await testCase.setupCase?.(accessor, simulationWorkspace);
|
||||
const mockProgressReporter = new SpyChatResponseStream();
|
||||
log(`> Query "${testCase.question}"\n`);
|
||||
|
||||
const parsedQuery = await parseQueryForScenarioTest(accessor, testCase, simulationWorkspace);
|
||||
const participantId = (parsedQuery.participantName && getChatParticipantIdFromName(parsedQuery.participantName)) ?? '';
|
||||
const request: ChatRequest = { prompt: parsedQuery.query, references: parsedQuery.variables, command: parsedQuery.command, location: ChatLocation.Panel, location2: undefined, attempt: 0, enableCommandDetection: false, isParticipantDetected: false, toolReferences: parsedQuery.toolReferences, toolInvocationToken: undefined as never, model: null!, tools: new Map(), id: '1' };
|
||||
if (testCase.tools) {
|
||||
for (const [toolName, shouldUse] of Object.entries(testCase.tools)) {
|
||||
request.tools.set(getContributedToolName(toolName), shouldUse);
|
||||
const parsedQuery = await parseQueryForScenarioTest(accessor, testCase, simulationWorkspace);
|
||||
const participantId = (parsedQuery.participantName && getChatParticipantIdFromName(parsedQuery.participantName)) ?? '';
|
||||
const request: ChatRequest = { prompt: parsedQuery.query, references: parsedQuery.variables, command: parsedQuery.command, location: ChatLocation.Panel, location2: undefined, attempt: 0, enableCommandDetection: false, isParticipantDetected: false, toolReferences: parsedQuery.toolReferences, toolInvocationToken: undefined as never, model: null!, tools: new Map(), id: '1' };
|
||||
if (testCase.tools) {
|
||||
for (const [toolName, shouldUse] of Object.entries(testCase.tools)) {
|
||||
request.tools.set(getContributedToolName(toolName), shouldUse);
|
||||
}
|
||||
}
|
||||
}
|
||||
const interactiveSession = accessor.get(IInstantiationService).createInstance(
|
||||
ChatParticipantRequestHandler,
|
||||
history,
|
||||
request,
|
||||
mockProgressReporter,
|
||||
CancellationToken.None,
|
||||
{
|
||||
agentId: participantId,
|
||||
agentName: parsedQuery.participantName || '',
|
||||
intentId: (!parsedQuery.participantName && parsedQuery.command) ? parsedQuery.command :
|
||||
parsedQuery.command ? agentsToCommands[parsedQuery.participantName as Intent]![parsedQuery.command] :
|
||||
parsedQuery.participantName,
|
||||
},
|
||||
Event.None,
|
||||
);
|
||||
const result = await interactiveSession.getResult();
|
||||
assert.ok(!result.errorDetails, result.errorDetails?.message);
|
||||
const interactiveSession = accessor.get(IInstantiationService).createInstance(
|
||||
ChatParticipantRequestHandler,
|
||||
history,
|
||||
request,
|
||||
mockProgressReporter,
|
||||
CancellationToken.None,
|
||||
{
|
||||
agentId: participantId,
|
||||
agentName: parsedQuery.participantName || '',
|
||||
intentId: (!parsedQuery.participantName && parsedQuery.command) ? parsedQuery.command :
|
||||
parsedQuery.command ? agentsToCommands[parsedQuery.participantName as Intent]![parsedQuery.command] :
|
||||
parsedQuery.participantName,
|
||||
},
|
||||
Event.None,
|
||||
);
|
||||
const result = await interactiveSession.getResult();
|
||||
assert.ok(!result.errorDetails, result.errorDetails?.message);
|
||||
|
||||
history.push(new ChatRequestTurn(request.prompt, request.command, [...request.references], getChatParticipantIdFromName(participantId), []));
|
||||
history.push(new ChatResponseTurn(mockProgressReporter.items.filter(x => x instanceof ChatResponseMarkdownPart || x instanceof ChatResponseAnchorPart), result, participantId, request.command));
|
||||
history.push(new ChatRequestTurn(request.prompt, request.command, [...request.references], getChatParticipantIdFromName(participantId), []));
|
||||
history.push(new ChatResponseTurn(mockProgressReporter.items.filter(x => x instanceof ChatResponseMarkdownPart || x instanceof ChatResponseAnchorPart), result, participantId, request.command));
|
||||
|
||||
testCase.answer = mockProgressReporter.currentProgress;
|
||||
testCase.answer = mockProgressReporter.currentProgress;
|
||||
|
||||
const turn = interactiveSession.conversation.getLatestTurn();
|
||||
const fullResponse = turn?.responseMessage?.message ?? '';
|
||||
const turn = interactiveSession.conversation.getLatestTurn();
|
||||
const fullResponse = turn?.responseMessage?.message ?? '';
|
||||
|
||||
accessor.get(ISimulationTestRuntime).setOutcome({
|
||||
kind: 'answer',
|
||||
content: fullResponse
|
||||
});
|
||||
accessor.get(ISimulationTestRuntime).setOutcome({
|
||||
kind: 'answer',
|
||||
content: fullResponse
|
||||
});
|
||||
|
||||
// Use the evaluator passed to us to evaluate if the response is correct
|
||||
log(`## Response:\n${fullResponse}\n`);
|
||||
log(`## Commands:\n`);
|
||||
const commands = mockProgressReporter.commandButtons;
|
||||
for (const command of commands) {
|
||||
log(`- ${JSON.stringify(command)}\n`);
|
||||
}
|
||||
// Use the evaluator passed to us to evaluate if the response is correct
|
||||
log(`## Response:\n${fullResponse}\n`);
|
||||
log(`## Commands:\n`);
|
||||
const commands = mockProgressReporter.commandButtons;
|
||||
for (const command of commands) {
|
||||
log(`- ${JSON.stringify(command)}\n`);
|
||||
}
|
||||
|
||||
if (scenario[i].applyChatCodeBlocks) {
|
||||
const codeBlocks = turn?.getMetadata(CodeBlocksMetadata)?.codeBlocks ?? [];
|
||||
const testRuntime = accessor.get(ISimulationTestRuntime);
|
||||
if (scenario[i].applyChatCodeBlocks) {
|
||||
const codeBlocks = turn?.getMetadata(CodeBlocksMetadata)?.codeBlocks ?? [];
|
||||
const testRuntime = accessor.get(ISimulationTestRuntime);
|
||||
|
||||
if (codeBlocks.length !== 0) {
|
||||
const codeMapper = accessor.get(IInstantiationService).createInstance(CodeMapper);
|
||||
const changedDocs: Map<string, { document: TextDocument; originalContent: string; postContent: string }> = new Map();
|
||||
if (codeBlocks.length !== 0) {
|
||||
const codeMapper = accessor.get(IInstantiationService).createInstance(CodeMapper);
|
||||
const changedDocs: Map<string, { document: TextDocument; originalContent: string; postContent: string }> = new Map();
|
||||
|
||||
// Apply Code Block Changes
|
||||
let codeBlockApplyErrorDetails: ChatErrorDetails | undefined = undefined;
|
||||
for (const codeBlock of codeBlocks) {
|
||||
const prevDocument = simulationWorkspace.activeTextEditor?.document!;
|
||||
// Set the active document if the code resource has a uri
|
||||
if (codeBlock.resource) {
|
||||
simulationWorkspace.setCurrentDocument(codeBlock.resource);
|
||||
}
|
||||
const editor = accessor.get(ITabsAndEditorsService).activeTextEditor!;
|
||||
const codeMap = codeBlock.code;
|
||||
const document = simulationWorkspace.activeTextEditor!.document;
|
||||
const documentContext = IDocumentContext.fromEditor(editor);
|
||||
const workspacePath = simulationWorkspace.getFilePath(document.uri);
|
||||
// Apply Code Block Changes
|
||||
let codeBlockApplyErrorDetails: ChatErrorDetails | undefined = undefined;
|
||||
for (const codeBlock of codeBlocks) {
|
||||
const prevDocument = simulationWorkspace.activeTextEditor?.document!;
|
||||
// Set the active document if the code resource has a uri
|
||||
if (codeBlock.resource) {
|
||||
simulationWorkspace.setCurrentDocument(codeBlock.resource);
|
||||
}
|
||||
const editor = accessor.get(ITabsAndEditorsService).activeTextEditor!;
|
||||
const codeMap = codeBlock.code;
|
||||
const document = simulationWorkspace.activeTextEditor!.document;
|
||||
const documentContext = IDocumentContext.fromEditor(editor);
|
||||
const workspacePath = simulationWorkspace.getFilePath(document.uri);
|
||||
|
||||
const previousTextContent = document.getText();
|
||||
const response: MappedEditsResponseStream = {
|
||||
textEdit(target, edits) {
|
||||
simulationWorkspace.applyEdits(target, Array.isArray(edits) ? edits : [edits]);
|
||||
},
|
||||
notebookEdit(target, edits) {
|
||||
simulationWorkspace.applyNotebookEdits(target, Array.isArray(edits) ? edits : [edits]);
|
||||
},
|
||||
};
|
||||
const input: ICodeMapperExistingDocument = { createNew: false, codeBlock: codeMap, uri: document.uri, markdownBeforeBlock: undefined, existingDocument: documentContext.document };
|
||||
const result = await codeMapper.mapCode(input, response, undefined, CancellationToken.None);
|
||||
|
||||
if (!result) {
|
||||
codeBlockApplyErrorDetails = {
|
||||
message: `Code block changes failed to apply to ${document.uri.toString()}`,
|
||||
const previousTextContent = document.getText();
|
||||
const response: MappedEditsResponseStream = {
|
||||
textEdit(target, edits) {
|
||||
simulationWorkspace.applyEdits(target, Array.isArray(edits) ? edits : [edits]);
|
||||
},
|
||||
notebookEdit(target, edits) {
|
||||
simulationWorkspace.applyNotebookEdits(target, Array.isArray(edits) ? edits : [edits]);
|
||||
},
|
||||
};
|
||||
break;
|
||||
}
|
||||
const input: ICodeMapperExistingDocument = { createNew: false, codeBlock: codeMap, uri: document.uri, markdownBeforeBlock: undefined, existingDocument: documentContext.document };
|
||||
const result = await codeMapper.mapCode(input, response, undefined, CancellationToken.None);
|
||||
|
||||
if (result.errorDetails) {
|
||||
result.errorDetails.message = `Code block changes failed to apply to ${document.uri.toString()}:\n${result.errorDetails.message}`;
|
||||
codeBlockApplyErrorDetails = result.errorDetails;
|
||||
break;
|
||||
}
|
||||
if (!result) {
|
||||
codeBlockApplyErrorDetails = {
|
||||
message: `Code block changes failed to apply to ${document.uri.toString()}`,
|
||||
};
|
||||
break;
|
||||
}
|
||||
|
||||
const postEditTextContent = editor.document.getText();
|
||||
if (previousTextContent !== postEditTextContent) {
|
||||
const previousChange = changedDocs.get(workspacePath);
|
||||
if (previousChange) {
|
||||
previousChange.postContent = postEditTextContent;
|
||||
changedDocs.set(workspacePath, previousChange);
|
||||
} else {
|
||||
changedDocs.set(workspacePath, { document, originalContent: previousTextContent, postContent: postEditTextContent });
|
||||
if (result.errorDetails) {
|
||||
result.errorDetails.message = `Code block changes failed to apply to ${document.uri.toString()}:\n${result.errorDetails.message}`;
|
||||
codeBlockApplyErrorDetails = result.errorDetails;
|
||||
break;
|
||||
}
|
||||
|
||||
const postEditTextContent = editor.document.getText();
|
||||
if (previousTextContent !== postEditTextContent) {
|
||||
const previousChange = changedDocs.get(workspacePath);
|
||||
if (previousChange) {
|
||||
previousChange.postContent = postEditTextContent;
|
||||
changedDocs.set(workspacePath, previousChange);
|
||||
} else {
|
||||
changedDocs.set(workspacePath, { document, originalContent: previousTextContent, postContent: postEditTextContent });
|
||||
}
|
||||
}
|
||||
|
||||
if (prevDocument) {
|
||||
simulationWorkspace.setCurrentDocument(prevDocument.uri);
|
||||
}
|
||||
}
|
||||
|
||||
if (prevDocument) {
|
||||
simulationWorkspace.setCurrentDocument(prevDocument.uri);
|
||||
}
|
||||
}
|
||||
// Log the changed files
|
||||
const changedFilePaths: IWorkspaceStateFile[] = [];
|
||||
if (!codeBlockApplyErrorDetails && changedDocs.size > 0) {
|
||||
const seenDoc = new Set<string>();
|
||||
for (const [workspacePath, changes] of changedDocs.entries()) {
|
||||
if (seenDoc.has(workspacePath)) {
|
||||
continue;
|
||||
}
|
||||
seenDoc.add(workspacePath);
|
||||
|
||||
// Log the changed files
|
||||
const changedFilePaths: IWorkspaceStateFile[] = [];
|
||||
if (!codeBlockApplyErrorDetails && changedDocs.size > 0) {
|
||||
const seenDoc = new Set<string>();
|
||||
for (const [workspacePath, changes] of changedDocs.entries()) {
|
||||
if (seenDoc.has(workspacePath)) {
|
||||
continue;
|
||||
if (isNotebook(changes.document.uri)) {
|
||||
await testRuntime.writeFile(workspacePath + '.txt', changes.originalContent, INLINE_INITIAL_DOC_TAG); // using .txt instead of real file extension to avoid breaking automation scripts
|
||||
|
||||
changedFilePaths.push({
|
||||
workspacePath,
|
||||
relativeDiskPath: await testRuntime.writeFile(workspacePath, changes.postContent, INLINE_CHANGED_DOC_TAG),
|
||||
languageId: changes.document.languageId
|
||||
});
|
||||
} else {
|
||||
await testRuntime.writeFile(workspacePath + '.txt', changes.originalContent, INLINE_INITIAL_DOC_TAG); // using .txt instead of real file extension to avoid breaking automation scripts
|
||||
|
||||
changedFilePaths.push({
|
||||
workspacePath,
|
||||
relativeDiskPath: await testRuntime.writeFile(workspacePath, changes.postContent, INLINE_CHANGED_DOC_TAG),
|
||||
languageId: changes.document.languageId
|
||||
});
|
||||
}
|
||||
}
|
||||
seenDoc.add(workspacePath);
|
||||
|
||||
if (isNotebook(changes.document.uri)) {
|
||||
await testRuntime.writeFile(workspacePath + '.txt', changes.originalContent, INLINE_INITIAL_DOC_TAG); // using .txt instead of real file extension to avoid breaking automation scripts
|
||||
|
||||
changedFilePaths.push({
|
||||
workspacePath,
|
||||
relativeDiskPath: await testRuntime.writeFile(workspacePath, changes.postContent, INLINE_CHANGED_DOC_TAG),
|
||||
languageId: changes.document.languageId
|
||||
});
|
||||
} else {
|
||||
await testRuntime.writeFile(workspacePath + '.txt', changes.originalContent, INLINE_INITIAL_DOC_TAG); // using .txt instead of real file extension to avoid breaking automation scripts
|
||||
|
||||
changedFilePaths.push({
|
||||
workspacePath,
|
||||
relativeDiskPath: await testRuntime.writeFile(workspacePath, changes.postContent, INLINE_CHANGED_DOC_TAG),
|
||||
languageId: changes.document.languageId
|
||||
});
|
||||
}
|
||||
testRuntime.setOutcome({
|
||||
kind: 'edit',
|
||||
files: changedFilePaths.map(f => ({ srcUri: f.workspacePath, post: f.relativeDiskPath }))
|
||||
});
|
||||
} else if (codeBlockApplyErrorDetails) {
|
||||
testRuntime.setOutcome({
|
||||
kind: 'failed',
|
||||
error: codeBlockApplyErrorDetails.message,
|
||||
hitContentFilter: codeBlockApplyErrorDetails.responseIsFiltered ?? false,
|
||||
critical: false
|
||||
});
|
||||
}
|
||||
|
||||
testRuntime.setOutcome({
|
||||
kind: 'edit',
|
||||
files: changedFilePaths.map(f => ({ srcUri: f.workspacePath, post: f.relativeDiskPath }))
|
||||
});
|
||||
} else if (codeBlockApplyErrorDetails) {
|
||||
testRuntime.setOutcome({
|
||||
kind: 'failed',
|
||||
error: codeBlockApplyErrorDetails.message,
|
||||
hitContentFilter: codeBlockApplyErrorDetails.responseIsFiltered ?? false,
|
||||
critical: false
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const evaluatedResponse = await evaluator(
|
||||
accessor,
|
||||
testCase.question,
|
||||
mockProgressReporter.currentProgress,
|
||||
fullResponse,
|
||||
turn,
|
||||
i,
|
||||
commands,
|
||||
mockProgressReporter.confirmations,
|
||||
mockProgressReporter.fileTrees,
|
||||
);
|
||||
assert.ok(evaluatedResponse.success, evaluatedResponse.errorMessage);
|
||||
const evaluatedResponse = await evaluator(
|
||||
accessor,
|
||||
testCase.question,
|
||||
mockProgressReporter.currentProgress,
|
||||
fullResponse,
|
||||
turn,
|
||||
i,
|
||||
commands,
|
||||
mockProgressReporter.confirmations,
|
||||
mockProgressReporter.fileTrees,
|
||||
);
|
||||
assert.ok(evaluatedResponse.success, evaluatedResponse.errorMessage);
|
||||
}
|
||||
} finally {
|
||||
disposables.dispose();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -64,11 +64,12 @@ export function setupSimulationWorkspace(testingServiceCollection: TestingServic
|
||||
return workspace;
|
||||
}
|
||||
|
||||
export async function teardownSimulationWorkspace(accessor: ITestingServicesAccessor, _workbench: SimulationWorkspace): Promise<void> {
|
||||
export async function teardownSimulationWorkspace(accessor: ITestingServicesAccessor, workbench: SimulationWorkspace): Promise<void> {
|
||||
const ls = accessor.get(ILanguageFeaturesService);
|
||||
if (ls instanceof SimulationLanguageFeaturesService) {
|
||||
await ls.teardown();
|
||||
}
|
||||
workbench.dispose();
|
||||
}
|
||||
|
||||
function isDeserializedWorkspaceStateBasedScenario(scenario: IScenario): scenario is IDeserializedWorkspaceStateBasedScenario {
|
||||
|
||||
Reference in New Issue
Block a user