diff --git a/src/vs/workbench/contrib/chat/browser/chat.contribution.ts b/src/vs/workbench/contrib/chat/browser/chat.contribution.ts index 492a1f8cf38..f22dae961cb 100644 --- a/src/vs/workbench/contrib/chat/browser/chat.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chat.contribution.ts @@ -68,6 +68,7 @@ import { Extensions as JSONExtensions, IJSONContributionRegistry } from '../../. import { IPromptsService } from '../common/promptSyntax/service/promptsService.js'; import { PromptsService } from '../common/promptSyntax/service/promptsServiceImpl.js'; import { LanguageModelToolsExtensionPointHandler } from '../common/tools/languageModelToolsContribution.js'; +import './telemetry/chatModelCountTelemetry.js'; import { BuiltinToolsContribution } from '../common/tools/builtinTools/tools.js'; import { RenameToolContribution } from './tools/renameTool.js'; import { UsagesToolContribution } from './tools/usagesTool.js'; @@ -1735,55 +1736,6 @@ class ChatForegroundSessionCountContribution extends Disposable implements IWork } } -type ChatModelsAtStartupEvent = { - totalModels: number; - modelsOpenInWidgets: number; - backgroundModels: number; -}; - -type ChatModelsAtStartupClassification = { - totalModels: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Total number of live chat models at startup.' }; - modelsOpenInWidgets: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of chat models that are open in a chat widget or editor.' }; - backgroundModels: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of chat models kept alive in the background without a widget.' }; - owner: 'roblourens'; - comment: 'Tracks chat model counts at startup.'; -}; - -class ChatModelsAtStartupTelemetry extends Disposable implements IWorkbenchContribution { - - static readonly ID = 'workbench.contrib.chatModelsAtStartupTelemetry'; - - constructor( - @IChatService private readonly chatService: IChatService, - @IChatWidgetService private readonly chatWidgetService: IChatWidgetService, - @ITelemetryService private readonly telemetryService: ITelemetryService, - ) { - super(); - void this.logTelemetry(); - } - - private async logTelemetry(): Promise { - const snapshot = this.chatService.getChatModelReferenceDebugInfo(); - - let modelsOpenInWidgets = 0; - let backgroundModels = 0; - - for (const model of snapshot.models) { - if (this.chatWidgetService.getWidgetBySessionResource(model.sessionResource)) { - modelsOpenInWidgets++; - } else { - backgroundModels++; - } - } - - this.telemetryService.publicLog2('chat.modelsAtStartup', { - totalModels: snapshot.totalModels, - modelsOpenInWidgets, - backgroundModels, - }); - } -} - /** * Given builtin and custom modes, returns only the custom mode IDs that should have actions registered. @@ -1991,7 +1943,6 @@ registerWorkbenchContribution2(UsagesToolContribution.ID, UsagesToolContribution registerWorkbenchContribution2(RenameToolContribution.ID, RenameToolContribution, WorkbenchPhase.BlockRestore); registerWorkbenchContribution2(ChatAgentSettingContribution.ID, ChatAgentSettingContribution, WorkbenchPhase.AfterRestored); registerWorkbenchContribution2(ChatForegroundSessionCountContribution.ID, ChatForegroundSessionCountContribution, WorkbenchPhase.AfterRestored); -registerWorkbenchContribution2(ChatModelsAtStartupTelemetry.ID, ChatModelsAtStartupTelemetry, WorkbenchPhase.AfterRestored); registerWorkbenchContribution2(ChatAgentActionsContribution.ID, ChatAgentActionsContribution, WorkbenchPhase.Eventually); registerWorkbenchContribution2(HookSchemaAssociationContribution.ID, HookSchemaAssociationContribution, WorkbenchPhase.AfterRestored); registerWorkbenchContribution2(ToolReferenceNamesContribution.ID, ToolReferenceNamesContribution, WorkbenchPhase.AfterRestored); diff --git a/src/vs/workbench/contrib/chat/browser/telemetry/chatModelCountTelemetry.ts b/src/vs/workbench/contrib/chat/browser/telemetry/chatModelCountTelemetry.ts new file mode 100644 index 00000000000..658bfb1bf68 --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/telemetry/chatModelCountTelemetry.ts @@ -0,0 +1,127 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from '../../../../../base/common/lifecycle.js'; +import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; +import { IWorkbenchContribution, WorkbenchPhase, registerWorkbenchContribution2 } from '../../../../common/contributions.js'; +import { IChatService } from '../../common/chatService/chatService.js'; +import { ChatAgentLocation } from '../../common/constants.js'; +import { IChatWidgetService } from '../chat.js'; + +type ChatModelCountEvent = { + totalModels: number; + modelsOpenInWidgets: number; + backgroundModels: number; + backgroundModels_modifiedEditsKeepAlive: number; + backgroundModels_requestInProgressKeepAlive: number; + backgroundModels_otherHolders: number; +}; + +type ChatModelCountClassification = { + totalModels: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Total number of live chat models.' }; + modelsOpenInWidgets: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of chat models that are open in a chat widget or editor.' }; + backgroundModels: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of chat models with no open widget.' }; + backgroundModels_modifiedEditsKeepAlive: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of background models held alive by the ChatModel#modifiedEditsKeepAlive reference (has pending edits).' }; + backgroundModels_requestInProgressKeepAlive: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of background models held alive by the ChatModel#requestInProgressKeepAlive reference (request is running).' }; + backgroundModels_otherHolders: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of background models with unrecognized holders (potential leaks).' }; +}; + +type ChatModelsAtStartupClassification = ChatModelCountClassification & { + owner: 'roblourens'; + comment: 'Tracks chat model counts at startup.'; +}; + +type ChatModelCreatedEvent = ChatModelCountEvent & { + newModelLocation: string; +}; + +type ChatModelCreatedClassification = ChatModelCountClassification & { + newModelLocation: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ChatAgentLocation of the newly created chat model.' }; + owner: 'roblourens'; + comment: 'Tracks chat model counts each time a new chat model is created, to detect accumulation of background sessions over the lifetime of a window.'; +}; + +/** + * Logs telemetry about how many chat models are live at two moments: + * 1. At startup, after sessions with pending edits have been revived. + * 2. Each time a new chat model is created (skipping the very first one). + * + * Both events share the same model-count snapshot logic to track background + * session accumulation. + */ +export class ChatModelCountTelemetry extends Disposable implements IWorkbenchContribution { + + static readonly ID = 'workbench.contrib.chatModelCountTelemetry'; + + constructor( + @IChatService private readonly chatService: IChatService, + @IChatWidgetService private readonly chatWidgetService: IChatWidgetService, + @ITelemetryService private readonly telemetryService: ITelemetryService, + ) { + super(); + this.logStartupTelemetry(); + this._register(this.chatService.onDidCreateModel(model => this.onDidCreateModel(model.initialLocation))); + } + + private logStartupTelemetry(): void { + this.telemetryService.publicLog2('chat.modelsAtStartup', this.getSnapshot()); + } + + private onDidCreateModel(newModelLocation: ChatAgentLocation): void { + const snapshot = this.getSnapshot(); + + // Skip the trivial case of the very first chat model in the window + if (snapshot.totalModels <= 1) { + return; + } + + this.telemetryService.publicLog2('chat.modelCreatedStats', { + ...snapshot, + newModelLocation, + }); + } + + private getSnapshot(): ChatModelCountEvent { + const snapshot = this.chatService.getChatModelReferenceDebugInfo(); + + let modelsOpenInWidgets = 0; + let backgroundModels = 0; + let backgroundModels_modifiedEditsKeepAlive = 0; + let backgroundModels_requestInProgressKeepAlive = 0; + let backgroundModels_otherHolders = 0; + + for (const model of snapshot.models) { + if (this.chatWidgetService.getWidgetBySessionResource(model.sessionResource)) { + modelsOpenInWidgets++; + } else { + backgroundModels++; + let hasOther = false; + for (const { holder } of model.holders) { + if (holder === 'ChatModel#modifiedEditsKeepAlive') { + backgroundModels_modifiedEditsKeepAlive++; + } else if (holder === 'ChatModel#requestInProgressKeepAlive') { + backgroundModels_requestInProgressKeepAlive++; + } else { + hasOther = true; + } + } + if (hasOther) { + backgroundModels_otherHolders++; + } + } + } + + return { + totalModels: snapshot.totalModels, + modelsOpenInWidgets, + backgroundModels, + backgroundModels_modifiedEditsKeepAlive, + backgroundModels_requestInProgressKeepAlive, + backgroundModels_otherHolders, + }; + } +} + +registerWorkbenchContribution2(ChatModelCountTelemetry.ID, ChatModelCountTelemetry, WorkbenchPhase.AfterRestored);