diff --git a/extensions/copilot/src/extension/conversation/vscode-node/chatParticipants.ts b/extensions/copilot/src/extension/conversation/vscode-node/chatParticipants.ts index 564b9ae6850..b4554505387 100644 --- a/extensions/copilot/src/extension/conversation/vscode-node/chatParticipants.ts +++ b/extensions/copilot/src/extension/conversation/vscode-node/chatParticipants.ts @@ -14,6 +14,7 @@ import { IExperimentationService } from '../../../platform/telemetry/common/null import { DisposableStore, IDisposable } from '../../../util/vs/base/common/lifecycle'; import { autorun } from '../../../util/vs/base/common/observableInternal'; import { URI } from '../../../util/vs/base/common/uri'; +import { generateUuid } from '../../../util/vs/base/common/uuid'; import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; import { ChatRequest } from '../../../vscodeTypes'; import { Intent, agentsToCommands } from '../../common/constants'; @@ -235,8 +236,14 @@ Learn more about [GitHub Copilot](https://docs.github.com/copilot/using-github-c // The user is starting an interaction with the chat this.interactionService.startInteraction(); + // Generate a shared telemetry message ID on the first turn only — subsequent turns have no + // categorization event to join and ChatTelemetryBuilder will generate its own ID. + const telemetryMessageId = context.history.length === 0 ? generateUuid() : undefined; + // Categorize the first prompt (fire-and-forget) - this.promptCategorizerService.categorizePrompt(request, context); + if (telemetryMessageId !== undefined) { + this.promptCategorizerService.categorizePrompt(request, context, telemetryMessageId); + } const defaultIntentId = typeof defaultIntentIdOrGetter === 'function' ? defaultIntentIdOrGetter(request) : @@ -248,7 +255,7 @@ Learn more about [GitHub Copilot](https://docs.github.com/copilot/using-github-c commandsForAgent[request.command] : defaultIntentId; - const handler = this.instantiationService.createInstance(ChatParticipantRequestHandler, context.history, request, stream, token, { agentName: name, agentId: id, intentId }, () => context.yieldRequested); + const handler = this.instantiationService.createInstance(ChatParticipantRequestHandler, context.history, request, stream, token, { agentName: name, agentId: id, intentId }, () => context.yieldRequested, telemetryMessageId); return await handler.getResult(); }; } diff --git a/extensions/copilot/src/extension/conversation/vscode-node/test/interactiveSessionProvider.telemetry.test.ts b/extensions/copilot/src/extension/conversation/vscode-node/test/interactiveSessionProvider.telemetry.test.ts index c9729b01d98..228f9d2f3cd 100644 --- a/extensions/copilot/src/extension/conversation/vscode-node/test/interactiveSessionProvider.telemetry.test.ts +++ b/extensions/copilot/src/extension/conversation/vscode-node/test/interactiveSessionProvider.telemetry.test.ts @@ -35,7 +35,8 @@ suite('Conversation telemetry tests - Integration tests', function () { stream, token, { agentName: '', agentId: '' }, - () => false); + () => false, + undefined); await session.getResult(); // and throw away the result }); assert.ok(allEvents(messages)); diff --git a/extensions/copilot/src/extension/inlineChat/vscode-node/inlineChatCommands.ts b/extensions/copilot/src/extension/inlineChat/vscode-node/inlineChatCommands.ts index 4ee7dde4ddd..db154d77a12 100644 --- a/extensions/copilot/src/extension/inlineChat/vscode-node/inlineChatCommands.ts +++ b/extensions/copilot/src/extension/inlineChat/vscode-node/inlineChatCommands.ts @@ -382,7 +382,7 @@ function fetchSuggestion(accessor: ServicesAccessor, thread: vscode.CommentThrea agentId: getChatParticipantIdFromName(editorAgentName), agentName: editorAgentName, intentId: request.command, - }, () => false); + }, () => false, undefined); const result = await requestHandler.getResult(); if (result.errorDetails) { throw new Error(result.errorDetails.message); diff --git a/extensions/copilot/src/extension/intents/node/test/agentSummarizeCommand.spec.ts b/extensions/copilot/src/extension/intents/node/test/agentSummarizeCommand.spec.ts index 806d4807be6..70f1519d4e6 100644 --- a/extensions/copilot/src/extension/intents/node/test/agentSummarizeCommand.spec.ts +++ b/extensions/copilot/src/extension/intents/node/test/agentSummarizeCommand.spec.ts @@ -79,6 +79,7 @@ describe('AgentIntent /summarize command', () => { undefined, true, request, + undefined, ); const result = await intent.handleRequest( diff --git a/extensions/copilot/src/extension/prompt/node/chatParticipantRequestHandler.ts b/extensions/copilot/src/extension/prompt/node/chatParticipantRequestHandler.ts index b8a295308c0..2eff528394d 100644 --- a/extensions/copilot/src/extension/prompt/node/chatParticipantRequestHandler.ts +++ b/extensions/copilot/src/extension/prompt/node/chatParticipantRequestHandler.ts @@ -73,6 +73,7 @@ export class ChatParticipantRequestHandler { private readonly token: CancellationToken, private readonly chatAgentArgs: IChatAgentArgs, private readonly yieldRequested: () => boolean, + telemetryMessageId: string | undefined, @IInstantiationService private readonly _instantiationService: IInstantiationService, @IEndpointProvider private readonly _endpointProvider: IEndpointProvider, @ICommandService private readonly _commandService: ICommandService, @@ -117,7 +118,8 @@ export class ChatParticipantRequestHandler { actualSessionId, this.documentContext, turns.length === 0, - this.request + this.request, + telemetryMessageId ); const latestTurn = Turn.fromRequest( diff --git a/extensions/copilot/src/extension/prompt/node/chatParticipantTelemetry.ts b/extensions/copilot/src/extension/prompt/node/chatParticipantTelemetry.ts index 45fabd754c2..98c2c97d18f 100644 --- a/extensions/copilot/src/extension/prompt/node/chatParticipantTelemetry.ts +++ b/extensions/copilot/src/extension/prompt/node/chatParticipantTelemetry.ts @@ -12,6 +12,7 @@ import { isAutoModel } from '../../../platform/endpoint/node/autoChatEndpoint'; import { ILanguageDiagnosticsService } from '../../../platform/languages/common/languageDiagnosticsService'; import { IChatEndpoint } from '../../../platform/networking/common/networking'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; +import { TelemetryData as PlatformTelemetryData } from '../../../platform/telemetry/common/telemetryData'; import { isNotebookCellOrNotebookChatInput } from '../../../util/common/notebooks'; import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; import { isBYOKModel } from '../../byok/node/openAIEndpoint'; @@ -29,7 +30,7 @@ import { IToolCall, IToolCallRound } from '../common/intents'; import { IDocumentContext } from './documentContext'; import { IIntent, TelemetryData } from './intents'; import { RepoInfoTelemetry } from './repoInfoTelemetry'; -import { ConversationalBaseTelemetryData, createTelemetryWithId, extendUserMessageTelemetryData, getCodeBlocks, sendModelMessageTelemetry, sendOffTopicMessageTelemetry, sendUserActionTelemetry, sendUserMessageTelemetry } from './telemetry'; +import { ConversationalBaseTelemetryData, ConversationalTelemetryData, createTelemetryWithId, extendUserMessageTelemetryData, getCodeBlocks, sendModelMessageTelemetry, sendOffTopicMessageTelemetry, sendUserActionTelemetry, sendUserMessageTelemetry } from './telemetry'; // #region: internal telemetry for responses @@ -206,7 +207,7 @@ type RequestInlineTelemetryMeasurements = RequestTelemetryMeasurements & { export class ChatTelemetryBuilder { - public readonly baseUserTelemetry: ConversationalBaseTelemetryData = createTelemetryWithId(); + public readonly baseUserTelemetry: ConversationalBaseTelemetryData; private readonly _repoInfoTelemetry: RepoInfoTelemetry; @@ -220,8 +221,12 @@ export class ChatTelemetryBuilder { private readonly _documentContext: IDocumentContext | undefined, private readonly _firstTurn: boolean, private readonly _request: vscode.ChatRequest, + telemetryMessageId: string | undefined, @IInstantiationService private readonly instantiationService: IInstantiationService, ) { + this.baseUserTelemetry = telemetryMessageId + ? new ConversationalTelemetryData(PlatformTelemetryData.createAndMarkAsIssued({ messageId: telemetryMessageId })) + : createTelemetryWithId(); // Repo info telemetry is held here as the begin event should be sent only by the first PanelChatTelemetry instance created for a user request. // and a new PanelChatTelemetry instance is created per step in the request. this._repoInfoTelemetry = this.instantiationService.createInstance(RepoInfoTelemetry, this.baseUserTelemetry.properties.messageId); diff --git a/extensions/copilot/src/extension/prompt/node/promptCategorizer.ts b/extensions/copilot/src/extension/prompt/node/promptCategorizer.ts index 9794447cba5..9676a8f8fb4 100644 --- a/extensions/copilot/src/extension/prompt/node/promptCategorizer.ts +++ b/extensions/copilot/src/extension/prompt/node/promptCategorizer.ts @@ -33,8 +33,10 @@ export interface IPromptCategorizerService { * This runs as a fire-and-forget operation and sends results to telemetry. * Only runs for panel location, first attempt, non-subagent requests. * Requires telemetry to be enabled and experiment flag to be set. + * + * @param telemetryMessageId The extension-generated request ID (shared with panel.request telemetry) */ - categorizePrompt(request: vscode.ChatRequest, context: vscode.ChatContext): void; + categorizePrompt(request: vscode.ChatRequest, context: vscode.ChatContext, telemetryMessageId: string): void; } // Categorization outcome values for telemetry @@ -128,7 +130,7 @@ export class PromptCategorizerService implements IPromptCategorizerService { @ICopilotTokenStore private readonly copilotTokenStore: ICopilotTokenStore, ) { } - categorizePrompt(request: vscode.ChatRequest, context: vscode.ChatContext): void { + categorizePrompt(request: vscode.ChatRequest, context: vscode.ChatContext, telemetryMessageId: string): void { // Always enable for internal users; external users require experiment flag const isInternal = this.copilotTokenStore.copilotToken?.isInternal === true; if (!isInternal && !this.experimentationService.getTreatmentVariable(EXP_FLAG_PROMPT_CATEGORIZATION)) { @@ -152,12 +154,12 @@ export class PromptCategorizerService implements IPromptCategorizerService { } // Fire and forget - don't await - this._categorizePromptAsync(request, context).catch(err => { + this._categorizePromptAsync(request, context, telemetryMessageId).catch(err => { this.logService.error(`[PromptCategorizer] Error categorizing prompt: ${err instanceof Error ? err.message : String(err)}`); }); } - private async _categorizePromptAsync(request: vscode.ChatRequest, _context: vscode.ChatContext): Promise { + private async _categorizePromptAsync(request: vscode.ChatRequest, _context: vscode.ChatContext, telemetryMessageId: string): Promise { const startTime = Date.now(); let outcome: typeof CATEGORIZATION_OUTCOMES[keyof typeof CATEGORIZATION_OUTCOMES] = CATEGORIZATION_OUTCOMES.ERROR; let errorDetail = ''; @@ -284,7 +286,8 @@ export class PromptCategorizerService implements IPromptCategorizerService { "owner": "digitarald", "comment": "Classifies agent requests for understanding user intent and response quality", "sessionId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The chat session identifier" }, - "requestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The unique request identifier within the session" }, + "requestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The extension-generated request identifier, matches panel.request requestId" }, + "vscodeRequestId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The VS Code chat request id, for joining with VS Code telemetry events" }, "modeName": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The chat mode name being used" }, "currentLanguage": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The language ID of the active editor" }, "outcome": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Classification outcome: empty string for success, partialClassification for recovered core fields, or error kind (timeout, requestFailed, noToolCall, parseError, invalidClassification, error)" }, @@ -304,7 +307,8 @@ export class PromptCategorizerService implements IPromptCategorizerService { 'promptCategorization', { sessionId: request.sessionId ?? '', - requestId: request.id ?? '', + requestId: telemetryMessageId, + vscodeRequestId: request.id ?? '', modeName: request.modeInstructions2?.name, currentLanguage: currentLanguage ?? '', outcome, @@ -334,7 +338,8 @@ export class PromptCategorizerService implements IPromptCategorizerService { 'promptCategorization', { sessionId: request.sessionId ?? '', - requestId: request.id ?? '', + requestId: telemetryMessageId, + vscodeRequestId: request.id ?? '', modeName: request.modeInstructions2?.name, currentLanguage: currentLanguage ?? '', outcome, diff --git a/extensions/copilot/src/extension/prompt/node/test/defaultIntentRequestHandler.spec.ts b/extensions/copilot/src/extension/prompt/node/test/defaultIntentRequestHandler.spec.ts index c5b94aaf34d..cc03d2f3b5d 100644 --- a/extensions/copilot/src/extension/prompt/node/test/defaultIntentRequestHandler.spec.ts +++ b/extensions/copilot/src/extension/prompt/node/test/defaultIntentRequestHandler.spec.ts @@ -161,7 +161,7 @@ suite('defaultIntentRequestHandler', () => { CancellationToken.None, undefined, ChatLocation.Panel, - instaService.createInstance(ChatTelemetryBuilder, Date.now(), sessionId, undefined, turns.length > 1, request), + instaService.createInstance(ChatTelemetryBuilder, Date.now(), sessionId, undefined, turns.length > 1, request, undefined), { maxToolCallIterations }, undefined, ); diff --git a/extensions/copilot/src/extension/test/vscode-node/sanity.sanity-test.ts b/extensions/copilot/src/extension/test/vscode-node/sanity.sanity-test.ts index 04fbaa7f8d6..24f443c120e 100644 --- a/extensions/copilot/src/extension/test/vscode-node/sanity.sanity-test.ts +++ b/extensions/copilot/src/extension/test/vscode-node/sanity.sanity-test.ts @@ -77,7 +77,7 @@ suite('Copilot Chat Sanity Test', function () { try { conversationFeature.activated = true; let stream = new SpyChatResponseStream(); - let interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('Write me a for loop in javascript'), stream, fakeToken, { agentName: '', agentId: '', intentId: '' }, () => false); + let interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('Write me a for loop in javascript'), stream, fakeToken, { agentName: '', agentId: '', intentId: '' }, () => false, undefined); await interactiveSession.getResult(); @@ -85,7 +85,7 @@ suite('Copilot Chat Sanity Test', function () { const oldText = stream.currentProgress; stream = new SpyChatResponseStream(); - interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('Can you make it in typescript instead'), stream, fakeToken, { agentName: '', agentId: '', intentId: '' }, () => false); + interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('Can you make it in typescript instead'), stream, fakeToken, { agentName: '', agentId: '', intentId: '' }, () => false, undefined); const result2 = await interactiveSession.getResult(); assert.ok(stream.currentProgress, 'Expected progress after second request'); @@ -117,7 +117,7 @@ suite('Copilot Chat Sanity Test', function () { let stream = new SpyChatResponseStream(); const testRequest = new TestChatRequest(`You must use the get_errors tool to check the window for errors. It may fail, that's ok, just testing, don't retry.`); testRequest.tools.set(ContributedToolName.GetErrors, true); - let interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], testRequest, stream, fakeToken, { agentName: '', agentId: '', intentId: Intent.Agent }, () => false); + let interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], testRequest, stream, fakeToken, { agentName: '', agentId: '', intentId: Intent.Agent }, () => false, undefined); const onWillInvokeTool = Event.toPromise(toolsService.onWillInvokeTool); const getResultPromise = interactiveSession.getResult(); @@ -128,7 +128,7 @@ suite('Copilot Chat Sanity Test', function () { const oldText = stream.currentProgress; stream = new SpyChatResponseStream(); - interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('And what is 1+1'), stream, fakeToken, { agentName: '', agentId: '', intentId: Intent.Agent }, () => false); + interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('And what is 1+1'), stream, fakeToken, { agentName: '', agentId: '', intentId: Intent.Agent }, () => false, undefined); const result2 = await interactiveSession.getResult(); assert.ok(stream.currentProgress, 'Expected progress after second request'); @@ -152,7 +152,7 @@ suite('Copilot Chat Sanity Test', function () { try { conversationFeature.activated = true; const progressReport = new SpyChatResponseStream(); - const interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('What is a fibonacci sequence?'), progressReport, fakeToken, { agentName: '', agentId: '', intentId: 'explain' }, () => false); + const interactiveSession = instaService.createInstance(ChatParticipantRequestHandler, [], new TestChatRequest('What is a fibonacci sequence?'), progressReport, fakeToken, { agentName: '', agentId: '', intentId: 'explain' }, () => false, undefined); // Ask a `/explain` question await interactiveSession.getResult(); diff --git a/extensions/copilot/test/e2e/scenarioTest.ts b/extensions/copilot/test/e2e/scenarioTest.ts index 7b9f110f19c..a0c17a91ffd 100644 --- a/extensions/copilot/test/e2e/scenarioTest.ts +++ b/extensions/copilot/test/e2e/scenarioTest.ts @@ -93,6 +93,7 @@ export function generateScenarioTestRunner(scenario: Scenario, evaluator: Scenar parsedQuery.participantName, }, () => false, + undefined, ); const result = await interactiveSession.getResult(); assert.ok(!result.errorDetails, result.errorDetails?.message); diff --git a/extensions/copilot/test/simulation/inlineChatSimulator.ts b/extensions/copilot/test/simulation/inlineChatSimulator.ts index 3b6a9f3eb32..e2ed4957732 100644 --- a/extensions/copilot/test/simulation/inlineChatSimulator.ts +++ b/extensions/copilot/test/simulation/inlineChatSimulator.ts @@ -490,7 +490,7 @@ export async function simulateEditingScenario( intentId: request.command }; - const requestHandler = instaService.createInstance(ChatParticipantRequestHandler, history, request, stream, CancellationToken.None, agentArgs, () => false); + const requestHandler = instaService.createInstance(ChatParticipantRequestHandler, history, request, stream, CancellationToken.None, agentArgs, () => false, undefined); const result = await requestHandler.getResult(); history.push(new ChatRequestTurn(request.prompt, request.command, [...request.references], '', [])); history.push(new ChatResponseTurn([new ChatResponseMarkdownPart(markdownChunks.join(''))], result, ''));