diff --git a/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts b/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts index 622739872c8..2dd7e74abfa 100644 --- a/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts +++ b/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import { RequestMetadata, RequestType } from '@vscode/copilot-api'; -import * as l10n from '@vscode/l10n'; import { OpenAI, Raw } from '@vscode/prompt-tsx'; import type { CancellationToken } from 'vscode'; import { ITokenizer, TokenizerType } from '../../../util/common/tokenizer'; @@ -30,11 +29,12 @@ import { ITelemetryService, TelemetryProperties } from '../../telemetry/common/t import { TelemetryData } from '../../telemetry/common/telemetryData'; import { ITokenizerProvider } from '../../tokenizer/node/tokenizer'; import { ICAPIClientService } from '../common/capiClient'; -import { isGeminiFamily, modelSupportsContextEditing, modelSupportsToolSearch } from '../common/chatModelCapabilities'; +import { isAnthropicFamily, isGeminiFamily, modelSupportsContextEditing, modelSupportsToolSearch } from '../common/chatModelCapabilities'; import { IDomainService } from '../common/domainService'; import { CustomModel, IChatModelInformation, ModelSupportedEndpoint } from '../common/endpointProvider'; import { createMessagesRequestBody, processResponseFromMessagesEndpoint } from './messagesApi'; import { createResponsesRequestBody, getResponsesApiCompactionThreshold, processResponseFromChatEndpoint } from './responsesApi'; +import { filterHistoryImages } from './imageLimits'; /** * The default processor for the stream format from CAPI @@ -288,13 +288,10 @@ export class ChatEndpoint implements IChatEndpoint { } createRequestBody(options: ICreateEndpointBodyOptions): IEndpointBody { - // Validate image count if endpoint has max_prompt_images limit (Gemini only for now) - if (isGeminiFamily(this) && this.maxPromptImages !== undefined) { - const imageCount = this.countImages(options.messages, this.maxPromptImages); - if (imageCount > this.maxPromptImages) { - const errorMsg = l10n.t('Too many images in request: {0} images provided, but the model supports a maximum of {1} images.', imageCount, this.maxPromptImages); - throw new Error(errorMsg); - } + // Determine per-model image limit for APIs with known restrictions + const imageLimit = this.getImageLimit(); + if (imageLimit !== undefined) { + options = { ...options, messages: this.validateAndFilterImages(options.messages, imageLimit) }; } if (this.useResponsesApi) { @@ -309,22 +306,27 @@ export class ChatEndpoint implements IChatEndpoint { } } - private countImages(messages: Raw.ChatMessage[], maxAllowed?: number): number { - let imageCount = 0; - for (const message of messages) { - if (Array.isArray(message.content)) { - for (const part of message.content) { - if (part.type === Raw.ChatCompletionContentPartKind.Image) { - imageCount++; - // Early exit if we've already exceeded the limit - if (maxAllowed !== undefined && imageCount > maxAllowed) { - return imageCount; - } - } - } - } + /** + * Returns the model-specific image limit, or `undefined` if no limit applies. + * Anthropic Messages API allows up to 20 images per request; Gemini allows up to 10. + * These are hardcoded based on API documentation rather than model metadata to + * avoid being clamped by unreliable server-provided values. + */ + private getImageLimit(): number | undefined { + if (this.useMessagesApi && isAnthropicFamily(this)) { + return 20; } - return imageCount; + if (isGeminiFamily(this)) { + return 10; + } + return undefined; + } + + /** + * Thin wrapper around {@link filterHistoryImages} retained for test ergonomics. + */ + private validateAndFilterImages(messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] { + return filterHistoryImages(messages, maxImages); } protected getCompletionsCallback(): RawMessageConversionCallback | undefined { diff --git a/extensions/copilot/src/platform/endpoint/node/imageLimits.ts b/extensions/copilot/src/platform/endpoint/node/imageLimits.ts new file mode 100644 index 00000000000..77dcf6111d4 --- /dev/null +++ b/extensions/copilot/src/platform/endpoint/node/imageLimits.ts @@ -0,0 +1,128 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as l10n from '@vscode/l10n'; +import { Raw } from '@vscode/prompt-tsx'; + +/** + * Model-facing placeholder substituted for dropped history images. + * Intentionally not localized — this text is sent to the model, not the user. + */ +const IMAGE_PLACEHOLDER_TEXT = '[Image omitted from conversation history due to model limit.]'; + +/** + * Silently drops the oldest images from history when the total number of images + * in the conversation exceeds `maxImages`. Images belonging to the current turn + * (the last user message and anything after it, e.g. recent tool results) are + * always preserved. + * + * If the current turn alone exceeds the limit, throws a localized error rather + * than sending a request we know will be rejected with an opaque server error. + * + * @returns A (possibly filtered) copy of messages. The original array is never mutated. + */ +export function filterHistoryImages(messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] { + // Anchor the current turn at the last user message; anything at or after this + // index is treated as "current turn" and its images are never filtered. + let lastUserIdx = -1; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === Raw.ChatRole.User) { + lastUserIdx = i; + break; + } + } + + // Corner case: no user message at all (e.g. system-only history). Treat the + // last message as the current turn so we still filter earlier images. + if (lastUserIdx === -1 && messages.length > 0) { + lastUserIdx = messages.length - 1; + } + + // Count images in the current turn (the last user message and anything after it). + let currentTurnImages = 0; + for (let i = Math.max(lastUserIdx, 0); i < messages.length; i++) { + const content = messages[i].content; + if (!Array.isArray(content)) { + continue; + } + for (const part of content) { + if (part.type === Raw.ChatCompletionContentPartKind.Image) { + currentTurnImages++; + } + } + } + + // Count total images across all messages + let totalImages = 0; + for (const message of messages) { + if (Array.isArray(message.content)) { + for (const part of message.content) { + if (part.type === Raw.ChatCompletionContentPartKind.Image) { + totalImages++; + } + } + } + } + + // No filtering needed if total is within the limit + if (totalImages <= maxImages) { + return messages; + } + + // Fail fast with a clear, localized error when the current turn alone exceeds + // the limit — otherwise we'd send a request the server will reject with an + // opaque error. Silent history filtering is only safe when dropping history + // images can bring the total down to the limit. + if (currentTurnImages > maxImages) { + throw new Error(l10n.t('Too many images in request: {0} images provided, but the model supports a maximum of {1} images.', currentTurnImages, maxImages)); + } + + // Walk backward through history (before the current turn), keeping the + // most recent images and replacing the oldest with placeholders. + let historyBudget = maxImages - currentTurnImages; + + // Collect keep/drop decisions by walking backward through history + const historyImageDecisions = new Map(); // "msgIdx:partIdx" -> keep + for (let i = lastUserIdx - 1; i >= 0; i--) { + if (!Array.isArray(messages[i].content)) { + continue; + } + for (let j = messages[i].content.length - 1; j >= 0; j--) { + if (messages[i].content[j].type === Raw.ChatCompletionContentPartKind.Image) { + const key = `${i}:${j}`; + if (historyBudget > 0) { + historyImageDecisions.set(key, true); + historyBudget--; + } else { + historyImageDecisions.set(key, false); + } + } + } + } + + // Build filtered messages, replacing dropped images with text placeholders + return messages.map((message, msgIdx) => { + if (msgIdx >= lastUserIdx) { + return message; + } + if (!Array.isArray(message.content)) { + return message; + } + if (!message.content.some(p => p.type === Raw.ChatCompletionContentPartKind.Image)) { + return message; + } + return { + ...message, + content: message.content.map((part, partIdx) => { + if (part.type !== Raw.ChatCompletionContentPartKind.Image) { + return part; + } + if (historyImageDecisions.get(`${msgIdx}:${partIdx}`)) { + return part; + } + return { type: Raw.ChatCompletionContentPartKind.Text, text: IMAGE_PLACEHOLDER_TEXT }; + }) + }; + }); +} diff --git a/extensions/copilot/src/platform/endpoint/node/test/copilotChatEndpoint.spec.ts b/extensions/copilot/src/platform/endpoint/node/test/copilotChatEndpoint.spec.ts index ed1b914eac7..f9b3c73996c 100644 --- a/extensions/copilot/src/platform/endpoint/node/test/copilotChatEndpoint.spec.ts +++ b/extensions/copilot/src/platform/endpoint/node/test/copilotChatEndpoint.spec.ts @@ -13,7 +13,7 @@ import { DefaultsOnlyConfigurationService } from '../../../configuration/common/ import { InMemoryConfigurationService } from '../../../configuration/test/common/inMemoryConfigurationService'; import { ICAPIClientService } from '../../../endpoint/common/capiClient'; import { IDomainService } from '../../../endpoint/common/domainService'; -import { IChatModelInformation } from '../../../endpoint/common/endpointProvider'; +import { IChatModelInformation, ModelSupportedEndpoint } from '../../../endpoint/common/endpointProvider'; import { IEnvService } from '../../../env/common/envService'; import { ILogService } from '../../../log/common/logService'; import { IFetcherService } from '../../../networking/common/fetcherService'; @@ -246,15 +246,23 @@ describe('ChatEndpoint - Image Count Validation', () => { mockServices = createMockServices(); }); - const createImageMessage = (): Raw.ChatMessage => ({ + const createImageMessage = (imageCount: number = 1): Raw.ChatMessage => ({ role: Raw.ChatRole.User, content: [ { type: Raw.ChatCompletionContentPartKind.Text, text: 'What is in this image?' }, - { type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'data:image/png;base64,test' } } + ...Array.from({ length: imageCount }, () => ({ + type: Raw.ChatCompletionContentPartKind.Image as const, + imageUrl: { url: 'data:image/png;base64,test' } + })) ] }); - const createGeminiModelMetadata = (maxPromptImages: number): IChatModelInformation => { + const createAssistantMessage = (): Raw.ChatMessage => ({ + role: Raw.ChatRole.Assistant, + content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'I see an image.' }] + }); + + const createGeminiModelMetadata = (maxPromptImages?: number): IChatModelInformation => { const baseMetadata = createNonAnthropicModelMetadata('gemini-3'); return { ...baseMetadata, @@ -266,19 +274,30 @@ describe('ChatEndpoint - Image Count Validation', () => { }, limits: { ...baseMetadata.capabilities.limits, - vision: { - max_prompt_images: maxPromptImages - } + ...(maxPromptImages !== undefined ? { vision: { max_prompt_images: maxPromptImages } } : {}) } } }; }; - it('should throw error when image count exceeds maxPromptImages', () => { - const modelMetadata = createGeminiModelMetadata(2); + const createAnthropicMessagesModelMetadata = (): IChatModelInformation => { + const baseMetadata = createNonAnthropicModelMetadata('claude-sonnet-4'); + return { + ...baseMetadata, + supported_endpoints: [ModelSupportedEndpoint.Messages], + capabilities: { + ...baseMetadata.capabilities, + supports: { + ...baseMetadata.capabilities.supports, + vision: true + } + } + }; + }; - const endpoint = new ChatEndpoint( - modelMetadata, + const createEndpoint = (metadata: IChatModelInformation) => + new ChatEndpoint( + metadata, mockServices.domainService, mockServices.chatMLFetcher, mockServices.tokenizerProvider, @@ -289,30 +308,147 @@ describe('ChatEndpoint - Image Count Validation', () => { mockServices.logService ); - // Create 3 messages each with 1 image (total 3 images) - const options = createTestOptions([createImageMessage(), createImageMessage(), createImageMessage()]); + const countImages = (messages: Raw.ChatMessage[]): number => { + let count = 0; + for (const msg of messages) { + if (Array.isArray(msg.content)) { + for (const part of msg.content) { + if (part.type === Raw.ChatCompletionContentPartKind.Image) { + count++; + } + } + } + } + return count; + }; - expect(() => endpoint.createRequestBody(options)).toThrow(/Too many images in request/); + // Exercises the private `validateAndFilterImages` method directly so we can + // assert on the filtered messages without being blocked by downstream mocks. + const filterImages = (endpoint: ChatEndpoint, messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] => { + return (endpoint as unknown as { validateAndFilterImages(m: Raw.ChatMessage[], n: number): Raw.ChatMessage[] }) + .validateAndFilterImages(messages, maxImages); + }; + + describe('Gemini image limits', () => { + it('should allow requests within image limit', () => { + const endpoint = createEndpoint(createGeminiModelMetadata(5)); + const messages = [createImageMessage(), createImageMessage()]; + const options = createTestOptions(messages); + expect(() => endpoint.createRequestBody(options)).not.toThrow(); + // Input is within limit — messages should be returned untouched. + expect(filterImages(endpoint, messages, 5)).toBe(messages); + }); + + it('should silently filter history images when total exceeds limit', () => { + const endpoint = createEndpoint(createGeminiModelMetadata(3)); + // 2 history user messages with 1 image each + current user message with 2 images = 4 total > 3 limit + const messages = [ + createImageMessage(), + createAssistantMessage(), + createImageMessage(), + createAssistantMessage(), + createImageMessage(2), + ]; + expect(() => endpoint.createRequestBody(createTestOptions(messages))).not.toThrow(); + const filtered = filterImages(endpoint, messages, 3); + // Total image parts in the filtered output must not exceed the limit. + expect(countImages(filtered)).toBeLessThanOrEqual(3); + // Current user message (last) must retain all 2 of its images. + expect(countImages([filtered[filtered.length - 1]])).toBe(2); + // Original messages must not be mutated. + expect(countImages(messages)).toBe(4); + }); }); - it('should allow requests within image limit', () => { - const modelMetadata = createGeminiModelMetadata(5); + describe('Anthropic Messages API image limits', () => { + it('should allow requests within image limit', () => { + const endpoint = createEndpoint(createAnthropicMessagesModelMetadata()); + const messages = [createImageMessage(5)]; + // Within limit — filter must not alter the messages. + expect(filterImages(endpoint, messages, 20)).toBe(messages); + }); - const endpoint = new ChatEndpoint( - modelMetadata, - mockServices.domainService, - mockServices.chatMLFetcher, - mockServices.tokenizerProvider, - mockServices.instantiationService, - mockServices.configurationService, - mockServices.expService, - mockServices.chatWebSocketService, - mockServices.logService - ); + it('should silently filter history images when total exceeds limit', () => { + const endpoint = createEndpoint(createAnthropicMessagesModelMetadata()); + // Build history with 18 images + current message with 5 images = 23 total > 20 limit + const messages: Raw.ChatMessage[] = []; + for (let i = 0; i < 18; i++) { + messages.push(createImageMessage()); + messages.push(createAssistantMessage()); + } + messages.push(createImageMessage(5)); + const filtered = filterImages(endpoint, messages, 20); + expect(countImages(filtered)).toBeLessThanOrEqual(20); + // Current user message must retain all 5 of its images. + expect(countImages([filtered[filtered.length - 1]])).toBe(5); + // Original messages must not be mutated. + expect(countImages(messages)).toBe(23); + }); + }); - // Create 2 messages each with 1 image (total 2 images) - const options = createTestOptions([createImageMessage(), createImageMessage()]); + describe('non-limited models', () => { + it('should not apply image limits to non-Anthropic non-Gemini models', () => { + const metadata = createNonAnthropicModelMetadata('gpt-4o'); + const endpoint = createEndpoint(metadata); + // 25 images should not throw for a non-limited model + const options = createTestOptions([createImageMessage(25)]); + expect(() => endpoint.createRequestBody(options)).not.toThrow(); + }); + }); - expect(() => endpoint.createRequestBody(options)).not.toThrow(); + describe('edge cases', () => { + it('should filter tool-result images in history the same as user images', () => { + const endpoint = createEndpoint(createGeminiModelMetadata(2)); + const toolResultImage: Raw.ChatMessage = { + role: Raw.ChatRole.Tool, + toolCallId: 'tool-1', + content: [ + { type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'https://example.com/tool.png' } } + ] + }; + // 2 tool-result images in history + 1 current user image = 3 total > 2 limit + const messages: Raw.ChatMessage[] = [ + toolResultImage, + createAssistantMessage(), + toolResultImage, + createAssistantMessage(), + createImageMessage(1), + ]; + const filtered = filterImages(endpoint, messages, 2); + expect(countImages(filtered)).toBeLessThanOrEqual(2); + // Original messages must not be mutated. + expect(countImages(messages)).toBe(3); + }); + + it('should ignore an overly-restrictive server-provided maxPromptImages and use the hardcoded Gemini limit of 10', () => { + // Server reports max_prompt_images: 1 but the true Gemini limit is 10. + // 2 images in the current turn must not throw. + const endpoint = createEndpoint(createGeminiModelMetadata(1)); + const options = createTestOptions([createImageMessage(2)]); + expect(() => endpoint.createRequestBody(options)).not.toThrow(); + }); + + it('should throw using the hardcoded Gemini limit of 10 when the current turn exceeds it', () => { + const endpoint = createEndpoint(createGeminiModelMetadata(1)); + const options = createTestOptions([createImageMessage(11)]); + expect(() => endpoint.createRequestBody(options)).toThrow(/maximum of 10 images/); + }); + + it('should throw using the hardcoded Anthropic Messages limit of 20 when the current turn exceeds it', () => { + const endpoint = createEndpoint(createAnthropicMessagesModelMetadata()); + const options = createTestOptions([createImageMessage(21)]); + expect(() => endpoint.createRequestBody(options)).toThrow(/maximum of 20 images/); + }); + + it('should throw a clear error when the current turn alone exceeds the limit', () => { + const endpoint = createEndpoint(createGeminiModelMetadata(2)); + // Current user message has 5 images, limit is 2. History has 1 image. + const messages = [ + createImageMessage(), + createAssistantMessage(), + createImageMessage(5), + ]; + expect(() => filterImages(endpoint, messages, 2)).toThrow(/Too many images/); + }); }); }); \ No newline at end of file diff --git a/extensions/copilot/src/platform/endpoint/node/test/imageLimits.spec.ts b/extensions/copilot/src/platform/endpoint/node/test/imageLimits.spec.ts new file mode 100644 index 00000000000..964744d4e4a --- /dev/null +++ b/extensions/copilot/src/platform/endpoint/node/test/imageLimits.spec.ts @@ -0,0 +1,146 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Raw } from '@vscode/prompt-tsx'; +import { describe, expect, it } from 'vitest'; +import { filterHistoryImages } from '../imageLimits'; + +const createUserImageMessage = (imageCount: number = 1): Raw.ChatMessage => ({ + role: Raw.ChatRole.User, + content: [ + { type: Raw.ChatCompletionContentPartKind.Text, text: 'What is in this image?' }, + ...Array.from({ length: imageCount }, () => ({ + type: Raw.ChatCompletionContentPartKind.Image as const, + imageUrl: { url: 'data:image/png;base64,test' } + })) + ] +}); + +const createAssistantMessage = (): Raw.ChatMessage => ({ + role: Raw.ChatRole.Assistant, + content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'I see an image.' }] +}); + +const createToolImageMessage = (): Raw.ChatMessage => ({ + role: Raw.ChatRole.Tool, + toolCallId: 'tool-1', + content: [ + { type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'https://example.com/tool.png' } } + ] +}); + +const countImages = (messages: Raw.ChatMessage[]): number => { + let count = 0; + for (const msg of messages) { + if (Array.isArray(msg.content)) { + for (const part of msg.content) { + if (part.type === Raw.ChatCompletionContentPartKind.Image) { + count++; + } + } + } + } + return count; +}; + +describe('filterHistoryImages', () => { + it('returns the original array by reference when within the limit', () => { + const messages = [createUserImageMessage(), createUserImageMessage()]; + expect(filterHistoryImages(messages, 5)).toBe(messages); + }); + + it('silently filters oldest history images when total exceeds the limit', () => { + // 2 history user messages with 1 image each + current user message with 2 images = 4 total > 3 limit + const messages = [ + createUserImageMessage(), + createAssistantMessage(), + createUserImageMessage(), + createAssistantMessage(), + createUserImageMessage(2), + ]; + const filtered = filterHistoryImages(messages, 3); + expect(countImages(filtered)).toBeLessThanOrEqual(3); + // Current user message (last) must retain all 2 of its images. + expect(countImages([filtered[filtered.length - 1]])).toBe(2); + // Original messages must not be mutated. + expect(countImages(messages)).toBe(4); + }); + + it('replaces dropped images with a text placeholder', () => { + const messages = [ + createUserImageMessage(), + createAssistantMessage(), + createUserImageMessage(1), + ]; + const filtered = filterHistoryImages(messages, 1); + const droppedMessage = filtered[0]; + if (!Array.isArray(droppedMessage.content)) { + throw new Error('expected array content'); + } + const placeholder = droppedMessage.content.find(p => p.type === Raw.ChatCompletionContentPartKind.Text && p.text.includes('Image omitted')); + expect(placeholder).toBeDefined(); + }); + + it('filters tool-result images in history the same as user images', () => { + // 2 tool-result images in history + 1 current user image = 3 total > 2 limit + const messages: Raw.ChatMessage[] = [ + createToolImageMessage(), + createAssistantMessage(), + createToolImageMessage(), + createAssistantMessage(), + createUserImageMessage(1), + ]; + const filtered = filterHistoryImages(messages, 2); + expect(countImages(filtered)).toBeLessThanOrEqual(2); + // Original messages must not be mutated. + expect(countImages(messages)).toBe(3); + }); + + it('throws a clear error including the per-model limit when the current turn alone exceeds it', () => { + // Current user message has 11 images. The error must mention the exact + // model-scoped limit (10 for Gemini, 20 for Anthropic Messages API). + const messages = [createUserImageMessage(11)]; + expect(() => filterHistoryImages(messages, 10)).toThrow(/11 images provided.*maximum of 10 images/); + + const many = [createUserImageMessage(25)]; + expect(() => filterHistoryImages(many, 20)).toThrow(/25 images provided.*maximum of 20 images/); + }); + + it('handles conversations with no user message by treating the last message as current', () => { + const messages: Raw.ChatMessage[] = [ + createToolImageMessage(), + createToolImageMessage(), + createToolImageMessage(), + ]; + const filtered = filterHistoryImages(messages, 1); + // Last message preserved; earlier tool-result images filtered. + expect(countImages(filtered)).toBeLessThanOrEqual(1); + expect(countImages([filtered[filtered.length - 1]])).toBe(1); + }); + + it('does not mutate the original messages array or its contents', () => { + // History has 2 images, current turn has 1, limit is 1 → history filters silently. + const messages = [ + createUserImageMessage(), + createAssistantMessage(), + createUserImageMessage(), + createAssistantMessage(), + createUserImageMessage(1), + ]; + const snapshot = JSON.stringify(messages); + filterHistoryImages(messages, 1); + expect(JSON.stringify(messages)).toBe(snapshot); + }); + + it('passes through messages with non-array content', () => { + const messages: Raw.ChatMessage[] = [ + { role: Raw.ChatRole.System, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'system' }] }, + createUserImageMessage(2), + ]; + // Total = 2 images, within limit of 2 → returned unchanged. + const filtered = filterHistoryImages(messages, 2); + expect(filtered).toBe(messages); + }); +});