better handling of too many images in chat (#310437)

* better handling of too many images in chat

* address review: handle no-user-message fallback, assert filter output, cover tool-role images

* factor filterHistoryImages into a pure helper with direct tests

* restore validateAndFilterImages delegate so existing tests are unchanged

* address comments, throw early
This commit is contained in:
Justin Chen
2026-04-16 01:32:09 -07:00
committed by GitHub
parent 602484ad5f
commit c47cdd79dd
4 changed files with 466 additions and 54 deletions

View File

@@ -3,7 +3,6 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { RequestMetadata, RequestType } from '@vscode/copilot-api';
import * as l10n from '@vscode/l10n';
import { OpenAI, Raw } from '@vscode/prompt-tsx';
import type { CancellationToken } from 'vscode';
import { ITokenizer, TokenizerType } from '../../../util/common/tokenizer';
@@ -30,11 +29,12 @@ import { ITelemetryService, TelemetryProperties } from '../../telemetry/common/t
import { TelemetryData } from '../../telemetry/common/telemetryData';
import { ITokenizerProvider } from '../../tokenizer/node/tokenizer';
import { ICAPIClientService } from '../common/capiClient';
import { isGeminiFamily, modelSupportsContextEditing, modelSupportsToolSearch } from '../common/chatModelCapabilities';
import { isAnthropicFamily, isGeminiFamily, modelSupportsContextEditing, modelSupportsToolSearch } from '../common/chatModelCapabilities';
import { IDomainService } from '../common/domainService';
import { CustomModel, IChatModelInformation, ModelSupportedEndpoint } from '../common/endpointProvider';
import { createMessagesRequestBody, processResponseFromMessagesEndpoint } from './messagesApi';
import { createResponsesRequestBody, getResponsesApiCompactionThreshold, processResponseFromChatEndpoint } from './responsesApi';
import { filterHistoryImages } from './imageLimits';
/**
* The default processor for the stream format from CAPI
@@ -288,13 +288,10 @@ export class ChatEndpoint implements IChatEndpoint {
}
createRequestBody(options: ICreateEndpointBodyOptions): IEndpointBody {
// Validate image count if endpoint has max_prompt_images limit (Gemini only for now)
if (isGeminiFamily(this) && this.maxPromptImages !== undefined) {
const imageCount = this.countImages(options.messages, this.maxPromptImages);
if (imageCount > this.maxPromptImages) {
const errorMsg = l10n.t('Too many images in request: {0} images provided, but the model supports a maximum of {1} images.', imageCount, this.maxPromptImages);
throw new Error(errorMsg);
}
// Determine per-model image limit for APIs with known restrictions
const imageLimit = this.getImageLimit();
if (imageLimit !== undefined) {
options = { ...options, messages: this.validateAndFilterImages(options.messages, imageLimit) };
}
if (this.useResponsesApi) {
@@ -309,22 +306,27 @@ export class ChatEndpoint implements IChatEndpoint {
}
}
private countImages(messages: Raw.ChatMessage[], maxAllowed?: number): number {
let imageCount = 0;
for (const message of messages) {
if (Array.isArray(message.content)) {
for (const part of message.content) {
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
imageCount++;
// Early exit if we've already exceeded the limit
if (maxAllowed !== undefined && imageCount > maxAllowed) {
return imageCount;
}
}
}
}
/**
* Returns the model-specific image limit, or `undefined` if no limit applies.
* Anthropic Messages API allows up to 20 images per request; Gemini allows up to 10.
* These are hardcoded based on API documentation rather than model metadata to
* avoid being clamped by unreliable server-provided values.
*/
private getImageLimit(): number | undefined {
if (this.useMessagesApi && isAnthropicFamily(this)) {
return 20;
}
return imageCount;
if (isGeminiFamily(this)) {
return 10;
}
return undefined;
}
/**
* Thin wrapper around {@link filterHistoryImages} retained for test ergonomics.
*/
private validateAndFilterImages(messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] {
return filterHistoryImages(messages, maxImages);
}
protected getCompletionsCallback(): RawMessageConversionCallback | undefined {

View File

@@ -0,0 +1,128 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as l10n from '@vscode/l10n';
import { Raw } from '@vscode/prompt-tsx';
/**
* Model-facing placeholder substituted for dropped history images.
* Intentionally not localized — this text is sent to the model, not the user.
*/
const IMAGE_PLACEHOLDER_TEXT = '[Image omitted from conversation history due to model limit.]';
/**
* Silently drops the oldest images from history when the total number of images
* in the conversation exceeds `maxImages`. Images belonging to the current turn
* (the last user message and anything after it, e.g. recent tool results) are
* always preserved.
*
* If the current turn alone exceeds the limit, throws a localized error rather
* than sending a request we know will be rejected with an opaque server error.
*
* @returns A (possibly filtered) copy of messages. The original array is never mutated.
*/
export function filterHistoryImages(messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] {
// Anchor the current turn at the last user message; anything at or after this
// index is treated as "current turn" and its images are never filtered.
let lastUserIdx = -1;
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i].role === Raw.ChatRole.User) {
lastUserIdx = i;
break;
}
}
// Corner case: no user message at all (e.g. system-only history). Treat the
// last message as the current turn so we still filter earlier images.
if (lastUserIdx === -1 && messages.length > 0) {
lastUserIdx = messages.length - 1;
}
// Count images in the current turn (the last user message and anything after it).
let currentTurnImages = 0;
for (let i = Math.max(lastUserIdx, 0); i < messages.length; i++) {
const content = messages[i].content;
if (!Array.isArray(content)) {
continue;
}
for (const part of content) {
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
currentTurnImages++;
}
}
}
// Count total images across all messages
let totalImages = 0;
for (const message of messages) {
if (Array.isArray(message.content)) {
for (const part of message.content) {
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
totalImages++;
}
}
}
}
// No filtering needed if total is within the limit
if (totalImages <= maxImages) {
return messages;
}
// Fail fast with a clear, localized error when the current turn alone exceeds
// the limit — otherwise we'd send a request the server will reject with an
// opaque error. Silent history filtering is only safe when dropping history
// images can bring the total down to the limit.
if (currentTurnImages > maxImages) {
throw new Error(l10n.t('Too many images in request: {0} images provided, but the model supports a maximum of {1} images.', currentTurnImages, maxImages));
}
// Walk backward through history (before the current turn), keeping the
// most recent images and replacing the oldest with placeholders.
let historyBudget = maxImages - currentTurnImages;
// Collect keep/drop decisions by walking backward through history
const historyImageDecisions = new Map<string, boolean>(); // "msgIdx:partIdx" -> keep
for (let i = lastUserIdx - 1; i >= 0; i--) {
if (!Array.isArray(messages[i].content)) {
continue;
}
for (let j = messages[i].content.length - 1; j >= 0; j--) {
if (messages[i].content[j].type === Raw.ChatCompletionContentPartKind.Image) {
const key = `${i}:${j}`;
if (historyBudget > 0) {
historyImageDecisions.set(key, true);
historyBudget--;
} else {
historyImageDecisions.set(key, false);
}
}
}
}
// Build filtered messages, replacing dropped images with text placeholders
return messages.map((message, msgIdx) => {
if (msgIdx >= lastUserIdx) {
return message;
}
if (!Array.isArray(message.content)) {
return message;
}
if (!message.content.some(p => p.type === Raw.ChatCompletionContentPartKind.Image)) {
return message;
}
return {
...message,
content: message.content.map((part, partIdx) => {
if (part.type !== Raw.ChatCompletionContentPartKind.Image) {
return part;
}
if (historyImageDecisions.get(`${msgIdx}:${partIdx}`)) {
return part;
}
return { type: Raw.ChatCompletionContentPartKind.Text, text: IMAGE_PLACEHOLDER_TEXT };
})
};
});
}

View File

@@ -13,7 +13,7 @@ import { DefaultsOnlyConfigurationService } from '../../../configuration/common/
import { InMemoryConfigurationService } from '../../../configuration/test/common/inMemoryConfigurationService';
import { ICAPIClientService } from '../../../endpoint/common/capiClient';
import { IDomainService } from '../../../endpoint/common/domainService';
import { IChatModelInformation } from '../../../endpoint/common/endpointProvider';
import { IChatModelInformation, ModelSupportedEndpoint } from '../../../endpoint/common/endpointProvider';
import { IEnvService } from '../../../env/common/envService';
import { ILogService } from '../../../log/common/logService';
import { IFetcherService } from '../../../networking/common/fetcherService';
@@ -246,15 +246,23 @@ describe('ChatEndpoint - Image Count Validation', () => {
mockServices = createMockServices();
});
const createImageMessage = (): Raw.ChatMessage => ({
const createImageMessage = (imageCount: number = 1): Raw.ChatMessage => ({
role: Raw.ChatRole.User,
content: [
{ type: Raw.ChatCompletionContentPartKind.Text, text: 'What is in this image?' },
{ type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'data:image/png;base64,test' } }
...Array.from({ length: imageCount }, () => ({
type: Raw.ChatCompletionContentPartKind.Image as const,
imageUrl: { url: 'data:image/png;base64,test' }
}))
]
});
const createGeminiModelMetadata = (maxPromptImages: number): IChatModelInformation => {
const createAssistantMessage = (): Raw.ChatMessage => ({
role: Raw.ChatRole.Assistant,
content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'I see an image.' }]
});
const createGeminiModelMetadata = (maxPromptImages?: number): IChatModelInformation => {
const baseMetadata = createNonAnthropicModelMetadata('gemini-3');
return {
...baseMetadata,
@@ -266,19 +274,30 @@ describe('ChatEndpoint - Image Count Validation', () => {
},
limits: {
...baseMetadata.capabilities.limits,
vision: {
max_prompt_images: maxPromptImages
}
...(maxPromptImages !== undefined ? { vision: { max_prompt_images: maxPromptImages } } : {})
}
}
};
};
it('should throw error when image count exceeds maxPromptImages', () => {
const modelMetadata = createGeminiModelMetadata(2);
const createAnthropicMessagesModelMetadata = (): IChatModelInformation => {
const baseMetadata = createNonAnthropicModelMetadata('claude-sonnet-4');
return {
...baseMetadata,
supported_endpoints: [ModelSupportedEndpoint.Messages],
capabilities: {
...baseMetadata.capabilities,
supports: {
...baseMetadata.capabilities.supports,
vision: true
}
}
};
};
const endpoint = new ChatEndpoint(
modelMetadata,
const createEndpoint = (metadata: IChatModelInformation) =>
new ChatEndpoint(
metadata,
mockServices.domainService,
mockServices.chatMLFetcher,
mockServices.tokenizerProvider,
@@ -289,30 +308,147 @@ describe('ChatEndpoint - Image Count Validation', () => {
mockServices.logService
);
// Create 3 messages each with 1 image (total 3 images)
const options = createTestOptions([createImageMessage(), createImageMessage(), createImageMessage()]);
const countImages = (messages: Raw.ChatMessage[]): number => {
let count = 0;
for (const msg of messages) {
if (Array.isArray(msg.content)) {
for (const part of msg.content) {
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
count++;
}
}
}
}
return count;
};
expect(() => endpoint.createRequestBody(options)).toThrow(/Too many images in request/);
// Exercises the private `validateAndFilterImages` method directly so we can
// assert on the filtered messages without being blocked by downstream mocks.
const filterImages = (endpoint: ChatEndpoint, messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] => {
return (endpoint as unknown as { validateAndFilterImages(m: Raw.ChatMessage[], n: number): Raw.ChatMessage[] })
.validateAndFilterImages(messages, maxImages);
};
describe('Gemini image limits', () => {
it('should allow requests within image limit', () => {
const endpoint = createEndpoint(createGeminiModelMetadata(5));
const messages = [createImageMessage(), createImageMessage()];
const options = createTestOptions(messages);
expect(() => endpoint.createRequestBody(options)).not.toThrow();
// Input is within limit — messages should be returned untouched.
expect(filterImages(endpoint, messages, 5)).toBe(messages);
});
it('should silently filter history images when total exceeds limit', () => {
const endpoint = createEndpoint(createGeminiModelMetadata(3));
// 2 history user messages with 1 image each + current user message with 2 images = 4 total > 3 limit
const messages = [
createImageMessage(),
createAssistantMessage(),
createImageMessage(),
createAssistantMessage(),
createImageMessage(2),
];
expect(() => endpoint.createRequestBody(createTestOptions(messages))).not.toThrow();
const filtered = filterImages(endpoint, messages, 3);
// Total image parts in the filtered output must not exceed the limit.
expect(countImages(filtered)).toBeLessThanOrEqual(3);
// Current user message (last) must retain all 2 of its images.
expect(countImages([filtered[filtered.length - 1]])).toBe(2);
// Original messages must not be mutated.
expect(countImages(messages)).toBe(4);
});
});
it('should allow requests within image limit', () => {
const modelMetadata = createGeminiModelMetadata(5);
describe('Anthropic Messages API image limits', () => {
it('should allow requests within image limit', () => {
const endpoint = createEndpoint(createAnthropicMessagesModelMetadata());
const messages = [createImageMessage(5)];
// Within limit — filter must not alter the messages.
expect(filterImages(endpoint, messages, 20)).toBe(messages);
});
const endpoint = new ChatEndpoint(
modelMetadata,
mockServices.domainService,
mockServices.chatMLFetcher,
mockServices.tokenizerProvider,
mockServices.instantiationService,
mockServices.configurationService,
mockServices.expService,
mockServices.chatWebSocketService,
mockServices.logService
);
it('should silently filter history images when total exceeds limit', () => {
const endpoint = createEndpoint(createAnthropicMessagesModelMetadata());
// Build history with 18 images + current message with 5 images = 23 total > 20 limit
const messages: Raw.ChatMessage[] = [];
for (let i = 0; i < 18; i++) {
messages.push(createImageMessage());
messages.push(createAssistantMessage());
}
messages.push(createImageMessage(5));
const filtered = filterImages(endpoint, messages, 20);
expect(countImages(filtered)).toBeLessThanOrEqual(20);
// Current user message must retain all 5 of its images.
expect(countImages([filtered[filtered.length - 1]])).toBe(5);
// Original messages must not be mutated.
expect(countImages(messages)).toBe(23);
});
});
// Create 2 messages each with 1 image (total 2 images)
const options = createTestOptions([createImageMessage(), createImageMessage()]);
describe('non-limited models', () => {
it('should not apply image limits to non-Anthropic non-Gemini models', () => {
const metadata = createNonAnthropicModelMetadata('gpt-4o');
const endpoint = createEndpoint(metadata);
// 25 images should not throw for a non-limited model
const options = createTestOptions([createImageMessage(25)]);
expect(() => endpoint.createRequestBody(options)).not.toThrow();
});
});
expect(() => endpoint.createRequestBody(options)).not.toThrow();
describe('edge cases', () => {
it('should filter tool-result images in history the same as user images', () => {
const endpoint = createEndpoint(createGeminiModelMetadata(2));
const toolResultImage: Raw.ChatMessage = {
role: Raw.ChatRole.Tool,
toolCallId: 'tool-1',
content: [
{ type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'https://example.com/tool.png' } }
]
};
// 2 tool-result images in history + 1 current user image = 3 total > 2 limit
const messages: Raw.ChatMessage[] = [
toolResultImage,
createAssistantMessage(),
toolResultImage,
createAssistantMessage(),
createImageMessage(1),
];
const filtered = filterImages(endpoint, messages, 2);
expect(countImages(filtered)).toBeLessThanOrEqual(2);
// Original messages must not be mutated.
expect(countImages(messages)).toBe(3);
});
it('should ignore an overly-restrictive server-provided maxPromptImages and use the hardcoded Gemini limit of 10', () => {
// Server reports max_prompt_images: 1 but the true Gemini limit is 10.
// 2 images in the current turn must not throw.
const endpoint = createEndpoint(createGeminiModelMetadata(1));
const options = createTestOptions([createImageMessage(2)]);
expect(() => endpoint.createRequestBody(options)).not.toThrow();
});
it('should throw using the hardcoded Gemini limit of 10 when the current turn exceeds it', () => {
const endpoint = createEndpoint(createGeminiModelMetadata(1));
const options = createTestOptions([createImageMessage(11)]);
expect(() => endpoint.createRequestBody(options)).toThrow(/maximum of 10 images/);
});
it('should throw using the hardcoded Anthropic Messages limit of 20 when the current turn exceeds it', () => {
const endpoint = createEndpoint(createAnthropicMessagesModelMetadata());
const options = createTestOptions([createImageMessage(21)]);
expect(() => endpoint.createRequestBody(options)).toThrow(/maximum of 20 images/);
});
it('should throw a clear error when the current turn alone exceeds the limit', () => {
const endpoint = createEndpoint(createGeminiModelMetadata(2));
// Current user message has 5 images, limit is 2. History has 1 image.
const messages = [
createImageMessage(),
createAssistantMessage(),
createImageMessage(5),
];
expect(() => filterImages(endpoint, messages, 2)).toThrow(/Too many images/);
});
});
});

View File

@@ -0,0 +1,146 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { Raw } from '@vscode/prompt-tsx';
import { describe, expect, it } from 'vitest';
import { filterHistoryImages } from '../imageLimits';
const createUserImageMessage = (imageCount: number = 1): Raw.ChatMessage => ({
role: Raw.ChatRole.User,
content: [
{ type: Raw.ChatCompletionContentPartKind.Text, text: 'What is in this image?' },
...Array.from({ length: imageCount }, () => ({
type: Raw.ChatCompletionContentPartKind.Image as const,
imageUrl: { url: 'data:image/png;base64,test' }
}))
]
});
const createAssistantMessage = (): Raw.ChatMessage => ({
role: Raw.ChatRole.Assistant,
content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'I see an image.' }]
});
const createToolImageMessage = (): Raw.ChatMessage => ({
role: Raw.ChatRole.Tool,
toolCallId: 'tool-1',
content: [
{ type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'https://example.com/tool.png' } }
]
});
const countImages = (messages: Raw.ChatMessage[]): number => {
let count = 0;
for (const msg of messages) {
if (Array.isArray(msg.content)) {
for (const part of msg.content) {
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
count++;
}
}
}
}
return count;
};
describe('filterHistoryImages', () => {
it('returns the original array by reference when within the limit', () => {
const messages = [createUserImageMessage(), createUserImageMessage()];
expect(filterHistoryImages(messages, 5)).toBe(messages);
});
it('silently filters oldest history images when total exceeds the limit', () => {
// 2 history user messages with 1 image each + current user message with 2 images = 4 total > 3 limit
const messages = [
createUserImageMessage(),
createAssistantMessage(),
createUserImageMessage(),
createAssistantMessage(),
createUserImageMessage(2),
];
const filtered = filterHistoryImages(messages, 3);
expect(countImages(filtered)).toBeLessThanOrEqual(3);
// Current user message (last) must retain all 2 of its images.
expect(countImages([filtered[filtered.length - 1]])).toBe(2);
// Original messages must not be mutated.
expect(countImages(messages)).toBe(4);
});
it('replaces dropped images with a text placeholder', () => {
const messages = [
createUserImageMessage(),
createAssistantMessage(),
createUserImageMessage(1),
];
const filtered = filterHistoryImages(messages, 1);
const droppedMessage = filtered[0];
if (!Array.isArray(droppedMessage.content)) {
throw new Error('expected array content');
}
const placeholder = droppedMessage.content.find(p => p.type === Raw.ChatCompletionContentPartKind.Text && p.text.includes('Image omitted'));
expect(placeholder).toBeDefined();
});
it('filters tool-result images in history the same as user images', () => {
// 2 tool-result images in history + 1 current user image = 3 total > 2 limit
const messages: Raw.ChatMessage[] = [
createToolImageMessage(),
createAssistantMessage(),
createToolImageMessage(),
createAssistantMessage(),
createUserImageMessage(1),
];
const filtered = filterHistoryImages(messages, 2);
expect(countImages(filtered)).toBeLessThanOrEqual(2);
// Original messages must not be mutated.
expect(countImages(messages)).toBe(3);
});
it('throws a clear error including the per-model limit when the current turn alone exceeds it', () => {
// Current user message has 11 images. The error must mention the exact
// model-scoped limit (10 for Gemini, 20 for Anthropic Messages API).
const messages = [createUserImageMessage(11)];
expect(() => filterHistoryImages(messages, 10)).toThrow(/11 images provided.*maximum of 10 images/);
const many = [createUserImageMessage(25)];
expect(() => filterHistoryImages(many, 20)).toThrow(/25 images provided.*maximum of 20 images/);
});
it('handles conversations with no user message by treating the last message as current', () => {
const messages: Raw.ChatMessage[] = [
createToolImageMessage(),
createToolImageMessage(),
createToolImageMessage(),
];
const filtered = filterHistoryImages(messages, 1);
// Last message preserved; earlier tool-result images filtered.
expect(countImages(filtered)).toBeLessThanOrEqual(1);
expect(countImages([filtered[filtered.length - 1]])).toBe(1);
});
it('does not mutate the original messages array or its contents', () => {
// History has 2 images, current turn has 1, limit is 1 → history filters silently.
const messages = [
createUserImageMessage(),
createAssistantMessage(),
createUserImageMessage(),
createAssistantMessage(),
createUserImageMessage(1),
];
const snapshot = JSON.stringify(messages);
filterHistoryImages(messages, 1);
expect(JSON.stringify(messages)).toBe(snapshot);
});
it('passes through messages with non-array content', () => {
const messages: Raw.ChatMessage[] = [
{ role: Raw.ChatRole.System, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'system' }] },
createUserImageMessage(2),
];
// Total = 2 images, within limit of 2 → returned unchanged.
const filtered = filterHistoryImages(messages, 2);
expect(filtered).toBe(messages);
});
});