mirror of
https://github.com/microsoft/vscode.git
synced 2026-07-10 16:58:55 -05:00
better handling of too many images in chat (#310437)
* better handling of too many images in chat * address review: handle no-user-message fallback, assert filter output, cover tool-role images * factor filterHistoryImages into a pure helper with direct tests * restore validateAndFilterImages delegate so existing tests are unchanged * address comments, throw early
This commit is contained in:
@@ -3,7 +3,6 @@
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
import { RequestMetadata, RequestType } from '@vscode/copilot-api';
|
||||
import * as l10n from '@vscode/l10n';
|
||||
import { OpenAI, Raw } from '@vscode/prompt-tsx';
|
||||
import type { CancellationToken } from 'vscode';
|
||||
import { ITokenizer, TokenizerType } from '../../../util/common/tokenizer';
|
||||
@@ -30,11 +29,12 @@ import { ITelemetryService, TelemetryProperties } from '../../telemetry/common/t
|
||||
import { TelemetryData } from '../../telemetry/common/telemetryData';
|
||||
import { ITokenizerProvider } from '../../tokenizer/node/tokenizer';
|
||||
import { ICAPIClientService } from '../common/capiClient';
|
||||
import { isGeminiFamily, modelSupportsContextEditing, modelSupportsToolSearch } from '../common/chatModelCapabilities';
|
||||
import { isAnthropicFamily, isGeminiFamily, modelSupportsContextEditing, modelSupportsToolSearch } from '../common/chatModelCapabilities';
|
||||
import { IDomainService } from '../common/domainService';
|
||||
import { CustomModel, IChatModelInformation, ModelSupportedEndpoint } from '../common/endpointProvider';
|
||||
import { createMessagesRequestBody, processResponseFromMessagesEndpoint } from './messagesApi';
|
||||
import { createResponsesRequestBody, getResponsesApiCompactionThreshold, processResponseFromChatEndpoint } from './responsesApi';
|
||||
import { filterHistoryImages } from './imageLimits';
|
||||
|
||||
/**
|
||||
* The default processor for the stream format from CAPI
|
||||
@@ -288,13 +288,10 @@ export class ChatEndpoint implements IChatEndpoint {
|
||||
}
|
||||
|
||||
createRequestBody(options: ICreateEndpointBodyOptions): IEndpointBody {
|
||||
// Validate image count if endpoint has max_prompt_images limit (Gemini only for now)
|
||||
if (isGeminiFamily(this) && this.maxPromptImages !== undefined) {
|
||||
const imageCount = this.countImages(options.messages, this.maxPromptImages);
|
||||
if (imageCount > this.maxPromptImages) {
|
||||
const errorMsg = l10n.t('Too many images in request: {0} images provided, but the model supports a maximum of {1} images.', imageCount, this.maxPromptImages);
|
||||
throw new Error(errorMsg);
|
||||
}
|
||||
// Determine per-model image limit for APIs with known restrictions
|
||||
const imageLimit = this.getImageLimit();
|
||||
if (imageLimit !== undefined) {
|
||||
options = { ...options, messages: this.validateAndFilterImages(options.messages, imageLimit) };
|
||||
}
|
||||
|
||||
if (this.useResponsesApi) {
|
||||
@@ -309,22 +306,27 @@ export class ChatEndpoint implements IChatEndpoint {
|
||||
}
|
||||
}
|
||||
|
||||
private countImages(messages: Raw.ChatMessage[], maxAllowed?: number): number {
|
||||
let imageCount = 0;
|
||||
for (const message of messages) {
|
||||
if (Array.isArray(message.content)) {
|
||||
for (const part of message.content) {
|
||||
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
|
||||
imageCount++;
|
||||
// Early exit if we've already exceeded the limit
|
||||
if (maxAllowed !== undefined && imageCount > maxAllowed) {
|
||||
return imageCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Returns the model-specific image limit, or `undefined` if no limit applies.
|
||||
* Anthropic Messages API allows up to 20 images per request; Gemini allows up to 10.
|
||||
* These are hardcoded based on API documentation rather than model metadata to
|
||||
* avoid being clamped by unreliable server-provided values.
|
||||
*/
|
||||
private getImageLimit(): number | undefined {
|
||||
if (this.useMessagesApi && isAnthropicFamily(this)) {
|
||||
return 20;
|
||||
}
|
||||
return imageCount;
|
||||
if (isGeminiFamily(this)) {
|
||||
return 10;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin wrapper around {@link filterHistoryImages} retained for test ergonomics.
|
||||
*/
|
||||
private validateAndFilterImages(messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] {
|
||||
return filterHistoryImages(messages, maxImages);
|
||||
}
|
||||
|
||||
protected getCompletionsCallback(): RawMessageConversionCallback | undefined {
|
||||
|
||||
128
extensions/copilot/src/platform/endpoint/node/imageLimits.ts
Normal file
128
extensions/copilot/src/platform/endpoint/node/imageLimits.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
import * as l10n from '@vscode/l10n';
|
||||
import { Raw } from '@vscode/prompt-tsx';
|
||||
|
||||
/**
|
||||
* Model-facing placeholder substituted for dropped history images.
|
||||
* Intentionally not localized — this text is sent to the model, not the user.
|
||||
*/
|
||||
const IMAGE_PLACEHOLDER_TEXT = '[Image omitted from conversation history due to model limit.]';
|
||||
|
||||
/**
|
||||
* Silently drops the oldest images from history when the total number of images
|
||||
* in the conversation exceeds `maxImages`. Images belonging to the current turn
|
||||
* (the last user message and anything after it, e.g. recent tool results) are
|
||||
* always preserved.
|
||||
*
|
||||
* If the current turn alone exceeds the limit, throws a localized error rather
|
||||
* than sending a request we know will be rejected with an opaque server error.
|
||||
*
|
||||
* @returns A (possibly filtered) copy of messages. The original array is never mutated.
|
||||
*/
|
||||
export function filterHistoryImages(messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] {
|
||||
// Anchor the current turn at the last user message; anything at or after this
|
||||
// index is treated as "current turn" and its images are never filtered.
|
||||
let lastUserIdx = -1;
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
if (messages[i].role === Raw.ChatRole.User) {
|
||||
lastUserIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Corner case: no user message at all (e.g. system-only history). Treat the
|
||||
// last message as the current turn so we still filter earlier images.
|
||||
if (lastUserIdx === -1 && messages.length > 0) {
|
||||
lastUserIdx = messages.length - 1;
|
||||
}
|
||||
|
||||
// Count images in the current turn (the last user message and anything after it).
|
||||
let currentTurnImages = 0;
|
||||
for (let i = Math.max(lastUserIdx, 0); i < messages.length; i++) {
|
||||
const content = messages[i].content;
|
||||
if (!Array.isArray(content)) {
|
||||
continue;
|
||||
}
|
||||
for (const part of content) {
|
||||
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
|
||||
currentTurnImages++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Count total images across all messages
|
||||
let totalImages = 0;
|
||||
for (const message of messages) {
|
||||
if (Array.isArray(message.content)) {
|
||||
for (const part of message.content) {
|
||||
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
|
||||
totalImages++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No filtering needed if total is within the limit
|
||||
if (totalImages <= maxImages) {
|
||||
return messages;
|
||||
}
|
||||
|
||||
// Fail fast with a clear, localized error when the current turn alone exceeds
|
||||
// the limit — otherwise we'd send a request the server will reject with an
|
||||
// opaque error. Silent history filtering is only safe when dropping history
|
||||
// images can bring the total down to the limit.
|
||||
if (currentTurnImages > maxImages) {
|
||||
throw new Error(l10n.t('Too many images in request: {0} images provided, but the model supports a maximum of {1} images.', currentTurnImages, maxImages));
|
||||
}
|
||||
|
||||
// Walk backward through history (before the current turn), keeping the
|
||||
// most recent images and replacing the oldest with placeholders.
|
||||
let historyBudget = maxImages - currentTurnImages;
|
||||
|
||||
// Collect keep/drop decisions by walking backward through history
|
||||
const historyImageDecisions = new Map<string, boolean>(); // "msgIdx:partIdx" -> keep
|
||||
for (let i = lastUserIdx - 1; i >= 0; i--) {
|
||||
if (!Array.isArray(messages[i].content)) {
|
||||
continue;
|
||||
}
|
||||
for (let j = messages[i].content.length - 1; j >= 0; j--) {
|
||||
if (messages[i].content[j].type === Raw.ChatCompletionContentPartKind.Image) {
|
||||
const key = `${i}:${j}`;
|
||||
if (historyBudget > 0) {
|
||||
historyImageDecisions.set(key, true);
|
||||
historyBudget--;
|
||||
} else {
|
||||
historyImageDecisions.set(key, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build filtered messages, replacing dropped images with text placeholders
|
||||
return messages.map((message, msgIdx) => {
|
||||
if (msgIdx >= lastUserIdx) {
|
||||
return message;
|
||||
}
|
||||
if (!Array.isArray(message.content)) {
|
||||
return message;
|
||||
}
|
||||
if (!message.content.some(p => p.type === Raw.ChatCompletionContentPartKind.Image)) {
|
||||
return message;
|
||||
}
|
||||
return {
|
||||
...message,
|
||||
content: message.content.map((part, partIdx) => {
|
||||
if (part.type !== Raw.ChatCompletionContentPartKind.Image) {
|
||||
return part;
|
||||
}
|
||||
if (historyImageDecisions.get(`${msgIdx}:${partIdx}`)) {
|
||||
return part;
|
||||
}
|
||||
return { type: Raw.ChatCompletionContentPartKind.Text, text: IMAGE_PLACEHOLDER_TEXT };
|
||||
})
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import { DefaultsOnlyConfigurationService } from '../../../configuration/common/
|
||||
import { InMemoryConfigurationService } from '../../../configuration/test/common/inMemoryConfigurationService';
|
||||
import { ICAPIClientService } from '../../../endpoint/common/capiClient';
|
||||
import { IDomainService } from '../../../endpoint/common/domainService';
|
||||
import { IChatModelInformation } from '../../../endpoint/common/endpointProvider';
|
||||
import { IChatModelInformation, ModelSupportedEndpoint } from '../../../endpoint/common/endpointProvider';
|
||||
import { IEnvService } from '../../../env/common/envService';
|
||||
import { ILogService } from '../../../log/common/logService';
|
||||
import { IFetcherService } from '../../../networking/common/fetcherService';
|
||||
@@ -246,15 +246,23 @@ describe('ChatEndpoint - Image Count Validation', () => {
|
||||
mockServices = createMockServices();
|
||||
});
|
||||
|
||||
const createImageMessage = (): Raw.ChatMessage => ({
|
||||
const createImageMessage = (imageCount: number = 1): Raw.ChatMessage => ({
|
||||
role: Raw.ChatRole.User,
|
||||
content: [
|
||||
{ type: Raw.ChatCompletionContentPartKind.Text, text: 'What is in this image?' },
|
||||
{ type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'data:image/png;base64,test' } }
|
||||
...Array.from({ length: imageCount }, () => ({
|
||||
type: Raw.ChatCompletionContentPartKind.Image as const,
|
||||
imageUrl: { url: 'data:image/png;base64,test' }
|
||||
}))
|
||||
]
|
||||
});
|
||||
|
||||
const createGeminiModelMetadata = (maxPromptImages: number): IChatModelInformation => {
|
||||
const createAssistantMessage = (): Raw.ChatMessage => ({
|
||||
role: Raw.ChatRole.Assistant,
|
||||
content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'I see an image.' }]
|
||||
});
|
||||
|
||||
const createGeminiModelMetadata = (maxPromptImages?: number): IChatModelInformation => {
|
||||
const baseMetadata = createNonAnthropicModelMetadata('gemini-3');
|
||||
return {
|
||||
...baseMetadata,
|
||||
@@ -266,19 +274,30 @@ describe('ChatEndpoint - Image Count Validation', () => {
|
||||
},
|
||||
limits: {
|
||||
...baseMetadata.capabilities.limits,
|
||||
vision: {
|
||||
max_prompt_images: maxPromptImages
|
||||
}
|
||||
...(maxPromptImages !== undefined ? { vision: { max_prompt_images: maxPromptImages } } : {})
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
it('should throw error when image count exceeds maxPromptImages', () => {
|
||||
const modelMetadata = createGeminiModelMetadata(2);
|
||||
const createAnthropicMessagesModelMetadata = (): IChatModelInformation => {
|
||||
const baseMetadata = createNonAnthropicModelMetadata('claude-sonnet-4');
|
||||
return {
|
||||
...baseMetadata,
|
||||
supported_endpoints: [ModelSupportedEndpoint.Messages],
|
||||
capabilities: {
|
||||
...baseMetadata.capabilities,
|
||||
supports: {
|
||||
...baseMetadata.capabilities.supports,
|
||||
vision: true
|
||||
}
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const endpoint = new ChatEndpoint(
|
||||
modelMetadata,
|
||||
const createEndpoint = (metadata: IChatModelInformation) =>
|
||||
new ChatEndpoint(
|
||||
metadata,
|
||||
mockServices.domainService,
|
||||
mockServices.chatMLFetcher,
|
||||
mockServices.tokenizerProvider,
|
||||
@@ -289,30 +308,147 @@ describe('ChatEndpoint - Image Count Validation', () => {
|
||||
mockServices.logService
|
||||
);
|
||||
|
||||
// Create 3 messages each with 1 image (total 3 images)
|
||||
const options = createTestOptions([createImageMessage(), createImageMessage(), createImageMessage()]);
|
||||
const countImages = (messages: Raw.ChatMessage[]): number => {
|
||||
let count = 0;
|
||||
for (const msg of messages) {
|
||||
if (Array.isArray(msg.content)) {
|
||||
for (const part of msg.content) {
|
||||
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return count;
|
||||
};
|
||||
|
||||
expect(() => endpoint.createRequestBody(options)).toThrow(/Too many images in request/);
|
||||
// Exercises the private `validateAndFilterImages` method directly so we can
|
||||
// assert on the filtered messages without being blocked by downstream mocks.
|
||||
const filterImages = (endpoint: ChatEndpoint, messages: Raw.ChatMessage[], maxImages: number): Raw.ChatMessage[] => {
|
||||
return (endpoint as unknown as { validateAndFilterImages(m: Raw.ChatMessage[], n: number): Raw.ChatMessage[] })
|
||||
.validateAndFilterImages(messages, maxImages);
|
||||
};
|
||||
|
||||
describe('Gemini image limits', () => {
|
||||
it('should allow requests within image limit', () => {
|
||||
const endpoint = createEndpoint(createGeminiModelMetadata(5));
|
||||
const messages = [createImageMessage(), createImageMessage()];
|
||||
const options = createTestOptions(messages);
|
||||
expect(() => endpoint.createRequestBody(options)).not.toThrow();
|
||||
// Input is within limit — messages should be returned untouched.
|
||||
expect(filterImages(endpoint, messages, 5)).toBe(messages);
|
||||
});
|
||||
|
||||
it('should silently filter history images when total exceeds limit', () => {
|
||||
const endpoint = createEndpoint(createGeminiModelMetadata(3));
|
||||
// 2 history user messages with 1 image each + current user message with 2 images = 4 total > 3 limit
|
||||
const messages = [
|
||||
createImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createImageMessage(2),
|
||||
];
|
||||
expect(() => endpoint.createRequestBody(createTestOptions(messages))).not.toThrow();
|
||||
const filtered = filterImages(endpoint, messages, 3);
|
||||
// Total image parts in the filtered output must not exceed the limit.
|
||||
expect(countImages(filtered)).toBeLessThanOrEqual(3);
|
||||
// Current user message (last) must retain all 2 of its images.
|
||||
expect(countImages([filtered[filtered.length - 1]])).toBe(2);
|
||||
// Original messages must not be mutated.
|
||||
expect(countImages(messages)).toBe(4);
|
||||
});
|
||||
});
|
||||
|
||||
it('should allow requests within image limit', () => {
|
||||
const modelMetadata = createGeminiModelMetadata(5);
|
||||
describe('Anthropic Messages API image limits', () => {
|
||||
it('should allow requests within image limit', () => {
|
||||
const endpoint = createEndpoint(createAnthropicMessagesModelMetadata());
|
||||
const messages = [createImageMessage(5)];
|
||||
// Within limit — filter must not alter the messages.
|
||||
expect(filterImages(endpoint, messages, 20)).toBe(messages);
|
||||
});
|
||||
|
||||
const endpoint = new ChatEndpoint(
|
||||
modelMetadata,
|
||||
mockServices.domainService,
|
||||
mockServices.chatMLFetcher,
|
||||
mockServices.tokenizerProvider,
|
||||
mockServices.instantiationService,
|
||||
mockServices.configurationService,
|
||||
mockServices.expService,
|
||||
mockServices.chatWebSocketService,
|
||||
mockServices.logService
|
||||
);
|
||||
it('should silently filter history images when total exceeds limit', () => {
|
||||
const endpoint = createEndpoint(createAnthropicMessagesModelMetadata());
|
||||
// Build history with 18 images + current message with 5 images = 23 total > 20 limit
|
||||
const messages: Raw.ChatMessage[] = [];
|
||||
for (let i = 0; i < 18; i++) {
|
||||
messages.push(createImageMessage());
|
||||
messages.push(createAssistantMessage());
|
||||
}
|
||||
messages.push(createImageMessage(5));
|
||||
const filtered = filterImages(endpoint, messages, 20);
|
||||
expect(countImages(filtered)).toBeLessThanOrEqual(20);
|
||||
// Current user message must retain all 5 of its images.
|
||||
expect(countImages([filtered[filtered.length - 1]])).toBe(5);
|
||||
// Original messages must not be mutated.
|
||||
expect(countImages(messages)).toBe(23);
|
||||
});
|
||||
});
|
||||
|
||||
// Create 2 messages each with 1 image (total 2 images)
|
||||
const options = createTestOptions([createImageMessage(), createImageMessage()]);
|
||||
describe('non-limited models', () => {
|
||||
it('should not apply image limits to non-Anthropic non-Gemini models', () => {
|
||||
const metadata = createNonAnthropicModelMetadata('gpt-4o');
|
||||
const endpoint = createEndpoint(metadata);
|
||||
// 25 images should not throw for a non-limited model
|
||||
const options = createTestOptions([createImageMessage(25)]);
|
||||
expect(() => endpoint.createRequestBody(options)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
expect(() => endpoint.createRequestBody(options)).not.toThrow();
|
||||
describe('edge cases', () => {
|
||||
it('should filter tool-result images in history the same as user images', () => {
|
||||
const endpoint = createEndpoint(createGeminiModelMetadata(2));
|
||||
const toolResultImage: Raw.ChatMessage = {
|
||||
role: Raw.ChatRole.Tool,
|
||||
toolCallId: 'tool-1',
|
||||
content: [
|
||||
{ type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'https://example.com/tool.png' } }
|
||||
]
|
||||
};
|
||||
// 2 tool-result images in history + 1 current user image = 3 total > 2 limit
|
||||
const messages: Raw.ChatMessage[] = [
|
||||
toolResultImage,
|
||||
createAssistantMessage(),
|
||||
toolResultImage,
|
||||
createAssistantMessage(),
|
||||
createImageMessage(1),
|
||||
];
|
||||
const filtered = filterImages(endpoint, messages, 2);
|
||||
expect(countImages(filtered)).toBeLessThanOrEqual(2);
|
||||
// Original messages must not be mutated.
|
||||
expect(countImages(messages)).toBe(3);
|
||||
});
|
||||
|
||||
it('should ignore an overly-restrictive server-provided maxPromptImages and use the hardcoded Gemini limit of 10', () => {
|
||||
// Server reports max_prompt_images: 1 but the true Gemini limit is 10.
|
||||
// 2 images in the current turn must not throw.
|
||||
const endpoint = createEndpoint(createGeminiModelMetadata(1));
|
||||
const options = createTestOptions([createImageMessage(2)]);
|
||||
expect(() => endpoint.createRequestBody(options)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should throw using the hardcoded Gemini limit of 10 when the current turn exceeds it', () => {
|
||||
const endpoint = createEndpoint(createGeminiModelMetadata(1));
|
||||
const options = createTestOptions([createImageMessage(11)]);
|
||||
expect(() => endpoint.createRequestBody(options)).toThrow(/maximum of 10 images/);
|
||||
});
|
||||
|
||||
it('should throw using the hardcoded Anthropic Messages limit of 20 when the current turn exceeds it', () => {
|
||||
const endpoint = createEndpoint(createAnthropicMessagesModelMetadata());
|
||||
const options = createTestOptions([createImageMessage(21)]);
|
||||
expect(() => endpoint.createRequestBody(options)).toThrow(/maximum of 20 images/);
|
||||
});
|
||||
|
||||
it('should throw a clear error when the current turn alone exceeds the limit', () => {
|
||||
const endpoint = createEndpoint(createGeminiModelMetadata(2));
|
||||
// Current user message has 5 images, limit is 2. History has 1 image.
|
||||
const messages = [
|
||||
createImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createImageMessage(5),
|
||||
];
|
||||
expect(() => filterImages(endpoint, messages, 2)).toThrow(/Too many images/);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,146 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { Raw } from '@vscode/prompt-tsx';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { filterHistoryImages } from '../imageLimits';
|
||||
|
||||
const createUserImageMessage = (imageCount: number = 1): Raw.ChatMessage => ({
|
||||
role: Raw.ChatRole.User,
|
||||
content: [
|
||||
{ type: Raw.ChatCompletionContentPartKind.Text, text: 'What is in this image?' },
|
||||
...Array.from({ length: imageCount }, () => ({
|
||||
type: Raw.ChatCompletionContentPartKind.Image as const,
|
||||
imageUrl: { url: 'data:image/png;base64,test' }
|
||||
}))
|
||||
]
|
||||
});
|
||||
|
||||
const createAssistantMessage = (): Raw.ChatMessage => ({
|
||||
role: Raw.ChatRole.Assistant,
|
||||
content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'I see an image.' }]
|
||||
});
|
||||
|
||||
const createToolImageMessage = (): Raw.ChatMessage => ({
|
||||
role: Raw.ChatRole.Tool,
|
||||
toolCallId: 'tool-1',
|
||||
content: [
|
||||
{ type: Raw.ChatCompletionContentPartKind.Image, imageUrl: { url: 'https://example.com/tool.png' } }
|
||||
]
|
||||
});
|
||||
|
||||
const countImages = (messages: Raw.ChatMessage[]): number => {
|
||||
let count = 0;
|
||||
for (const msg of messages) {
|
||||
if (Array.isArray(msg.content)) {
|
||||
for (const part of msg.content) {
|
||||
if (part.type === Raw.ChatCompletionContentPartKind.Image) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return count;
|
||||
};
|
||||
|
||||
describe('filterHistoryImages', () => {
|
||||
it('returns the original array by reference when within the limit', () => {
|
||||
const messages = [createUserImageMessage(), createUserImageMessage()];
|
||||
expect(filterHistoryImages(messages, 5)).toBe(messages);
|
||||
});
|
||||
|
||||
it('silently filters oldest history images when total exceeds the limit', () => {
|
||||
// 2 history user messages with 1 image each + current user message with 2 images = 4 total > 3 limit
|
||||
const messages = [
|
||||
createUserImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createUserImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createUserImageMessage(2),
|
||||
];
|
||||
const filtered = filterHistoryImages(messages, 3);
|
||||
expect(countImages(filtered)).toBeLessThanOrEqual(3);
|
||||
// Current user message (last) must retain all 2 of its images.
|
||||
expect(countImages([filtered[filtered.length - 1]])).toBe(2);
|
||||
// Original messages must not be mutated.
|
||||
expect(countImages(messages)).toBe(4);
|
||||
});
|
||||
|
||||
it('replaces dropped images with a text placeholder', () => {
|
||||
const messages = [
|
||||
createUserImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createUserImageMessage(1),
|
||||
];
|
||||
const filtered = filterHistoryImages(messages, 1);
|
||||
const droppedMessage = filtered[0];
|
||||
if (!Array.isArray(droppedMessage.content)) {
|
||||
throw new Error('expected array content');
|
||||
}
|
||||
const placeholder = droppedMessage.content.find(p => p.type === Raw.ChatCompletionContentPartKind.Text && p.text.includes('Image omitted'));
|
||||
expect(placeholder).toBeDefined();
|
||||
});
|
||||
|
||||
it('filters tool-result images in history the same as user images', () => {
|
||||
// 2 tool-result images in history + 1 current user image = 3 total > 2 limit
|
||||
const messages: Raw.ChatMessage[] = [
|
||||
createToolImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createToolImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createUserImageMessage(1),
|
||||
];
|
||||
const filtered = filterHistoryImages(messages, 2);
|
||||
expect(countImages(filtered)).toBeLessThanOrEqual(2);
|
||||
// Original messages must not be mutated.
|
||||
expect(countImages(messages)).toBe(3);
|
||||
});
|
||||
|
||||
it('throws a clear error including the per-model limit when the current turn alone exceeds it', () => {
|
||||
// Current user message has 11 images. The error must mention the exact
|
||||
// model-scoped limit (10 for Gemini, 20 for Anthropic Messages API).
|
||||
const messages = [createUserImageMessage(11)];
|
||||
expect(() => filterHistoryImages(messages, 10)).toThrow(/11 images provided.*maximum of 10 images/);
|
||||
|
||||
const many = [createUserImageMessage(25)];
|
||||
expect(() => filterHistoryImages(many, 20)).toThrow(/25 images provided.*maximum of 20 images/);
|
||||
});
|
||||
|
||||
it('handles conversations with no user message by treating the last message as current', () => {
|
||||
const messages: Raw.ChatMessage[] = [
|
||||
createToolImageMessage(),
|
||||
createToolImageMessage(),
|
||||
createToolImageMessage(),
|
||||
];
|
||||
const filtered = filterHistoryImages(messages, 1);
|
||||
// Last message preserved; earlier tool-result images filtered.
|
||||
expect(countImages(filtered)).toBeLessThanOrEqual(1);
|
||||
expect(countImages([filtered[filtered.length - 1]])).toBe(1);
|
||||
});
|
||||
|
||||
it('does not mutate the original messages array or its contents', () => {
|
||||
// History has 2 images, current turn has 1, limit is 1 → history filters silently.
|
||||
const messages = [
|
||||
createUserImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createUserImageMessage(),
|
||||
createAssistantMessage(),
|
||||
createUserImageMessage(1),
|
||||
];
|
||||
const snapshot = JSON.stringify(messages);
|
||||
filterHistoryImages(messages, 1);
|
||||
expect(JSON.stringify(messages)).toBe(snapshot);
|
||||
});
|
||||
|
||||
it('passes through messages with non-array content', () => {
|
||||
const messages: Raw.ChatMessage[] = [
|
||||
{ role: Raw.ChatRole.System, content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'system' }] },
|
||||
createUserImageMessage(2),
|
||||
];
|
||||
// Total = 2 images, within limit of 2 → returned unchanged.
|
||||
const filtered = filterHistoryImages(messages, 2);
|
||||
expect(filtered).toBe(messages);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user