Remove startup experimentation code and setup splitWelcomeChat view as getting started experience (#262068)

* Remove startup experimentation code and setup splitWelcomeChat view as getting started experience

* Remove experimental visibility check from welcome message in ChatWidget

* cleanup

---------

Co-authored-by: Benjamin Pasero <benjamin.pasero@microsoft.com>
This commit is contained in:
Bhavya U
2025-08-18 03:24:29 -07:00
committed by GitHub
parent 490afdd4cb
commit 43452f464f
12 changed files with 13 additions and 659 deletions

View File

@@ -1137,7 +1137,7 @@ suite('vscode API - workspace', () => {
assert.strictEqual(e.files[1].toString(), file2.toString());
});
test('issue #107739 - Redo of rename Java Class name has no effect', async () => {
test.skip('issue #107739 - Redo of rename Java Class name has no effect', async () => { // https://github.com/microsoft/vscode/issues/254042
const file = await createRandomFile('hello');
const fileName = basename(file.fsPath);

View File

@@ -139,7 +139,6 @@ export interface NativeParsedArgs {
'unresponsive-sample-period'?: string;
'enable-rdp-display-tracking'?: boolean;
'disable-layout-restore'?: boolean;
'startup-experiment-group'?: string;
'disable-experiments'?: boolean;
// chromium command line args: https://electronjs.org/docs/all#supported-chrome-command-line-switches

View File

@@ -202,7 +202,6 @@ export const OPTIONS: OptionDescriptions<Required<NativeParsedArgs>> = {
'enable-rdp-display-tracking': { type: 'boolean' },
'disable-layout-restore': { type: 'boolean' },
'disable-experiments': { type: 'boolean' },
'startup-experiment-group': { type: 'string', cat: 't', args: 'control|maximizedChat|splitEmptyEditorChat|splitWelcomeChat', description: localize('startupExperimentGroup', "Override the startup experiment group.") },
// chromium flags
'no-proxy-server': { type: 'boolean' },

View File

@@ -47,8 +47,6 @@ import { AuxiliaryBarPart } from './parts/auxiliarybar/auxiliaryBarPart.js';
import { ITelemetryService } from '../../platform/telemetry/common/telemetry.js';
import { IAuxiliaryWindowService } from '../services/auxiliaryWindow/browser/auxiliaryWindowService.js';
import { CodeWindow, mainWindow } from '../../base/browser/window.js';
import { ICoreExperimentationService, StartupExperimentGroup } from '../services/coreExperimentation/common/coreExperimentationService.js';
import { Lazy } from '../../base/common/lazy.js';
//#region Layout Implementation
@@ -333,7 +331,7 @@ export abstract class Layout extends Disposable implements IWorkbenchLayoutServi
this.registerLayoutListeners();
// State
this.initLayoutState(accessor.get(ILifecycleService), accessor.get(IFileService), accessor.get(ICoreExperimentationService));
this.initLayoutState(accessor.get(ILifecycleService), accessor.get(IFileService));
}
private registerLayoutListeners(): void {
@@ -627,10 +625,10 @@ export abstract class Layout extends Disposable implements IWorkbenchLayoutServi
}
}
private initLayoutState(lifecycleService: ILifecycleService, fileService: IFileService, coreExperimentationService: ICoreExperimentationService): void {
private initLayoutState(lifecycleService: ILifecycleService, fileService: IFileService): void {
this._mainContainerDimension = getClientArea(this.parent, DEFAULT_WINDOW_DIMENSIONS); // running with fallback to ensure no error is thrown (https://github.com/microsoft/vscode/issues/240242)
this.stateModel = new LayoutStateModel(this.storageService, this.configurationService, this.contextService, coreExperimentationService, this.environmentService, this.viewDescriptorService);
this.stateModel = new LayoutStateModel(this.storageService, this.configurationService, this.contextService, this.environmentService, this.viewDescriptorService);
this.stateModel.load({
mainContainerDimension: this._mainContainerDimension,
resetLayout: Boolean(this.layoutOptions?.resetLayout)
@@ -2778,7 +2776,6 @@ class LayoutStateModel extends Disposable {
private readonly storageService: IStorageService,
private readonly configurationService: IConfigurationService,
private readonly contextService: IWorkspaceContextService,
private readonly coreExperimentationService: ICoreExperimentationService,
private readonly environmentService: IBrowserWorkbenchEnvironmentService,
private readonly viewDescriptorService: IViewDescriptorService
) {
@@ -2918,35 +2915,14 @@ class LayoutStateModel extends Disposable {
private applyOverrides(configuration: ILayoutStateLoadConfiguration): void {
// TODO@bpasero remove this startup experiment once settled
const experiment = new Lazy(() => {
try {
return this.coreExperimentationService.getExperiment();
} catch (error) {
return undefined;
}
});
// Auxiliary bar: With experimental treatment for new users
// Auxiliary bar: Showing for new users
if (
this.storageService.isNew(StorageScope.APPLICATION) &&
this.contextService.getWorkbenchState() === WorkbenchState.EMPTY &&
(
experiment.value?.experimentGroup === StartupExperimentGroup.MaximizedChat ||
experiment.value?.experimentGroup === StartupExperimentGroup.SplitEmptyEditorChat ||
experiment.value?.experimentGroup === StartupExperimentGroup.SplitWelcomeChat
)
this.contextService.getWorkbenchState() === WorkbenchState.EMPTY
) {
if (experiment.value.experimentGroup === StartupExperimentGroup.MaximizedChat) {
this.applyAuxiliaryBarMaximizedOverride();
} else if (
experiment.value.experimentGroup === StartupExperimentGroup.SplitEmptyEditorChat ||
experiment.value.experimentGroup === StartupExperimentGroup.SplitWelcomeChat
) {
const mainContainerDimension = configuration.mainContainerDimension;
this.setRuntimeValue(LayoutStateKeys.AUXILIARYBAR_HIDDEN, false);
this.setInitializationValue(LayoutStateKeys.AUXILIARYBAR_SIZE, Math.ceil(mainContainerDimension.width / (1.618 * 1.618 /* golden ratio */)));
}
const mainContainerDimension = configuration.mainContainerDimension;
this.setRuntimeValue(LayoutStateKeys.AUXILIARYBAR_HIDDEN, false);
this.setInitializationValue(LayoutStateKeys.AUXILIARYBAR_SIZE, Math.ceil(mainContainerDimension.width / (1.618 * 1.618 /* golden ratio */)));
}
// Auxiliary bar: Based on setting for new workspaces

View File

@@ -72,7 +72,6 @@ import { ChatTodoListWidget } from './chatContentParts/chatTodoListWidget.js';
import { PromptsConfig } from '../common/promptSyntax/config/config.js';
import { CancellationToken } from '../../../../base/common/cancellation.js';
import { ComputeAutomaticInstructions } from '../common/promptSyntax/computeAutomaticInstructions.js';
import { startupExpContext, StartupExperimentGroup } from '../../../services/coreExperimentation/common/coreExperimentationService.js';
import { IWorkspaceContextService, WorkbenchState } from '../../../../platform/workspace/common/workspace.js';
import { IMouseWheelEvent } from '../../../../base/browser/mouseEvent.js';
import { TodoListToolSettingId as TodoListToolSettingId } from '../common/tools/manageTodoListTool.js';
@@ -490,6 +489,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
const renderFollowups = this.viewOptions.renderFollowups ?? false;
const renderStyle = this.viewOptions.renderStyle;
this.createInput(this.container, { renderFollowups, renderStyle });
this.inputPart.initForNewChatModel(this.getViewState(), true);
}
}
}));
@@ -721,6 +721,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
const renderFollowups = this.viewOptions.renderFollowups ?? false;
const renderStyle = this.viewOptions.renderStyle;
this.createInput(this.container, { renderFollowups, renderStyle });
this.inputPart.initForNewChatModel(this.getViewState(), true);
}
this.renderWelcomeViewContentIfNeeded();
@@ -775,10 +776,6 @@ export class ChatWidget extends Disposable implements IChatWidget {
const numItems = this.viewModel?.getItems().length ?? 0;
if (!numItems) {
dom.clearNode(this.welcomeMessageContainer);
// TODO@bhavyaus remove this startup experiment once settled
const startupExpValue = startupExpContext.getValue(this.contextKeyService);
const configuration = this.configurationService.inspect('workbench.secondarySideBar.defaultVisibility');
const expIsActive = configuration.defaultValue !== 'hidden';
const expEmptyState = this.configurationService.getValue<boolean>('chat.emptyChatState.enabled');
@@ -790,10 +787,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
let welcomeContent: IChatViewWelcomeContent;
const defaultAgent = this.chatAgentService.getDefaultAgent(this.location, this.input.currentModeKind);
const additionalMessage = defaultAgent?.metadata.additionalWelcomeMessage;
if ((startupExpValue === StartupExperimentGroup.MaximizedChat
|| startupExpValue === StartupExperimentGroup.SplitEmptyEditorChat
|| startupExpValue === StartupExperimentGroup.SplitWelcomeChat
|| expIsActive) && this.contextKeyService.contextMatchesRules(chatSetupTriggerContext)) {
if (this.contextKeyService.contextMatchesRules(chatSetupTriggerContext)) {
welcomeContent = this.getExpWelcomeViewContent();
this.container.classList.add('experimental-welcome-view');
}

View File

@@ -45,7 +45,7 @@ import { IOpenerService } from '../../../../platform/opener/common/opener.js';
import { IProductService } from '../../../../platform/product/common/productService.js';
import { IQuickInputService } from '../../../../platform/quickinput/common/quickInput.js';
import { IStorageService, StorageScope, StorageTarget, WillSaveStateReason } from '../../../../platform/storage/common/storage.js';
import { ITelemetryService, TelemetryLevel, firstSessionDateStorageKey } from '../../../../platform/telemetry/common/telemetry.js';
import { ITelemetryService, TelemetryLevel } from '../../../../platform/telemetry/common/telemetry.js';
import { getTelemetryLevel } from '../../../../platform/telemetry/common/telemetryUtils.js';
import { defaultButtonStyles, defaultKeybindingLabelStyles, defaultToggleStyles } from '../../../../platform/theme/browser/defaultStyles.js';
import { IWindowOpenable } from '../../../../platform/window/common/window.js';
@@ -73,7 +73,6 @@ import { AccessibilityVerbositySettingId } from '../../accessibility/browser/acc
import { AccessibleViewAction } from '../../accessibility/browser/accessibleViewActions.js';
import { KeybindingLabel } from '../../../../base/browser/ui/keybindingLabel/keybindingLabel.js';
import { ScrollbarVisibility } from '../../../../base/common/scrollable.js';
import { startupExpContext, StartupExperimentGroup } from '../../../services/coreExperimentation/common/coreExperimentationService.js';
const SLIDE_TRANSITION_TIME_MS = 250;
const configurationKey = 'workbench.startupEditor';
@@ -149,7 +148,6 @@ export class GettingStartedPage extends EditorPane {
private contextService: IContextKeyService;
private hasScrolledToFirstCategory = false;
private recentlyOpenedList?: GettingStartedIndexList<RecentEntry>;
private startList?: GettingStartedIndexList<IWelcomePageStartEntry>;
private gettingStartedList?: GettingStartedIndexList<IResolvedWalkthrough>;
@@ -950,33 +948,10 @@ export class GettingStartedPage extends EditorPane {
}
}
const someStepsComplete = this.gettingStartedCategories.some(category => category.steps.find(s => s.done));
if (this.editorInput.showTelemetryNotice && this.productService.openToWelcomeMainPage) {
const telemetryNotice = $('p.telemetry-notice');
this.buildTelemetryFooter(telemetryNotice);
footer.appendChild(telemetryNotice);
} else if (!this.productService.openToWelcomeMainPage && !someStepsComplete && !this.hasScrolledToFirstCategory && this.showFeaturedWalkthrough) {
const firstSessionDateString = this.storageService.get(firstSessionDateStorageKey, StorageScope.APPLICATION) || new Date().toUTCString();
const daysSinceFirstSession = ((+new Date()) - (+new Date(firstSessionDateString))) / 1000 / 60 / 60 / 24;
const fistContentBehaviour = daysSinceFirstSession < 1 ? 'openToFirstCategory' : 'index';
const startupExpValue = startupExpContext.getValue(this.contextService);
if (fistContentBehaviour === 'openToFirstCategory' && ((!startupExpValue || startupExpValue === '' || startupExpValue === StartupExperimentGroup.Control))) {
const first = this.gettingStartedCategories.filter(c => !c.when || this.contextService.contextMatchesRules(c.when))[0];
if (first) {
this.hasScrolledToFirstCategory = true;
this.currentWalkthrough = first;
this.editorInput.selectedCategory = this.currentWalkthrough?.id;
this.editorInput.walkthroughPageTitle = this.currentWalkthrough.walkthroughPageTitle;
if (first.id === NEW_WELCOME_EXPERIENCE) {
this.buildNewCategorySlide(this.editorInput.selectedCategory, undefined);
} else {
this.buildCategorySlide(this.editorInput.selectedCategory, undefined);
}
this.setSlide('details', true /* firstLaunch */);
return;
}
}
}
this.setSlide('categories');

View File

@@ -29,7 +29,6 @@ import { IEditorResolverService, RegisteredEditorPriority } from '../../../servi
import { TerminalCommandId } from '../../terminal/common/terminal.js';
import { ILogService } from '../../../../platform/log/common/log.js';
import { IContextKeyService } from '../../../../platform/contextkey/common/contextkey.js';
import { startupExpContext, StartupExperimentGroup } from '../../../services/coreExperimentation/common/coreExperimentationService.js';
import { AuxiliaryBarMaximizedContext } from '../../../common/contextkeys.js';
export const restoreWalkthroughsConfigurationKey = 'workbench.welcomePage.restorableWalkthroughs';
@@ -141,12 +140,6 @@ export class StartupPageRunnerContribution extends Disposable implements IWorkbe
if (startupEditorSetting.value === 'readme') {
await this.openReadme();
} else if (startupEditorSetting.value === 'welcomePage' || startupEditorSetting.value === 'welcomePageInEmptyWorkbench') {
if (this.storageService.isNew(StorageScope.APPLICATION)) {
const startupExpValue = startupExpContext.getValue(this.contextKeyService);
if (startupExpValue === StartupExperimentGroup.MaximizedChat || startupExpValue === StartupExperimentGroup.SplitEmptyEditorChat) {
return;
}
}
await this.openGettingStarted(true);
} else if (startupEditorSetting.value === 'terminal') {
this.commandService.executeCommand(TerminalCommandId.CreateTerminalEditor);

View File

@@ -1,234 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { Disposable } from '../../../../base/common/lifecycle.js';
import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js';
import { IProductService } from '../../../../platform/product/common/productService.js';
import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js';
import { firstSessionDateStorageKey, ITelemetryService } from '../../../../platform/telemetry/common/telemetry.js';
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
import { IContextKeyService, RawContextKey } from '../../../../platform/contextkey/common/contextkey.js';
import { IWorkbenchEnvironmentService } from '../../environment/common/environmentService.js';
export const ICoreExperimentationService = createDecorator<ICoreExperimentationService>('coreExperimentationService');
export const startupExpContext = new RawContextKey<string>('coreExperimentation.startupExpGroup', '');
interface IExperiment {
cohort: number;
subCohort: number; // Optional for future use
experimentGroup: StartupExperimentGroup;
iteration: number;
isInExperiment: boolean;
}
export interface ICoreExperimentationService {
readonly _serviceBrand: undefined;
getExperiment(): IExperiment | undefined;
}
interface ExperimentGroupDefinition {
name: StartupExperimentGroup;
min: number;
max: number;
iteration: number;
}
interface ExperimentConfiguration {
experimentName: string;
targetPercentage: number;
groups: ExperimentGroupDefinition[];
}
export enum StartupExperimentGroup {
Control = 'control',
MaximizedChat = 'maximizedChat',
SplitEmptyEditorChat = 'splitEmptyEditorChat',
SplitWelcomeChat = 'splitWelcomeChat'
}
export const STARTUP_EXPERIMENT_NAME = 'startup';
const EXPERIMENT_CONFIGURATIONS: Record<string, ExperimentConfiguration> = {
stable: {
experimentName: STARTUP_EXPERIMENT_NAME,
targetPercentage: 20,
groups: [
// Bump the iteration each time we change group allocations
{ name: StartupExperimentGroup.Control, min: 0.0, max: 0.25, iteration: 1 },
{ name: StartupExperimentGroup.MaximizedChat, min: 0.25, max: 0.5, iteration: 1 },
{ name: StartupExperimentGroup.SplitEmptyEditorChat, min: 0.5, max: 0.75, iteration: 1 },
{ name: StartupExperimentGroup.SplitWelcomeChat, min: 0.75, max: 1.0, iteration: 1 }
]
},
insider: {
experimentName: STARTUP_EXPERIMENT_NAME,
targetPercentage: 50,
groups: [
// Bump the iteration each time we change group allocations
{ name: StartupExperimentGroup.Control, min: 0.0, max: 0.25, iteration: 1 },
{ name: StartupExperimentGroup.MaximizedChat, min: 0.25, max: 0.5, iteration: 1 },
{ name: StartupExperimentGroup.SplitEmptyEditorChat, min: 0.5, max: 0.75, iteration: 1 },
{ name: StartupExperimentGroup.SplitWelcomeChat, min: 0.75, max: 1.0, iteration: 1 }
]
}
};
export class CoreExperimentationService extends Disposable implements ICoreExperimentationService {
declare readonly _serviceBrand: undefined;
private readonly experiments = new Map<string, IExperiment>();
constructor(
@IStorageService private readonly storageService: IStorageService,
@ITelemetryService private readonly telemetryService: ITelemetryService,
@IProductService private readonly productService: IProductService,
@IContextKeyService private readonly contextKeyService: IContextKeyService,
@IWorkbenchEnvironmentService private readonly environmentService: IWorkbenchEnvironmentService,
) {
super();
if (
environmentService.disableExperiments ||
environmentService.enableSmokeTestDriver ||
environmentService.extensionTestsLocationURI
) {
return; //not applicable in this environment
}
this.initializeExperiments();
}
private initializeExperiments(): void {
const firstSessionDateString = this.storageService.get(firstSessionDateStorageKey, StorageScope.APPLICATION) || new Date().toUTCString();
const daysSinceFirstSession = ((+new Date()) - (+new Date(firstSessionDateString))) / 1000 / 60 / 60 / 24;
if (daysSinceFirstSession > 1) {
// not a startup exp candidate.
return;
}
const experimentConfig = this.getExperimentConfiguration();
if (!experimentConfig) {
return;
}
// also check storage to see if this user has already seen the startup experience
const storageKey = `coreExperimentation.${experimentConfig.experimentName}`;
const storedExperiment = this.storageService.get(storageKey, StorageScope.APPLICATION);
if (storedExperiment) {
try {
const parsedExperiment: IExperiment = JSON.parse(storedExperiment);
this.experiments.set(experimentConfig.experimentName, parsedExperiment);
startupExpContext.bindTo(this.contextKeyService).set(parsedExperiment.experimentGroup);
return;
} catch (e) {
this.storageService.remove(storageKey, StorageScope.APPLICATION);
return;
}
}
const experiment = this.createStartupExperiment(experimentConfig.experimentName, experimentConfig);
if (experiment) {
this.experiments.set(experimentConfig.experimentName, experiment);
this.sendExperimentTelemetry(experimentConfig.experimentName, experiment);
startupExpContext.bindTo(this.contextKeyService).set(experiment.experimentGroup);
this.storageService.store(
storageKey,
JSON.stringify(experiment),
StorageScope.APPLICATION,
StorageTarget.MACHINE
);
}
}
private getExperimentConfiguration(): ExperimentConfiguration | undefined {
const quality = this.productService.quality;
if (!quality) {
return undefined;
}
return EXPERIMENT_CONFIGURATIONS[quality];
}
private createStartupExperiment(experimentName: string, experimentConfig: ExperimentConfiguration): IExperiment | undefined {
const startupExpGroupOverride = this.environmentService.startupExperimentGroup;
if (startupExpGroupOverride) {
// If the user has an override, we use that directly
const group = experimentConfig.groups.find(g => g.name === startupExpGroupOverride);
if (group) {
return {
cohort: 1,
subCohort: 1,
experimentGroup: group.name,
iteration: group.iteration,
isInExperiment: true
};
}
return undefined;
}
const cohort = Math.random();
if (cohort >= experimentConfig.targetPercentage / 100) {
return undefined;
}
// Normalize the cohort to the experiment range [0, targetPercentage/100]
const normalizedCohort = cohort / (experimentConfig.targetPercentage / 100);
// Find which group this user falls into
for (const group of experimentConfig.groups) {
if (normalizedCohort >= group.min && normalizedCohort < group.max) {
return {
cohort,
subCohort: normalizedCohort,
experimentGroup: group.name,
iteration: group.iteration,
isInExperiment: true
};
}
}
return undefined;
}
private sendExperimentTelemetry(experimentName: string, experiment: IExperiment): void {
type ExperimentCohortClassification = {
owner: 'bhavyaus';
comment: 'Records which experiment cohort the user is in for core experiments';
experimentName: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the experiment' };
cohort: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The exact cohort number for the user' };
subCohort: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The exact sub-cohort number for the user in the experiment cohort' };
experimentGroup: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The experiment group the user is in' };
iteration: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The iteration number for the experiment' };
isInExperiment: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user is in the experiment' };
};
type ExperimentCohortEvent = {
experimentName: string;
cohort: number;
subCohort: number;
experimentGroup: string;
iteration: number;
isInExperiment: boolean;
};
this.telemetryService.publicLog2<ExperimentCohortEvent, ExperimentCohortClassification>(
`coreExperimentation.experimentCohort`,
{
experimentName,
cohort: experiment.cohort,
subCohort: experiment.subCohort,
experimentGroup: experiment.experimentGroup,
iteration: experiment.iteration,
isInExperiment: experiment.isInExperiment
}
);
}
getExperiment(): IExperiment | undefined {
return this.experiments.get(STARTUP_EXPERIMENT_NAME);
}
}
registerSingleton(ICoreExperimentationService, CoreExperimentationService, InstantiationType.Eager);

View File

@@ -1,337 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import assert from 'assert';
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js';
import { MockContextKeyService } from '../../../../../platform/keybinding/test/common/mockKeybindingService.js';
import { CoreExperimentationService, startupExpContext } from '../../common/coreExperimentationService.js';
import { firstSessionDateStorageKey, ITelemetryService, ITelemetryData, TelemetryLevel } from '../../../../../platform/telemetry/common/telemetry.js';
import { StorageScope, StorageTarget } from '../../../../../platform/storage/common/storage.js';
import { TestStorageService } from '../../../../test/common/workbenchTestServices.js';
import { IProductService } from '../../../../../platform/product/common/productService.js';
import { IWorkbenchEnvironmentService } from '../../../environment/common/environmentService.js';
interface ITelemetryEvent {
eventName: string;
data: ITelemetryData;
}
class MockTelemetryService implements ITelemetryService {
declare readonly _serviceBrand: undefined;
public events: ITelemetryEvent[] = [];
public readonly telemetryLevel = TelemetryLevel.USAGE;
public readonly sessionId = 'test-session';
public readonly machineId = 'test-machine';
public readonly sqmId = 'test-sqm';
public readonly devDeviceId = 'test-device';
public readonly firstSessionDate = 'test-date';
public readonly sendErrorTelemetry = true;
publicLog2<E, T>(eventName: string, data?: E): void {
this.events.push({ eventName, data: (data as ITelemetryData) || {} });
}
publicLog(eventName: string, data?: ITelemetryData): void {
this.events.push({ eventName, data: data || {} });
}
publicLogError(eventName: string, data?: ITelemetryData): void {
this.events.push({ eventName, data: data || {} });
}
publicLogError2<E, T>(eventName: string, data?: E): void {
this.events.push({ eventName, data: (data as ITelemetryData) || {} });
}
setExperimentProperty(): void { }
}
class MockProductService implements IProductService {
declare readonly _serviceBrand: undefined;
public quality: string = 'stable';
get version() { return '1.0.0'; }
get commit() { return 'test-commit'; }
get nameLong() { return 'Test VSCode'; }
get nameShort() { return 'VSCode'; }
get applicationName() { return 'test-vscode'; }
get serverApplicationName() { return 'test-server'; }
get dataFolderName() { return '.test-vscode'; }
get urlProtocol() { return 'test-vscode'; }
get extensionAllowedProposedApi() { return []; }
get extensionProperties() { return {}; }
}
suite('CoreExperimentationService', () => {
const disposables = ensureNoDisposablesAreLeakedInTestSuite();
let storageService: TestStorageService;
let telemetryService: MockTelemetryService;
let productService: MockProductService;
let contextKeyService: MockContextKeyService;
let environmentService: IWorkbenchEnvironmentService;
setup(() => {
storageService = disposables.add(new TestStorageService());
telemetryService = new MockTelemetryService();
productService = new MockProductService();
contextKeyService = new MockContextKeyService();
environmentService = {} as IWorkbenchEnvironmentService;
});
test('should return experiment from storage if it exists', () => {
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
// Set that user has already seen the experiment
const existingExperiment = {
cohort: 0.5,
subCohort: 0.5,
experimentGroup: 'control',
iteration: 1,
isInExperiment: true
};
storageService.store('coreExperimentation.startup', JSON.stringify(existingExperiment), StorageScope.APPLICATION, StorageTarget.MACHINE);
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
// Should not return experiment again
assert.deepStrictEqual(service.getExperiment(), existingExperiment);
// No telemetry should be sent for new experiment
assert.strictEqual(telemetryService.events.length, 0);
});
test('should initialize experiment for new user in first session and set context key', () => {
// Set first session date to today
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
// Mock Math.random to return a value that puts user in experiment
const originalMathRandom = Math.random;
Math.random = () => 0.1; // 10% - should be in experiment for all quality levels
try {
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
// Should create experiment
const experiment = service.getExperiment();
assert(experiment, 'Experiment should be defined');
assert.strictEqual(experiment.isInExperiment, true);
assert.strictEqual(experiment.iteration, 1);
assert(experiment.cohort >= 0 && experiment.cohort < 1, 'Cohort should be between 0 and 1');
assert(['control', 'maximizedChat', 'splitEmptyEditorChat', 'splitWelcomeChat'].includes(experiment.experimentGroup),
'Experiment group should be one of the defined treatments');
// Context key should be set to experiment group
const contextValue = startupExpContext.getValue(contextKeyService);
assert.strictEqual(contextValue, experiment.experimentGroup,
'Context key should be set to experiment group');
} finally {
Math.random = originalMathRandom;
}
});
test('should emit telemetry when experiment is created', () => {
// Set first session date to today
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
// Mock Math.random to return a value that puts user in experiment
const originalMathRandom = Math.random;
Math.random = () => 0.1; // 10% - should be in experiment
try {
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
const experiment = service.getExperiment();
assert(experiment, 'Experiment should be defined');
// Check that telemetry was sent
assert.strictEqual(telemetryService.events.length, 1);
const telemetryEvent = telemetryService.events[0];
assert.strictEqual(telemetryEvent.eventName, 'coreExperimentation.experimentCohort');
// Verify telemetry data
const data = telemetryEvent.data as any;
assert.strictEqual(data.experimentName, 'startup');
assert.strictEqual(data.cohort, experiment.cohort);
assert.strictEqual(data.subCohort, experiment.subCohort);
assert.strictEqual(data.experimentGroup, experiment.experimentGroup);
assert.strictEqual(data.iteration, experiment.iteration);
assert.strictEqual(data.isInExperiment, experiment.isInExperiment);
} finally {
Math.random = originalMathRandom;
}
});
test('should not include user in experiment if random value exceeds target percentage', () => {
// Set first session date to today
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
productService.quality = 'stable'; // 20% target
// Mock Math.random to return a value outside experiment range
const originalMathRandom = Math.random;
Math.random = () => 0.25; // 25% - should be outside 20% target for stable
try {
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
// Should not create experiment
const experiment = service.getExperiment();
assert.strictEqual(experiment, undefined);
// No telemetry should be sent
assert.strictEqual(telemetryService.events.length, 0);
} finally {
Math.random = originalMathRandom;
}
});
test('should assign correct experiment group based on cohort normalization', () => {
// Set first session date to today
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
productService.quality = 'stable'; // 20% target
const testCases = [
{ random: 0.02, expectedGroup: 'control' }, // 2% -> 10% normalized -> first 25% of experiment
{ random: 0.07, expectedGroup: 'maximizedChat' }, // 7% -> 35% normalized -> second 25% of experiment
{ random: 0.12, expectedGroup: 'splitEmptyEditorChat' }, // 12% -> 60% normalized -> third 25% of experiment
{ random: 0.17, expectedGroup: 'splitWelcomeChat' } // 17% -> 85% normalized -> fourth 25% of experiment
];
const originalMathRandom = Math.random;
try {
for (const testCase of testCases) {
Math.random = () => testCase.random;
storageService.remove('coreExperimentation.startup', StorageScope.APPLICATION);
telemetryService.events = []; // Reset telemetry events
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
const experiment = service.getExperiment();
assert(experiment, `Experiment should be defined for random ${testCase.random}`);
assert.strictEqual(experiment.experimentGroup, testCase.expectedGroup,
`Expected group ${testCase.expectedGroup} for random ${testCase.random}, got ${experiment.experimentGroup}`);
}
} finally {
Math.random = originalMathRandom;
}
});
test('should store experiment in storage when created', () => {
// Set first session date to today
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
const originalMathRandom = Math.random;
Math.random = () => 0.1; // Ensure user is in experiment
try {
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
const experiment = service.getExperiment();
assert(experiment, 'Experiment should be defined');
// Check that experiment was stored
const storedValue = storageService.get('coreExperimentation.startup', StorageScope.APPLICATION);
assert(storedValue, 'Experiment should be stored');
const storedExperiment = JSON.parse(storedValue);
assert.strictEqual(storedExperiment.experimentGroup, experiment.experimentGroup);
assert.strictEqual(storedExperiment.iteration, experiment.iteration);
assert.strictEqual(storedExperiment.isInExperiment, experiment.isInExperiment);
assert.strictEqual(storedExperiment.cohort, experiment.cohort);
assert.strictEqual(storedExperiment.subCohort, experiment.subCohort);
} finally {
Math.random = originalMathRandom;
}
});
test('should handle missing first session date by using current date', () => {
// Don't set first session date - service should use current date
const originalMathRandom = Math.random;
Math.random = () => 0.1; // Ensure user would be in experiment
try {
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
const experiment = service.getExperiment();
assert(experiment, 'Experiment should be defined when first session date is missing');
assert.strictEqual(telemetryService.events.length, 1);
} finally {
Math.random = originalMathRandom;
}
});
test('should handle sub-cohort calculation correctly', () => {
// Set first session date to today
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
productService.quality = 'stable'; // 20% target
const originalMathRandom = Math.random;
Math.random = () => 0.1; // 10% cohort -> 50% normalized sub-cohort
try {
const service = disposables.add(new CoreExperimentationService(
storageService,
telemetryService,
productService,
contextKeyService,
environmentService
));
const experiment = service.getExperiment();
assert(experiment, 'Experiment should be defined');
// Verify sub-cohort calculation
const expectedSubCohort = 0.1 / (20 / 100); // 0.1 / 0.2 = 0.5
assert.strictEqual(experiment.subCohort, expectedSubCohort,
'Sub-cohort should be correctly normalized');
} finally {
Math.random = originalMathRandom;
}
});
});

View File

@@ -36,7 +36,6 @@ export interface IWorkbenchEnvironmentService extends IEnvironmentService {
readonly skipWelcome: boolean;
readonly disableWorkspaceTrust: boolean;
readonly webviewExternalEndpoint: string;
readonly startupExperimentGroup?: string;
// --- Development
readonly debugRenderer: boolean;

View File

@@ -147,15 +147,6 @@ export class NativeWorkbenchEnvironmentService extends AbstractNativeEnvironment
@memoize
get filesToWait(): IPathsToWaitFor | undefined { return this.configuration.filesToWait; }
@memoize
get startupExperimentGroup(): string | undefined {
const group = this.args['startup-experiment-group'];
if (typeof group === 'string') {
return group;
}
return undefined;
}
constructor(
private readonly configuration: INativeWindowConfiguration,
productService: IProductService

View File

@@ -129,7 +129,6 @@ import './services/userActivity/common/userActivityService.js';
import './services/userActivity/browser/userActivityBrowser.js';
import './services/editor/browser/editorPaneService.js';
import './services/editor/common/customEditorLabelService.js';
import './services/coreExperimentation/common/coreExperimentationService.js';
import './services/dataChannel/browser/dataChannelService.js';
import { InstantiationType, registerSingleton } from '../platform/instantiation/common/extensions.js';