mirror of
https://github.com/microsoft/vscode.git
synced 2026-04-16 22:14:20 -05:00
Remove startup experimentation code and setup splitWelcomeChat view as getting started experience (#262068)
* Remove startup experimentation code and setup splitWelcomeChat view as getting started experience * Remove experimental visibility check from welcome message in ChatWidget * cleanup --------- Co-authored-by: Benjamin Pasero <benjamin.pasero@microsoft.com>
This commit is contained in:
@@ -1137,7 +1137,7 @@ suite('vscode API - workspace', () => {
|
||||
assert.strictEqual(e.files[1].toString(), file2.toString());
|
||||
});
|
||||
|
||||
test('issue #107739 - Redo of rename Java Class name has no effect', async () => {
|
||||
test.skip('issue #107739 - Redo of rename Java Class name has no effect', async () => { // https://github.com/microsoft/vscode/issues/254042
|
||||
const file = await createRandomFile('hello');
|
||||
const fileName = basename(file.fsPath);
|
||||
|
||||
|
||||
@@ -139,7 +139,6 @@ export interface NativeParsedArgs {
|
||||
'unresponsive-sample-period'?: string;
|
||||
'enable-rdp-display-tracking'?: boolean;
|
||||
'disable-layout-restore'?: boolean;
|
||||
'startup-experiment-group'?: string;
|
||||
'disable-experiments'?: boolean;
|
||||
|
||||
// chromium command line args: https://electronjs.org/docs/all#supported-chrome-command-line-switches
|
||||
|
||||
@@ -202,7 +202,6 @@ export const OPTIONS: OptionDescriptions<Required<NativeParsedArgs>> = {
|
||||
'enable-rdp-display-tracking': { type: 'boolean' },
|
||||
'disable-layout-restore': { type: 'boolean' },
|
||||
'disable-experiments': { type: 'boolean' },
|
||||
'startup-experiment-group': { type: 'string', cat: 't', args: 'control|maximizedChat|splitEmptyEditorChat|splitWelcomeChat', description: localize('startupExperimentGroup', "Override the startup experiment group.") },
|
||||
|
||||
// chromium flags
|
||||
'no-proxy-server': { type: 'boolean' },
|
||||
|
||||
@@ -47,8 +47,6 @@ import { AuxiliaryBarPart } from './parts/auxiliarybar/auxiliaryBarPart.js';
|
||||
import { ITelemetryService } from '../../platform/telemetry/common/telemetry.js';
|
||||
import { IAuxiliaryWindowService } from '../services/auxiliaryWindow/browser/auxiliaryWindowService.js';
|
||||
import { CodeWindow, mainWindow } from '../../base/browser/window.js';
|
||||
import { ICoreExperimentationService, StartupExperimentGroup } from '../services/coreExperimentation/common/coreExperimentationService.js';
|
||||
import { Lazy } from '../../base/common/lazy.js';
|
||||
|
||||
//#region Layout Implementation
|
||||
|
||||
@@ -333,7 +331,7 @@ export abstract class Layout extends Disposable implements IWorkbenchLayoutServi
|
||||
this.registerLayoutListeners();
|
||||
|
||||
// State
|
||||
this.initLayoutState(accessor.get(ILifecycleService), accessor.get(IFileService), accessor.get(ICoreExperimentationService));
|
||||
this.initLayoutState(accessor.get(ILifecycleService), accessor.get(IFileService));
|
||||
}
|
||||
|
||||
private registerLayoutListeners(): void {
|
||||
@@ -627,10 +625,10 @@ export abstract class Layout extends Disposable implements IWorkbenchLayoutServi
|
||||
}
|
||||
}
|
||||
|
||||
private initLayoutState(lifecycleService: ILifecycleService, fileService: IFileService, coreExperimentationService: ICoreExperimentationService): void {
|
||||
private initLayoutState(lifecycleService: ILifecycleService, fileService: IFileService): void {
|
||||
this._mainContainerDimension = getClientArea(this.parent, DEFAULT_WINDOW_DIMENSIONS); // running with fallback to ensure no error is thrown (https://github.com/microsoft/vscode/issues/240242)
|
||||
|
||||
this.stateModel = new LayoutStateModel(this.storageService, this.configurationService, this.contextService, coreExperimentationService, this.environmentService, this.viewDescriptorService);
|
||||
this.stateModel = new LayoutStateModel(this.storageService, this.configurationService, this.contextService, this.environmentService, this.viewDescriptorService);
|
||||
this.stateModel.load({
|
||||
mainContainerDimension: this._mainContainerDimension,
|
||||
resetLayout: Boolean(this.layoutOptions?.resetLayout)
|
||||
@@ -2778,7 +2776,6 @@ class LayoutStateModel extends Disposable {
|
||||
private readonly storageService: IStorageService,
|
||||
private readonly configurationService: IConfigurationService,
|
||||
private readonly contextService: IWorkspaceContextService,
|
||||
private readonly coreExperimentationService: ICoreExperimentationService,
|
||||
private readonly environmentService: IBrowserWorkbenchEnvironmentService,
|
||||
private readonly viewDescriptorService: IViewDescriptorService
|
||||
) {
|
||||
@@ -2918,35 +2915,14 @@ class LayoutStateModel extends Disposable {
|
||||
|
||||
private applyOverrides(configuration: ILayoutStateLoadConfiguration): void {
|
||||
|
||||
// TODO@bpasero remove this startup experiment once settled
|
||||
const experiment = new Lazy(() => {
|
||||
try {
|
||||
return this.coreExperimentationService.getExperiment();
|
||||
} catch (error) {
|
||||
return undefined;
|
||||
}
|
||||
});
|
||||
|
||||
// Auxiliary bar: With experimental treatment for new users
|
||||
// Auxiliary bar: Showing for new users
|
||||
if (
|
||||
this.storageService.isNew(StorageScope.APPLICATION) &&
|
||||
this.contextService.getWorkbenchState() === WorkbenchState.EMPTY &&
|
||||
(
|
||||
experiment.value?.experimentGroup === StartupExperimentGroup.MaximizedChat ||
|
||||
experiment.value?.experimentGroup === StartupExperimentGroup.SplitEmptyEditorChat ||
|
||||
experiment.value?.experimentGroup === StartupExperimentGroup.SplitWelcomeChat
|
||||
)
|
||||
this.contextService.getWorkbenchState() === WorkbenchState.EMPTY
|
||||
) {
|
||||
if (experiment.value.experimentGroup === StartupExperimentGroup.MaximizedChat) {
|
||||
this.applyAuxiliaryBarMaximizedOverride();
|
||||
} else if (
|
||||
experiment.value.experimentGroup === StartupExperimentGroup.SplitEmptyEditorChat ||
|
||||
experiment.value.experimentGroup === StartupExperimentGroup.SplitWelcomeChat
|
||||
) {
|
||||
const mainContainerDimension = configuration.mainContainerDimension;
|
||||
this.setRuntimeValue(LayoutStateKeys.AUXILIARYBAR_HIDDEN, false);
|
||||
this.setInitializationValue(LayoutStateKeys.AUXILIARYBAR_SIZE, Math.ceil(mainContainerDimension.width / (1.618 * 1.618 /* golden ratio */)));
|
||||
}
|
||||
const mainContainerDimension = configuration.mainContainerDimension;
|
||||
this.setRuntimeValue(LayoutStateKeys.AUXILIARYBAR_HIDDEN, false);
|
||||
this.setInitializationValue(LayoutStateKeys.AUXILIARYBAR_SIZE, Math.ceil(mainContainerDimension.width / (1.618 * 1.618 /* golden ratio */)));
|
||||
}
|
||||
|
||||
// Auxiliary bar: Based on setting for new workspaces
|
||||
|
||||
@@ -72,7 +72,6 @@ import { ChatTodoListWidget } from './chatContentParts/chatTodoListWidget.js';
|
||||
import { PromptsConfig } from '../common/promptSyntax/config/config.js';
|
||||
import { CancellationToken } from '../../../../base/common/cancellation.js';
|
||||
import { ComputeAutomaticInstructions } from '../common/promptSyntax/computeAutomaticInstructions.js';
|
||||
import { startupExpContext, StartupExperimentGroup } from '../../../services/coreExperimentation/common/coreExperimentationService.js';
|
||||
import { IWorkspaceContextService, WorkbenchState } from '../../../../platform/workspace/common/workspace.js';
|
||||
import { IMouseWheelEvent } from '../../../../base/browser/mouseEvent.js';
|
||||
import { TodoListToolSettingId as TodoListToolSettingId } from '../common/tools/manageTodoListTool.js';
|
||||
@@ -490,6 +489,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
|
||||
const renderFollowups = this.viewOptions.renderFollowups ?? false;
|
||||
const renderStyle = this.viewOptions.renderStyle;
|
||||
this.createInput(this.container, { renderFollowups, renderStyle });
|
||||
this.inputPart.initForNewChatModel(this.getViewState(), true);
|
||||
}
|
||||
}
|
||||
}));
|
||||
@@ -721,6 +721,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
|
||||
const renderFollowups = this.viewOptions.renderFollowups ?? false;
|
||||
const renderStyle = this.viewOptions.renderStyle;
|
||||
this.createInput(this.container, { renderFollowups, renderStyle });
|
||||
this.inputPart.initForNewChatModel(this.getViewState(), true);
|
||||
}
|
||||
|
||||
this.renderWelcomeViewContentIfNeeded();
|
||||
@@ -775,10 +776,6 @@ export class ChatWidget extends Disposable implements IChatWidget {
|
||||
const numItems = this.viewModel?.getItems().length ?? 0;
|
||||
if (!numItems) {
|
||||
dom.clearNode(this.welcomeMessageContainer);
|
||||
// TODO@bhavyaus remove this startup experiment once settled
|
||||
const startupExpValue = startupExpContext.getValue(this.contextKeyService);
|
||||
const configuration = this.configurationService.inspect('workbench.secondarySideBar.defaultVisibility');
|
||||
const expIsActive = configuration.defaultValue !== 'hidden';
|
||||
|
||||
const expEmptyState = this.configurationService.getValue<boolean>('chat.emptyChatState.enabled');
|
||||
|
||||
@@ -790,10 +787,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
|
||||
let welcomeContent: IChatViewWelcomeContent;
|
||||
const defaultAgent = this.chatAgentService.getDefaultAgent(this.location, this.input.currentModeKind);
|
||||
const additionalMessage = defaultAgent?.metadata.additionalWelcomeMessage;
|
||||
if ((startupExpValue === StartupExperimentGroup.MaximizedChat
|
||||
|| startupExpValue === StartupExperimentGroup.SplitEmptyEditorChat
|
||||
|| startupExpValue === StartupExperimentGroup.SplitWelcomeChat
|
||||
|| expIsActive) && this.contextKeyService.contextMatchesRules(chatSetupTriggerContext)) {
|
||||
if (this.contextKeyService.contextMatchesRules(chatSetupTriggerContext)) {
|
||||
welcomeContent = this.getExpWelcomeViewContent();
|
||||
this.container.classList.add('experimental-welcome-view');
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ import { IOpenerService } from '../../../../platform/opener/common/opener.js';
|
||||
import { IProductService } from '../../../../platform/product/common/productService.js';
|
||||
import { IQuickInputService } from '../../../../platform/quickinput/common/quickInput.js';
|
||||
import { IStorageService, StorageScope, StorageTarget, WillSaveStateReason } from '../../../../platform/storage/common/storage.js';
|
||||
import { ITelemetryService, TelemetryLevel, firstSessionDateStorageKey } from '../../../../platform/telemetry/common/telemetry.js';
|
||||
import { ITelemetryService, TelemetryLevel } from '../../../../platform/telemetry/common/telemetry.js';
|
||||
import { getTelemetryLevel } from '../../../../platform/telemetry/common/telemetryUtils.js';
|
||||
import { defaultButtonStyles, defaultKeybindingLabelStyles, defaultToggleStyles } from '../../../../platform/theme/browser/defaultStyles.js';
|
||||
import { IWindowOpenable } from '../../../../platform/window/common/window.js';
|
||||
@@ -73,7 +73,6 @@ import { AccessibilityVerbositySettingId } from '../../accessibility/browser/acc
|
||||
import { AccessibleViewAction } from '../../accessibility/browser/accessibleViewActions.js';
|
||||
import { KeybindingLabel } from '../../../../base/browser/ui/keybindingLabel/keybindingLabel.js';
|
||||
import { ScrollbarVisibility } from '../../../../base/common/scrollable.js';
|
||||
import { startupExpContext, StartupExperimentGroup } from '../../../services/coreExperimentation/common/coreExperimentationService.js';
|
||||
|
||||
const SLIDE_TRANSITION_TIME_MS = 250;
|
||||
const configurationKey = 'workbench.startupEditor';
|
||||
@@ -149,7 +148,6 @@ export class GettingStartedPage extends EditorPane {
|
||||
|
||||
private contextService: IContextKeyService;
|
||||
|
||||
private hasScrolledToFirstCategory = false;
|
||||
private recentlyOpenedList?: GettingStartedIndexList<RecentEntry>;
|
||||
private startList?: GettingStartedIndexList<IWelcomePageStartEntry>;
|
||||
private gettingStartedList?: GettingStartedIndexList<IResolvedWalkthrough>;
|
||||
@@ -950,33 +948,10 @@ export class GettingStartedPage extends EditorPane {
|
||||
}
|
||||
}
|
||||
|
||||
const someStepsComplete = this.gettingStartedCategories.some(category => category.steps.find(s => s.done));
|
||||
if (this.editorInput.showTelemetryNotice && this.productService.openToWelcomeMainPage) {
|
||||
const telemetryNotice = $('p.telemetry-notice');
|
||||
this.buildTelemetryFooter(telemetryNotice);
|
||||
footer.appendChild(telemetryNotice);
|
||||
} else if (!this.productService.openToWelcomeMainPage && !someStepsComplete && !this.hasScrolledToFirstCategory && this.showFeaturedWalkthrough) {
|
||||
const firstSessionDateString = this.storageService.get(firstSessionDateStorageKey, StorageScope.APPLICATION) || new Date().toUTCString();
|
||||
const daysSinceFirstSession = ((+new Date()) - (+new Date(firstSessionDateString))) / 1000 / 60 / 60 / 24;
|
||||
const fistContentBehaviour = daysSinceFirstSession < 1 ? 'openToFirstCategory' : 'index';
|
||||
const startupExpValue = startupExpContext.getValue(this.contextService);
|
||||
|
||||
if (fistContentBehaviour === 'openToFirstCategory' && ((!startupExpValue || startupExpValue === '' || startupExpValue === StartupExperimentGroup.Control))) {
|
||||
const first = this.gettingStartedCategories.filter(c => !c.when || this.contextService.contextMatchesRules(c.when))[0];
|
||||
if (first) {
|
||||
this.hasScrolledToFirstCategory = true;
|
||||
this.currentWalkthrough = first;
|
||||
this.editorInput.selectedCategory = this.currentWalkthrough?.id;
|
||||
this.editorInput.walkthroughPageTitle = this.currentWalkthrough.walkthroughPageTitle;
|
||||
if (first.id === NEW_WELCOME_EXPERIENCE) {
|
||||
this.buildNewCategorySlide(this.editorInput.selectedCategory, undefined);
|
||||
} else {
|
||||
this.buildCategorySlide(this.editorInput.selectedCategory, undefined);
|
||||
}
|
||||
this.setSlide('details', true /* firstLaunch */);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.setSlide('categories');
|
||||
|
||||
@@ -29,7 +29,6 @@ import { IEditorResolverService, RegisteredEditorPriority } from '../../../servi
|
||||
import { TerminalCommandId } from '../../terminal/common/terminal.js';
|
||||
import { ILogService } from '../../../../platform/log/common/log.js';
|
||||
import { IContextKeyService } from '../../../../platform/contextkey/common/contextkey.js';
|
||||
import { startupExpContext, StartupExperimentGroup } from '../../../services/coreExperimentation/common/coreExperimentationService.js';
|
||||
import { AuxiliaryBarMaximizedContext } from '../../../common/contextkeys.js';
|
||||
|
||||
export const restoreWalkthroughsConfigurationKey = 'workbench.welcomePage.restorableWalkthroughs';
|
||||
@@ -141,12 +140,6 @@ export class StartupPageRunnerContribution extends Disposable implements IWorkbe
|
||||
if (startupEditorSetting.value === 'readme') {
|
||||
await this.openReadme();
|
||||
} else if (startupEditorSetting.value === 'welcomePage' || startupEditorSetting.value === 'welcomePageInEmptyWorkbench') {
|
||||
if (this.storageService.isNew(StorageScope.APPLICATION)) {
|
||||
const startupExpValue = startupExpContext.getValue(this.contextKeyService);
|
||||
if (startupExpValue === StartupExperimentGroup.MaximizedChat || startupExpValue === StartupExperimentGroup.SplitEmptyEditorChat) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
await this.openGettingStarted(true);
|
||||
} else if (startupEditorSetting.value === 'terminal') {
|
||||
this.commandService.executeCommand(TerminalCommandId.CreateTerminalEditor);
|
||||
|
||||
@@ -1,234 +0,0 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { Disposable } from '../../../../base/common/lifecycle.js';
|
||||
import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js';
|
||||
import { IProductService } from '../../../../platform/product/common/productService.js';
|
||||
import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js';
|
||||
import { firstSessionDateStorageKey, ITelemetryService } from '../../../../platform/telemetry/common/telemetry.js';
|
||||
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
||||
import { IContextKeyService, RawContextKey } from '../../../../platform/contextkey/common/contextkey.js';
|
||||
import { IWorkbenchEnvironmentService } from '../../environment/common/environmentService.js';
|
||||
|
||||
export const ICoreExperimentationService = createDecorator<ICoreExperimentationService>('coreExperimentationService');
|
||||
export const startupExpContext = new RawContextKey<string>('coreExperimentation.startupExpGroup', '');
|
||||
|
||||
interface IExperiment {
|
||||
cohort: number;
|
||||
subCohort: number; // Optional for future use
|
||||
experimentGroup: StartupExperimentGroup;
|
||||
iteration: number;
|
||||
isInExperiment: boolean;
|
||||
}
|
||||
|
||||
export interface ICoreExperimentationService {
|
||||
readonly _serviceBrand: undefined;
|
||||
getExperiment(): IExperiment | undefined;
|
||||
}
|
||||
|
||||
interface ExperimentGroupDefinition {
|
||||
name: StartupExperimentGroup;
|
||||
min: number;
|
||||
max: number;
|
||||
iteration: number;
|
||||
}
|
||||
|
||||
interface ExperimentConfiguration {
|
||||
experimentName: string;
|
||||
targetPercentage: number;
|
||||
groups: ExperimentGroupDefinition[];
|
||||
}
|
||||
|
||||
export enum StartupExperimentGroup {
|
||||
Control = 'control',
|
||||
MaximizedChat = 'maximizedChat',
|
||||
SplitEmptyEditorChat = 'splitEmptyEditorChat',
|
||||
SplitWelcomeChat = 'splitWelcomeChat'
|
||||
}
|
||||
|
||||
export const STARTUP_EXPERIMENT_NAME = 'startup';
|
||||
|
||||
const EXPERIMENT_CONFIGURATIONS: Record<string, ExperimentConfiguration> = {
|
||||
stable: {
|
||||
experimentName: STARTUP_EXPERIMENT_NAME,
|
||||
targetPercentage: 20,
|
||||
groups: [
|
||||
// Bump the iteration each time we change group allocations
|
||||
{ name: StartupExperimentGroup.Control, min: 0.0, max: 0.25, iteration: 1 },
|
||||
{ name: StartupExperimentGroup.MaximizedChat, min: 0.25, max: 0.5, iteration: 1 },
|
||||
{ name: StartupExperimentGroup.SplitEmptyEditorChat, min: 0.5, max: 0.75, iteration: 1 },
|
||||
{ name: StartupExperimentGroup.SplitWelcomeChat, min: 0.75, max: 1.0, iteration: 1 }
|
||||
]
|
||||
},
|
||||
insider: {
|
||||
experimentName: STARTUP_EXPERIMENT_NAME,
|
||||
targetPercentage: 50,
|
||||
groups: [
|
||||
// Bump the iteration each time we change group allocations
|
||||
{ name: StartupExperimentGroup.Control, min: 0.0, max: 0.25, iteration: 1 },
|
||||
{ name: StartupExperimentGroup.MaximizedChat, min: 0.25, max: 0.5, iteration: 1 },
|
||||
{ name: StartupExperimentGroup.SplitEmptyEditorChat, min: 0.5, max: 0.75, iteration: 1 },
|
||||
{ name: StartupExperimentGroup.SplitWelcomeChat, min: 0.75, max: 1.0, iteration: 1 }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
export class CoreExperimentationService extends Disposable implements ICoreExperimentationService {
|
||||
declare readonly _serviceBrand: undefined;
|
||||
|
||||
private readonly experiments = new Map<string, IExperiment>();
|
||||
|
||||
constructor(
|
||||
@IStorageService private readonly storageService: IStorageService,
|
||||
@ITelemetryService private readonly telemetryService: ITelemetryService,
|
||||
@IProductService private readonly productService: IProductService,
|
||||
@IContextKeyService private readonly contextKeyService: IContextKeyService,
|
||||
@IWorkbenchEnvironmentService private readonly environmentService: IWorkbenchEnvironmentService,
|
||||
) {
|
||||
super();
|
||||
|
||||
if (
|
||||
environmentService.disableExperiments ||
|
||||
environmentService.enableSmokeTestDriver ||
|
||||
environmentService.extensionTestsLocationURI
|
||||
) {
|
||||
return; //not applicable in this environment
|
||||
}
|
||||
|
||||
this.initializeExperiments();
|
||||
}
|
||||
|
||||
private initializeExperiments(): void {
|
||||
|
||||
const firstSessionDateString = this.storageService.get(firstSessionDateStorageKey, StorageScope.APPLICATION) || new Date().toUTCString();
|
||||
const daysSinceFirstSession = ((+new Date()) - (+new Date(firstSessionDateString))) / 1000 / 60 / 60 / 24;
|
||||
if (daysSinceFirstSession > 1) {
|
||||
// not a startup exp candidate.
|
||||
return;
|
||||
}
|
||||
|
||||
const experimentConfig = this.getExperimentConfiguration();
|
||||
if (!experimentConfig) {
|
||||
return;
|
||||
}
|
||||
|
||||
// also check storage to see if this user has already seen the startup experience
|
||||
const storageKey = `coreExperimentation.${experimentConfig.experimentName}`;
|
||||
const storedExperiment = this.storageService.get(storageKey, StorageScope.APPLICATION);
|
||||
if (storedExperiment) {
|
||||
try {
|
||||
const parsedExperiment: IExperiment = JSON.parse(storedExperiment);
|
||||
this.experiments.set(experimentConfig.experimentName, parsedExperiment);
|
||||
startupExpContext.bindTo(this.contextKeyService).set(parsedExperiment.experimentGroup);
|
||||
return;
|
||||
} catch (e) {
|
||||
this.storageService.remove(storageKey, StorageScope.APPLICATION);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const experiment = this.createStartupExperiment(experimentConfig.experimentName, experimentConfig);
|
||||
if (experiment) {
|
||||
this.experiments.set(experimentConfig.experimentName, experiment);
|
||||
this.sendExperimentTelemetry(experimentConfig.experimentName, experiment);
|
||||
startupExpContext.bindTo(this.contextKeyService).set(experiment.experimentGroup);
|
||||
this.storageService.store(
|
||||
storageKey,
|
||||
JSON.stringify(experiment),
|
||||
StorageScope.APPLICATION,
|
||||
StorageTarget.MACHINE
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private getExperimentConfiguration(): ExperimentConfiguration | undefined {
|
||||
const quality = this.productService.quality;
|
||||
if (!quality) {
|
||||
return undefined;
|
||||
}
|
||||
return EXPERIMENT_CONFIGURATIONS[quality];
|
||||
}
|
||||
|
||||
private createStartupExperiment(experimentName: string, experimentConfig: ExperimentConfiguration): IExperiment | undefined {
|
||||
const startupExpGroupOverride = this.environmentService.startupExperimentGroup;
|
||||
if (startupExpGroupOverride) {
|
||||
// If the user has an override, we use that directly
|
||||
const group = experimentConfig.groups.find(g => g.name === startupExpGroupOverride);
|
||||
if (group) {
|
||||
return {
|
||||
cohort: 1,
|
||||
subCohort: 1,
|
||||
experimentGroup: group.name,
|
||||
iteration: group.iteration,
|
||||
isInExperiment: true
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const cohort = Math.random();
|
||||
|
||||
if (cohort >= experimentConfig.targetPercentage / 100) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Normalize the cohort to the experiment range [0, targetPercentage/100]
|
||||
const normalizedCohort = cohort / (experimentConfig.targetPercentage / 100);
|
||||
|
||||
// Find which group this user falls into
|
||||
for (const group of experimentConfig.groups) {
|
||||
if (normalizedCohort >= group.min && normalizedCohort < group.max) {
|
||||
return {
|
||||
cohort,
|
||||
subCohort: normalizedCohort,
|
||||
experimentGroup: group.name,
|
||||
iteration: group.iteration,
|
||||
isInExperiment: true
|
||||
};
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private sendExperimentTelemetry(experimentName: string, experiment: IExperiment): void {
|
||||
type ExperimentCohortClassification = {
|
||||
owner: 'bhavyaus';
|
||||
comment: 'Records which experiment cohort the user is in for core experiments';
|
||||
experimentName: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the experiment' };
|
||||
cohort: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The exact cohort number for the user' };
|
||||
subCohort: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The exact sub-cohort number for the user in the experiment cohort' };
|
||||
experimentGroup: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The experiment group the user is in' };
|
||||
iteration: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The iteration number for the experiment' };
|
||||
isInExperiment: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user is in the experiment' };
|
||||
};
|
||||
|
||||
type ExperimentCohortEvent = {
|
||||
experimentName: string;
|
||||
cohort: number;
|
||||
subCohort: number;
|
||||
experimentGroup: string;
|
||||
iteration: number;
|
||||
isInExperiment: boolean;
|
||||
};
|
||||
|
||||
this.telemetryService.publicLog2<ExperimentCohortEvent, ExperimentCohortClassification>(
|
||||
`coreExperimentation.experimentCohort`,
|
||||
{
|
||||
experimentName,
|
||||
cohort: experiment.cohort,
|
||||
subCohort: experiment.subCohort,
|
||||
experimentGroup: experiment.experimentGroup,
|
||||
iteration: experiment.iteration,
|
||||
isInExperiment: experiment.isInExperiment
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
getExperiment(): IExperiment | undefined {
|
||||
return this.experiments.get(STARTUP_EXPERIMENT_NAME);
|
||||
}
|
||||
}
|
||||
|
||||
registerSingleton(ICoreExperimentationService, CoreExperimentationService, InstantiationType.Eager);
|
||||
@@ -1,337 +0,0 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import assert from 'assert';
|
||||
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js';
|
||||
import { MockContextKeyService } from '../../../../../platform/keybinding/test/common/mockKeybindingService.js';
|
||||
import { CoreExperimentationService, startupExpContext } from '../../common/coreExperimentationService.js';
|
||||
import { firstSessionDateStorageKey, ITelemetryService, ITelemetryData, TelemetryLevel } from '../../../../../platform/telemetry/common/telemetry.js';
|
||||
import { StorageScope, StorageTarget } from '../../../../../platform/storage/common/storage.js';
|
||||
import { TestStorageService } from '../../../../test/common/workbenchTestServices.js';
|
||||
import { IProductService } from '../../../../../platform/product/common/productService.js';
|
||||
import { IWorkbenchEnvironmentService } from '../../../environment/common/environmentService.js';
|
||||
|
||||
interface ITelemetryEvent {
|
||||
eventName: string;
|
||||
data: ITelemetryData;
|
||||
}
|
||||
|
||||
class MockTelemetryService implements ITelemetryService {
|
||||
declare readonly _serviceBrand: undefined;
|
||||
|
||||
public events: ITelemetryEvent[] = [];
|
||||
public readonly telemetryLevel = TelemetryLevel.USAGE;
|
||||
public readonly sessionId = 'test-session';
|
||||
public readonly machineId = 'test-machine';
|
||||
public readonly sqmId = 'test-sqm';
|
||||
public readonly devDeviceId = 'test-device';
|
||||
public readonly firstSessionDate = 'test-date';
|
||||
public readonly sendErrorTelemetry = true;
|
||||
|
||||
publicLog2<E, T>(eventName: string, data?: E): void {
|
||||
this.events.push({ eventName, data: (data as ITelemetryData) || {} });
|
||||
}
|
||||
|
||||
publicLog(eventName: string, data?: ITelemetryData): void {
|
||||
this.events.push({ eventName, data: data || {} });
|
||||
}
|
||||
|
||||
publicLogError(eventName: string, data?: ITelemetryData): void {
|
||||
this.events.push({ eventName, data: data || {} });
|
||||
}
|
||||
|
||||
publicLogError2<E, T>(eventName: string, data?: E): void {
|
||||
this.events.push({ eventName, data: (data as ITelemetryData) || {} });
|
||||
}
|
||||
|
||||
setExperimentProperty(): void { }
|
||||
}
|
||||
|
||||
class MockProductService implements IProductService {
|
||||
declare readonly _serviceBrand: undefined;
|
||||
|
||||
public quality: string = 'stable';
|
||||
|
||||
get version() { return '1.0.0'; }
|
||||
get commit() { return 'test-commit'; }
|
||||
get nameLong() { return 'Test VSCode'; }
|
||||
get nameShort() { return 'VSCode'; }
|
||||
get applicationName() { return 'test-vscode'; }
|
||||
get serverApplicationName() { return 'test-server'; }
|
||||
get dataFolderName() { return '.test-vscode'; }
|
||||
get urlProtocol() { return 'test-vscode'; }
|
||||
get extensionAllowedProposedApi() { return []; }
|
||||
get extensionProperties() { return {}; }
|
||||
}
|
||||
|
||||
suite('CoreExperimentationService', () => {
|
||||
const disposables = ensureNoDisposablesAreLeakedInTestSuite();
|
||||
|
||||
let storageService: TestStorageService;
|
||||
let telemetryService: MockTelemetryService;
|
||||
let productService: MockProductService;
|
||||
let contextKeyService: MockContextKeyService;
|
||||
let environmentService: IWorkbenchEnvironmentService;
|
||||
|
||||
setup(() => {
|
||||
storageService = disposables.add(new TestStorageService());
|
||||
telemetryService = new MockTelemetryService();
|
||||
productService = new MockProductService();
|
||||
contextKeyService = new MockContextKeyService();
|
||||
environmentService = {} as IWorkbenchEnvironmentService;
|
||||
});
|
||||
|
||||
test('should return experiment from storage if it exists', () => {
|
||||
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
|
||||
// Set that user has already seen the experiment
|
||||
const existingExperiment = {
|
||||
cohort: 0.5,
|
||||
subCohort: 0.5,
|
||||
experimentGroup: 'control',
|
||||
iteration: 1,
|
||||
isInExperiment: true
|
||||
};
|
||||
storageService.store('coreExperimentation.startup', JSON.stringify(existingExperiment), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
// Should not return experiment again
|
||||
assert.deepStrictEqual(service.getExperiment(), existingExperiment);
|
||||
|
||||
// No telemetry should be sent for new experiment
|
||||
assert.strictEqual(telemetryService.events.length, 0);
|
||||
});
|
||||
|
||||
test('should initialize experiment for new user in first session and set context key', () => {
|
||||
// Set first session date to today
|
||||
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
|
||||
// Mock Math.random to return a value that puts user in experiment
|
||||
const originalMathRandom = Math.random;
|
||||
Math.random = () => 0.1; // 10% - should be in experiment for all quality levels
|
||||
|
||||
try {
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
// Should create experiment
|
||||
const experiment = service.getExperiment();
|
||||
assert(experiment, 'Experiment should be defined');
|
||||
assert.strictEqual(experiment.isInExperiment, true);
|
||||
assert.strictEqual(experiment.iteration, 1);
|
||||
assert(experiment.cohort >= 0 && experiment.cohort < 1, 'Cohort should be between 0 and 1');
|
||||
assert(['control', 'maximizedChat', 'splitEmptyEditorChat', 'splitWelcomeChat'].includes(experiment.experimentGroup),
|
||||
'Experiment group should be one of the defined treatments');
|
||||
|
||||
// Context key should be set to experiment group
|
||||
const contextValue = startupExpContext.getValue(contextKeyService);
|
||||
assert.strictEqual(contextValue, experiment.experimentGroup,
|
||||
'Context key should be set to experiment group');
|
||||
} finally {
|
||||
Math.random = originalMathRandom;
|
||||
}
|
||||
});
|
||||
|
||||
test('should emit telemetry when experiment is created', () => {
|
||||
// Set first session date to today
|
||||
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
|
||||
// Mock Math.random to return a value that puts user in experiment
|
||||
const originalMathRandom = Math.random;
|
||||
Math.random = () => 0.1; // 10% - should be in experiment
|
||||
|
||||
try {
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
const experiment = service.getExperiment();
|
||||
assert(experiment, 'Experiment should be defined');
|
||||
|
||||
// Check that telemetry was sent
|
||||
assert.strictEqual(telemetryService.events.length, 1);
|
||||
const telemetryEvent = telemetryService.events[0];
|
||||
assert.strictEqual(telemetryEvent.eventName, 'coreExperimentation.experimentCohort');
|
||||
// Verify telemetry data
|
||||
const data = telemetryEvent.data as any;
|
||||
assert.strictEqual(data.experimentName, 'startup');
|
||||
assert.strictEqual(data.cohort, experiment.cohort);
|
||||
assert.strictEqual(data.subCohort, experiment.subCohort);
|
||||
assert.strictEqual(data.experimentGroup, experiment.experimentGroup);
|
||||
assert.strictEqual(data.iteration, experiment.iteration);
|
||||
assert.strictEqual(data.isInExperiment, experiment.isInExperiment);
|
||||
} finally {
|
||||
Math.random = originalMathRandom;
|
||||
}
|
||||
});
|
||||
|
||||
test('should not include user in experiment if random value exceeds target percentage', () => {
|
||||
// Set first session date to today
|
||||
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
productService.quality = 'stable'; // 20% target
|
||||
|
||||
// Mock Math.random to return a value outside experiment range
|
||||
const originalMathRandom = Math.random;
|
||||
Math.random = () => 0.25; // 25% - should be outside 20% target for stable
|
||||
|
||||
try {
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
// Should not create experiment
|
||||
const experiment = service.getExperiment();
|
||||
assert.strictEqual(experiment, undefined);
|
||||
|
||||
// No telemetry should be sent
|
||||
assert.strictEqual(telemetryService.events.length, 0);
|
||||
} finally {
|
||||
Math.random = originalMathRandom;
|
||||
}
|
||||
});
|
||||
|
||||
test('should assign correct experiment group based on cohort normalization', () => {
|
||||
// Set first session date to today
|
||||
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
productService.quality = 'stable'; // 20% target
|
||||
|
||||
const testCases = [
|
||||
{ random: 0.02, expectedGroup: 'control' }, // 2% -> 10% normalized -> first 25% of experiment
|
||||
{ random: 0.07, expectedGroup: 'maximizedChat' }, // 7% -> 35% normalized -> second 25% of experiment
|
||||
{ random: 0.12, expectedGroup: 'splitEmptyEditorChat' }, // 12% -> 60% normalized -> third 25% of experiment
|
||||
{ random: 0.17, expectedGroup: 'splitWelcomeChat' } // 17% -> 85% normalized -> fourth 25% of experiment
|
||||
];
|
||||
|
||||
const originalMathRandom = Math.random;
|
||||
|
||||
try {
|
||||
for (const testCase of testCases) {
|
||||
Math.random = () => testCase.random;
|
||||
storageService.remove('coreExperimentation.startup', StorageScope.APPLICATION);
|
||||
telemetryService.events = []; // Reset telemetry events
|
||||
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
const experiment = service.getExperiment();
|
||||
assert(experiment, `Experiment should be defined for random ${testCase.random}`);
|
||||
assert.strictEqual(experiment.experimentGroup, testCase.expectedGroup,
|
||||
`Expected group ${testCase.expectedGroup} for random ${testCase.random}, got ${experiment.experimentGroup}`);
|
||||
}
|
||||
} finally {
|
||||
Math.random = originalMathRandom;
|
||||
}
|
||||
});
|
||||
|
||||
test('should store experiment in storage when created', () => {
|
||||
// Set first session date to today
|
||||
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
|
||||
const originalMathRandom = Math.random;
|
||||
Math.random = () => 0.1; // Ensure user is in experiment
|
||||
|
||||
try {
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
const experiment = service.getExperiment();
|
||||
assert(experiment, 'Experiment should be defined');
|
||||
|
||||
// Check that experiment was stored
|
||||
const storedValue = storageService.get('coreExperimentation.startup', StorageScope.APPLICATION);
|
||||
assert(storedValue, 'Experiment should be stored');
|
||||
|
||||
const storedExperiment = JSON.parse(storedValue);
|
||||
assert.strictEqual(storedExperiment.experimentGroup, experiment.experimentGroup);
|
||||
assert.strictEqual(storedExperiment.iteration, experiment.iteration);
|
||||
assert.strictEqual(storedExperiment.isInExperiment, experiment.isInExperiment);
|
||||
assert.strictEqual(storedExperiment.cohort, experiment.cohort);
|
||||
assert.strictEqual(storedExperiment.subCohort, experiment.subCohort);
|
||||
} finally {
|
||||
Math.random = originalMathRandom;
|
||||
}
|
||||
});
|
||||
|
||||
test('should handle missing first session date by using current date', () => {
|
||||
// Don't set first session date - service should use current date
|
||||
const originalMathRandom = Math.random;
|
||||
Math.random = () => 0.1; // Ensure user would be in experiment
|
||||
|
||||
try {
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
const experiment = service.getExperiment();
|
||||
assert(experiment, 'Experiment should be defined when first session date is missing');
|
||||
assert.strictEqual(telemetryService.events.length, 1);
|
||||
} finally {
|
||||
Math.random = originalMathRandom;
|
||||
}
|
||||
});
|
||||
|
||||
test('should handle sub-cohort calculation correctly', () => {
|
||||
// Set first session date to today
|
||||
storageService.store(firstSessionDateStorageKey, new Date().toUTCString(), StorageScope.APPLICATION, StorageTarget.MACHINE);
|
||||
productService.quality = 'stable'; // 20% target
|
||||
|
||||
const originalMathRandom = Math.random;
|
||||
Math.random = () => 0.1; // 10% cohort -> 50% normalized sub-cohort
|
||||
|
||||
try {
|
||||
const service = disposables.add(new CoreExperimentationService(
|
||||
storageService,
|
||||
telemetryService,
|
||||
productService,
|
||||
contextKeyService,
|
||||
environmentService
|
||||
));
|
||||
|
||||
const experiment = service.getExperiment();
|
||||
assert(experiment, 'Experiment should be defined');
|
||||
|
||||
// Verify sub-cohort calculation
|
||||
const expectedSubCohort = 0.1 / (20 / 100); // 0.1 / 0.2 = 0.5
|
||||
assert.strictEqual(experiment.subCohort, expectedSubCohort,
|
||||
'Sub-cohort should be correctly normalized');
|
||||
} finally {
|
||||
Math.random = originalMathRandom;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -36,7 +36,6 @@ export interface IWorkbenchEnvironmentService extends IEnvironmentService {
|
||||
readonly skipWelcome: boolean;
|
||||
readonly disableWorkspaceTrust: boolean;
|
||||
readonly webviewExternalEndpoint: string;
|
||||
readonly startupExperimentGroup?: string;
|
||||
|
||||
// --- Development
|
||||
readonly debugRenderer: boolean;
|
||||
|
||||
@@ -147,15 +147,6 @@ export class NativeWorkbenchEnvironmentService extends AbstractNativeEnvironment
|
||||
@memoize
|
||||
get filesToWait(): IPathsToWaitFor | undefined { return this.configuration.filesToWait; }
|
||||
|
||||
@memoize
|
||||
get startupExperimentGroup(): string | undefined {
|
||||
const group = this.args['startup-experiment-group'];
|
||||
if (typeof group === 'string') {
|
||||
return group;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly configuration: INativeWindowConfiguration,
|
||||
productService: IProductService
|
||||
|
||||
@@ -129,7 +129,6 @@ import './services/userActivity/common/userActivityService.js';
|
||||
import './services/userActivity/browser/userActivityBrowser.js';
|
||||
import './services/editor/browser/editorPaneService.js';
|
||||
import './services/editor/common/customEditorLabelService.js';
|
||||
import './services/coreExperimentation/common/coreExperimentationService.js';
|
||||
import './services/dataChannel/browser/dataChannelService.js';
|
||||
|
||||
import { InstantiationType, registerSingleton } from '../platform/instantiation/common/extensions.js';
|
||||
|
||||
Reference in New Issue
Block a user