From be5540eb52d31ce5cf07de3d57a152b2641fa4de Mon Sep 17 00:00:00 2001 From: Kyle Cutler <67761731+kycutler@users.noreply.github.com> Date: Wed, 13 May 2026 15:44:46 -0700 Subject: [PATCH] Add element to chat via context menu (#316321) * Add element to chat via context menu * feedback --- .../electron-browser/preload-browserView.ts | 38 ++++++++++++-- .../browserView/electron-main/browserView.ts | 5 -- .../browserViewElementInspector.ts | 52 ++++++++++++++----- .../electron-main/browserViewMainService.ts | 11 +++- .../contrib/browserView/common/browserView.ts | 10 ++++ 5 files changed, 93 insertions(+), 23 deletions(-) diff --git a/src/vs/platform/browserView/electron-browser/preload-browserView.ts b/src/vs/platform/browserView/electron-browser/preload-browserView.ts index 4bd3276745c..c280e4aca59 100644 --- a/src/vs/platform/browserView/electron-browser/preload-browserView.ts +++ b/src/vs/platform/browserView/electron-browser/preload-browserView.ts @@ -112,7 +112,7 @@ }); }); - const globals = { + const isolatedHelpers = { /** * Get the currently selected text in the page. */ @@ -128,11 +128,43 @@ } }; + let contextMenuTargetRef: WeakRef | undefined; + window.addEventListener('contextmenu', (event) => { + const target = event.target; + if (target instanceof Element) { + contextMenuTargetRef = new WeakRef(target); + } else { + contextMenuTargetRef = undefined; + } + }, { capture: true }); + + const mainWorldHelpers = { + getElement(id: string, clear: boolean = true): Element | null { + switch (id) { + case 'active': + // eslint-disable-next-line no-restricted-syntax + return document.activeElement; + case 'context-menu-target': { + const element = contextMenuTargetRef?.deref() ?? null; + if (clear) { + contextMenuTargetRef = undefined; + } + return element; + } + default: + return null; + } + } + }; + try { // Use `contextBridge` APIs to expose globals to the same isolated world where this preload script runs (worldId 999). - // The globals object will be recursively frozen (and for functions also proxied) by Electron to prevent + // The isolatedHelpers object will be recursively frozen (and for functions also proxied) by Electron to prevent // modification within the given context. - contextBridge.exposeInIsolatedWorld(999, 'browserViewAPI', globals); + contextBridge.exposeInIsolatedWorld(999, 'browserViewAPI', isolatedHelpers); + // Expose helpers on `window.__vscode_helpers` in the page's main world + // for CDP `Runtime.evaluate` (which runs against the main world) to use. + contextBridge.exposeInMainWorld('__vscode_helpers', mainWorldHelpers); } catch (error) { console.error(error); } diff --git a/src/vs/platform/browserView/electron-main/browserView.ts b/src/vs/platform/browserView/electron-main/browserView.ts index 265482b6b49..e1ae26c8e26 100644 --- a/src/vs/platform/browserView/electron-main/browserView.ts +++ b/src/vs/platform/browserView/electron-main/browserView.ts @@ -346,11 +346,6 @@ export class BrowserView extends Disposable { }); const onCommandKeydown = (_event: unknown, keyEvent: IBrowserViewKeyDownEvent) => { - // Intercept Ctrl/Cmd+Enter during element selection to pick the focused element. - if (this.inspector.isElementSelectionActive && keyEvent.key === 'Enter' && (keyEvent.ctrlKey || keyEvent.metaKey)) { - void this.inspector.pickFocusedElement(); - return; - } this._onDidKeyCommand.fire(keyEvent); }; diff --git a/src/vs/platform/browserView/electron-main/browserViewElementInspector.ts b/src/vs/platform/browserView/electron-main/browserViewElementInspector.ts index 0c9c72a4432..b5aff998899 100644 --- a/src/vs/platform/browserView/electron-main/browserViewElementInspector.ts +++ b/src/vs/platform/browserView/electron-main/browserViewElementInspector.ts @@ -37,6 +37,22 @@ interface ILayoutMetricsResult { }; } +export interface IElementHandle { + addToChat(): Promise; +} + +/** + * Well-known ids understood by `__vscode_helpers.getElement(id)` in + * `preload-browserView.ts`. Any other string is treated as the id of a + * dynamically tracked element. + */ +export const enum BrowserViewInspectElementId { + /** The page's `document.activeElement`. */ + Active = 'active', + /** The element targeted by the most recent `contextmenu` event. */ + ContextMenuTarget = 'context-menu-target', +} + function useScopedDisposal() { const store = new DisposableStore() as DisposableStore & { [Symbol.dispose](): void }; store[Symbol.dispose] = () => store.dispose(); @@ -170,31 +186,39 @@ export class BrowserViewElementInspector extends Disposable { } /** - * Fire a selection event for the currently focused element. - * Only effective when element selection is active. + * Resolve a handle to the element identified by `id`. + * + * `id` is interpreted by `__vscode_helpers.getElement(id, clear)` + * in the page preload (see {@link BrowserViewSelectedElementId} for + * well-known values). Returns `undefined` if no element is found. + * + * @param clear When true (default) the preload also clears any underlying + * tracking ref it held for this id, so subsequent calls won't return the + * same element. Pass `false` for a non-consuming peek. */ - async pickFocusedElement(): Promise { - if (!this._elementSelectionActive) { - return; - } - + async getElementHandle(id: string, clear: boolean = true): Promise { const connection = await this._connectionPromise; await connection.sendCommand('Runtime.enable'); const { result } = await connection.sendCommand('Runtime.evaluate', { - expression: 'document.activeElement', + expression: `window.__vscode_helpers?.getElement(${JSON.stringify(id)}, ${clear})`, returnByValue: false, }) as { result: { objectId?: string } }; if (!result?.objectId) { - return; + return undefined; } - const nodeData = await extractNodeData(connection, { objectId: result.objectId }); - this._onDidSelectElement.fire({ - ...nodeData, - url: this.browser.getURL() - }); + const objectId = result.objectId; + return { + addToChat: async () => { + const nodeData = await extractNodeData(connection, { objectId }); + this._onDidSelectElement.fire({ + ...nodeData, + url: this.browser.getURL() + }); + } + }; } async getVisualViewportScale(): Promise { diff --git a/src/vs/platform/browserView/electron-main/browserViewMainService.ts b/src/vs/platform/browserView/electron-main/browserViewMainService.ts index a30efa2733a..edd84720a91 100644 --- a/src/vs/platform/browserView/electron-main/browserViewMainService.ts +++ b/src/vs/platform/browserView/electron-main/browserViewMainService.ts @@ -20,6 +20,7 @@ import { ITelemetryService } from '../../telemetry/common/telemetry.js'; import { localize } from '../../../nls.js'; import { INativeHostMainService } from '../../native/electron-main/nativeHostMainService.js'; import { htmlAttributeEncodeValue } from '../../../base/common/strings.js'; +import { BrowserViewInspectElementId } from './browserViewElementInspector.js'; export const IBrowserViewMainService = createDecorator('browserViewMainService'); @@ -371,7 +372,7 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa return view; } - private showContextMenu(view: BrowserView, params: Electron.ContextMenuParams): void { + private async showContextMenu(view: BrowserView, params: Electron.ContextMenuParams): Promise { const win = view.getElectronWindow(); if (!win) { return; @@ -380,6 +381,8 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa if (webContents.isDestroyed()) { return; } + + const inspectTarget = await view.inspector.getElementHandle(BrowserViewInspectElementId.ContextMenuTarget); const menu = new Menu(); if (params.linkURL) { @@ -469,6 +472,12 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa } menu.append(new MenuItem({ type: 'separator' })); + if (inspectTarget) { + menu.append(new MenuItem({ + label: localize('browser.contextMenu.addElementToChat', 'Add Element to Chat'), + click: () => inspectTarget.addToChat() + })); + } menu.append(new MenuItem({ label: localize('browser.contextMenu.inspect', 'Inspect'), click: () => webContents.inspectElement(params.x, params.y) diff --git a/src/vs/workbench/contrib/browserView/common/browserView.ts b/src/vs/workbench/contrib/browserView/common/browserView.ts index 88dfae3d1a3..14fcb2ddc27 100644 --- a/src/vs/workbench/contrib/browserView/common/browserView.ts +++ b/src/vs/workbench/contrib/browserView/common/browserView.ts @@ -86,6 +86,13 @@ type IntegratedBrowserShareWithAgentClassification = { comment: 'Tracks user choices around sharing browser content with agents'; }; +type IntegratedBrowserAddElementToChatStartEvent = {}; + +type IntegratedBrowserAddElementToChatStartClassification = { + owner: 'jruales'; + comment: 'The user initiated an Add Element to Chat action in Integrated Browser.'; +}; + /** * View state stored in editor options when opening a browser view. */ @@ -381,6 +388,9 @@ export class BrowserViewModel extends Disposable implements IBrowserViewModel { })); this._register(this.onDidChangeElementSelectionActive(active => { + if (active) { + this.telemetryService.publicLog2('integratedBrowser.addElementToChat.start', {}); + } this._isElementSelectionActive = active; }));