Add element to chat via context menu (#316321)

* Add element to chat via context menu

* feedback

src/vs/platform/browserView/electron-browser/preload-browserView.ts

@@ -112,7 +112,7 @@
 		});
 	});
 
-	const globals = {
+	const isolatedHelpers = {
 		/**
 		 * Get the currently selected text in the page.
 		 */
@@ -128,11 +128,43 @@
 		}
 	};
 
+	let contextMenuTargetRef: WeakRef<Element> | undefined;
+	window.addEventListener('contextmenu', (event) => {
+		const target = event.target;
+		if (target instanceof Element) {
+			contextMenuTargetRef = new WeakRef(target);
+		} else {
+			contextMenuTargetRef = undefined;
+		}
+	}, { capture: true });
+
+	const mainWorldHelpers = {
+		getElement(id: string, clear: boolean = true): Element | null {
+			switch (id) {
+				case 'active':
+					// eslint-disable-next-line no-restricted-syntax
+					return document.activeElement;
+				case 'context-menu-target': {
+					const element = contextMenuTargetRef?.deref() ?? null;
+					if (clear) {
+						contextMenuTargetRef = undefined;
+					}
+					return element;
+				}
+				default:
+					return null;
+			}
+		}
+	};
+
 	try {
 		// Use `contextBridge` APIs to expose globals to the same isolated world where this preload script runs (worldId 999).
-		// The globals object will be recursively frozen (and for functions also proxied) by Electron to prevent
+		// The isolatedHelpers object will be recursively frozen (and for functions also proxied) by Electron to prevent
 		// modification within the given context.
-		contextBridge.exposeInIsolatedWorld(999, 'browserViewAPI', globals);
+		contextBridge.exposeInIsolatedWorld(999, 'browserViewAPI', isolatedHelpers);
+		// Expose helpers on `window.__vscode_helpers` in the page's main world
+		// for CDP `Runtime.evaluate` (which runs against the main world) to use.
+		contextBridge.exposeInMainWorld('__vscode_helpers', mainWorldHelpers);
 	} catch (error) {
 		console.error(error);
 	}

src/vs/platform/browserView/electron-main/browserView.ts

@@ -346,11 +346,6 @@ export class BrowserView extends Disposable {
 		});
 
 		const onCommandKeydown = (_event: unknown, keyEvent: IBrowserViewKeyDownEvent) => {
-			// Intercept Ctrl/Cmd+Enter during element selection to pick the focused element.
-			if (this.inspector.isElementSelectionActive && keyEvent.key === 'Enter' && (keyEvent.ctrlKey || keyEvent.metaKey)) {
-				void this.inspector.pickFocusedElement();
-				return;
-			}
 			this._onDidKeyCommand.fire(keyEvent);
 		};
 

src/vs/platform/browserView/electron-main/browserViewElementInspector.ts

@@ -37,6 +37,22 @@ interface ILayoutMetricsResult {
 	};
 }
 
+export interface IElementHandle {
+	addToChat(): Promise<void>;
+}
+
+/**
+ * Well-known ids understood by `__vscode_helpers.getElement(id)` in
+ * `preload-browserView.ts`. Any other string is treated as the id of a
+ * dynamically tracked element.
+ */
+export const enum BrowserViewInspectElementId {
+	/** The page's `document.activeElement`. */
+	Active = 'active',
+	/** The element targeted by the most recent `contextmenu` event. */
+	ContextMenuTarget = 'context-menu-target',
+}
+
 function useScopedDisposal() {
 	const store = new DisposableStore() as DisposableStore & { [Symbol.dispose](): void };
 	store[Symbol.dispose] = () => store.dispose();
@@ -170,31 +186,39 @@ export class BrowserViewElementInspector extends Disposable {
 	}
 
 	/**
-	 * Fire a selection event for the currently focused element.
-	 * Only effective when element selection is active.
+	 * Resolve a handle to the element identified by `id`.
+	 *
+	 * `id` is interpreted by `__vscode_helpers.getElement(id, clear)`
+	 * in the page preload (see {@link BrowserViewSelectedElementId} for
+	 * well-known values). Returns `undefined` if no element is found.
+	 *
+	 * @param clear When true (default) the preload also clears any underlying
+	 * tracking ref it held for this id, so subsequent calls won't return the
+	 * same element. Pass `false` for a non-consuming peek.
 	 */
-	async pickFocusedElement(): Promise<void> {
-		if (!this._elementSelectionActive) {
-			return;
-		}
-
+	async getElementHandle(id: string, clear: boolean = true): Promise<IElementHandle | undefined> {
 		const connection = await this._connectionPromise;
 
 		await connection.sendCommand('Runtime.enable');
 		const { result } = await connection.sendCommand('Runtime.evaluate', {
-			expression: 'document.activeElement',
+			expression: `window.__vscode_helpers?.getElement(${JSON.stringify(id)}, ${clear})`,
 			returnByValue: false,
 		}) as { result: { objectId?: string } };
 
 		if (!result?.objectId) {
-			return;
+			return undefined;
 		}
 
-		const nodeData = await extractNodeData(connection, { objectId: result.objectId });
-		this._onDidSelectElement.fire({
-			...nodeData,
-			url: this.browser.getURL()
-		});
+		const objectId = result.objectId;
+		return {
+			addToChat: async () => {
+				const nodeData = await extractNodeData(connection, { objectId });
+				this._onDidSelectElement.fire({
+					...nodeData,
+					url: this.browser.getURL()
+				});
+			}
+		};
 	}
 
 	async getVisualViewportScale(): Promise<number> {

src/vs/platform/browserView/electron-main/browserViewMainService.ts

@@ -20,6 +20,7 @@ import { ITelemetryService } from '../../telemetry/common/telemetry.js';
 import { localize } from '../../../nls.js';
 import { INativeHostMainService } from '../../native/electron-main/nativeHostMainService.js';
 import { htmlAttributeEncodeValue } from '../../../base/common/strings.js';
+import { BrowserViewInspectElementId } from './browserViewElementInspector.js';
 
 export const IBrowserViewMainService = createDecorator<IBrowserViewMainService>('browserViewMainService');
 
@@ -371,7 +372,7 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa
 		return view;
 	}
 
-	private showContextMenu(view: BrowserView, params: Electron.ContextMenuParams): void {
+	private async showContextMenu(view: BrowserView, params: Electron.ContextMenuParams): Promise<void> {
 		const win = view.getElectronWindow();
 		if (!win) {
 			return;
@@ -380,6 +381,8 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa
 		if (webContents.isDestroyed()) {
 			return;
 		}
+
+		const inspectTarget = await view.inspector.getElementHandle(BrowserViewInspectElementId.ContextMenuTarget);
 		const menu = new Menu();
 
 		if (params.linkURL) {
@@ -469,6 +472,12 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa
 		}
 
 		menu.append(new MenuItem({ type: 'separator' }));
+		if (inspectTarget) {
+			menu.append(new MenuItem({
+				label: localize('browser.contextMenu.addElementToChat', 'Add Element to Chat'),
+				click: () => inspectTarget.addToChat()
+			}));
+		}
 		menu.append(new MenuItem({
 			label: localize('browser.contextMenu.inspect', 'Inspect'),
 			click: () => webContents.inspectElement(params.x, params.y)

src/vs/workbench/contrib/browserView/common/browserView.ts

@@ -86,6 +86,13 @@ type IntegratedBrowserShareWithAgentClassification = {
 	comment: 'Tracks user choices around sharing browser content with agents';
 };
 
+type IntegratedBrowserAddElementToChatStartEvent = {};
+
+type IntegratedBrowserAddElementToChatStartClassification = {
+	owner: 'jruales';
+	comment: 'The user initiated an Add Element to Chat action in Integrated Browser.';
+};
+
 /**
  * View state stored in editor options when opening a browser view.
  */
@@ -381,6 +388,9 @@ export class BrowserViewModel extends Disposable implements IBrowserViewModel {
 		}));
 
 		this._register(this.onDidChangeElementSelectionActive(active => {
+			if (active) {
+				this.telemetryService.publicLog2<IntegratedBrowserAddElementToChatStartEvent, IntegratedBrowserAddElementToChatStartClassification>('integratedBrowser.addElementToChat.start', {});
+			}
 			this._isElementSelectionActive = active;
 		}));