Provide encoding-related APIs for editor extensions (#824) (#240804)

extensions/vscode-api-tests/src/singlefolder-tests/workspace.test.ts

@@ -1311,7 +1311,7 @@ suite('vscode API - workspace', () => {
 		return deleteFile(file);
 	}
 
-	test('text document encodings', async () => {
+	test('encoding: text document encodings', async () => {
 		const uri1 = await createRandomFile();
 		const uri2 = await createRandomFile(new Uint8Array([0xEF, 0xBB, 0xBF]) /* UTF-8 with BOM */);
 		const uri3 = await createRandomFile(new Uint8Array([0xFF, 0xFE]) /* UTF-16 LE BOM */);
@@ -1333,7 +1333,66 @@ suite('vscode API - workspace', () => {
 		assert.strictEqual(doc5.encoding, 'utf8');
 	});
 
-	test('fs.decode', async function () {
+	test('encoding: openTextDocument', async () => {
+		const uri1 = await createRandomFile();
+
+		let doc1 = await vscode.workspace.openTextDocument(uri1, { encoding: 'cp1252' });
+		assert.strictEqual(doc1.encoding, 'cp1252');
+
+		let listener: vscode.Disposable | undefined;
+		const documentChangePromise = new Promise<void>(resolve => {
+			listener = vscode.workspace.onDidChangeTextDocument(e => {
+				if (e.document.uri.toString() === uri1.toString()) {
+					resolve();
+				}
+			});
+		});
+
+		doc1 = await vscode.workspace.openTextDocument(uri1, { encoding: 'utf16le' });
+		assert.strictEqual(doc1.encoding, 'utf16le');
+		await documentChangePromise;
+
+		const doc2 = await vscode.workspace.openTextDocument({ encoding: 'utf16be' });
+		assert.strictEqual(doc2.encoding, 'utf16be');
+
+		const doc3 = await vscode.workspace.openTextDocument({ content: 'Hello World', encoding: 'utf16le' });
+		assert.strictEqual(doc3.encoding, 'utf16le');
+
+		listener?.dispose();
+	});
+
+	test('encoding: openTextDocument - throws for dirty documents', async () => {
+		const uri1 = await createRandomFile();
+
+		const doc1 = await vscode.workspace.openTextDocument(uri1, { encoding: 'cp1252' });
+
+		const edit = new vscode.WorkspaceEdit();
+		edit.insert(doc1.uri, new vscode.Position(0, 0), 'Hello World');
+		await vscode.workspace.applyEdit(edit);
+		assert.strictEqual(doc1.isDirty, true);
+
+		let err;
+		try {
+			await vscode.workspace.decode(new Uint8Array([0, 0, 0, 0]), doc1.uri);
+		} catch (e) {
+			err = e;
+		}
+		assert.ok(err);
+	});
+
+	test('encoding: openTextDocument - multiple requests with different encoding work', async () => {
+		const uri1 = await createRandomFile();
+
+		const doc1P = vscode.workspace.openTextDocument(uri1);
+		const doc2P = vscode.workspace.openTextDocument(uri1, { encoding: 'cp1252' });
+
+		const [doc1, doc2] = await Promise.all([doc1P, doc2P]);
+
+		assert.strictEqual(doc1.encoding, 'cp1252');
+		assert.strictEqual(doc2.encoding, 'cp1252');
+	});
+
+	test('encoding: decode', async function () {
 		const uri = root.with({ path: posix.join(root.path, 'file.txt') });
 
 		// without setting
@@ -1375,7 +1434,7 @@ suite('vscode API - workspace', () => {
 		assert.ok(err);
 	});
 
-	test('fs.encode', async function () {
+	test('encoding: encode', async function () {
 		const uri = root.with({ path: posix.join(root.path, 'file.txt') });
 
 		// without setting

src/vs/workbench/api/browser/mainThreadDocuments.ts

@@ -12,7 +12,7 @@ import { IModelService } from '../../../editor/common/services/model.js';
 import { ITextModelService } from '../../../editor/common/services/resolverService.js';
 import { IFileService, FileOperation } from '../../../platform/files/common/files.js';
 import { ExtHostContext, ExtHostDocumentsShape, MainThreadDocumentsShape } from '../common/extHost.protocol.js';
-import { ITextFileEditorModel, ITextFileService } from '../../services/textfile/common/textfiles.js';
+import { EncodingMode, ITextFileEditorModel, ITextFileService, TextFileResolveReason } from '../../services/textfile/common/textfiles.js';
 import { IUntitledTextEditorModel } from '../../services/untitled/common/untitledTextEditorModel.js';
 import { IWorkbenchEnvironmentService } from '../../services/environment/common/environmentService.js';
 import { toLocalResource, extUri, IExtUri } from '../../../base/common/resources.js';
@@ -219,7 +219,7 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
 		return Boolean(target);
 	}
 
-	async $tryOpenDocument(uriData: UriComponents): Promise<URI> {
+	async $tryOpenDocument(uriData: UriComponents, options?: { encoding?: string }): Promise<URI> {
 		const inputUri = URI.revive(uriData);
 		if (!inputUri.scheme || !(inputUri.fsPath || inputUri.authority)) {
 			throw new ErrorNoTelemetry(`Invalid uri. Scheme and authority or path must be set.`);
@@ -230,11 +230,11 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
 		let promise: Promise<URI>;
 		switch (canonicalUri.scheme) {
 			case Schemas.untitled:
-				promise = this._handleUntitledScheme(canonicalUri);
+				promise = this._handleUntitledScheme(canonicalUri, options);
 				break;
 			case Schemas.file:
 			default:
-				promise = this._handleAsResourceInput(canonicalUri);
+				promise = this._handleAsResourceInput(canonicalUri, options);
 				break;
 		}
 
@@ -255,31 +255,40 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
 		}
 	}
 
-	$tryCreateDocument(options?: { language?: string; content?: string }): Promise<URI> {
-		return this._doCreateUntitled(undefined, options ? options.language : undefined, options ? options.content : undefined);
+	$tryCreateDocument(options?: { language?: string; content?: string; encoding?: string }): Promise<URI> {
+		return this._doCreateUntitled(undefined, options);
 	}
 
-	private async _handleAsResourceInput(uri: URI): Promise<URI> {
+	private async _handleAsResourceInput(uri: URI, options?: { encoding?: string }): Promise<URI> {
+		if (options?.encoding) {
+			const model = await this._textFileService.files.resolve(uri, { encoding: options.encoding, reason: TextFileResolveReason.REFERENCE });
+			if (model.isDirty()) {
+				throw new ErrorNoTelemetry(`Cannot re-open a dirty text document with different encoding. Save it first.`);
+			}
+			await model.setEncoding(options.encoding, EncodingMode.Decode);
+		}
+
 		const ref = await this._textModelResolverService.createModelReference(uri);
 		this._modelReferenceCollection.add(uri, ref, ref.object.textEditorModel.getValueLength());
 		return ref.object.textEditorModel.uri;
 	}
 
-	private async _handleUntitledScheme(uri: URI): Promise<URI> {
+	private async _handleUntitledScheme(uri: URI, options?: { encoding?: string }): Promise<URI> {
 		const asLocalUri = toLocalResource(uri, this._environmentService.remoteAuthority, this._pathService.defaultUriScheme);
 		const exists = await this._fileService.exists(asLocalUri);
 		if (exists) {
 			// don't create a new file ontop of an existing file
 			return Promise.reject(new Error('file already exists'));
 		}
-		return await this._doCreateUntitled(Boolean(uri.path) ? uri : undefined);
+		return await this._doCreateUntitled(Boolean(uri.path) ? uri : undefined, options);
 	}
 
-	private async _doCreateUntitled(associatedResource?: URI, languageId?: string, initialValue?: string): Promise<URI> {
+	private async _doCreateUntitled(associatedResource?: URI, options?: { language?: string; content?: string; encoding?: string }): Promise<URI> {
 		const model = this._textFileService.untitled.create({
 			associatedResource,
-			languageId,
-			initialValue
+			languageId: options?.language,
+			initialValue: options?.content,
+			encoding: options?.encoding
 		});
 		const resource = model.resource;
 		const ref = await this._textModelResolverService.createModelReference(resource);

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -1024,10 +1024,14 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			set textDocuments(value) {
 				throw new errors.ReadonlyError('textDocuments');
 			},
-			openTextDocument(uriOrFileNameOrOptions?: vscode.Uri | string | { language?: string; content?: string }) {
+			openTextDocument(uriOrFileNameOrOptions?: vscode.Uri | string | { language?: string; content?: string; encoding?: string }, options?: { encoding?: string }) {
 				let uriPromise: Thenable<URI>;
 
-				const options = uriOrFileNameOrOptions as { language?: string; content?: string };
+				options = (options ?? uriOrFileNameOrOptions) as ({ language?: string; content?: string; encoding?: string } | undefined);
+				if (typeof options?.encoding === 'string') {
+					checkProposedApiEnabled(extension, 'textDocumentEncoding');
+				}
+
 				if (typeof uriOrFileNameOrOptions === 'string') {
 					uriPromise = Promise.resolve(URI.file(uriOrFileNameOrOptions));
 				} else if (URI.isUri(uriOrFileNameOrOptions)) {
@@ -1043,7 +1047,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 					if (uri.scheme === Schemas.vscodeRemote && !uri.authority) {
 						extHostApiDeprecation.report('workspace.openTextDocument', extension, `A URI of 'vscode-remote' scheme requires an authority.`);
 					}
-					return extHostDocuments.ensureDocumentData(uri).then(documentData => {
+					return extHostDocuments.ensureDocumentData(uri, options).then(documentData => {
 						return documentData.document;
 					});
 				});

src/vs/workbench/api/common/extHost.protocol.ts

@@ -238,8 +238,8 @@ export interface MainThreadDocumentContentProvidersShape extends IDisposable {
 }
 
 export interface MainThreadDocumentsShape extends IDisposable {
-	$tryCreateDocument(options?: { language?: string; content?: string }): Promise<UriComponents>;
-	$tryOpenDocument(uri: UriComponents): Promise<UriComponents>;
+	$tryCreateDocument(options?: { language?: string; content?: string; encoding?: string }): Promise<UriComponents>;
+	$tryOpenDocument(uri: UriComponents, options?: { encoding?: string }): Promise<UriComponents>;
 	$trySaveDocument(uri: UriComponents): Promise<boolean>;
 }
 

src/vs/workbench/api/common/extHostDocuments.ts

@@ -76,16 +76,16 @@ export class ExtHostDocuments implements ExtHostDocumentsShape {
 		return data.document;
 	}
 
-	public ensureDocumentData(uri: URI): Promise<ExtHostDocumentData> {
+	public ensureDocumentData(uri: URI, options?: { encoding?: string }): Promise<ExtHostDocumentData> {
 
 		const cached = this._documentsAndEditors.getDocument(uri);
-		if (cached) {
+		if (cached && (!options?.encoding || cached.document.encoding === options.encoding)) {
 			return Promise.resolve(cached);
 		}
 
 		let promise = this._documentLoader.get(uri.toString());
 		if (!promise) {
-			promise = this._proxy.$tryOpenDocument(uri).then(uriData => {
+			promise = this._proxy.$tryOpenDocument(uri, options).then(uriData => {
 				this._documentLoader.delete(uri.toString());
 				const canonicalUri = URI.revive(uriData);
 				return assertIsDefined(this._documentsAndEditors.getDocument(canonicalUri));
@@ -94,12 +94,21 @@ export class ExtHostDocuments implements ExtHostDocumentsShape {
 				return Promise.reject(err);
 			});
 			this._documentLoader.set(uri.toString(), promise);
+		} else {
+			if (options?.encoding) {
+				promise = promise.then(data => {
+					if (data.document.encoding !== options.encoding) {
+						return this.ensureDocumentData(uri, options);
+					}
+					return data;
+				});
+			}
 		}
 
 		return promise;
 	}
 
-	public createDocumentData(options?: { language?: string; content?: string }): Promise<URI> {
+	public createDocumentData(options?: { language?: string; content?: string; encoding?: string }): Promise<URI> {
 		return this._proxy.$tryCreateDocument(options).then(data => URI.revive(data));
 	}
 

src/vscode-dts/vscode.proposed.textDocumentEncoding.d.ts

@@ -30,6 +30,91 @@ declare module 'vscode' {
 
 	export namespace workspace {
 
+		/**
+		 * Opens a document. Will return early if this document is already open. Otherwise
+		 * the document is loaded and the {@link workspace.onDidOpenTextDocument didOpen}-event fires.
+		 *
+		 * The document is denoted by an {@link Uri}. Depending on the {@link Uri.scheme scheme} the
+		 * following rules apply:
+		 * * `file`-scheme: Open a file on disk (`openTextDocument(Uri.file(path))`). Will be rejected if the file
+		 * does not exist or cannot be loaded.
+		 * * `untitled`-scheme: Open a blank untitled file with associated path (`openTextDocument(Uri.file(path).with({ scheme: 'untitled' }))`).
+		 * The language will be derived from the file name.
+		 * * For all other schemes contributed {@link TextDocumentContentProvider text document content providers} and
+		 * {@link FileSystemProvider file system providers} are consulted.
+		 *
+		 * *Note* that the lifecycle of the returned document is owned by the editor and not by the extension. That means an
+		 * {@linkcode workspace.onDidCloseTextDocument onDidClose}-event can occur at any time after opening it.
+		 *
+		 * @throws This method will throw an error when an existing text document with the provided uri is dirty.
+		 *
+		 * @param uri Identifies the resource to open.
+		 * @param options Options to control how the document will be opened.
+		 * @returns A promise that resolves to a {@link TextDocument document}.
+		 */
+		export function openTextDocument(uri: Uri, options?: {
+			/**
+			 * The {@link TextDocument.encoding encoding} of the document to use
+			 * for decoding the underlying buffer to text. If omitted, the encoding
+			 * will be guessed based on the file content and/or the editor settings.
+			 *
+			 * See {@link TextDocument.encoding} for more information about valid
+			 * values for encoding.
+			 *
+			 * *Note* that opening a text document that was already opened with a
+			 * different encoding has the potential of changing the text contents of
+			 * the text document.
+			 */
+			encoding?: string;
+		}): Thenable<TextDocument>;
+
+		/**
+		 * A short-hand for `openTextDocument(Uri.file(path))`.
+		 *
+		 * @see {@link workspace.openTextDocument}
+		 * @param path A path of a file on disk.
+		 * @param options Options to control how the document will be opened.
+		 * @returns A promise that resolves to a {@link TextDocument document}.
+		 */
+		export function openTextDocument(path: string, options?: {
+			/**
+			 * The {@link TextDocument.encoding encoding} of the document to use
+			 * for decoding the underlying buffer to text. If omitted, the encoding
+			 * will be guessed based on the file content and/or the editor settings.
+			 *
+			 * See {@link TextDocument.encoding} for more information about valid
+			 * values for encoding.
+			 *
+			 * *Note* that opening a text document that was already opened with a
+			 * different encoding has the potential of changing the text contents of
+			 * the text document.
+			 */
+			encoding?: string;
+		}): Thenable<TextDocument>;
+
+		/**
+		 * Opens an untitled text document. The editor will prompt the user for a file
+		 * path when the document is to be saved. The `options` parameter allows to
+		 * specify the *language*, *encoding* and/or the *content* of the document.
+		 *
+		 * @param options Options to control how the document will be created.
+		 * @returns A promise that resolves to a {@link TextDocument document}.
+		 */
+		export function openTextDocument(options?: {
+			/**
+			 * The {@link TextDocument.languageId language} of the document.
+			 */
+			language?: string;
+			/**
+			 * The initial contents of the document.
+			 */
+			content?: string;
+			/**
+			 * The {@link TextDocument.encoding encoding} of the document.
+			 */
+			encoding?: string;
+		}): Thenable<TextDocument>;
+
 		/**
 		 * Decodes the content from a `Uint8Array` to a `string`.
 		 *
@@ -42,10 +127,11 @@ declare module 'vscode' {
 		 * @param content The content to decode as a `Uint8Array`.
 		 * @param uri The URI that represents the file. This information
 		 * is used to figure out the encoding related configuration for the file.
-		 * @param options Allows to explicitly pick the encoding to use.
+		 * @param options Allows to explicitly pick the encoding to use. See {@link TextDocument.encoding}
+		 * for more information about valid values for encoding.
 		 * @returns A thenable that resolves to the decoded `string`.
 		 */
-		export function decode(content: Uint8Array, uri: Uri | undefined, options?: { readonly encoding: string }): Thenable<string>;
+		export function decode(content: Uint8Array, uri: Uri | undefined, options?: { encoding: string }): Thenable<string>;
 
 		/**
 		 * Encodes the content of a `string` to a `Uint8Array`.
@@ -56,9 +142,10 @@ declare module 'vscode' {
 		 * @param content The content to decode as a `string`.
 		 * @param uri The URI that represents the file. This information
 		 * is used to figure out the encoding related configuration for the file.
-		 * @param options Allows to explicitly pick the encoding to use.
+		 * @param options Allows to explicitly pick the encoding to use. See {@link TextDocument.encoding}
+		 * for more information about valid values for encoding.
 		 * @returns A thenable that resolves to the encoded `Uint8Array`.
 		 */
-		export function encode(content: string, uri: Uri | undefined, options?: { readonly encoding: string }): Thenable<Uint8Array>;
+		export function encode(content: string, uri: Uri | undefined, options?: { encoding: string }): Thenable<Uint8Array>;
 	}
 }