From e14a7ca0bc4adac8ed16b978d6bd4f025c7849cd Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Wed, 6 Jul 2016 00:28:24 -0700 Subject: [PATCH] initial support for compressing responses --- src/harness/harness.ts | 4 ++ src/harness/harnessLanguageService.ts | 5 +- src/server/node.d.ts | 9 +++ src/server/protocol.d.ts | 2 + src/server/server.ts | 27 +++++++-- src/server/session.ts | 55 ++++++++++++++----- .../cases/unittests/cachingInServerLSHost.ts | 3 + tests/cases/unittests/session.ts | 19 ++++--- .../cases/unittests/tsserverProjectSystem.ts | 1 + 9 files changed, 94 insertions(+), 31 deletions(-) diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 6f895c37b19..8726bf69da5 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -70,6 +70,10 @@ namespace Utils { return Buffer ? Buffer.byteLength(s, encoding) : s.length; } + export function compress(s: string): any { + return Buffer ? new Buffer(s, "utf8") : { s, length: s.length }; + } + export function evalFile(fileContents: string, fileName: string, nodeContext?: any) { const environment = getExecutionEnvironment(); switch (environment) { diff --git a/src/harness/harnessLanguageService.ts b/src/harness/harnessLanguageService.ts index 3e431c73f9e..7ca3c832377 100644 --- a/src/harness/harnessLanguageService.ts +++ b/src/harness/harnessLanguageService.ts @@ -572,6 +572,8 @@ namespace Harness.LanguageService { this.writeMessage(message); } + writeCompressedData() { + } readFile(fileName: string): string { if (fileName.indexOf(Harness.Compiler.defaultLibFileName) >= 0) { @@ -690,7 +692,8 @@ namespace Harness.LanguageService { const server = new ts.server.Session(serverHost, { isCancellationRequested: () => false }, /*useOneInferredProject*/ false, - Buffer ? Buffer.byteLength : (string: string, encoding?: string) => string.length, + Utils.byteLength, + Utils.compress, process.hrtime, serverHost); // Fake the connection between the client and the server diff --git a/src/server/node.d.ts b/src/server/node.d.ts index 0bde0bb6602..8ac1f1f19b5 100644 --- a/src/server/node.d.ts +++ b/src/server/node.d.ts @@ -397,6 +397,15 @@ declare namespace NodeJS { } } +declare namespace NodeJS { + namespace zlib { + export interface GZip { + gzipSync(buf: Buffer): Buffer; + } + export function createGZip(): GZip; + } +} + declare namespace NodeJS { namespace fs { interface Stats { diff --git a/src/server/protocol.d.ts b/src/server/protocol.d.ts index 10b67d38ffe..3bef3e600eb 100644 --- a/src/server/protocol.d.ts +++ b/src/server/protocol.d.ts @@ -30,6 +30,8 @@ declare namespace ts.server.protocol { * Object containing arguments for the command */ arguments?: any; + + canCompressResponse?: boolean; } /** diff --git a/src/server/server.ts b/src/server/server.ts index c9f807778c1..1ac6a0c7cbb 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -4,8 +4,10 @@ /* tslint:disable:no-null-keyword */ namespace ts.server { + const readline: NodeJS.ReadLine = require("readline"); const fs: typeof NodeJS.fs = require("fs"); + const zlib: typeof NodeJS.zlib = require("zlib"); const rl = readline.createInterface({ input: process.stdin, @@ -13,6 +15,11 @@ namespace ts.server { terminal: false, }); + function compress(s: string): CompressedData { + const gzip = zlib.createGZip(); + return gzip.gzipSync(new Buffer(s, "utf8")); + } + class Logger implements ts.server.Logger { private fd = -1; private seq = 0; @@ -92,7 +99,7 @@ namespace ts.server { class IOSession extends Session { constructor(host: ServerHost, cancellationToken: HostCancellationToken, useSingleInferredProject: boolean, logger: ts.server.Logger) { - super(host, cancellationToken, useSingleInferredProject, Buffer.byteLength, process.hrtime, logger); + super(host, cancellationToken, useSingleInferredProject, Buffer.byteLength, compress, process.hrtime, logger); } exit() { @@ -260,15 +267,16 @@ namespace ts.server { const pollingWatchedFileSet = createPollingWatchedFileSet(); const logger = createLoggerFromEnv(); - const pending: string[] = []; + const pending: Buffer[] = []; let canWrite = true; - function writeMessage(s: string) { + + function writeMessage(buf: Buffer) { if (!canWrite) { - pending.push(s); + pending.push(buf); } else { canWrite = false; - process.stdout.write(new Buffer(s, "utf8"), setCanWriteFlagAndWriteMessageIfNecessary); + process.stdout.write(buf, setCanWriteFlagAndWriteMessageIfNecessary); } } @@ -279,10 +287,16 @@ namespace ts.server { } } + function writeCompressedData(prefix: string, compressed: CompressedData, suffix: string): void { + sys.write(prefix); + writeMessage(compressed); + sys.write(suffix); + } + const sys = ts.sys; // Override sys.write because fs.writeSync is not reliable on Node 4 - sys.write = (s: string) => writeMessage(s); + sys.write = (s: string) => writeMessage(new Buffer(s, "utf8")); sys.watchFile = (fileName, callback) => { const watchedFile = pollingWatchedFileSet.addFile(fileName, callback); return { @@ -294,6 +308,7 @@ namespace ts.server { sys.clearTimeout = clearTimeout; sys.setImmediate = setImmediate; sys.clearImmediate = clearImmediate; + sys.writeCompressedData = writeCompressedData; let cancellationToken: HostCancellationToken; try { diff --git a/src/server/session.ts b/src/server/session.ts index 7368eed1bc0..855097ef86b 100644 --- a/src/server/session.ts +++ b/src/server/session.ts @@ -10,6 +10,15 @@ namespace ts.server { stack?: string; } + export interface CompressedData { + __compressedDataTag: any; + length: number; + } + + export interface ServerHost { + writeCompressedData(prefix: string, data: CompressedData, suffix: string): void; + } + export function generateSpaces(n: number): string { if (!spaceCache[n]) { let strBuilder = ""; @@ -183,6 +192,7 @@ namespace ts.server { cancellationToken: HostCancellationToken, useSingleInferredProject: boolean, private byteLength: (buf: string, encoding?: string) => number, + private compress: (s: string) => CompressedData, private hrtime: (start?: number[]) => number[], private logger: Logger) { this.projectService = @@ -215,13 +225,27 @@ namespace ts.server { this.host.write(line + this.host.newLine); } - public send(msg: protocol.Message) { + private sendCompressedDataToClient(prefix: string, data: CompressedData) { + this.host.writeCompressedData(prefix, data, this.host.newLine); + } + + public send(msg: protocol.Message, canCompressResponse: boolean) { const json = JSON.stringify(msg); if (this.logger.isVerbose()) { this.logger.info(msg.type + ": " + json); } - this.sendLineToClient("Content-Length: " + (1 + this.byteLength(json, "utf8")) + - "\r\n\r\n" + json); + const len = this.byteLength(json, "utf8"); + if (len < 84000 || !canCompressResponse) { + this.sendLineToClient("Content-Length: " + (1 + this.byteLength(json, "utf8")) + "\r\n\r\n" + json); + } + else { + // TODO: measure time + const compressed = this.compress(json); + if (this.logger.isVerbose()) { + this.logger.info(`compressed message to ${compressed.length}`); + } + this.sendCompressedDataToClient(`Content-Length: ${compressed.length + 1}`, compressed); + } } public configFileDiagnosticEvent(triggerFile: string, configFile: string, diagnostics: ts.Diagnostic[]) { @@ -236,7 +260,7 @@ namespace ts.server { diagnostics: bakedDiags } }; - this.send(ev); + this.send(ev, /*canCompressResponse*/ false); } public event(info: any, eventName: string) { @@ -246,10 +270,10 @@ namespace ts.server { event: eventName, body: info, }; - this.send(ev); + this.send(ev, /*canCompressResponse*/ false); } - private response(info: any, cmdName: string, reqSeq = 0, errorMsg?: string) { + private response(info: any, cmdName: string, canCompressResponse: boolean, reqSeq = 0, errorMsg?: string) { const res: protocol.Response = { seq: 0, type: "response", @@ -263,11 +287,11 @@ namespace ts.server { else { res.message = errorMsg; } - this.send(res); + this.send(res, canCompressResponse); } - public output(body: any, commandName: string, requestSequence = 0, errorMessage?: string) { - this.response(body, commandName, requestSequence, errorMessage); + public output(body: any, commandName: string, canCompressResponse: boolean, requestSequence = 0, errorMessage?: string) { + this.response(body, commandName, canCompressResponse, requestSequence, errorMessage); } private getLocation(position: number, scriptInfo: ScriptInfo): protocol.Location { @@ -1030,7 +1054,7 @@ namespace ts.server { this.changeSeq++; // make sure no changes happen before this one is finished if (project.reloadScript(file)) { - this.output(undefined, CommandNames.Reload, reqSeq); + this.output(undefined, CommandNames.Reload, /*canCompressResponse*/ false, reqSeq); } } } @@ -1407,7 +1431,7 @@ namespace ts.server { }, [CommandNames.Configure]: (request: protocol.ConfigureRequest) => { this.projectService.setHostConfiguration(request.arguments); - this.output(undefined, CommandNames.Configure, request.seq); + this.output(undefined, CommandNames.Configure, /*canCompressResponse*/ false, request.seq); return this.notRequired(); }, [CommandNames.Reload]: (request: protocol.ReloadRequest) => { @@ -1473,7 +1497,7 @@ namespace ts.server { } else { this.projectService.log("Unrecognized JSON command: " + JSON.stringify(request)); - this.output(undefined, CommandNames.Unknown, request.seq, "Unrecognized JSON command: " + request.command); + this.output(undefined, CommandNames.Unknown, /*canCompressResponse*/ false, request.seq, "Unrecognized JSON command: " + request.command); return { responseRequired: false }; } } @@ -1501,22 +1525,23 @@ namespace ts.server { this.logger.msg(leader + ": " + elapsedMs.toFixed(4).toString(), "Perf"); } if (response) { - this.output(response, request.command, request.seq); + this.output(response, request.command, request.canCompressResponse, request.seq); } else if (responseRequired) { - this.output(undefined, request.command, request.seq, "No content available."); + this.output(undefined, request.command, /*canCompressResponse*/ false, request.seq, "No content available."); } } catch (err) { if (err instanceof OperationCanceledException) { // Handle cancellation exceptions - this.output({ canceled: true }, request.command, request.seq); + this.output({ canceled: true }, request.command, /*canCompressResponse*/ false, request.seq); return; } this.logError(err, message); this.output( undefined, request ? request.command : CommandNames.Unknown, + /*canCompressResponse*/ false, request ? request.seq : 0, "Error processing request. " + (err).message + "\n" + (err).stack); } diff --git a/tests/cases/unittests/cachingInServerLSHost.ts b/tests/cases/unittests/cachingInServerLSHost.ts index 888618fcefb..f3fb83e6ffc 100644 --- a/tests/cases/unittests/cachingInServerLSHost.ts +++ b/tests/cases/unittests/cachingInServerLSHost.ts @@ -29,6 +29,9 @@ namespace ts { writeFile: (path: string, data: string, writeByteOrderMark?: boolean) => { throw new Error("NYI"); }, + writeCompressedData() { + throw new Error("NYI"); + }, resolvePath: (path: string): string => { throw new Error("NYI"); }, diff --git a/tests/cases/unittests/session.ts b/tests/cases/unittests/session.ts index 90343f4699d..3eb928a0462 100644 --- a/tests/cases/unittests/session.ts +++ b/tests/cases/unittests/session.ts @@ -23,7 +23,8 @@ namespace ts.server { setTimeout(callback, ms, ...args) { return 0; }, clearTimeout(timeoutId) { }, setImmediate: () => 0, - clearImmediate() {} + clearImmediate() {}, + writeCompressedData() {} }; const nullCancellationToken: HostCancellationToken = { isCancellationRequested: () => false }; const mockLogger: Logger = { @@ -42,7 +43,7 @@ namespace ts.server { let lastSent: protocol.Message; beforeEach(() => { - session = new Session(mockHost, nullCancellationToken, /*useOneInferredProject*/ false, Utils.byteLength, process.hrtime, mockLogger); + session = new Session(mockHost, nullCancellationToken, /*useOneInferredProject*/ false, Utils.byteLength, Utils.compress, process.hrtime, mockLogger); session.send = (msg: protocol.Message) => { lastSent = msg; }; @@ -180,7 +181,7 @@ namespace ts.server { session.send = Session.prototype.send; assert(session.send); - expect(session.send(msg)).to.not.exist; + expect(session.send(msg, /*canCompressResponse*/ false)).to.not.exist; expect(lastWrittenToHost).to.equal(resultMsg); }); }); @@ -248,7 +249,7 @@ namespace ts.server { }; const command = "test"; - session.output(body, command); + session.output(body, command, /*canCompressResponse*/ false); expect(lastSent).to.deep.equal({ seq: 0, @@ -267,7 +268,7 @@ namespace ts.server { lastSent: protocol.Message; customHandler = "testhandler"; constructor() { - super(mockHost, nullCancellationToken, /*useOneInferredProject*/ false, Utils.byteLength, process.hrtime, mockLogger); + super(mockHost, nullCancellationToken, /*useOneInferredProject*/ false, Utils.byteLength, Utils.compress, process.hrtime, mockLogger); this.addProtocolHandler(this.customHandler, () => { return { response: undefined, responseRequired: true }; }); @@ -286,7 +287,7 @@ namespace ts.server { }; const command = "test"; - session.output(body, command); + session.output(body, command, /*canCompressResponse*/ false); expect(session.lastSent).to.deep.equal({ seq: 0, @@ -325,7 +326,7 @@ namespace ts.server { class InProcSession extends Session { private queue: protocol.Request[] = []; constructor(private client: InProcClient) { - super(mockHost, nullCancellationToken, /*useOneInferredProject*/ false, Utils.byteLength, process.hrtime, mockLogger); + super(mockHost, nullCancellationToken, /*useOneInferredProject*/ false, Utils.byteLength, Utils.compress, process.hrtime, mockLogger); this.addProtocolHandler("echo", (req: protocol.Request) => ({ response: req.arguments, responseRequired: true @@ -346,11 +347,11 @@ namespace ts.server { ({ response } = this.executeCommand(msg)); } catch (e) { - this.output(undefined, msg.command, msg.seq, e.toString()); + this.output(undefined, msg.command, /*canCompressResponse*/ false, msg.seq, e.toString()); return; } if (response) { - this.output(response, msg.command, msg.seq); + this.output(response, msg.command, /*canCompressResponse*/ false, msg.seq); } } diff --git a/tests/cases/unittests/tsserverProjectSystem.ts b/tests/cases/unittests/tsserverProjectSystem.ts index e883587d812..b5a5f3aef54 100644 --- a/tests/cases/unittests/tsserverProjectSystem.ts +++ b/tests/cases/unittests/tsserverProjectSystem.ts @@ -366,6 +366,7 @@ namespace ts { readonly getExecutingFilePath = () => this.executingFilePath; readonly getCurrentDirectory = () => this.currentDirectory; readonly writeFile = (path: string, content: string) => notImplemented(); + readonly writeCompressedData = () => notImplemented(); readonly write = (s: string) => notImplemented(); readonly createDirectory = (s: string) => notImplemented(); readonly exit = () => notImplemented();