microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-27 17:43:00 -05:00
Code Issues Packages Projects Releases 177 Wiki Activity

eng - make fetchWithRetry more robust (#184716)

Browse Source
This commit is contained in:
Benjamin Pasero
2023-06-09 14:05:38 +02:00
committed by GitHub
parent 9979f9cc3c
commit eac16e9b63
2 changed files with 62 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
build/lib/gulpRemoteSource.js
View File

@@ -34,38 +34,51 @@ function remote(urls, options) {
}));
}
exports.remote = remote;
async function fetchWithRetry(url, options, retries = 10, retryDelay = 250) {
async function fetchWithRetry(url, options, retries = 10, retryDelay = 1000) {
try {
let startTime = 0;
if (options.verbose) {
log(`Start fetching ${ansiColors.magenta(url)}${retries !== 10 ? `(${10 - retries} retry}` : ''}`);
startTime = new Date().getTime();
}
const response = await (0, node_fetch_1.default)(url, options.fetchOptions);
if (options.verbose) {
log(`Fetch completed: Status ${response.status}. Took ${ansiColors.magenta(`${new Date().getTime() - startTime} ms`)}`);
}
if (response.ok && (response.status >= 200 && response.status < 300)) {
// request must be piped out once created, or we'll get this error: "You cannot pipe after data has been emitted from the response."
const contents = options.buffer ? await response.buffer() : response.body.pipe(through2());
return new VinylFile({
cwd: '/',
base: options.base,
path: url,
contents
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30 * 1000);
try {
const response = await (0, node_fetch_1.default)(url, {
...options.fetchOptions,
signal: controller.signal /* Typings issue with lib.dom.d.ts */
});
if (options.verbose) {
log(`Fetch completed: Status ${response.status}. Took ${ansiColors.magenta(`${new Date().getTime() - startTime} ms`)}`);
}
if (response.ok && (response.status >= 200 && response.status < 300)) {
// request must be piped out once created, or we'll get this error: "You cannot pipe after data has been emitted from the response."
const contents = options.buffer ? await response.buffer() : response.body.pipe(through2());
if (options.buffer && options.verbose) {
log(`Fetched response body buffer: ${ansiColors.magenta(`${contents.byteLength} bytes`)}`);
}
return new VinylFile({
cwd: '/',
base: options.base,
path: url,
contents
});
}
throw new Error(`Request ${ansiColors.magenta(url)} failed with status code: ${response.status}`);
}
finally {
clearTimeout(timeout);
}
throw new Error(`Request ${ansiColors.magenta(url)} failed with status code: ${response.status}`);
}
catch (e) {
if (retries > 0) {
await new Promise(c => setTimeout(c, retryDelay));
return fetchWithRetry(url, options, retries - 1, retryDelay * 2);
}
if (options.verbose) {
log(`Fetching ${ansiColors.cyan(url)} failed: ${e}`);
}
if (retries > 0) {
await new Promise(resolve => setTimeout(resolve, retryDelay));
return fetchWithRetry(url, options, retries - 1, retryDelay);
}
throw e;
}
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ3VscFJlbW90ZVNvdXJjZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbImd1bHBSZW1vdGVTb3VyY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBOzs7Z0dBR2dHOzs7QUFFaEcsbUNBQW1DO0FBQ25DLDJDQUFnRDtBQUNoRCxtQ0FBbUM7QUFDbkMscUNBQXFDO0FBQ3JDLGlDQUFpQztBQUNqQywwQ0FBMEM7QUFTMUMsU0FBZ0IsTUFBTSxDQUFDLElBQXVCLEVBQUUsT0FBaUI7SUFDaEUsSUFBSSxPQUFPLEtBQUssU0FBUyxFQUFFO1FBQzFCLE9BQU8sR0FBRyxFQUFFLENBQUM7S0FDYjtJQUVELElBQUksT0FBTyxPQUFPLENBQUMsSUFBSSxLQUFLLFFBQVEsSUFBSSxPQUFPLENBQUMsSUFBSSxLQUFLLElBQUksRUFBRTtRQUM5RCxPQUFPLENBQUMsSUFBSSxHQUFHLEdBQUcsQ0FBQztLQUNuQjtJQUVELElBQUksT0FBTyxPQUFPLENBQUMsTUFBTSxLQUFLLFNBQVMsRUFBRTtRQUN4QyxPQUFPLENBQUMsTUFBTSxHQUFHLElBQUksQ0FBQztLQUN0QjtJQUVELElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxFQUFFO1FBQ3pCLElBQUksR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO0tBQ2Q7SUFFRCxPQUFPLEVBQUUsQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxHQUFHLENBQTJCLENBQUMsSUFBWSxFQUFFLEVBQUUsRUFBRSxFQUFFO1FBQ3BGLE1BQU0sR0FBRyxHQUFHLENBQUMsT0FBTyxDQUFDLElBQUksRUFBRSxJQUFJLENBQUMsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7UUFDMUMsY0FBYyxDQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUU7WUFDeEMsRUFBRSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsQ0FBQztRQUNyQixDQUFDLEVBQUUsS0FBSyxDQUFDLEVBQUU7WUFDVixFQUFFLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDWCxDQUFDLENBQUMsQ0FBQztJQUNKLENBQUMsQ0FBQyxDQUFDLENBQUM7QUFDTCxDQUFDO0FBekJELHdCQXlCQztBQUVELEtBQUssVUFBVSxjQUFjLENBQUMsR0FBVyxFQUFFLE9BQWlCLEVBQUUsT0FBTyxHQUFHLEVBQUUsRUFBRSxVQUFVLEdBQUcsR0FBRztJQUMzRixJQUFJO1FBQ0gsSUFBSSxTQUFTLEdBQUcsQ0FBQyxDQUFDO1FBQ2xCLElBQUksT0FBTyxDQUFDLE9BQU8sRUFBRTtZQUNwQixHQUFHLENBQUMsa0JBQWtCLFVBQVUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsT0FBTyxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLEdBQUcsT0FBTyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7WUFDbkcsU0FBUyxHQUFHLElBQUksSUFBSSxFQUFFLENBQUMsT0FBTyxFQUFFLENBQUM7U0FDakM7UUFDRCxNQUFNLFFBQVEsR0FBRyxNQUFNLElBQUEsb0JBQUssRUFBQyxHQUFHLEVBQUUsT0FBTyxDQUFDLFlBQVksQ0FBQyxDQUFDO1FBQ3hELElBQUksT0FBTyxDQUFDLE9BQU8sRUFBRTtZQUNwQixHQUFHLENBQUMsMkJBQTJCLFFBQVEsQ0FBQyxNQUFNLFVBQVUsVUFBVSxDQUFDLE9BQU8sQ0FBQyxHQUFHLElBQUksSUFBSSxFQUFFLENBQUMsT0FBTyxFQUFFLEdBQUcsU0FBUyxLQUFLLENBQUMsRUFBRSxDQUFDLENBQUM7U0FDeEg7UUFDRCxJQUFJLFFBQVEsQ0FBQyxFQUFFLElBQUksQ0FBQyxRQUFRLENBQUMsTUFBTSxJQUFJLEdBQUcsSUFBSSxRQUFRLENBQUMsTUFBTSxHQUFHLEdBQUcsQ0FBQyxFQUFFO1lBQ3JFLG9JQUFvSTtZQUNwSSxNQUFNLFFBQVEsR0FBRyxPQUFPLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxNQUFNLFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxDQUFDLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQztZQUMzRixPQUFPLElBQUksU0FBUyxDQUFDO2dCQUNwQixHQUFHLEVBQUUsR0FBRztnQkFDUixJQUFJLEVBQUUsT0FBTyxDQUFDLElBQUk7Z0JBQ2xCLElBQUksRUFBRSxHQUFHO2dCQUNULFFBQVE7YUFDUixDQUFDLENBQUM7U0FDSDtRQUNELE1BQU0sSUFBSSxLQUFLLENBQUMsV0FBVyxVQUFVLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyw2QkFBNkIsUUFBUSxDQUFDLE1BQU0sRUFBRSxDQUFDLENBQUM7S0FDbEc7SUFBQyxPQUFPLENBQUMsRUFBRTtRQUNYLElBQUksT0FBTyxHQUFHLENBQUMsRUFBRTtZQUNoQixNQUFNLElBQUksT0FBTyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsVUFBVSxDQUFDLENBQUMsRUFBRSxVQUFVLENBQUMsQ0FBQyxDQUFDO1lBQ2xELE9BQU8sY0FBYyxDQUFDLEdBQUcsRUFBRSxPQUFPLEVBQUUsT0FBTyxHQUFHLENBQUMsRUFBRSxVQUFVLEdBQUcsQ0FBQyxDQUFDLENBQUM7U0FDakU7UUFDRCxJQUFJLE9BQU8sQ0FBQyxPQUFPLEVBQUU7WUFDcEIsR0FBRyxDQUFDLFlBQVksVUFBVSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLEVBQUUsQ0FBQyxDQUFDO1NBQ3JEO1FBQ0QsTUFBTSxDQUFDLENBQUM7S0FDUjtBQUNGLENBQUMifQ==
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ3VscFJlbW90ZVNvdXJjZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbImd1bHBSZW1vdGVTb3VyY2UudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IjtBQUFBOzs7Z0dBR2dHOzs7QUFFaEcsbUNBQW1DO0FBQ25DLDJDQUFnRDtBQUNoRCxtQ0FBbUM7QUFDbkMscUNBQXFDO0FBQ3JDLGlDQUFpQztBQUNqQywwQ0FBMEM7QUFTMUMsU0FBZ0IsTUFBTSxDQUFDLElBQXVCLEVBQUUsT0FBaUI7SUFDaEUsSUFBSSxPQUFPLEtBQUssU0FBUyxFQUFFO1FBQzFCLE9BQU8sR0FBRyxFQUFFLENBQUM7S0FDYjtJQUVELElBQUksT0FBTyxPQUFPLENBQUMsSUFBSSxLQUFLLFFBQVEsSUFBSSxPQUFPLENBQUMsSUFBSSxLQUFLLElBQUksRUFBRTtRQUM5RCxPQUFPLENBQUMsSUFBSSxHQUFHLEdBQUcsQ0FBQztLQUNuQjtJQUVELElBQUksT0FBTyxPQUFPLENBQUMsTUFBTSxLQUFLLFNBQVMsRUFBRTtRQUN4QyxPQUFPLENBQUMsTUFBTSxHQUFHLElBQUksQ0FBQztLQUN0QjtJQUVELElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxFQUFFO1FBQ3pCLElBQUksR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO0tBQ2Q7SUFFRCxPQUFPLEVBQUUsQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxDQUFDLEVBQUUsQ0FBQyxHQUFHLENBQTJCLENBQUMsSUFBWSxFQUFFLEVBQUUsRUFBRSxFQUFFO1FBQ3BGLE1BQU0sR0FBRyxHQUFHLENBQUMsT0FBTyxDQUFDLElBQUksRUFBRSxJQUFJLENBQUMsQ0FBQyxJQUFJLENBQUMsRUFBRSxDQUFDLENBQUM7UUFDMUMsY0FBYyxDQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLEVBQUU7WUFDeEMsRUFBRSxDQUFDLFNBQVMsRUFBRSxJQUFJLENBQUMsQ0FBQztRQUNyQixDQUFDLEVBQUUsS0FBSyxDQUFDLEVBQUU7WUFDVixFQUFFLENBQUMsS0FBSyxDQUFDLENBQUM7UUFDWCxDQUFDLENBQUMsQ0FBQztJQUNKLENBQUMsQ0FBQyxDQUFDLENBQUM7QUFDTCxDQUFDO0FBekJELHdCQXlCQztBQUVELEtBQUssVUFBVSxjQUFjLENBQUMsR0FBVyxFQUFFLE9BQWlCLEVBQUUsT0FBTyxHQUFHLEVBQUUsRUFBRSxVQUFVLEdBQUcsSUFBSTtJQUM1RixJQUFJO1FBQ0gsSUFBSSxTQUFTLEdBQUcsQ0FBQyxDQUFDO1FBQ2xCLElBQUksT0FBTyxDQUFDLE9BQU8sRUFBRTtZQUNwQixHQUFHLENBQUMsa0JBQWtCLFVBQVUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsT0FBTyxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUFFLEdBQUcsT0FBTyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUM7WUFDbkcsU0FBUyxHQUFHLElBQUksSUFBSSxFQUFFLENBQUMsT0FBTyxFQUFFLENBQUM7U0FDakM7UUFDRCxNQUFNLFVBQVUsR0FBRyxJQUFJLGVBQWUsRUFBRSxDQUFDO1FBQ3pDLE1BQU0sT0FBTyxHQUFHLFVBQVUsQ0FBQyxHQUFHLEVBQUUsQ0FBQyxVQUFVLENBQUMsS0FBSyxFQUFFLEVBQUUsRUFBRSxHQUFHLElBQUksQ0FBQyxDQUFDO1FBQ2hFLElBQUk7WUFDSCxNQUFNLFFBQVEsR0FBRyxNQUFNLElBQUEsb0JBQUssRUFBQyxHQUFHLEVBQUU7Z0JBQ2pDLEdBQUcsT0FBTyxDQUFDLFlBQVk7Z0JBQ3ZCLE1BQU0sRUFBRSxVQUFVLENBQUMsTUFBYSxDQUFDLHFDQUFxQzthQUN0RSxDQUFDLENBQUM7WUFDSCxJQUFJLE9BQU8sQ0FBQyxPQUFPLEVBQUU7Z0JBQ3BCLEdBQUcsQ0FBQywyQkFBMkIsUUFBUSxDQUFDLE1BQU0sVUFBVSxVQUFVLENBQUMsT0FBTyxDQUFDLEdBQUcsSUFBSSxJQUFJLEVBQUUsQ0FBQyxPQUFPLEVBQUUsR0FBRyxTQUFTLEtBQUssQ0FBQyxFQUFFLENBQUMsQ0FBQzthQUN4SDtZQUNELElBQUksUUFBUSxDQUFDLEVBQUUsSUFBSSxDQUFDLFFBQVEsQ0FBQyxNQUFNLElBQUksR0FBRyxJQUFJLFFBQVEsQ0FBQyxNQUFNLEdBQUcsR0FBRyxDQUFDLEVBQUU7Z0JBQ3JFLG9JQUFvSTtnQkFDcEksTUFBTSxRQUFRLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsTUFBTSxRQUFRLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUM7Z0JBQzNGLElBQUksT0FBTyxDQUFDLE1BQU0sSUFBSSxPQUFPLENBQUMsT0FBTyxFQUFFO29CQUN0QyxHQUFHLENBQUMsaUNBQWlDLFVBQVUsQ0FBQyxPQUFPLENBQUMsR0FBSSxRQUFtQixDQUFDLFVBQVUsUUFBUSxDQUFDLEVBQUUsQ0FBQyxDQUFDO2lCQUN2RztnQkFDRCxPQUFPLElBQUksU0FBUyxDQUFDO29CQUNwQixHQUFHLEVBQUUsR0FBRztvQkFDUixJQUFJLEVBQUUsT0FBTyxDQUFDLElBQUk7b0JBQ2xCLElBQUksRUFBRSxHQUFHO29CQUNULFFBQVE7aUJBQ1IsQ0FBQyxDQUFDO2FBQ0g7WUFDRCxNQUFNLElBQUksS0FBSyxDQUFDLFdBQVcsVUFBVSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsNkJBQTZCLFFBQVEsQ0FBQyxNQUFNLEVBQUUsQ0FBQyxDQUFDO1NBQ2xHO2dCQUFTO1lBQ1QsWUFBWSxDQUFDLE9BQU8sQ0FBQyxDQUFDO1NBQ3RCO0tBQ0Q7SUFBQyxPQUFPLENBQUMsRUFBRTtRQUNYLElBQUksT0FBTyxDQUFDLE9BQU8sRUFBRTtZQUNwQixHQUFHLENBQUMsWUFBWSxVQUFVLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsRUFBRSxDQUFDLENBQUM7U0FDckQ7UUFDRCxJQUFJLE9BQU8sR0FBRyxDQUFDLEVBQUU7WUFDaEIsTUFBTSxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDLFVBQVUsQ0FBQyxPQUFPLEVBQUUsVUFBVSxDQUFDLENBQUMsQ0FBQztZQUM5RCxPQUFPLGNBQWMsQ0FBQyxHQUFHLEVBQUUsT0FBTyxFQUFFLE9BQU8sR0FBRyxDQUFDLEVBQUUsVUFBVSxDQUFDLENBQUM7U0FDN0Q7UUFDRCxNQUFNLENBQUMsQ0FBQztLQUNSO0FBQ0YsQ0FBQyJ9

48
build/lib/gulpRemoteSource.ts
View File

@@ -44,36 +44,48 @@ export function remote(urls: string[] | string, options: IOptions): es.ThroughSt
}));
}
async function fetchWithRetry(url: string, options: IOptions, retries = 10, retryDelay = 250): Promise<VinylFile> {
async function fetchWithRetry(url: string, options: IOptions, retries = 10, retryDelay = 1000): Promise<VinylFile> {
try {
let startTime = 0;
if (options.verbose) {
log(`Start fetching ${ansiColors.magenta(url)}${retries !== 10 ? `(${10 - retries} retry}` : ''}`);
startTime = new Date().getTime();
}
const response = await fetch(url, options.fetchOptions);
if (options.verbose) {
log(`Fetch completed: Status ${response.status}. Took ${ansiColors.magenta(`${new Date().getTime() - startTime} ms`)}`);
}
if (response.ok && (response.status >= 200 && response.status < 300)) {
// request must be piped out once created, or we'll get this error: "You cannot pipe after data has been emitted from the response."
const contents = options.buffer ? await response.buffer() : response.body.pipe(through2());
return new VinylFile({
cwd: '/',
base: options.base,
path: url,
contents
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30 * 1000);
try {
const response = await fetch(url, {
...options.fetchOptions,
signal: controller.signal as any /* Typings issue with lib.dom.d.ts */
});
if (options.verbose) {
log(`Fetch completed: Status ${response.status}. Took ${ansiColors.magenta(`${new Date().getTime() - startTime} ms`)}`);
}
if (response.ok && (response.status >= 200 && response.status < 300)) {
// request must be piped out once created, or we'll get this error: "You cannot pipe after data has been emitted from the response."
const contents = options.buffer ? await response.buffer() : response.body.pipe(through2());
if (options.buffer && options.verbose) {
log(`Fetched response body buffer: ${ansiColors.magenta(`${(contents as Buffer).byteLength} bytes`)}`);
}
return new VinylFile({
cwd: '/',
base: options.base,
path: url,
contents
});
}
throw new Error(`Request ${ansiColors.magenta(url)} failed with status code: ${response.status}`);
} finally {
clearTimeout(timeout);
}
throw new Error(`Request ${ansiColors.magenta(url)} failed with status code: ${response.status}`);
} catch (e) {
if (retries > 0) {
await new Promise(c => setTimeout(c, retryDelay));
return fetchWithRetry(url, options, retries - 1, retryDelay * 2);
}
if (options.verbose) {
log(`Fetching ${ansiColors.cyan(url)} failed: ${e}`);
}
if (retries > 0) {
await new Promise(resolve => setTimeout(resolve, retryDelay));
return fetchWithRetry(url, options, retries - 1, retryDelay);
}
throw e;
}
}