-
Notifications
You must be signed in to change notification settings - Fork 3.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: download browsers as TAR #34033
base: main
Are you sure you want to change the base?
Changes from 19 commits
a9b40bd
0b139f1
98d6138
657e435
c5ab0c5
3b72c3b
411cf29
d880a22
f39ae50
f071337
45b84b7
a497b6b
b048d2a
affff4d
879afb1
16ff2df
eaf6a52
220e28b
0ccbef1
30c5ec6
e2b1500
25494cb
2224fd2
41a552f
dce95a5
2749367
7744cc5
c717419
a2d214f
5086a56
e24a0d7
746bb68
c8b2ce0
6fb5355
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,6 +19,10 @@ | |
import { httpRequest } from '../../utils/network'; | ||
import { ManualPromise } from '../../utils/manualPromise'; | ||
import { extract } from '../../zipBundle'; | ||
import tar from '../../utils/tar'; | ||
Check failure on line 22 in packages/playwright-core/src/server/registry/oopDownloadBrowserMain.ts GitHub Actions / docs & lint
|
||
import type http from 'http'; | ||
import { pipeline } from 'stream/promises'; | ||
import { createBrotliDecompress } from 'zlib'; | ||
|
||
export type DownloadParams = { | ||
title: string; | ||
|
@@ -104,11 +108,72 @@ | |
} | ||
} | ||
|
||
async function throwUnexpectedResponseError(response: http.IncomingMessage) { | ||
let body = ''; | ||
try { | ||
await new Promise<void>((resolve, reject) => { | ||
response | ||
.on('data', chunk => body += chunk) | ||
.on('end', resolve) | ||
.on('error', reject); | ||
}); | ||
} catch (error) { | ||
body += error; | ||
} | ||
|
||
response.resume(); // consume response data to free up memory | ||
|
||
throw new Error(`server returned code ${response.statusCode} body '${body}'`); | ||
} | ||
|
||
async function downloadAndExtractBrotli(options: DownloadParams) { | ||
const response = await new Promise<http.IncomingMessage>((resolve, reject) => httpRequest({ | ||
url: options.url, | ||
headers: { | ||
'User-Agent': options.userAgent, | ||
}, | ||
timeout: options.connectionTimeout, | ||
}, resolve, reject)); | ||
|
||
log(`-- response status code: ${response.statusCode}`); | ||
if (response.statusCode !== 200) | ||
await throwUnexpectedResponseError(response); | ||
|
||
const totalBytes = parseInt(response.headers['content-length'] || '0', 10); | ||
log(`-- total bytes: ${totalBytes}`); | ||
|
||
let downloadedBytes = 0; | ||
response.on('data', chunk => { | ||
downloadedBytes += chunk.length; | ||
progress(downloadedBytes, totalBytes); | ||
}); | ||
|
||
await pipeline( | ||
response, | ||
createBrotliDecompress(), | ||
tar.extract(options.browserDirectory) | ||
); | ||
|
||
if (downloadedBytes !== totalBytes) | ||
throw new Error(`size mismatch, file size: ${downloadedBytes}, expected size: ${totalBytes}`); | ||
|
||
log(`-- download complete, size: ${downloadedBytes}`); | ||
} | ||
|
||
async function main(options: DownloadParams) { | ||
await downloadFile(options); | ||
log(`SUCCESS downloading ${options.title}`); | ||
log(`extracting archive`); | ||
await extract(options.zipPath, { dir: options.browserDirectory }); | ||
if (options.url.endsWith('.tar.br')) { | ||
try { | ||
await downloadAndExtractBrotli(options); | ||
} catch (error) { | ||
throw new Error(`Download failed. URL: ${options.url}`, { cause: error }); | ||
} | ||
log(`SUCCESS downloading and extracting ${options.title}`); | ||
} else { | ||
await downloadFile(options); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm seeing different error handling code in this branch, including explicit checks for ECONNRESET. Is walking away from them intended? Should we do both changes at a time? I'd be more comfortable with leaving the download code as is and swapping piping into file with piping into broti. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The new branch is intended to be as similar as possible, while also making the code a little more linear. The Let me see if I can refactor it to make the change less spooky. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've refactored it so we can reuse the existing download function. Good pointer, thanks! |
||
log(`SUCCESS downloading ${options.title}`); | ||
log(`extracting archive`); | ||
await extract(options.zipPath, { dir: options.browserDirectory }); | ||
} | ||
if (options.executablePath) { | ||
log(`fixing permissions at ${options.executablePath}`); | ||
await fs.promises.chmod(options.executablePath, 0o755); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
This directory contains a modified copy of the `tar-stream` library that's used exclusively to extract TAR files. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make sure all the third party files are under the third_party folder and corresponding license files are provided beside the files. Make sure they end up in third party list or in a distributed bundle There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done! See |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
whats the motivation for changing this?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
1010
doesn't have.tar.br
, and1010
is identical to1011
in functionality