diff --git a/bazel/browsers/update_script/BUILD.bazel b/bazel/browsers/update_script/BUILD.bazel new file mode 100644 index 00000000..1293e2cc --- /dev/null +++ b/bazel/browsers/update_script/BUILD.bazel @@ -0,0 +1,36 @@ +load("//tools:defaults.bzl", "esbuild", "ts_library") +load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary") + +package(default_visibility = ["//visibility:private"]) + +ts_library( + name = "update_script_lib", + testonly = True, + srcs = glob(["*.ts"]), + deps = [ + "//ng-dev/utils", + "@npm//@google-cloud/storage", + "@npm//@types/node", + "@npm//@types/tmp", + "@npm//@types/yargs", + "@npm//node-fetch", + "@npm//tmp", + "@npm//yargs", + ], +) + +esbuild( + name = "cli_script", + testonly = True, + entry_point = ":index.ts", + format = "iife", + deps = [":update_script_lib"], +) + +nodejs_binary( + name = "update-script", + testonly = True, + data = [":cli_script"], + entry_point = "cli_script.js", + templated_args = ["--nobazel_run_linker"], +) diff --git a/bazel/browsers/update_script/README.md b/bazel/browsers/update_script/README.md new file mode 100644 index 00000000..285128fc --- /dev/null +++ b/bazel/browsers/update_script/README.md @@ -0,0 +1,4 @@ +# Browser Update Script + +Forked from +https://github.com/angular/dev-infra/tree/1ad20ef9dd457de967252283c1a968b0d702d0ae/bazel/browsers/update-script. diff --git a/bazel/browsers/update_script/browser-artifact.mts b/bazel/browsers/update_script/browser-artifact.mts new file mode 100644 index 00000000..7830095c --- /dev/null +++ b/bazel/browsers/update_script/browser-artifact.mts @@ -0,0 +1,69 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Browser} from './browser.mjs'; +import * as path from 'path'; + +/** Type describing possible artifact types for browser downloads. */ +export type ArtifactType = 'driver-bin' | 'browser-bin'; + +/** Set of known artifact extensions, including chained extensions for gzipped files. */ +const KNOWN_EXTENSIONS = new Set(['zip', 'tar.gz', 'tar.bz2', 'dmg']); + +/** Class describing an artifact for a browser. */ +export class BrowserArtifact { + constructor( + /** Instance of the browser this artifact exists for. */ + public browser: Browser, + /** Type of the artifact. */ + public type: ArtifactType, + /** URL for downloading the artifact. */ + public downloadUrl: string, + /** Extension of the artifact. If unspecified, derived from the download URL. */ + public extension: string = detectArtifactExtension(downloadUrl), + ) {} +} + +/** + * Gets the extension of a given artifact file, excluding the dot/period. + * + * Since artifact download URLs can use chained extensions as for + * example with `.tar.gz`, we will need to keep track of known extensions + * and start looking with the first dot/period we discover. + */ +export function detectArtifactExtension(filePath: string) { + let tmpPath: string = filePath; + let extension: string = ''; + let currentPart: string = ''; + + // Iterate from the end of the path, finding the largest possible + // extension substring, accounting for cases like `a/b.tmp/file.tar.gz`. + while ((currentPart = path.extname(tmpPath)) !== '') { + // An extension needs to be a continuous set of alphanumeric characters. This is a rather + // strict requirement as technically extensions could contain e.g. `dashes`. In our case + // this strictness is acceptable though as we don't expect such extensions and it makes + // this extension detection logic more correct. e.g. the logic would not incorrectly + // detect an extension for `firefox-97.0-linux.tar.gz` to `0-linux.tar.gz`. + if (!/^\.[a-zA-Z0-9]+$/.test(currentPart)) { + break; + } + + extension = currentPart + extension; + tmpPath = path.basename(tmpPath, currentPart); + } + + // Strip off the leading period/dot from the extension. + // If there is no extension, this string would remain empty. + extension = extension.substring(1); + + if (KNOWN_EXTENSIONS.has(extension)) { + return extension; + } + + throw new Error(`Unable to find known extension for file path: ${filePath}`); +} diff --git a/bazel/browsers/update_script/browser.mts b/bazel/browsers/update_script/browser.mts new file mode 100644 index 00000000..87c55213 --- /dev/null +++ b/bazel/browsers/update_script/browser.mts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {BrowserArtifact, ArtifactType} from './browser-artifact.mjs'; +import {Platform} from './platform.mjs'; + +/** Interface describing a browser. */ +export interface Browser { + name: string; + revision: T; + supports(platform: Platform): boolean; + getArtifact(platform: Platform, type: ArtifactType): BrowserArtifact; +} diff --git a/bazel/browsers/update_script/chromium.mts b/bazel/browsers/update_script/chromium.mts new file mode 100644 index 00000000..669d7abe --- /dev/null +++ b/bazel/browsers/update_script/chromium.mts @@ -0,0 +1,82 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Browser} from './browser.mjs'; +import {ArtifactType, BrowserArtifact} from './browser-artifact.mjs'; +import {Platform} from './platform.mjs'; + +const cloudStorageArchiveUrl = + 'https://storage.googleapis.com/chromium-browser-snapshots/{platform}/{revision}/{file}'; + +const cloudStorageHeadRevisionUrl = `https://storage.googleapis.com/chromium-browser-snapshots/{platform}/LAST_CHANGE`; + +/** + * Map a platform to the platfrom key used by the Chromium snapshot buildbot. + * See: https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html. + */ +const PlatformSnapshotNameMap = { + [Platform.LINUX_X64]: 'Linux_x64', + [Platform.MAC_X64]: 'Mac', + [Platform.MAC_ARM64]: 'Mac_Arm', + [Platform.WINDOWS_X64]: 'Win', +}; + +/** Maps a browser platform to the snapshot archive file containing the browser binary. */ +const PlatformBrowserArchiveMap = { + [Platform.LINUX_X64]: 'chrome-linux.zip', + [Platform.MAC_X64]: 'chrome-mac.zip', + [Platform.MAC_ARM64]: 'chrome-mac.zip', + [Platform.WINDOWS_X64]: 'chrome-win.zip', +}; + +/** Maps a browser platform to the archive file containing the driver. */ +const PlatformDriverArchiveMap = { + [Platform.LINUX_X64]: 'chromedriver_linux64.zip', + [Platform.MAC_X64]: 'chromedriver_mac64.zip', + [Platform.MAC_ARM64]: 'chromedriver_mac64.zip', + [Platform.WINDOWS_X64]: 'chromedriver_win32.zip', +}; + +/** List of supported platforms for the Chromium binaries. */ +const supportedPlatforms = new Set([ + Platform.LINUX_X64, + Platform.MAC_X64, + Platform.MAC_ARM64, + Platform.WINDOWS_X64, +]); + +/** Class providing necessary information for the chromium browser. */ +export class Chromium implements Browser { + name = 'chromium'; + + constructor(public revision: number) {} + + supports(platform: Platform): boolean { + return supportedPlatforms.has(platform); + } + + getArtifact(platform: Platform, type: ArtifactType): BrowserArtifact { + return new BrowserArtifact(this, type, this.getDownloadUrl(platform, type)); + } + + getDownloadUrl(platform: Platform, type: ArtifactType): string { + return Chromium.getDownloadArtifactUrl(this.revision, platform, type); + } + + static getDownloadArtifactUrl(revision: number, platform: Platform, type: ArtifactType): string { + const archiveMap = type === 'driver-bin' ? PlatformDriverArchiveMap : PlatformBrowserArchiveMap; + return cloudStorageArchiveUrl + .replace('{platform}', PlatformSnapshotNameMap[platform]) + .replace('{revision}', `${revision}`) + .replace('{file}', archiveMap[platform]); + } + + static getLatestRevisionUrl(platform: Platform) { + return cloudStorageHeadRevisionUrl.replace('{platform}', PlatformSnapshotNameMap[platform]); + } +} diff --git a/bazel/browsers/update_script/find-revision-chromium.mts b/bazel/browsers/update_script/find-revision-chromium.mts new file mode 100644 index 00000000..ed6fdd37 --- /dev/null +++ b/bazel/browsers/update_script/find-revision-chromium.mts @@ -0,0 +1,189 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +/** + * @fileoverview + * Script that fetches the latest revision currently in the "stable" channel of Chromium. + * It then checks if build artifacts on the CDN exist for that revision. If there are missing + * build artifacts, it looks for more recent revisions, starting from the determined revision + * in the stable channel, and checks if those have build artifacts. This allows us to determine + * a Chromium revision that is as close as possible to the "stable" channel and we have CDN + * artifacts available for each supported platform. + * + * This is needed because Chromium does not build artifacts for every revision. See: + * https://github.com/puppeteer/puppeteer/issues/2567#issuecomment-393436282 + * + * Note: An explicit revision can be specified as command line argument. This allows + * for finding snapshot builds if a revision is already known. e.g. consider a case + * where a specific Chromium bug (needed for the Angular org) is fixed but is ahead + * of the current revision in the stable channel. We still may want to update Chromium + * to a revision ahead of the specified revision for which snapshot builds exist. + */ + +import {createHash} from 'crypto'; +import fetch from 'node-fetch'; +import {Spinner} from '../../../ng-dev/utils/spinner.js'; +import {ArtifactType} from './browser-artifact.mjs'; +import {Chromium} from './chromium.mjs'; +import {Platform} from './platform.mjs'; + +/** + * Entry-point for the script, finding a revision which has snapshot builds for all platforms. + * If an explicit start revision has been specified, this function looks for a closest + * revision that is available for all platforms. If none has been specified, we look for + * a revision that is as close as possible to the revision in the stable release channel. + */ +export async function findLatestRevisionForAllPlatforms( + explicitStartRevision: number | undefined, +): Promise { + const availableRevision = + explicitStartRevision === undefined + ? await findClosestStableRevisionForAllPlatforms() + : await findClosestAscendingRevisionForAllPlatforms(explicitStartRevision); + + if (availableRevision === null) { + console.error('Could not find a revision for which builds are available for all platforms.'); + process.exit(1); + } + + const browser = new Chromium(availableRevision); + + console.info('Found a revision for which builds are available for all platforms.'); + console.info('Printing the URLs and archive checksums:'); + console.info(); + // Note: We cannot extract the Chromium version and commit automatically because + // this requires an actual browser resolving a manual `window.open` redirect. + console.info('Release Info:', await getReleaseInfoUrlForRevision(availableRevision)); + console.info('Click on the link above to determine the Chromium version number.'); + console.info(); + + for (const platformName of Object.keys(Platform)) { + const platform = Platform[platformName as keyof typeof Platform]; + + console.info(`${platformName}: `.padEnd(10), browser.getDownloadUrl(platform, 'browser-bin')); + console.info( + ' '.repeat(15), + await getSha256ChecksumForPlatform(browser, platform, 'browser-bin'), + ); + console.info(' '.repeat(10), browser.getDownloadUrl(platform, 'driver-bin')); + console.info( + ' '.repeat(15), + await getSha256ChecksumForPlatform(browser, platform, 'driver-bin'), + ); + console.info(); + } +} + +/** + * Finds a Chromium revision which is as close as possible to the revision currently + * in the stable release channel, and for which snapshot builds exist for all platforms. + */ +async function findClosestStableRevisionForAllPlatforms(): Promise { + const stableBaseRevision = await getStableChromiumRevision(); + + // Note: We look for revisions with snapshot builds for every platform by searching in + // ascending order because going back to older revisions would mean that we use a revision + // which might miss fixes that have landed before the determined "stable" revision has been + // released. Note that searching for a revision is ascending order is technically also not + // ideal because it may contain new regressions, or new APIs, but either way is not ideal here. + // It seems better to use a more up-to-date revision, rather than relying on code that has + // already been fixed, but we'd accidentally use it then. + return findClosestAscendingRevisionForAllPlatforms(stableBaseRevision); +} + +/** + * Finds a Chromium revision in ascending order which is as close as possible to + * the specified revision and has snapshot builds for all platforms. + */ +async function findClosestAscendingRevisionForAllPlatforms( + startRevision: number, +): Promise { + return lookForRevisionWithBuildsForAllPlatforms(startRevision, await getHeadChromiumRevision()); +} + +/** + * Looks for revision within the specified revision range for which builds exist for + * every platform. This is needed because there are no builds available for every + * revision that lands within Chromium. More details can be found here: + * https://github.com/puppeteer/puppeteer/issues/2567#issuecomment-393436282. + */ +async function lookForRevisionWithBuildsForAllPlatforms( + startRevision: number, + toRevision: number, +): Promise { + const spinner = new Spinner('Looking for revision build.'); + const increment = toRevision >= startRevision ? 1 : -1; + + for (let i = startRevision; i !== toRevision; i += increment) { + spinner.update(`Checking: r${i}`); + + const checks = await Promise.all( + Object.values(Platform).map((p) => isRevisionAvailableForPlatform(i, p)), + ); + + // If the current revision is available for all platforms, stop + // searching and return the current revision. + if (checks.every((isAvailable) => isAvailable === true)) { + spinner.complete(); + console.log(` √ Found revision: r${i}`); + return i; + } + } + spinner.complete(); + console.log(' ✘ No revision found.'); + return null; +} + +/** Checks if the specified revision is available for the given platform. */ +async function isRevisionAvailableForPlatform( + revision: number, + platform: Platform, +): Promise { + // Look for the `driver` archive as this is smaller and faster to check. + const response = await fetch(Chromium.getDownloadArtifactUrl(revision, platform, 'driver-bin')); + return response.ok && response.status === 200; +} + +/** Gets the latest revision currently in the `stable` release channel of Chromium. */ +async function getStableChromiumRevision(): Promise { + // Endpoint is maintained by the Chromium team and can be consulted for determining + // the current latest revision in stable channel. + // https://github.com/googlechromelabs/chrome-for-testing/ + const response = await fetch( + `https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json`, + ); + const revisionStr = ((await response.json()) as any).channels.Stable.revision; + return Number(revisionStr); +} + +/** Gets the Chromium release information page URL for a given revision. */ +async function getReleaseInfoUrlForRevision(revision: number): Promise { + // This is a site used and maintained by the Chromium team. + // https://chromium.googlesource.com/chromium/chromium/+/refs/heads/trunk/tools/omahaproxy.py. + return `https://storage.googleapis.com/chromium-find-releases-static/index.html#r${revision}`; +} + +/** Determines the latest Chromium revision available in the CDN. */ +async function getHeadChromiumRevision(): Promise { + const responses = await Promise.all( + Object.values(Platform).map((p) => fetch(Chromium.getLatestRevisionUrl(p))), + ); + const revisions = await Promise.all(responses.map(async (r) => Number(await r.text()))); + return Math.max(...revisions); +} + +/** Gets the SHA256 checksum for the platform archive of a given chromium instance. */ +async function getSha256ChecksumForPlatform( + browser: Chromium, + platform: Platform, + artifactType: ArtifactType, +): Promise { + const response = await fetch(browser.getDownloadUrl(platform, artifactType)); + const binaryContent = Buffer.from(await response.arrayBuffer()); + return createHash('sha256').update(binaryContent).digest('hex'); +} diff --git a/bazel/browsers/update_script/firefox.mts b/bazel/browsers/update_script/firefox.mts new file mode 100644 index 00000000..b4d1b844 --- /dev/null +++ b/bazel/browsers/update_script/firefox.mts @@ -0,0 +1,75 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {ArtifactType, BrowserArtifact} from './browser-artifact.mjs'; +import {Browser} from './browser.mjs'; +import {Platform} from './platform.mjs'; +import {detectArtifactExtension} from './browser-artifact.mjs'; + +const downloadLinuxUrls = { + 'browser-bin': + 'https://ftp.mozilla.org/pub/firefox/releases/{version}/linux-x86_64/en-US/firefox-{version}.tar.bz2', + 'driver-bin': + 'https://github.com/mozilla/geckodriver/releases/download/v{version}/geckodriver-v{version}-linux64.tar.gz', +}; + +const downloadMacOsX64Urls = { + 'browser-bin': + 'https://ftp.mozilla.org/pub/firefox/releases/{version}/mac/en-US/Firefox {version}.dmg', + 'driver-bin': + 'https://github.com/mozilla/geckodriver/releases/download/v{version}/geckodriver-v{version}-macos.tar.gz', +}; + +const downloadMacOsArm64Urls = { + 'browser-bin': + 'https://ftp.mozilla.org/pub/firefox/releases/{version}/mac/en-US/Firefox {version}.dmg', + 'driver-bin': + 'https://github.com/mozilla/geckodriver/releases/download/v{version}/geckodriver-v{version}-macos-aarch64.tar.gz', +}; + +/** Class providing necessary information for the firefox browser. */ +export class Firefox implements Browser { + name = 'firefox'; + + constructor(public revision: string, public driverVersion: string) {} + + supports(platform: Platform): boolean { + return ( + platform === Platform.LINUX_X64 || + platform === Platform.MAC_X64 || + platform === Platform.MAC_ARM64 + ); + } + + getArtifact(platform: Platform, archiveType: ArtifactType): BrowserArtifact { + const urlSet = this._getUrlSetForPlatform(platform); + const baseUrl = urlSet[archiveType]; + const downloadUrl = baseUrl.replace( + /\{version}/g, + // Depending on browser, or driver being requested, substitute the associated version. + archiveType === 'browser-bin' ? this.revision : this.driverVersion, + ); + + // Note that for the artifact extension we will consult the non-substituted base URL + // as the substituted version like `97.0.tar.bz2` would throw off the detection. + return new BrowserArtifact(this, archiveType, downloadUrl, detectArtifactExtension(baseUrl)); + } + + private _getUrlSetForPlatform(platform: Platform): Record { + switch (platform) { + case Platform.LINUX_X64: + return downloadLinuxUrls; + case Platform.MAC_X64: + return downloadMacOsX64Urls; + case Platform.MAC_ARM64: + return downloadMacOsArm64Urls; + default: + throw Error(`Unexpected platform "${platform}" without Firefox support.`); + } + } +} diff --git a/bazel/browsers/update_script/index.mts b/bazel/browsers/update_script/index.mts new file mode 100644 index 00000000..e746b2b5 --- /dev/null +++ b/bazel/browsers/update_script/index.mts @@ -0,0 +1,57 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Storage} from '@google-cloud/storage'; +import yargs from 'yargs'; +import {Chromium} from './chromium.mjs'; +import {findLatestRevisionForAllPlatforms} from './find-revision-chromium.mjs'; +import {Firefox} from './firefox.mjs'; +import {uploadBrowserArtifactsToMirror} from './upload-mirror.js'; + +async function main() { + await yargs(process.argv.slice(2)) + .strict() + .help() + .scriptName('') + .demandCommand() + .command( + 'find-latest-chromium-revision [start-revision]', + 'Finds the latest stable revision for Chromium with artifacts available for all platforms.', + (args) => args.positional('startRevision', {type: 'number'}), + (args) => findLatestRevisionForAllPlatforms(args.startRevision), + ) + .command('upload-to-mirror', 'Upload browser binaries to the dev-infra cloud mirror', (args) => + args + .demandCommand() + .command( + 'chromium ', + 'Push Chromium artifacts', + (cArgs) => cArgs.positional('revision', {type: 'number', demandOption: true}), + (cArgs) => uploadBrowserArtifactsToMirror(new Storage(), new Chromium(cArgs.revision)), + ) + .command( + 'firefox ', + 'Push Firefox artifacts', + (fArgs) => + fArgs + .positional('browserVersion', {type: 'string', demandOption: true}) + .positional('driverVersion', {type: 'string', demandOption: true}), + (fArgs) => + uploadBrowserArtifactsToMirror( + new Storage(), + new Firefox(fArgs.browserVersion, fArgs.driverVersion), + ), + ), + ) + .parseAsync(); +} + +main().catch((e) => { + console.log(e); + process.exitCode = 1; +}); diff --git a/bazel/browsers/update_script/platform.mts b/bazel/browsers/update_script/platform.mts new file mode 100644 index 00000000..fd43ab44 --- /dev/null +++ b/bazel/browsers/update_script/platform.mts @@ -0,0 +1,14 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +export enum Platform { + LINUX_X64 = 'linux_x64', + MAC_X64 = 'mac_x64', + MAC_ARM64 = 'mac_arm64', + WINDOWS_X64 = 'windows_x64', +} diff --git a/bazel/browsers/update_script/upload-mirror.mts b/bazel/browsers/update_script/upload-mirror.mts new file mode 100644 index 00000000..b9537214 --- /dev/null +++ b/bazel/browsers/update_script/upload-mirror.mts @@ -0,0 +1,125 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {Bucket, Storage, File} from '@google-cloud/storage'; +import {Browser} from './browser.mjs'; +import {Platform} from './platform.mjs'; +import {createTmpDir, downloadFileThroughStreaming} from './utils.mjs'; + +import * as path from 'path'; +import {BrowserArtifact} from './browser-artifact.mjs'; + +/** Name of the Google Cloud Storage bucket for the browser mirror. */ +const MIRROR_BUCKET_NAME = 'dev-infra-mirror'; + +/** Gets the directory in the mirror bucket for a given browser instance. */ +export function getMirrorDirectoryForBrowserInstance(browser: Browser): string { + return `${browser.name}/${browser.revision}`; +} + +/** Gets the destination file path for a given browser artifact. */ +export function getDestinationFilePath(artifact: BrowserArtifact, platform: Platform): string { + const versionMirrorDir = getMirrorDirectoryForBrowserInstance(artifact.browser); + return `${versionMirrorDir}/${platform}/${artifact.type}.${artifact.extension}`; +} + +/** + * Uploads a browser platform artifact to the browser mirror. + * + * @throws {Error} An error if the artifact already exists in the mirror. + */ +export async function uploadArtifactToMirror( + bucket: Bucket, + artifact: BrowserArtifact, + platform: Platform, + sourceFile: string, +): Promise { + const [file] = await bucket.upload(sourceFile, { + destination: getDestinationFilePath(artifact, platform), + public: true, + }); + + return file; +} + +/** + * Helper function that takes an authenticated instance of the Google Cloud Storage API + * and a browser instance. The artifacts (both driver and browser itself) will be + * downloaded and re-uploaded to the mirror bucket in the given GCP instance. + */ +export async function uploadBrowserArtifactsToMirror(storage: Storage, browser: Browser) { + const bucket = storage.bucket(MIRROR_BUCKET_NAME); + const tmpDir = await createTmpDir({template: `${browser.name}-${browser.revision}-XXXXXX`}); + const downloadTasks: Promise<{ + platform: Platform; + filePath: string; + artifact: BrowserArtifact; + }>[] = []; + + for (const platform of Object.values(Platform)) { + if (!browser.supports(platform)) { + continue; + } + + const driverArtifact = browser.getArtifact(platform, 'driver-bin'); + const browserArtifact = browser.getArtifact(platform, 'browser-bin'); + const driverTmpPath = path.join(tmpDir, `${platform}-driver.${driverArtifact.extension}`); + const browserTmpPath = path.join(tmpDir, `${platform}-browser.${browserArtifact.extension}`); + + // We use the driver artifact (which is usually much smaller) to run a quick + // sanity check upstream to ensure that the artifact does not yet exist upstream. + const testDestinationFile = getDestinationFilePath(driverArtifact, platform); + // Note that we cannot check directly for the directory to exist since GCP does + // not support this. Hence we need to run this check for the actual files instead. + if ((await bucket.file(testDestinationFile).exists())[0]) { + throw Error('Revision is already in the mirror. Remove the artifacts if you want to retry.'); + } + + downloadTasks.push( + downloadFileThroughStreaming(browserArtifact.downloadUrl, browserTmpPath) + .then(() => console.info(`✅ Downloaded: ${browser.name} - ${platform} browser.`)) + .then(() => ({ + platform, + filePath: browserTmpPath, + artifact: browserArtifact, + })), + ); + + downloadTasks.push( + downloadFileThroughStreaming(driverArtifact.downloadUrl, driverTmpPath) + .then(() => console.info(`✅ Downloaded: ${browser.name} - ${platform} driver.`)) + .then(() => ({ + platform, + filePath: driverTmpPath, + artifact: driverArtifact, + })), + ); + } + + const tasks = await Promise.all(downloadTasks); + const uploadTasks: Promise<{platform: Platform; artifact: BrowserArtifact; file: File}>[] = []; + + console.info(); + console.info('Fetched all browser artifacts. Now uploading to mirror.'); + console.info(); + + for (const {platform, filePath, artifact} of tasks) { + uploadTasks.push( + uploadArtifactToMirror(bucket, artifact, platform, filePath).then((file) => { + console.log(`✅ Uploaded: ${platform} ${artifact.type}`); + console.log(` -> ${file.publicUrl()}`); + + return {platform, file, artifact}; + }), + ); + } + + await Promise.all(uploadTasks); + + console.info(`Uploaded ${browser.name} artifacts to the Google Cloud Storage mirror.`); +} diff --git a/bazel/browsers/update_script/utils.mts b/bazel/browsers/update_script/utils.mts new file mode 100644 index 00000000..be41c8cd --- /dev/null +++ b/bazel/browsers/update_script/utils.mts @@ -0,0 +1,44 @@ +/** + * @license + * Copyright Google LLC + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import * as fs from 'fs'; +import * as tmp from 'tmp'; +import fetch from 'node-fetch'; + +/** Creates a temporary directory with the given options. */ +export function createTmpDir(options: tmp.DirOptions): Promise { + return new Promise((resolve, reject) => { + tmp.dir(options, (err, name) => (err !== null ? reject(err) : resolve(name))); + }); +} + +/** + * Downloads a file and stores it at the given location. + * + * The file is downloaded asynchronously using streaming to avoid + * increasing acquired memory of the NodeJS process unnecessarily. + */ +export async function downloadFileThroughStreaming( + sourceUrl: string, + destinationPath: string, +): Promise { + return new Promise(async (resolve, reject) => { + const stream = (await fetch(sourceUrl)).body; + + if (stream === null) { + reject(); + return; + } + + const outStream = fs.createWriteStream(destinationPath); + + stream.on('error', (err) => reject(err)); + stream.on('close', () => resolve()); + stream.pipe(outStream); + }); +}