From a03808cde41dbfd75c585ad30484fbe955463d18 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Mon, 27 Jul 2020 16:07:22 -0700 Subject: [PATCH] devops: re-factor list-dependencies script to output per-browser results This refactors script to output per-browser package dependencies that could be easily copied over to the Github Action. References #2745 --- src/server/validateDependencies.ts | 131 ++++++------ .../inside_docker/list_dependencies.js | 197 +++++++++++++++--- 2 files changed, 231 insertions(+), 97 deletions(-) diff --git a/src/server/validateDependencies.ts b/src/server/validateDependencies.ts index b3b4c128a87180..d2b18244d1e74a 100644 --- a/src/server/validateDependencies.ts +++ b/src/server/validateDependencies.ts @@ -204,7 +204,6 @@ const LIBRARY_TO_PACKAGE_NAME_UBUNTU_18_04: { [s: string]: string} = { 'libpng16.so.16': 'libpng16-16', 'libsecret-1.so.0': 'libsecret-1-0', 'libsmime3.so': 'libnss3', - 'libssl3.so': 'libnss3', 'libvpx.so.5': 'libvpx5', 'libwayland-client.so.0': 'libwayland-client0', 'libwayland-egl.so.1': 'libwayland-egl1', @@ -221,85 +220,85 @@ const LIBRARY_TO_PACKAGE_NAME_UBUNTU_18_04: { [s: string]: string} = { }; const LIBRARY_TO_PACKAGE_NAME_UBUNTU_20_04: { [s: string]: string} = { - 'libglib-2.0.so.0': 'libglib2.0-0', - 'libX11.so.6': 'libx11-6', - 'libxcb.so.1': 'libxcb1', - 'libGL.so.1': 'libgl1', 'libEGL.so.1': 'libegl1', - 'libnotify.so.4': 'libnotify4', - 'libgdk_pixbuf-2.0.so.0': 'libgdk-pixbuf2.0-0', - 'libgio-2.0.so.0': 'libglib2.0-0', - 'libgobject-2.0.so.0': 'libglib2.0-0', - 'libvpx.so.6': 'libvpx6', - 'libopus.so.0': 'libopus0', - 'libxml2.so.2': 'libxml2', - 'libicui18n.so.66': 'libicu66', - 'libicuuc.so.66': 'libicu66', - 'libxslt.so.1': 'libxslt1.1', - 'libwoff2dec.so.1.0.2': 'libwoff1', + 'libGL.so.1': 'libgl1', + 'libX11-xcb.so.1': 'libx11-xcb1', + 'libX11.so.6': 'libx11-6', + 'libXcomposite.so.1': 'libxcomposite1', + 'libXcursor.so.1': 'libxcursor1', + 'libXdamage.so.1': 'libxdamage1', + 'libXext.so.6': 'libxext6', + 'libXfixes.so.3': 'libxfixes3', + 'libXi.so.6': 'libxi6', + 'libXrandr.so.2': 'libxrandr2', + 'libXrender.so.1': 'libxrender1', + 'libXt.so.6': 'libxt6', + 'libXtst.so.6': 'libxtst6', + 'libasound.so.2': 'libasound2', + 'libatk-1.0.so.0': 'libatk1.0-0', + 'libatk-bridge-2.0.so.0': 'libatk-bridge2.0-0', + 'libatspi.so.0': 'libatspi2.0-0', + 'libcairo-gobject.so.2': 'libcairo-gobject2', 'libcairo.so.2': 'libcairo2', + 'libcups.so.2': 'libcups2', + 'libdbus-1.so.3': 'libdbus-1-3', + 'libdbus-glib-1.so.2': 'libdbus-glib-1-2', + 'libdrm.so.2': 'libdrm2', + 'libenchant.so.1': 'libenchant1c2a', + 'libepoxy.so.0': 'libepoxy0', 'libfontconfig.so.1': 'libfontconfig1', 'libfreetype.so.6': 'libfreetype6', - 'libharfbuzz.so.0': 'libharfbuzz0b', - 'libharfbuzz-icu.so.0': 'libharfbuzz-icu0', + 'libgbm.so.1': 'libgbm1', + 'libgdk-3.so.0': 'libgtk-3-0', + 'libgdk-x11-2.0.so.0': 'libgtk2.0-0', + 'libgdk_pixbuf-2.0.so.0': 'libgdk-pixbuf2.0-0', + 'libgio-2.0.so.0': 'libglib2.0-0', + 'libglib-2.0.so.0': 'libglib2.0-0', + 'libgmodule-2.0.so.0': 'libglib2.0-0', + 'libgobject-2.0.so.0': 'libglib2.0-0', 'libgstapp-1.0.so.0': 'libgstreamer-plugins-base1.0-0', + 'libgstaudio-1.0.so.0': 'libgstreamer-plugins-base1.0-0', 'libgstbase-1.0.so.0': 'libgstreamer1.0-0', - 'libgstreamer-1.0.so.0': 'libgstreamer1.0-0', + 'libgstcodecparsers-1.0.so.0': 'libgstreamer-plugins-bad1.0-0', + 'libgstfft-1.0.so.0': 'libgstreamer-plugins-base1.0-0', + 'libgstgl-1.0.so.0': 'libgstreamer-gl1.0-0', 'libgstpbutils-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstaudio-1.0.so.0': 'libgstreamer-plugins-base1.0-0', + 'libgstreamer-1.0.so.0': 'libgstreamer1.0-0', 'libgsttag-1.0.so.0': 'libgstreamer-plugins-base1.0-0', 'libgstvideo-1.0.so.0': 'libgstreamer-plugins-base1.0-0', - 'libgstgl-1.0.so.0': 'libgstreamer-gl1.0-0', - 'libgstcodecparsers-1.0.so.0': 'libgstreamer-plugins-bad1.0-0', - 'libgstfft-1.0.so.0': 'libgstreamer-plugins-base1.0-0', + 'libgthread-2.0.so.0': 'libglib2.0-0', + 'libgtk-3.so.0': 'libgtk-3-0', + 'libgtk-x11-2.0.so.0': 'libgtk2.0-0', + 'libharfbuzz-icu.so.0': 'libharfbuzz-icu0', + 'libharfbuzz.so.0': 'libharfbuzz0b', + 'libhyphen.so.0': 'libhyphen0', + 'libicui18n.so.66': 'libicu66', + 'libicuuc.so.66': 'libicu66', 'libjpeg.so.8': 'libjpeg-turbo8', - 'libpng16.so.16': 'libpng16-16', + 'libnotify.so.4': 'libnotify4', + 'libnspr4.so': 'libnspr4', + 'libnss3.so': 'libnss3', + 'libnssutil3.so': 'libnss3', 'libopenjp2.so.7': 'libopenjp2-7', - 'libwebpdemux.so.2': 'libwebpdemux2', - 'libwebp.so.6': 'libwebp6', - 'libsoup-2.4.so.1': 'libsoup2.4-1', - 'libenchant.so.1': 'libenchant1c2a', - 'libgmodule-2.0.so.0': 'libglib2.0-0', - 'libsecret-1.so.0': 'libsecret-1-0', - 'libhyphen.so.0': 'libhyphen0', - 'libXcomposite.so.1': 'libxcomposite1', - 'libXdamage.so.1': 'libxdamage1', - 'libwayland-server.so.0': 'libwayland-server0', - 'libwayland-egl.so.1': 'libwayland-egl1', - 'libwayland-client.so.0': 'libwayland-client0', - 'libgtk-3.so.0': 'libgtk-3-0', - 'libgdk-3.so.0': 'libgtk-3-0', + 'libopus.so.0': 'libopus0', 'libpango-1.0.so.0': 'libpango-1.0-0', - 'libatk-1.0.so.0': 'libatk1.0-0', - 'libxkbcommon.so.0': 'libxkbcommon0', - 'libepoxy.so.0': 'libepoxy0', - 'libatk-bridge-2.0.so.0': 'libatk-bridge2.0-0', - 'libX11-xcb.so.1': 'libx11-xcb1', - 'libXcursor.so.1': 'libxcursor1', - 'libXext.so.6': 'libxext6', - 'libXfixes.so.3': 'libxfixes3', - 'libXi.so.6': 'libxi6', - 'libXrender.so.1': 'libxrender1', - 'libdbus-glib-1.so.2': 'libdbus-glib-1-2', - 'libdbus-1.so.3': 'libdbus-1-3', 'libpangocairo-1.0.so.0': 'libpangocairo-1.0-0', - 'libcairo-gobject.so.2': 'libcairo-gobject2', - 'libxcb-shm.so.0': 'libxcb-shm0', 'libpangoft2-1.0.so.0': 'libpangoft2-1.0-0', - 'libXt.so.6': 'libxt6', - 'libgthread-2.0.so.0': 'libglib2.0-0', - 'libgtk-x11-2.0.so.0': 'libgtk2.0-0', - 'libgdk-x11-2.0.so.0': 'libgtk2.0-0', - 'libnss3.so': 'libnss3', - 'libnssutil3.so': 'libnss3', + 'libpng16.so.16': 'libpng16-16', + 'libsecret-1.so.0': 'libsecret-1-0', 'libsmime3.so': 'libnss3', - 'libnspr4.so': 'libnspr4', + 'libsoup-2.4.so.1': 'libsoup2.4-1', + 'libvpx.so.6': 'libvpx6', + 'libwayland-client.so.0': 'libwayland-client0', + 'libwayland-egl.so.1': 'libwayland-egl1', + 'libwayland-server.so.0': 'libwayland-server0', + 'libwebp.so.6': 'libwebp6', + 'libwebpdemux.so.2': 'libwebpdemux2', + 'libwoff2dec.so.1.0.2': 'libwoff1', 'libxcb-dri3.so.0': 'libxcb-dri3-0', - 'libXtst.so.6': 'libxtst6', - 'libcups.so.2': 'libcups2', - 'libdrm.so.2': 'libdrm2', - 'libXrandr.so.2': 'libxrandr2', - 'libgbm.so.1': 'libgbm1', - 'libasound.so.2': 'libasound2', - 'libatspi.so.0': 'libatspi2.0-0', + 'libxcb-shm.so.0': 'libxcb-shm0', + 'libxcb.so.1': 'libxcb1', + 'libxkbcommon.so.0': 'libxkbcommon0', + 'libxml2.so.2': 'libxml2', + 'libxslt.so.1': 'libxslt1.1', }; diff --git a/utils/linux-browser-dependencies/inside_docker/list_dependencies.js b/utils/linux-browser-dependencies/inside_docker/list_dependencies.js index 0c1016735e0e78..f0824a8b23bf80 100644 --- a/utils/linux-browser-dependencies/inside_docker/list_dependencies.js +++ b/utils/linux-browser-dependencies/inside_docker/list_dependencies.js @@ -6,47 +6,157 @@ const path = require('path'); const {spawn} = require('child_process'); const browserPaths = require('playwright/lib/install/browserPaths.js'); +const readdirAsync = util.promisify(fs.readdir.bind(fs)); +const readFileAsync = util.promisify(fs.readFile.bind(fs)); + (async () => { + console.log('Working on:', await getDistributionName()); + console.log('Started at:', currentTime()); const allBrowsersPath = browserPaths.browsersPath(); - const {stdout} = await runCommand('find', [allBrowsersPath, '-executable', '-type', 'f']); - // lddPaths - files we want to run LDD against. - const lddPaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh')); - // List of all shared libraries missing. - const missingDeps = new Set(); - // Multimap: reverse-mapping from shared library to requiring file. - const depsToLddPaths = new Map(); - await Promise.all(lddPaths.map(async lddPath => { - const deps = await missingFileDependencies(lddPath); - for (const dep of deps) { - missingDeps.add(dep); - let depsToLdd = depsToLddPaths.get(dep); - if (!depsToLdd) { - depsToLdd = new Set(); - depsToLddPaths.set(dep, depsToLdd); - } - depsToLdd.add(lddPath); - } + const browserDescriptors = (await readdirAsync(allBrowsersPath)).filter(dir => !dir.startsWith('.')).map(dir => ({ + name: dir, + path: path.join(allBrowsersPath, dir), + // All files that we will try to inspect for missing dependencies. + filePaths: [], + // All libraries that are missing for the browser. + missingLibraries: new Set(), + // All packages required for the browser. + requiredPackages: new Set(), + // Libraries that we didn't find a package. + unresolvedLibraries: new Set(), })); - console.log(`==== MISSING DEPENDENCIES: ${missingDeps.size} ====`); - console.log([...missingDeps].sort().join('\n')); - console.log('{'); - for (const dep of missingDeps) { - const packages = await findPackages(dep); - if (packages.length === 0) { - console.log(` // UNRESOLVED: ${dep} `); - const depsToLdd = depsToLddPaths.get(dep); - for (const filePath of depsToLdd) - console.log(` // - required by ${filePath}`); + // Collect all missing libraries for all browsers. + const allMissingLibraries = new Set(); + for (const descriptor of browserDescriptors) { + const {stdout} = await runCommand('find', [descriptor.path, '-executable', '-type', 'f']); + descriptor.filePaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh')); + await Promise.all(descriptor.filePaths.map(async filePath => { + const missingLibraries = await missingFileDependencies(filePath); + for (const library of missingLibraries) { + descriptor.missingLibraries.add(library); + allMissingLibraries.add(library); + } + })); + } + + const libraryToPackage = new Map(); + const ambiguityLibraries = new Map(); + + // Map missing libraries to packages that could be installed to fulfill the dependency. + console.log(`Finding packages for ${allMissingLibraries.size} missing libraries...`); + + for (let i = 0, array = [...allMissingLibraries].sort(); i < allMissingLibraries.size; ++i) { + const library = array[i]; + const packages = await findPackages(library); + + const progress = `${i + 1}/${allMissingLibraries.size}`; + console.log(`${progress.padStart(7)}: ${library} => ${JSON.stringify(packages)}`); + + if (!packages.length) { + const browsersWithMissingLibrary = browserDescriptors.filter(d => d.missingLibraries.has(library)).map(d => d.name).join(', '); + const PADDING = ''.padStart(7) + ' '; + console.log(PADDING + `ERROR: failed to resolve '${library}' required by ${browsersWithMissingLibrary}`); } else if (packages.length === 1) { - console.log(` "${dep}": "${packages[0]}",`); + libraryToPackage.set(library, packages[0]); } else { - console.log(` "${dep}": ${JSON.stringify(packages)},`); + ambiguityLibraries.set(library, packages); } } + + console.log(''); + console.log(`Picking packages for ${ambiguityLibraries.size} libraries that have multiple package candidates`); + // Pick packages to install to fulfill missing libraries. + // + // This is a 2-step process: + // 1. Pick easy libraries by filtering out debug, test and dev packages. + // 2. After that, pick packages that we already picked before. + + // Step 1: pick libraries that are easy to pick. + const totalAmbiguityLibraries = ambiguityLibraries.size; + for (const [library, packages] of ambiguityLibraries) { + const package = pickPackage(library, packages); + if (!package) + continue; + libraryToPackage.set(library, package); + ambiguityLibraries.delete(library); + const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`; + console.log(`${progress.padStart(7)}: ${library} => ${package}`); + console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`); + } + // 2nd pass - prefer packages that we already picked. + const allUsedPackages = new Set(libraryToPackage.values()); + for (const [library, packages] of ambiguityLibraries) { + const package = packages.find(package => allUsedPackages.has(package)); + if (!package) + continue; + libraryToPackage.set(library, package); + ambiguityLibraries.delete(library); + const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`; + console.log(`${progress.padStart(7)}: ${library} => ${package}`); + console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`); + } + + // Report all ambiguities that were failed to resolve. + for (const [library, packages] of ambiguityLibraries) { + ambiguityLibraries.delete(library); + const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`; + console.log(`${progress.padStart(7)}: FAILED ${library} => ???`); + console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`); + } + + // For each browser build a list of packages to install. + for (const descriptor of browserDescriptors) { + for (const library of descriptor.missingLibraries) { + const package = libraryToPackage.get(library); + if (package) + descriptor.requiredPackages.add(package); + else + descriptor.unresolvedLibraries.add(library); + } + } + + // Formatting results. + console.log(''); + console.log(`----- Library to package name mapping -----`); + console.log('{'); + for (const [library, package] of libraryToPackage) + console.log(` "${library}": "${package}",`); console.log('}'); + + // Packages and unresolved libraries for every browser + for (const descriptor of browserDescriptors) { + console.log(''); + console.log(`======= ${descriptor.name}: required packages =======`); + const requiredPackages = [...descriptor.requiredPackages].sort(); + console.log(JSON.stringify(requiredPackages, null, 2)); + console.log(''); + console.log(`------- ${descriptor.name}: unresolved libraries -------`); + const unresolvedLibraries = [...descriptor.unresolvedLibraries].sort(); + console.log(JSON.stringify(unresolvedLibraries, null, 2)); + } + + const status = browserDescriptors.some(d => d.unresolvedLibraries.size) ? 'FAILED' : 'SUCCESS'; + console.log(` + ==================== + ${status} + ==================== + `); })(); +function pickPackage(library, packages) { + // Step 1: try to filter out debug, test and dev packages. + packages = packages.filter(package => !package.endsWith('-dbg') && !package.endsWith('-test') && !package.endsWith('-dev') && !package.endsWith('-mesa')); + if (packages.length === 1) + return packages[0]; + // Step 2: use library name to filter packages with the same name. + const prefix = library.split(/[-.]/).shift().toLowerCase(); + packages = packages.filter(package => package.toLowerCase().startsWith(prefix)); + if (packages.length === 1) + return packages[0]; + return null; +} + async function findPackages(libraryName) { const {stdout} = await runCommand('apt-file', ['search', libraryName]); if (!stdout.trim()) @@ -56,7 +166,9 @@ async function findPackages(libraryName) { } async function fileDependencies(filePath) { - const {stdout} = await lddAsync(filePath); + const {stdout, code} = await lddAsync(filePath); + if (code !== 0) + return []; const deps = stdout.split('\n').map(line => { line = line.trim(); const missing = line.includes('not found'); @@ -100,3 +212,26 @@ function runCommand(command, args, options = {}) { }); }); } + +async function getDistributionName() { + const osReleaseText = await readFileAsync('/etc/os-release', 'utf8'); + const fields = new Map(); + for (const line of osReleaseText.split('\n')) { + const tokens = line.split('='); + const name = tokens.shift(); + let value = tokens.join('=').trim(); + if (value.startsWith('"') && value.endsWith('"')) + value = value.substring(1, value.length - 1); + if (!name) + continue; + fields.set(name.toLowerCase(), value); + } + return fields.get('pretty_name') || ''; +} + +function currentTime() { + const date = new Date(); + const dateTimeFormat = new Intl.DateTimeFormat('en', { year: 'numeric', month: 'short', day: '2-digit' }); + const [{ value: month },,{ value: day },,{ value: year }] = dateTimeFormat .formatToParts(date ); + return `${month} ${day}, ${year}`; +}