Skip to content

Commit

Permalink
devops: re-factor list-dependencies script to output per-browser results
Browse files Browse the repository at this point in the history 
This refactors script to output per-browser package dependencies that
could be easily copied over to the Github Action.

References microsoft#2745
  • Loading branch information
@aslushnikov
aslushnikov committed Jul 28, 2020
1 parent 101dd3b commit 44d96a8
Show file tree
Hide file tree
Showing 4 changed files with 871 additions and 97 deletions.
131 changes: 65 additions & 66 deletions src/server/validateDependencies.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,6 @@ const LIBRARY_TO_PACKAGE_NAME_UBUNTU_18_04: { [s: string]: string} = {
'libpng16.so.16': 'libpng16-16',
'libsecret-1.so.0': 'libsecret-1-0',
'libsmime3.so': 'libnss3',
'libssl3.so': 'libnss3',
'libvpx.so.5': 'libvpx5',
'libwayland-client.so.0': 'libwayland-client0',
'libwayland-egl.so.1': 'libwayland-egl1',
Expand All @@ -221,85 +220,85 @@ const LIBRARY_TO_PACKAGE_NAME_UBUNTU_18_04: { [s: string]: string} = {
};

const LIBRARY_TO_PACKAGE_NAME_UBUNTU_20_04: { [s: string]: string} = {
'libglib-2.0.so.0': 'libglib2.0-0',
'libX11.so.6': 'libx11-6',
'libxcb.so.1': 'libxcb1',
'libGL.so.1': 'libgl1',
'libEGL.so.1': 'libegl1',
'libnotify.so.4': 'libnotify4',
'libgdk_pixbuf-2.0.so.0': 'libgdk-pixbuf2.0-0',
'libgio-2.0.so.0': 'libglib2.0-0',
'libgobject-2.0.so.0': 'libglib2.0-0',
'libvpx.so.6': 'libvpx6',
'libopus.so.0': 'libopus0',
'libxml2.so.2': 'libxml2',
'libicui18n.so.66': 'libicu66',
'libicuuc.so.66': 'libicu66',
'libxslt.so.1': 'libxslt1.1',
'libwoff2dec.so.1.0.2': 'libwoff1',
'libGL.so.1': 'libgl1',
'libX11-xcb.so.1': 'libx11-xcb1',
'libX11.so.6': 'libx11-6',
'libXcomposite.so.1': 'libxcomposite1',
'libXcursor.so.1': 'libxcursor1',
'libXdamage.so.1': 'libxdamage1',
'libXext.so.6': 'libxext6',
'libXfixes.so.3': 'libxfixes3',
'libXi.so.6': 'libxi6',
'libXrandr.so.2': 'libxrandr2',
'libXrender.so.1': 'libxrender1',
'libXt.so.6': 'libxt6',
'libXtst.so.6': 'libxtst6',
'libasound.so.2': 'libasound2',
'libatk-1.0.so.0': 'libatk1.0-0',
'libatk-bridge-2.0.so.0': 'libatk-bridge2.0-0',
'libatspi.so.0': 'libatspi2.0-0',
'libcairo-gobject.so.2': 'libcairo-gobject2',
'libcairo.so.2': 'libcairo2',
'libcups.so.2': 'libcups2',
'libdbus-1.so.3': 'libdbus-1-3',
'libdbus-glib-1.so.2': 'libdbus-glib-1-2',
'libdrm.so.2': 'libdrm2',
'libenchant.so.1': 'libenchant1c2a',
'libepoxy.so.0': 'libepoxy0',
'libfontconfig.so.1': 'libfontconfig1',
'libfreetype.so.6': 'libfreetype6',
'libharfbuzz.so.0': 'libharfbuzz0b',
'libharfbuzz-icu.so.0': 'libharfbuzz-icu0',
'libgbm.so.1': 'libgbm1',
'libgdk-3.so.0': 'libgtk-3-0',
'libgdk-x11-2.0.so.0': 'libgtk2.0-0',
'libgdk_pixbuf-2.0.so.0': 'libgdk-pixbuf2.0-0',
'libgio-2.0.so.0': 'libglib2.0-0',
'libglib-2.0.so.0': 'libglib2.0-0',
'libgmodule-2.0.so.0': 'libglib2.0-0',
'libgobject-2.0.so.0': 'libglib2.0-0',
'libgstapp-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgstaudio-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgstbase-1.0.so.0': 'libgstreamer1.0-0',
'libgstreamer-1.0.so.0': 'libgstreamer1.0-0',
'libgstcodecparsers-1.0.so.0': 'libgstreamer-plugins-bad1.0-0',
'libgstfft-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgstgl-1.0.so.0': 'libgstreamer-gl1.0-0',
'libgstpbutils-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgstaudio-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgstreamer-1.0.so.0': 'libgstreamer1.0-0',
'libgsttag-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgstvideo-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgstgl-1.0.so.0': 'libgstreamer-gl1.0-0',
'libgstcodecparsers-1.0.so.0': 'libgstreamer-plugins-bad1.0-0',
'libgstfft-1.0.so.0': 'libgstreamer-plugins-base1.0-0',
'libgthread-2.0.so.0': 'libglib2.0-0',
'libgtk-3.so.0': 'libgtk-3-0',
'libgtk-x11-2.0.so.0': 'libgtk2.0-0',
'libharfbuzz-icu.so.0': 'libharfbuzz-icu0',
'libharfbuzz.so.0': 'libharfbuzz0b',
'libhyphen.so.0': 'libhyphen0',
'libicui18n.so.66': 'libicu66',
'libicuuc.so.66': 'libicu66',
'libjpeg.so.8': 'libjpeg-turbo8',
'libpng16.so.16': 'libpng16-16',
'libnotify.so.4': 'libnotify4',
'libnspr4.so': 'libnspr4',
'libnss3.so': 'libnss3',
'libnssutil3.so': 'libnss3',
'libopenjp2.so.7': 'libopenjp2-7',
'libwebpdemux.so.2': 'libwebpdemux2',
'libwebp.so.6': 'libwebp6',
'libsoup-2.4.so.1': 'libsoup2.4-1',
'libenchant.so.1': 'libenchant1c2a',
'libgmodule-2.0.so.0': 'libglib2.0-0',
'libsecret-1.so.0': 'libsecret-1-0',
'libhyphen.so.0': 'libhyphen0',
'libXcomposite.so.1': 'libxcomposite1',
'libXdamage.so.1': 'libxdamage1',
'libwayland-server.so.0': 'libwayland-server0',
'libwayland-egl.so.1': 'libwayland-egl1',
'libwayland-client.so.0': 'libwayland-client0',
'libgtk-3.so.0': 'libgtk-3-0',
'libgdk-3.so.0': 'libgtk-3-0',
'libopus.so.0': 'libopus0',
'libpango-1.0.so.0': 'libpango-1.0-0',
'libatk-1.0.so.0': 'libatk1.0-0',
'libxkbcommon.so.0': 'libxkbcommon0',
'libepoxy.so.0': 'libepoxy0',
'libatk-bridge-2.0.so.0': 'libatk-bridge2.0-0',
'libX11-xcb.so.1': 'libx11-xcb1',
'libXcursor.so.1': 'libxcursor1',
'libXext.so.6': 'libxext6',
'libXfixes.so.3': 'libxfixes3',
'libXi.so.6': 'libxi6',
'libXrender.so.1': 'libxrender1',
'libdbus-glib-1.so.2': 'libdbus-glib-1-2',
'libdbus-1.so.3': 'libdbus-1-3',
'libpangocairo-1.0.so.0': 'libpangocairo-1.0-0',
'libcairo-gobject.so.2': 'libcairo-gobject2',
'libxcb-shm.so.0': 'libxcb-shm0',
'libpangoft2-1.0.so.0': 'libpangoft2-1.0-0',
'libXt.so.6': 'libxt6',
'libgthread-2.0.so.0': 'libglib2.0-0',
'libgtk-x11-2.0.so.0': 'libgtk2.0-0',
'libgdk-x11-2.0.so.0': 'libgtk2.0-0',
'libnss3.so': 'libnss3',
'libnssutil3.so': 'libnss3',
'libpng16.so.16': 'libpng16-16',
'libsecret-1.so.0': 'libsecret-1-0',
'libsmime3.so': 'libnss3',
'libnspr4.so': 'libnspr4',
'libsoup-2.4.so.1': 'libsoup2.4-1',
'libvpx.so.6': 'libvpx6',
'libwayland-client.so.0': 'libwayland-client0',
'libwayland-egl.so.1': 'libwayland-egl1',
'libwayland-server.so.0': 'libwayland-server0',
'libwebp.so.6': 'libwebp6',
'libwebpdemux.so.2': 'libwebpdemux2',
'libwoff2dec.so.1.0.2': 'libwoff1',
'libxcb-dri3.so.0': 'libxcb-dri3-0',
'libXtst.so.6': 'libxtst6',
'libcups.so.2': 'libcups2',
'libdrm.so.2': 'libdrm2',
'libXrandr.so.2': 'libxrandr2',
'libgbm.so.1': 'libgbm1',
'libasound.so.2': 'libasound2',
'libatspi.so.0': 'libatspi2.0-0',
'libxcb-shm.so.0': 'libxcb-shm0',
'libxcb.so.1': 'libxcb1',
'libxkbcommon.so.0': 'libxkbcommon0',
'libxml2.so.2': 'libxml2',
'libxslt.so.1': 'libxslt1.1',
};
197 changes: 166 additions & 31 deletions utils/linux-browser-dependencies/inside_docker/list_dependencies.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,47 +6,157 @@ const path = require('path');
const {spawn} = require('child_process');
const browserPaths = require('playwright/lib/install/browserPaths.js');

const readdirAsync = util.promisify(fs.readdir.bind(fs));
const readFileAsync = util.promisify(fs.readFile.bind(fs));

(async () => {
console.log('Working on:', await getDistributionName());
console.log('Started at:', currentTime());
const allBrowsersPath = browserPaths.browsersPath();
const {stdout} = await runCommand('find', [allBrowsersPath, '-executable', '-type', 'f']);
// lddPaths - files we want to run LDD against.
const lddPaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh'));
// List of all shared libraries missing.
const missingDeps = new Set();
// Multimap: reverse-mapping from shared library to requiring file.
const depsToLddPaths = new Map();
await Promise.all(lddPaths.map(async lddPath => {
const deps = await missingFileDependencies(lddPath);
for (const dep of deps) {
missingDeps.add(dep);
let depsToLdd = depsToLddPaths.get(dep);
if (!depsToLdd) {
depsToLdd = new Set();
depsToLddPaths.set(dep, depsToLdd);
}
depsToLdd.add(lddPath);
}
const browserDescriptors = (await readdirAsync(allBrowsersPath)).filter(dir => !dir.startsWith('.')).map(dir => ({
name: dir,
path: path.join(allBrowsersPath, dir),
// All files that we will try to inspect for missing dependencies.
filePaths: [],
// All libraries that are missing for the browser.
missingLibraries: new Set(),
// All packages required for the browser.
requiredPackages: new Set(),
// Libraries that we didn't find a package.
unresolvedLibraries: new Set(),
}));
console.log(`==== MISSING DEPENDENCIES: ${missingDeps.size} ====`);
console.log([...missingDeps].sort().join('\n'));

console.log('{');
for (const dep of missingDeps) {
const packages = await findPackages(dep);
if (packages.length === 0) {
console.log(` // UNRESOLVED: ${dep} `);
const depsToLdd = depsToLddPaths.get(dep);
for (const filePath of depsToLdd)
console.log(` // - required by ${filePath}`);
// Collect all missing libraries for all browsers.
const allMissingLibraries = new Set();
for (const descriptor of browserDescriptors) {
const {stdout} = await runCommand('find', [descriptor.path, '-executable', '-type', 'f']);
descriptor.filePaths = stdout.trim().split('\n').map(f => f.trim()).filter(filePath => !filePath.toLowerCase().endsWith('.sh'));
await Promise.all(descriptor.filePaths.map(async filePath => {
const missingLibraries = await missingFileDependencies(filePath);
for (const library of missingLibraries) {
descriptor.missingLibraries.add(library);
allMissingLibraries.add(library);
}
}));
}

const libraryToPackage = new Map();
const ambiguityLibraries = new Map();

// Map missing libraries to packages that could be installed to fulfill the dependency.
console.log(`Finding packages for ${allMissingLibraries.size} missing libraries...`);

for (let i = 0, array = [...allMissingLibraries].sort(); i < allMissingLibraries.size; ++i) {
const library = array[i];
const packages = await findPackages(library);

const progress = `${i + 1}/${allMissingLibraries.size}`;
console.log(`${progress.padStart(7)}: ${library} => ${JSON.stringify(packages)}`);

if (!packages.length) {
const browsersWithMissingLibrary = browserDescriptors.filter(d => d.missingLibraries.has(library)).map(d => d.name).join(', ');
const PADDING = ''.padStart(7) + ' ';
console.log(PADDING + `ERROR: failed to resolve '${library}' required by ${browsersWithMissingLibrary}`);
} else if (packages.length === 1) {
console.log(` "${dep}": "${packages[0]}",`);
libraryToPackage.set(library, packages[0]);
} else {
console.log(` "${dep}": ${JSON.stringify(packages)},`);
ambiguityLibraries.set(library, packages);
}
}

console.log('');
console.log(`Picking packages for ${ambiguityLibraries.size} libraries that have multiple package candidates`);
// Pick packages to install to fulfill missing libraries.
//
// This is a 2-step process:
// 1. Pick easy libraries by filtering out debug, test and dev packages.
// 2. After that, pick packages that we already picked before.

// Step 1: pick libraries that are easy to pick.
const totalAmbiguityLibraries = ambiguityLibraries.size;
for (const [library, packages] of ambiguityLibraries) {
const package = pickPackage(library, packages);
if (!package)
continue;
libraryToPackage.set(library, package);
ambiguityLibraries.delete(library);
const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
console.log(`${progress.padStart(7)}: ${library} => ${package}`);
console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
}
// 2nd pass - prefer packages that we already picked.
const allUsedPackages = new Set(libraryToPackage.values());
for (const [library, packages] of ambiguityLibraries) {
const package = packages.find(package => allUsedPackages.has(package));
if (!package)
continue;
libraryToPackage.set(library, package);
ambiguityLibraries.delete(library);
const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
console.log(`${progress.padStart(7)}: ${library} => ${package}`);
console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
}

// Report all ambiguities that were failed to resolve.
for (const [library, packages] of ambiguityLibraries) {
ambiguityLibraries.delete(library);
const progress = `${totalAmbiguityLibraries - ambiguityLibraries.size}/${totalAmbiguityLibraries}`;
console.log(`${progress.padStart(7)}: FAILED ${library} => ???`);
console.log(''.padStart(9) + `(note) packages are ${JSON.stringify(packages)}`);
}

// For each browser build a list of packages to install.
for (const descriptor of browserDescriptors) {
for (const library of descriptor.missingLibraries) {
const package = libraryToPackage.get(library);
if (package)
descriptor.requiredPackages.add(package);
else
descriptor.unresolvedLibraries.add(library);
}
}

// Formatting results.
console.log('');
console.log(`----- Library to package name mapping -----`);
console.log('{');
for (const [library, package] of libraryToPackage)
console.log(` "${library}": "${package}",`);
console.log('}');

// Packages and unresolved libraries for every browser
for (const descriptor of browserDescriptors) {
console.log('');
console.log(`======= ${descriptor.name}: required packages =======`);
const requiredPackages = [...descriptor.requiredPackages].sort();
console.log(JSON.stringify(requiredPackages, null, 2));
console.log('');
console.log(`------- ${descriptor.name}: unresolved libraries -------`);
const unresolvedLibraries = [...descriptor.unresolvedLibraries].sort();
console.log(JSON.stringify(unresolvedLibraries, null, 2));
}

const status = browserDescriptors.some(d => d.unresolvedLibraries.size) ? 'FAILED' : 'SUCCESS';
console.log(`
====================
${status}
====================
`);
})();

function pickPackage(library, packages) {
// Step 1: try to filter out debug, test and dev packages.
packages = packages.filter(package => !package.endsWith('-dbg') && !package.endsWith('-test') && !package.endsWith('-dev') && !package.endsWith('-mesa'));
if (packages.length === 1)
return packages[0];
// Step 2: use library name to filter packages with the same name.
const prefix = library.split(/[-.]/).shift().toLowerCase();
packages = packages.filter(package => package.toLowerCase().startsWith(prefix));
if (packages.length === 1)
return packages[0];
return null;
}

async function findPackages(libraryName) {
const {stdout} = await runCommand('apt-file', ['search', libraryName]);
if (!stdout.trim())
Expand All @@ -56,7 +166,9 @@ async function findPackages(libraryName) {
}

async function fileDependencies(filePath) {
const {stdout} = await lddAsync(filePath);
const {stdout, code} = await lddAsync(filePath);
if (code !== 0)
return [];
const deps = stdout.split('\n').map(line => {
line = line.trim();
const missing = line.includes('not found');
Expand Down Expand Up @@ -100,3 +212,26 @@ function runCommand(command, args, options = {}) {
});
});
}

async function getDistributionName() {
const osReleaseText = await readFileAsync('/etc/os-release', 'utf8');
const fields = new Map();
for (const line of osReleaseText.split('\n')) {
const tokens = line.split('=');
const name = tokens.shift();
let value = tokens.join('=').trim();
if (value.startsWith('"') && value.endsWith('"'))
value = value.substring(1, value.length - 1);
if (!name)
continue;
fields.set(name.toLowerCase(), value);
}
return fields.get('pretty_name') || '';
}

function currentTime() {
const date = new Date();
const dateTimeFormat = new Intl.DateTimeFormat('en', { year: 'numeric', month: 'short', day: '2-digit' });
const [{ value: month },,{ value: day },,{ value: year }] = dateTimeFormat .formatToParts(date );
return `${month} ${day}, ${year}`;
}
Loading

0 comments on commit 44d96a8

Please sign in to comment.