From 76321a09caa41cb33be550a09893c6c45d6133e7 Mon Sep 17 00:00:00 2001 From: IR0NSIGHT Date: Mon, 5 Feb 2024 15:35:53 +0100 Subject: [PATCH] add backend selection to fasteffekt, add verbose logging flag, refactor commandline app more cleaning up js runner and fixing bugs improve error logging when benchmarks fail --- src/javascript/compare/comparator.js | 110 ++++++++++++++++++--------- src/javascript/index.js | 62 +++++++++++---- 2 files changed, 120 insertions(+), 52 deletions(-) diff --git a/src/javascript/compare/comparator.js b/src/javascript/compare/comparator.js index b1f0f7ca..ebf79e83 100755 --- a/src/javascript/compare/comparator.js +++ b/src/javascript/compare/comparator.js @@ -1,21 +1,20 @@ /** * this file is the main benchmark-tool logic. * it has predefined shell commands that it executes, measures and logs - * called by index.js + * called by index.js */ -const { execSync } = require('child_process'); +const {execSync} = require('child_process'); const fs = require('fs'); // List of shell commands const commands = [ - ['permute', 'src/effekt/benchmark/permute.effekt', 'node src/javascript/Permute.js'], - ["nbody", "src/effekt/benchmark/nbody.effekt", "node src/javascript/Nbody.js"], - ['list', 'src/effekt/benchmark/list.effekt', 'node src/javascript/List.js'], - ["mandelbrot", "src/effekt/benchmark/mandelbrot.effekt", "node src/javascript/Mandelbrot.js"], - ["bounce","src/effekt/benchmark/bounce.effekt","node src/javascript/bounce.js"] - // Add more commands as needed - + ['permute', 'src/effekt/benchmark/permute.effekt', 'node src/javascript/Permute.js'], + ["nbody", "src/effekt/benchmark/nbody.effekt", "node src/javascript/Nbody.js"], + ['list', 'src/effekt/benchmark/list.effekt', 'node src/javascript/List.js'], + ["mandelbrot", "src/effekt/benchmark/mandelbrot.effekt", "node src/javascript/Mandelbrot.js"], + ["bounce", "src/effekt/benchmark/bounce.effekt", "node src/javascript/bounce.js"] + // Add more commands as needed ]; /** @@ -24,37 +23,68 @@ const commands = [ * @param {*} onOutput (commandOutput) => Void callback function */ const execute = (command, onOutput) => { - const output = execSync(command, { encoding: 'utf8' }); - onOutput(output.trim()) + const output = execSync(command, {encoding: 'utf8'}); + onOutput(output.trim()) } -function executeCommands(commands, isVerify) { - const outputs = []; - commands.forEach((command, index) => { - const performance = { name: command[0], effekt: {}, js: {} } - outputs.push(performance) +function effektCommand(backend, effektFile, amount, verifyArgs, name) { + return `effekt.sh -b ${effektFile} --backend ${backend} && ./out/${name} ${amount} ${verifyArgs}` +} - console.log("running benchmark:", performance.name) - const dirtyCd = "cd " + __dirname + " && cd ../../.. && " - const amount = " 3 " - const verifyArgs = isVerify ? " --verify" : "" +function executeCommands(commands, isVerify, backend, verbose) { + const outputs = []; + commands.forEach((command, index) => { + const performance = {name: command[0], effekt: {}, js: {}} + outputs.push(performance) - const [executableName, effektPath, jsCmd] = command - const effektCmd = dirtyCd + `effekt.sh -b ${effektPath} && ./out/${executableName} ` + amount + verifyArgs - execute(effektCmd, (time) => performance.effekt = time) + console.log("running benchmark:", performance.name) + const dirtyCd = `cd ${__dirname} && cd ../../..`; + const amount = "10" + const verifyArgs = isVerify ? "--verify" : "" - // run pure JS benchmark - const jsExecCmd = dirtyCd + jsCmd + amount + verifyArgs; - execute(jsExecCmd, (time) => performance.js = time) - }); + const [executableName, effektPath, jsCmd] = command + const effektCmd = [dirtyCd, effektCommand(backend, effektPath, amount, verifyArgs, executableName)].join(" && ") + if (verbose) + console.log(effektCmd); + try { + performance.command = effektCmd; + execute(effektCmd, (time) => performance.effekt = time) + } catch (error) { + console.error(`benchmark ${executableName} execution crashed in effekt. log in output file`) + performance.effekt = error; + } + // run pure JS benchmark + const jsExecCmd = [dirtyCd, jsCmd + ` ${amount} ${verifyArgs}`].join(" && "); + if (verbose) + console.log(jsExecCmd); + execute(jsExecCmd, (time) => performance.js = time) + }); - const analysis = outputs.map(mark => ({ ...mark, effekt: analyzeDurations(mark.effekt), js: analyzeDurations(mark.js) })) - const outputFile = "fasteffekt_results.json" - const resultString = JSON.stringify(analysis.map(perf => ({ ...perf, ratio: perf.effekt.sum / perf.js.sum })), null, 3) - fs.writeFileSync(outputFile, resultString); - console.log(`Verbose analysis saved to ${outputFile}`); - console.log("Mini analysis:",analysis.map(mark => ({ name: mark.name, effekt: mark.effekt.sum, js: mark.js.sum, ratio: mark.effekt.sum / mark.js.sum }))) + try { + let analysis = outputs.map(mark => ({ + name: mark.name, + effekt: analyzeDurations(mark.effekt), + js: analyzeDurations(mark.js) + })) + analysis = analysis.map(perf => ({...perf, ratio: perf.effekt.sum / perf.js.sum})) + console.log("Mini analysis:", analysis.map(mark => ({ + name: mark.name, + effekt: mark.effekt.sum, + js: mark.js.sum, + ratio: mark.effekt.sum / mark.js.sum + }))) + const outputFile = "fasteffekt_results.json" + console.log(`Verbose analysis saved to ${outputFile}`); + fs.writeFileSync(outputFile, JSON.stringify(analysis, null, 3)); + } catch { + let analysis = outputs + .map(mark => ([mark.command, mark.effekt].join("\n"))) + const errorlog = analysis.join("\n\n"); + const outputFile = "fasteffekt_error.txt" + fs.writeFileSync(outputFile, errorlog); + console.error("error occured: not all benchmarks returned readable values. See " + outputFile + " for detailed errors.") + } } /** @@ -62,14 +92,18 @@ function executeCommands(commands, isVerify) { * @param {string} durations : list of duration times for every benchmark run */ const analyzeDurations = (durations) => { - durations = JSON.parse(durations); - const sum = durations.reduce((accumulator, currentValue) => accumulator + currentValue, 0); - const avg = sum / durations.length + durations = JSON.parse(durations); + const sum = durations.reduce((accumulator, currentValue) => accumulator + currentValue, 0); + const avg = sum / durations.length - return { sum: sum, avg: avg, durations: durations } + return {sum: sum, avg: avg, durations: durations} } -const runAll = (isVerify) => executeCommands(commands, isVerify) +const runAll = (isVerify, backend, verbose) => { + if (verbose) + console.log(`run all benchmarks with backend=${backend}, small=${isVerify}`) + executeCommands(commands, isVerify, backend, verbose) +} module.exports = runAll // diff --git a/src/javascript/index.js b/src/javascript/index.js index f618e236..b75c7268 100755 --- a/src/javascript/index.js +++ b/src/javascript/index.js @@ -1,33 +1,67 @@ #!/usr/bin/env node -const { verify } = require('crypto'); +const {verify} = require('crypto'); const path = require('path'); const runAll = require("./compare/comparator") -const arg = process.argv.length > 2 ? process.argv[2].toLowerCase() : "" -const isHelp = (arg == "--help" || arg == "-h") +const knownBackends = ["js", "chez-lift", "llvm", "ml"]; +const passedArguments = { + help: false, + backend: "js", + isVerify: false, + version: false, + verbose: false +}; -if (isHelp) { - console.log(` +process.argv.slice(2).map(p => p.toLowerCase()).forEach(arg => { + if (arg == "--help" || arg == "-h") { + passedArguments.help = true; + } else if (arg.startsWith("--backend=")) { + const arr = arg.split("="); + if (arr.length == 2 && knownBackends.findIndex(p => p == arr[1]) != -1) { + passedArguments.backend = arr[1]; + return; + } + throw new Error(`backend parameter is incorrect. must be --backend=js or one of \n${knownBackends.join("\t")}`) + } else if (arg == "--small" || arg == "-s") { + passedArguments.isVerify = true; + } else if (arg == "--version" || arg == "-v") { + passedArguments.version = true; + } else if (arg == "--verbose") { + passedArguments.verbose = true; + } else { + throw new Error(`can not parse argument: ${arg}`) + } +}) + + +if (passedArguments.help) { + console.log(` fasteffekt - by Maximilian Marschall benchmarking the current install of the effekt language. will execute benchmarks in effekt and JS. outputs results to console and JSON file - execute all benchmarks: fasteffekt [--small] + execute all benchmarks: fasteffekt [--small] [backend] run with: options: --help, -h: documentation --small, -s: run minimal benchmark to verify they all work --version, -v: show version of fasteffekt + --verbose verbose logging + backend: which effekt backend to use. defaults to JS. passed directly to effekt.sh `) -} else if (arg == "--version" || arg == "-v") { - const packageJson = require('../../package.json'); - // Access the version field and log it - console.log('Package version:', packageJson.version); +} else if (passedArguments.version) { + const packageJson = require('../../package.json'); + // Access the version field and log it + console.log('Package version:', packageJson.version); } else { - const isVerify = (arg == "--small" || arg == "-s") - if (isVerify) - console.log("verify-mode:", isVerify) - runAll(isVerify ? "--verify" : "") + if (passedArguments.isVerify) + console.log("verify-mode:", passedArguments.isVerify) + if (passedArguments.verbose) { + console.log("verbose"); + } + console.log(`run for backend ${passedArguments.backend}`); + + runAll(passedArguments.isVerify ? "--verify" : "", passedArguments.backend, passedArguments.verbose); }