add backend selection to fasteffekt, add verbose logging flag, refact…

…or commandline app

more cleaning up js runner and fixing bugs

improve error logging when benchmarks fail

src/javascript/compare/comparator.js

@@ -1,21 +1,20 @@
 /**
  * this file is the main benchmark-tool logic.
  * it has predefined shell commands that it executes, measures and logs
- * called by index.js 
+ * called by index.js
  */
-const { execSync } = require('child_process');
+const {execSync} = require('child_process');
 const fs = require('fs');
 
 
 // List of shell commands
 const commands = [
-  ['permute', 'src/effekt/benchmark/permute.effekt', 'node src/javascript/Permute.js'],
-  ["nbody", "src/effekt/benchmark/nbody.effekt", "node src/javascript/Nbody.js"],
-  ['list', 'src/effekt/benchmark/list.effekt', 'node src/javascript/List.js'],
-  ["mandelbrot", "src/effekt/benchmark/mandelbrot.effekt", "node src/javascript/Mandelbrot.js"],
-  ["bounce","src/effekt/benchmark/bounce.effekt","node src/javascript/bounce.js"]
-  // Add more commands as needed
-
+    ['permute', 'src/effekt/benchmark/permute.effekt', 'node src/javascript/Permute.js'],
+    ["nbody", "src/effekt/benchmark/nbody.effekt", "node src/javascript/Nbody.js"],
+    ['list', 'src/effekt/benchmark/list.effekt', 'node src/javascript/List.js'],
+    ["mandelbrot", "src/effekt/benchmark/mandelbrot.effekt", "node src/javascript/Mandelbrot.js"],
+    ["bounce", "src/effekt/benchmark/bounce.effekt", "node src/javascript/bounce.js"]
+    // Add more commands as needed
 ];
 
 /**
@@ -24,52 +23,87 @@ const commands = [
  * @param {*} onOutput (commandOutput) => Void callback function
  */
 const execute = (command, onOutput) => {
-  const output = execSync(command, { encoding: 'utf8' });
-  onOutput(output.trim())
+    const output = execSync(command, {encoding: 'utf8'});
+    onOutput(output.trim())
 }
 
-function executeCommands(commands, isVerify) {
-  const outputs = [];
-  commands.forEach((command, index) => {
-    const performance = { name: command[0], effekt: {}, js: {} }
-    outputs.push(performance)
+function effektCommand(backend, effektFile, amount, verifyArgs, name) {
+    return `effekt.sh -b ${effektFile} --backend ${backend} && ./out/${name} ${amount} ${verifyArgs}`
+}
 
-    console.log("running benchmark:", performance.name)
-    const dirtyCd = "cd " + __dirname + " && cd ../../.. && "
-    const amount = " 3 "
-    const verifyArgs = isVerify ? " --verify" : ""
+function executeCommands(commands, isVerify, backend, verbose) {
+    const outputs = [];
+    commands.forEach((command, index) => {
+        const performance = {name: command[0], effekt: {}, js: {}}
+        outputs.push(performance)
 
-    const [executableName, effektPath, jsCmd] = command
-    const effektCmd = dirtyCd + `effekt.sh -b ${effektPath} && ./out/${executableName} ` + amount + verifyArgs
-    execute(effektCmd, (time) => performance.effekt = time)
+        console.log("running benchmark:", performance.name)
+        const dirtyCd = `cd ${__dirname} && cd ../../..`;
+        const amount = "10"
+        const verifyArgs = isVerify ? "--verify" : ""
 
-    // run pure JS benchmark 
-    const jsExecCmd = dirtyCd + jsCmd + amount + verifyArgs;
-    execute(jsExecCmd, (time) => performance.js = time)
-  });
+        const [executableName, effektPath, jsCmd] = command
+        const effektCmd = [dirtyCd, effektCommand(backend, effektPath, amount, verifyArgs, executableName)].join(" && ")
+        if (verbose)
+            console.log(effektCmd);
+        try {
+            performance.command = effektCmd;
+            execute(effektCmd, (time) => performance.effekt = time)
+        } catch (error) {
+            console.error(`benchmark ${executableName} execution crashed in effekt. log in output file`)
+            performance.effekt = error;
+        }
+        // run pure JS benchmark
+        const jsExecCmd = [dirtyCd, jsCmd + ` ${amount} ${verifyArgs}`].join(" && ");
+        if (verbose)
+            console.log(jsExecCmd);
+        execute(jsExecCmd, (time) => performance.js = time)
+    });
 
-  const analysis = outputs.map(mark => ({ ...mark, effekt: analyzeDurations(mark.effekt), js: analyzeDurations(mark.js) }))
 
-  const outputFile = "fasteffekt_results.json"
-  const resultString = JSON.stringify(analysis.map(perf => ({ ...perf, ratio: perf.effekt.sum / perf.js.sum })), null, 3)
-  fs.writeFileSync(outputFile, resultString);
-  console.log(`Verbose analysis saved to ${outputFile}`);
-  console.log("Mini analysis:",analysis.map(mark => ({ name: mark.name, effekt: mark.effekt.sum, js: mark.js.sum, ratio: mark.effekt.sum / mark.js.sum })))
+    try {
+        let analysis = outputs.map(mark => ({
+            name: mark.name,
+            effekt: analyzeDurations(mark.effekt),
+            js: analyzeDurations(mark.js)
+        }))
+        analysis = analysis.map(perf => ({...perf, ratio: perf.effekt.sum / perf.js.sum}))
+        console.log("Mini analysis:", analysis.map(mark => ({
+            name: mark.name,
+            effekt: mark.effekt.sum,
+            js: mark.js.sum,
+            ratio: mark.effekt.sum / mark.js.sum
+        })))
+        const outputFile = "fasteffekt_results.json"
+        console.log(`Verbose analysis saved to ${outputFile}`);
+        fs.writeFileSync(outputFile, JSON.stringify(analysis, null, 3));
+    } catch {
+        let analysis = outputs
+            .map(mark => ([mark.command, mark.effekt].join("\n")))
+        const errorlog = analysis.join("\n\n");
+        const outputFile = "fasteffekt_error.txt"
+        fs.writeFileSync(outputFile, errorlog);
+        console.error("error occured: not all benchmarks returned readable values. See " + outputFile + " for detailed errors.")
+    }
 }
 
 /**
  * analyze duration array
  * @param {string} durations : list of duration times for every benchmark run
  */
 const analyzeDurations = (durations) => {
-  durations = JSON.parse(durations);
-  const sum = durations.reduce((accumulator, currentValue) => accumulator + currentValue, 0);
-  const avg = sum / durations.length
+    durations = JSON.parse(durations);
+    const sum = durations.reduce((accumulator, currentValue) => accumulator + currentValue, 0);
+    const avg = sum / durations.length
 
-  return { sum: sum, avg: avg, durations: durations }
+    return {sum: sum, avg: avg, durations: durations}
 }
 
 
-const runAll = (isVerify) => executeCommands(commands, isVerify)
+const runAll = (isVerify, backend, verbose) => {
+    if (verbose)
+        console.log(`run all benchmarks with backend=${backend}, small=${isVerify}`)
+    executeCommands(commands, isVerify, backend, verbose)
+}
 module.exports = runAll
 // 

src/javascript/index.js

@@ -1,33 +1,67 @@
 #!/usr/bin/env node
 
-const { verify } = require('crypto');
+const {verify} = require('crypto');
 const path = require('path');
 const runAll = require("./compare/comparator")
 
-const arg = process.argv.length > 2 ? process.argv[2].toLowerCase() : ""
-const isHelp = (arg == "--help" || arg == "-h")
+const knownBackends = ["js", "chez-lift", "llvm", "ml"];
+const passedArguments = {
+    help: false,
+    backend: "js",
+    isVerify: false,
+    version: false,
+    verbose: false
+};
 
-if (isHelp) {
-  console.log(`
+process.argv.slice(2).map(p => p.toLowerCase()).forEach(arg => {
+    if (arg == "--help" || arg == "-h") {
+        passedArguments.help = true;
+    } else if (arg.startsWith("--backend=")) {
+        const arr = arg.split("=");
+        if (arr.length == 2 && knownBackends.findIndex(p => p == arr[1]) != -1) {
+            passedArguments.backend = arr[1];
+            return;
+        }
+        throw new Error(`backend parameter is incorrect. must be --backend=js or one of \n${knownBackends.join("\t")}`)
+    } else if (arg == "--small" || arg == "-s") {
+        passedArguments.isVerify = true;
+    } else if (arg == "--version" || arg == "-v") {
+        passedArguments.version = true;
+    } else if (arg == "--verbose") {
+        passedArguments.verbose = true;
+    } else {
+        throw new Error(`can not parse argument: ${arg}`)
+    }
+})
+
+
+if (passedArguments.help) {
+    console.log(`
   fasteffekt - by Maximilian Marschall
   benchmarking the current install of the effekt language.
   will execute benchmarks in effekt and JS.
   outputs results to console and JSON file
-  execute all benchmarks: fasteffekt [--small]
+  execute all benchmarks: fasteffekt [--small] [backend]
   run with: 
   options:
     --help, -h:      documentation
     --small, -s:     run minimal benchmark to verify they all work
     --version, -v:   show version of fasteffekt
+    --verbose        verbose logging
+    backend:         which effekt backend to use. defaults to JS. passed directly to effekt.sh
   `)
-} else if (arg == "--version" || arg == "-v") {
-  const packageJson = require('../../package.json');
-  // Access the version field and log it
-  console.log('Package version:', packageJson.version);
+} else if (passedArguments.version) {
+    const packageJson = require('../../package.json');
+    // Access the version field and log it
+    console.log('Package version:', packageJson.version);
 } else {
-  const isVerify = (arg == "--small" || arg == "-s")
-  if (isVerify)
-    console.log("verify-mode:", isVerify)
-  runAll(isVerify ? "--verify" : "")
+    if (passedArguments.isVerify)
+        console.log("verify-mode:", passedArguments.isVerify)
+    if (passedArguments.verbose) {
+        console.log("verbose");
+    }
+    console.log(`run for backend ${passedArguments.backend}`);
+
+    runAll(passedArguments.isVerify ? "--verify" : "", passedArguments.backend, passedArguments.verbose);
 }