diff --git a/JetStream.css b/JetStream.css index b510f02..c48cc3c 100644 --- a/JetStream.css +++ b/JetStream.css @@ -348,6 +348,13 @@ a.button { scroll-margin-bottom: 20vh; } +.benchmark .result.detail { + display: none; +} +.details .benchmark .result.detail { + display: inline-block; +} + .benchmark h4, .benchmark .result, .benchmark label, diff --git a/JetStreamDriver.js b/JetStreamDriver.js index 8b74ab2..26f500a 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -37,25 +37,8 @@ if (!JetStreamParams.prefetchResources) this.currentResolve = null; this.currentReject = null; -let showScoreDetails = false; -let categoryScores = null; - function displayCategoryScores() { - if (!categoryScores) - return; - - let scoreDetails = `
`; - for (let [category, scores] of categoryScores) { - scoreDetails += ` - ${uiFriendlyScore(geomeanScore(scores))} - - `; - } - scoreDetails += "
"; - let summaryElement = document.getElementById("result-summary"); - summaryElement.innerHTML += scoreDetails; - - categoryScores = null; + document.body.classList.add("details"); } function getIterationCount(plan) { @@ -81,19 +64,22 @@ function getWorstCaseCount(plan) { if (isInBrowser) { document.onkeydown = (keyboardEvent) => { const key = keyboardEvent.key; - if (key === "d" || key === "D") { - showScoreDetails = true; + if (key === "d" || key === "D") displayCategoryScores(); - } }; } -function mean(values) { +function sum(values) { console.assert(values instanceof Array); let sum = 0; for (let x of values) sum += x; - return sum / values.length; + return sum; +} + +function mean(values) { + const totalSum = sum(values) + return totalSum / values.length; } function geomeanScore(values) { @@ -135,9 +121,25 @@ function uiFriendlyScore(num) { } function uiFriendlyDuration(time) { - return `${time.toFixed(3)} ms`; + return `${time.toFixed(2)} ms`; +} + + +function shellFriendlyLabel(label) { + const namePadding = 40; + return `${label}:`.padEnd(namePadding); +} + +const valuePadding = 10; +function shellFriendlyDuration(time) { + return `${uiFriendlyDuration(time)}`.padStart(valuePadding); } +function shellFriendlyScore(time) { + return `${uiFriendlyScore(time)} # `.padStart(valuePadding); +} + + // TODO: Cleanup / remove / merge. This is only used for caching loads in the // non-browser setting. In the browser we use exclusively `loadCache`, // `loadBlob`, `doLoadBlob`, `prefetchResourcesForBrowser` etc., see below. @@ -226,7 +228,7 @@ class Driver { if (isInBrowser) document.getElementById("benchmark-total-time-score").innerHTML = uiFriendlyNumber(totalTime); else if (!JetStreamParams.dumpJSONResults) - console.log("Total time:", uiFriendlyNumber(totalTime)); + console.log("Total-Time:", uiFriendlyNumber(totalTime)); allScores.push(totalTime); } @@ -237,10 +239,13 @@ class Driver { allScores.push(score); } - categoryScores = new Map; + const categoryScores = new Map(); + const categoryTimes = new Map(); for (const benchmark of this.benchmarks) { for (let category of Object.keys(benchmark.subScores())) categoryScores.set(category, []); + for (let category of Object.keys(benchmark.subTimes())) + categoryTimes.set(category, []); } for (const benchmark of this.benchmarks) { @@ -249,25 +254,55 @@ class Driver { console.assert(value > 0, `Invalid ${benchmark.name} ${category} score: ${value}`); arr.push(value); } + for (let [category, value] of Object.entries(benchmark.subTimes())) { + const arr = categoryTimes.get(category); + console.assert(value > 0, `Invalid ${benchmark.name} ${category} time: ${value}`); + arr.push(value); + } } const totalScore = geomeanScore(allScores); console.assert(totalScore > 0, `Invalid total score: ${totalScore}`); if (isInBrowser) { + let summaryHtml = `
${uiFriendlyScore(totalScore)}
+ `; + summaryHtml += `
`; + for (let [category, scores] of categoryScores) { + summaryHtml += ` + ${uiFriendlyScore(geomeanScore(scores))} + + `; + } + summaryHtml += "
"; + for (let [category, times] of categoryTimes) { + summaryHtml += ` + ${uiFriendlyDuration(geomeanScore(times))} + + `; + } + summaryHtml += "
"; const summaryElement = document.getElementById("result-summary"); summaryElement.classList.add("done"); - summaryElement.innerHTML = `
${uiFriendlyScore(totalScore)}
- `; + summaryElement.innerHTML = summaryHtml; summaryElement.onclick = displayCategoryScores; - if (showScoreDetails) - displayCategoryScores(); statusElement.innerHTML = ""; } else if (!JetStreamParams.dumpJSONResults) { - console.log("\n"); - for (let [category, scores] of categoryScores) - console.log(`${category}: ${uiFriendlyScore(geomeanScore(scores))}`); - console.log("\nTotal Score: ", uiFriendlyScore(totalScore), "\n"); + console.log("Total:"); + for (let [category, scores] of categoryScores) { + console.log( + shellFriendlyLabel(`${category}-Score`), + shellFriendlyScore(geomeanScore(scores))); + } + for (let [category, times] of categoryTimes) { + console.log( + shellFriendlyLabel(`${category}-Time`), + shellFriendlyDuration(geomeanScore(times))); + } + console.log(""); + console.log(shellFriendlyLabel("Total-Score"), shellFriendlyScore(totalScore)); + console.log(shellFriendlyLabel("Total-Time"), shellFriendlyDuration(totalTime)); + console.log(""); } this.reportScoreToRunBenchmarkRunner(); @@ -284,33 +319,38 @@ class Driver { prepareToRun() { this.benchmarks.sort((a, b) => a.plan.name.toLowerCase() < b.plan.name.toLowerCase() ? 1 : -1); + if (!isInBrowser) + return; + let text = ""; for (const benchmark of this.benchmarks) { - const description = Object.keys(benchmark.subScores()); - description.push("Score"); + const scoreDescription = Object.keys(benchmark.allScores()); + const timeDescription = Object.keys(benchmark.allTimes()); - const scoreIds = benchmark.scoreIdentifiers(); + const scoreIds = benchmark.allScoreIdentifiers(); const overallScoreId = scoreIds.pop(); - - if (isInBrowser) { - text += - `
-

${benchmark.name} i

-

 

-

 

-

`; - for (let i = 0; i < scoreIds.length; i++) { - const scoreId = scoreIds[i]; - const label = description[i]; - text += ` ` - } - text += `

`; + const timeIds = benchmark.allTimeIdentifiers(); + + text += + `
+

${benchmark.name} i

+

 

+

 

+

`; + for (let i = 0; i < scoreIds.length; i++) { + const scoreId = scoreIds[i]; + const label = scoreDescription[i]; + text += ` ` } + text += "
"; + for (let i = 0; i < timeIds.length; i++) { + const timeId = timeIds[i]; + const label = timeDescription[i]; + text += ` ` + } + text += `

`; } - if (!isInBrowser) - return; - const timestamp = performance.now(); document.getElementById('jetstreams').style.backgroundImage = `url('jetstreams.svg?${timestamp}')`; const resultsTable = document.getElementById("results"); @@ -329,12 +369,13 @@ class Driver { if (!isInBrowser) return; - for (const id of benchmark.scoreIdentifiers()) { + for (const id of benchmark.allScoreIdentifiers()) document.getElementById(id).innerHTML = "error"; - const benchmarkResultsUI = document.getElementById(`benchmark-${benchmark.name}`); - benchmarkResultsUI.classList.remove("benchmark-running"); - benchmarkResultsUI.classList.add("benchmark-error"); - } + for (const id of benchmark.allTimeIdentifiers()) + document.getElementById(id).innerHTML = "error"; + const benchmarkResultsUI = document.getElementById(`benchmark-${benchmark.name}`); + benchmarkResultsUI.classList.remove("benchmark-running"); + benchmarkResultsUI.classList.add("benchmark-error"); } pushError(name, error) { @@ -731,16 +772,36 @@ class Benchmark { return geomeanScore(subScores); } + get totalTime() { + const subTimes = Object.values(this.subTimes()); + return sum(subTimes); + } + + get wallTime() { + return this.endTime - this.startTime; + } + subScores() { throw new Error("Subclasses need to implement this"); } + subTimes() { + throw new Error("Subclasses need to implement this"); + } + allScores() { const allScores = this.subScores(); allScores["Score"] = this.score; return allScores; } + allTimes() { + const allTimes = this.subTimes(); + allTimes["Wall"] = this.wallTime; + allTimes["Total"] = this.totalTime; + return allTimes; + } + get prerunCode() { return null; } get preIterationCode() { @@ -1001,7 +1062,7 @@ class Benchmark { this.preloads = Object.entries(this.plan.preload ?? {}); } - scoreIdentifiers() { + allScoreIdentifiers() { const ids = Object.keys(this.allScores()).map(name => this.scoreIdentifier(name)); return ids; } @@ -1010,6 +1071,15 @@ class Benchmark { return `results-cell-${this.name}-${scoreName}`; } + allTimeIdentifiers() { + const ids = Object.keys(this.allTimes()).map(name => this.timeIdentifier(name)); + return ids; + } + + timeIdentifier(scoreName) { + return `results-cell-${this.name}-${scoreName}-time`; + } + updateUIBeforeRun() { if (!JetStreamParams.dumpJSONResults) console.log(`Running ${this.name}:`); @@ -1022,30 +1092,56 @@ class Benchmark { resultsBenchmarkUI.classList.add("benchmark-running"); resultsBenchmarkUI.scrollIntoView({ block: "nearest" }); - for (const id of this.scoreIdentifiers()) + for (const id of this.allScoreIdentifiers()) + document.getElementById(id).innerHTML = "..."; + for (const id of this.allTimeIdentifiers()) document.getElementById(id).innerHTML = "..."; } updateUIAfterRun() { - const scoreEntries = Object.entries(this.allScores()); if (isInBrowser) - this.updateUIAfterRunInBrowser(scoreEntries); + this.updateUIAfterRunInBrowser(); if (JetStreamParams.dumpJSONResults) return; - this.updateConsoleAfterRun(scoreEntries); + this.updateConsoleAfterRun(); } - updateUIAfterRunInBrowser(scoreEntries) { + updateUIAfterRunInBrowser() { const benchmarkResultsUI = document.getElementById(`benchmark-${this.name}`); benchmarkResultsUI.classList.remove("benchmark-running"); benchmarkResultsUI.classList.add("benchmark-done"); - for (const [name, value] of scoreEntries) + for (const [name, value] of Object.entries(this.allScores())) document.getElementById(this.scoreIdentifier(name)).innerHTML = uiFriendlyScore(value); + for (const [name, value] of Object.entries(this.allTimes())) + document.getElementById(this.timeIdentifier(name)).innerHTML = uiFriendlyDuration(value); this.renderScatterPlot(); } + updateConsoleAfterRun() { + for (let [name, value] of Object.entries(this.allScores())) { + if (!name.endsWith("Score")) + name = `${name}-Score`; + + this.logMetric(name, shellFriendlyScore(value)); + } + for (let [name, value] of Object.entries(this.allTimes())) { + this.logMetric(`${name}-Time`, shellFriendlyDuration(value)); + } + if (JetStreamParams.RAMification) { + this.logMetric("Current Footprint", uiFriendlyNumber(this.currentFootprint)); + this.logMetric("Peak Footprint", uiFriendlyNumber(this.peakFootprint)); + } + console.log(""); + } + + logMetric(name, value) { + console.log( + shellFriendlyLabel(`${this.name} ${name}`), + value); + } + renderScatterPlot() { const plotContainer = document.getElementById(`plot-${this.name}`); if (!plotContainer || !this.results || this.results.length === 0) @@ -1073,17 +1169,6 @@ class Benchmark { } plotContainer.innerHTML = `${circlesSVG}`; } - - updateConsoleAfterRun(scoreEntries) { - for (let [name, value] of scoreEntries) { - console.log(` ${name}:`, uiFriendlyScore(value)); - } - if (JetStreamParams.RAMification) { - console.log(" Current Footprint:", uiFriendlyNumber(this.currentFootprint)); - console.log(" Peak Footprint:", uiFriendlyNumber(this.peakFootprint)); - } - console.log(" Wall-Time:", uiFriendlyDuration(this.endTime - this.startTime)); - } }; class GroupedBenchmark extends Benchmark { @@ -1167,6 +1252,22 @@ class GroupedBenchmark extends Benchmark { results[subScore] = geomeanScore(results[subScore]); return results; } + + subTimes() { + const results = {}; + + for (const benchmark of this.benchmarks) { + let times = benchmark.subTimes(); + for (let subTime in times) { + results[subTime] ??= []; + results[subTime].push(times[subTime]); + } + } + + for (let subTimes in results) + results[subTimes] = sum(results[subTimes]); + return results; + } }; class DefaultBenchmark extends Benchmark { @@ -1215,6 +1316,17 @@ class DefaultBenchmark extends Benchmark { scores["Average"] = this.averageScore; return scores; } + + subTimes() { + const times = { + "First": this.firstIterationTime, + }; + if (this.worstCaseCount) + times["Worst"] = this.worstTime; + if (this.iterations > 1) + times["Average"] = this.averageTime; + return times; + } } class AsyncBenchmark extends DefaultBenchmark { @@ -1379,6 +1491,13 @@ class WSLBenchmark extends Benchmark { }`; } + subTimes() { + return { + "Stdlib": this.stdlibTime, + "MainRun": this.mainRunTime, + }; + } + subScores() { return { "Stdlib": this.stdlibScore, @@ -1516,6 +1635,13 @@ class WasmLegacyBenchmark extends Benchmark { "Runtime": this.runScore, }; } + + subTimes() { + return { + "Startup": this.startupTime, + "Runtime": this.runTime, + }; + } }; function dotnetPreloads(type)