add Benchmark (pytest) benchmark result for 86dff1a36cbd620f2d73af763e949ec00d777239
diff --git a/dev/bench/data.js b/dev/bench/data.js new file mode 100644 index 0000000..edbc85c --- /dev/null +++ b/dev/bench/data.js
@@ -0,0 +1,112 @@ +window.BENCHMARK_DATA = { + "lastUpdate": 1719528536364, + "repoUrl": "https://github.com/MPACT-ORG/mpact-compiler", + "entries": { + "Benchmark": [ + { + "commit": { + "author": { + "name": "MPACT-ORG", + "username": "MPACT-ORG" + }, + "committer": { + "name": "MPACT-ORG", + "username": "MPACT-ORG" + }, + "id": "86dff1a36cbd620f2d73af763e949ec00d777239", + "message": "[mpact][benchmark] add regression benchmark to gh page", + "timestamp": "2024-06-27T22:22:10Z", + "url": "https://github.com/MPACT-ORG/mpact-compiler/pull/52/commits/86dff1a36cbd620f2d73af763e949ec00d777239" + }, + "date": 1719528535086, + "tool": "pytest", + "benches": [ + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mv_dense", + "value": 6669.427405418031, + "unit": "iter/sec", + "range": "stddev: 0.000005968668091106435", + "extra": "mean: 149.93790909061124 usec\nrounds: 2057" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mm_dense", + "value": 34.302140003556715, + "unit": "iter/sec", + "range": "stddev: 0.0003297513733362601", + "extra": "mean: 29.15270009090722 msec\nrounds: 33" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_add_dense", + "value": 5915.897194757258, + "unit": "iter/sec", + "range": "stddev: 0.00003919411579877867", + "extra": "mean: 169.03606791649665 usec\nrounds: 1973" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mul_dense", + "value": 6002.4939914783945, + "unit": "iter/sec", + "range": "stddev: 0.000027105251732015887", + "extra": "mean: 166.59741790990168 usec\nrounds: 3551" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_nop_dense", + "value": 948864.5687594158, + "unit": "iter/sec", + "range": "stddev: 1.8437976818208762e-7", + "extra": "mean: 1.0538911799683288 usec\nrounds: 144238" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_sddmm_dense", + "value": 32.142115430220215, + "unit": "iter/sec", + "range": "stddev: 0.000630869582999464", + "extra": "mean: 31.111829032254484 msec\nrounds: 31" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mv_sparse", + "value": 12377.336292065489, + "unit": "iter/sec", + "range": "stddev: 0.000005090184458909657", + "extra": "mean: 80.79282782686057 usec\nrounds: 3299" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mm_sparse", + "value": 20.396281430385955, + "unit": "iter/sec", + "range": "stddev: 0.0003606405379378096", + "extra": "mean: 49.02854490476979 msec\nrounds: 21" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_add_sparse", + "value": 210.87200147418721, + "unit": "iter/sec", + "range": "stddev: 0.0005598196294447149", + "extra": "mean: 4.742213252632355 msec\nrounds: 285" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mul_sparse", + "value": 189.26652258818748, + "unit": "iter/sec", + "range": "stddev: 0.00010650763765171751", + "extra": "mean: 5.2835545680512865 msec\nrounds: 169" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_nop_sparse", + "value": 1093738.9035266023, + "unit": "iter/sec", + "range": "stddev: 8.140592632863567e-8", + "extra": "mean: 914.2949901257589 nsec\nrounds: 177589" + }, + { + "name": "benchmark/python/benchmarks/regression_benchmark.py::test_sddmm_sparse", + "value": 21.33905041385192, + "unit": "iter/sec", + "range": "stddev: 0.002583893132946909", + "extra": "mean: 46.862441421051486 msec\nrounds: 19" + } + ] + } + ] + } +} \ No newline at end of file
diff --git a/dev/bench/index.html b/dev/bench/index.html new file mode 100644 index 0000000..6c88780 --- /dev/null +++ b/dev/bench/index.html
@@ -0,0 +1,281 @@ +<!DOCTYPE html> +<html> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1, user-scalable=yes" /> + <style> + html { + font-family: BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif; + -webkit-font-smoothing: antialiased; + background-color: #fff; + font-size: 16px; + } + body { + color: #4a4a4a; + margin: 8px; + font-size: 1em; + font-weight: 400; + } + header { + margin-bottom: 8px; + display: flex; + flex-direction: column; + } + main { + width: 100%; + display: flex; + flex-direction: column; + } + a { + color: #3273dc; + cursor: pointer; + text-decoration: none; + } + a:hover { + color: #000; + } + button { + color: #fff; + background-color: #3298dc; + border-color: transparent; + cursor: pointer; + text-align: center; + } + button:hover { + background-color: #2793da; + flex: none; + } + .spacer { + flex: auto; + } + .small { + font-size: 0.75rem; + } + footer { + margin-top: 16px; + display: flex; + align-items: center; + } + .header-label { + margin-right: 4px; + } + .benchmark-set { + margin: 8px 0; + width: 100%; + display: flex; + flex-direction: column; + } + .benchmark-title { + font-size: 3rem; + font-weight: 600; + word-break: break-word; + text-align: center; + } + .benchmark-graphs { + display: flex; + flex-direction: row; + justify-content: space-around; + align-items: center; + flex-wrap: wrap; + width: 100%; + } + .benchmark-chart { + max-width: 1000px; + } + </style> + <title>Benchmarks</title> + </head> + + <body> + <header id="header"> + <div class="header-item"> + <strong class="header-label">Last Update:</strong> + <span id="last-update"></span> + </div> + <div class="header-item"> + <strong class="header-label">Repository:</strong> + <a id="repository-link" rel="noopener"></a> + </div> + </header> + <main id="main"></main> + <footer> + <button id="dl-button">Download data as JSON</button> + <div class="spacer"></div> + <div class="small">Powered by <a rel="noopener" href="https://github.com/marketplace/actions/continuous-benchmark">github-action-benchmark</a></div> + </footer> + + <script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.2/dist/Chart.min.js"></script> + <script src="data.js"></script> + <script id="main-script"> + 'use strict'; + (function() { + // Colors from https://github.com/github/linguist/blob/master/lib/linguist/languages.yml + const toolColors = { + cargo: '#dea584', + go: '#00add8', + benchmarkjs: '#f1e05a', + benchmarkluau: '#000080', + pytest: '#3572a5', + googlecpp: '#f34b7d', + catch2: '#f34b7d', + julia: '#a270ba', + jmh: '#b07219', + benchmarkdotnet: '#178600', + customBiggerIsBetter: '#38ff38', + customSmallerIsBetter: '#ff3838', + _: '#333333' + }; + + function init() { + function collectBenchesPerTestCase(entries) { + const map = new Map(); + for (const entry of entries) { + const {commit, date, tool, benches} = entry; + for (const bench of benches) { + const result = { commit, date, tool, bench }; + const arr = map.get(bench.name); + if (arr === undefined) { + map.set(bench.name, [result]); + } else { + arr.push(result); + } + } + } + return map; + } + + const data = window.BENCHMARK_DATA; + + // Render header + document.getElementById('last-update').textContent = new Date(data.lastUpdate).toString(); + const repoLink = document.getElementById('repository-link'); + repoLink.href = data.repoUrl; + repoLink.textContent = data.repoUrl; + + // Render footer + document.getElementById('dl-button').onclick = () => { + const dataUrl = 'data:,' + JSON.stringify(data, null, 2); + const a = document.createElement('a'); + a.href = dataUrl; + a.download = 'benchmark_data.json'; + a.click(); + }; + + // Prepare data points for charts + return Object.keys(data.entries).map(name => ({ + name, + dataSet: collectBenchesPerTestCase(data.entries[name]), + })); + } + + function renderAllChars(dataSets) { + + function renderGraph(parent, name, dataset) { + const canvas = document.createElement('canvas'); + canvas.className = 'benchmark-chart'; + parent.appendChild(canvas); + + const color = toolColors[dataset.length > 0 ? dataset[0].tool : '_']; + const data = { + labels: dataset.map(d => d.commit.id.slice(0, 7)), + datasets: [ + { + label: name, + data: dataset.map(d => d.bench.value), + borderColor: color, + backgroundColor: color + '60', // Add alpha for #rrggbbaa + } + ], + }; + const options = { + scales: { + xAxes: [ + { + scaleLabel: { + display: true, + labelString: 'commit', + }, + } + ], + yAxes: [ + { + scaleLabel: { + display: true, + labelString: dataset.length > 0 ? dataset[0].bench.unit : '', + }, + ticks: { + beginAtZero: true, + } + } + ], + }, + tooltips: { + callbacks: { + afterTitle: items => { + const {index} = items[0]; + const data = dataset[index]; + return '\n' + data.commit.message + '\n\n' + data.commit.timestamp + ' committed by @' + data.commit.committer.username + '\n'; + }, + label: item => { + let label = item.value; + const { range, unit } = dataset[item.index].bench; + label += ' ' + unit; + if (range) { + label += ' (' + range + ')'; + } + return label; + }, + afterLabel: item => { + const { extra } = dataset[item.index].bench; + return extra ? '\n' + extra : ''; + } + } + }, + onClick: (_mouseEvent, activeElems) => { + if (activeElems.length === 0) { + return; + } + // XXX: Undocumented. How can we know the index? + const index = activeElems[0]._index; + const url = dataset[index].commit.url; + window.open(url, '_blank'); + }, + }; + + new Chart(canvas, { + type: 'line', + data, + options, + }); + } + + function renderBenchSet(name, benchSet, main) { + const setElem = document.createElement('div'); + setElem.className = 'benchmark-set'; + main.appendChild(setElem); + + const nameElem = document.createElement('h1'); + nameElem.className = 'benchmark-title'; + nameElem.textContent = name; + setElem.appendChild(nameElem); + + const graphsElem = document.createElement('div'); + graphsElem.className = 'benchmark-graphs'; + setElem.appendChild(graphsElem); + + for (const [benchName, benches] of benchSet.entries()) { + renderGraph(graphsElem, benchName, benches) + } + } + + const main = document.getElementById('main'); + for (const {name, dataSet} of dataSets) { + renderBenchSet(name, dataSet, main); + } + } + + renderAllChars(init()); // Start + })(); + </script> + </body> +</html>