add Benchmark (pytest) benchmark result for 86dff1a36cbd620f2d73af763e949ec00d777239
diff --git a/dev/bench/data.js b/dev/bench/data.js
new file mode 100644
index 0000000..edbc85c
--- /dev/null
+++ b/dev/bench/data.js
@@ -0,0 +1,112 @@
+window.BENCHMARK_DATA = {
+ "lastUpdate": 1719528536364,
+ "repoUrl": "https://github.com/MPACT-ORG/mpact-compiler",
+ "entries": {
+ "Benchmark": [
+ {
+ "commit": {
+ "author": {
+ "name": "MPACT-ORG",
+ "username": "MPACT-ORG"
+ },
+ "committer": {
+ "name": "MPACT-ORG",
+ "username": "MPACT-ORG"
+ },
+ "id": "86dff1a36cbd620f2d73af763e949ec00d777239",
+ "message": "[mpact][benchmark] add regression benchmark to gh page",
+ "timestamp": "2024-06-27T22:22:10Z",
+ "url": "https://github.com/MPACT-ORG/mpact-compiler/pull/52/commits/86dff1a36cbd620f2d73af763e949ec00d777239"
+ },
+ "date": 1719528535086,
+ "tool": "pytest",
+ "benches": [
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mv_dense",
+ "value": 6669.427405418031,
+ "unit": "iter/sec",
+ "range": "stddev: 0.000005968668091106435",
+ "extra": "mean: 149.93790909061124 usec\nrounds: 2057"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mm_dense",
+ "value": 34.302140003556715,
+ "unit": "iter/sec",
+ "range": "stddev: 0.0003297513733362601",
+ "extra": "mean: 29.15270009090722 msec\nrounds: 33"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_add_dense",
+ "value": 5915.897194757258,
+ "unit": "iter/sec",
+ "range": "stddev: 0.00003919411579877867",
+ "extra": "mean: 169.03606791649665 usec\nrounds: 1973"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mul_dense",
+ "value": 6002.4939914783945,
+ "unit": "iter/sec",
+ "range": "stddev: 0.000027105251732015887",
+ "extra": "mean: 166.59741790990168 usec\nrounds: 3551"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_nop_dense",
+ "value": 948864.5687594158,
+ "unit": "iter/sec",
+ "range": "stddev: 1.8437976818208762e-7",
+ "extra": "mean: 1.0538911799683288 usec\nrounds: 144238"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_sddmm_dense",
+ "value": 32.142115430220215,
+ "unit": "iter/sec",
+ "range": "stddev: 0.000630869582999464",
+ "extra": "mean: 31.111829032254484 msec\nrounds: 31"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mv_sparse",
+ "value": 12377.336292065489,
+ "unit": "iter/sec",
+ "range": "stddev: 0.000005090184458909657",
+ "extra": "mean: 80.79282782686057 usec\nrounds: 3299"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mm_sparse",
+ "value": 20.396281430385955,
+ "unit": "iter/sec",
+ "range": "stddev: 0.0003606405379378096",
+ "extra": "mean: 49.02854490476979 msec\nrounds: 21"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_add_sparse",
+ "value": 210.87200147418721,
+ "unit": "iter/sec",
+ "range": "stddev: 0.0005598196294447149",
+ "extra": "mean: 4.742213252632355 msec\nrounds: 285"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_mul_sparse",
+ "value": 189.26652258818748,
+ "unit": "iter/sec",
+ "range": "stddev: 0.00010650763765171751",
+ "extra": "mean: 5.2835545680512865 msec\nrounds: 169"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_nop_sparse",
+ "value": 1093738.9035266023,
+ "unit": "iter/sec",
+ "range": "stddev: 8.140592632863567e-8",
+ "extra": "mean: 914.2949901257589 nsec\nrounds: 177589"
+ },
+ {
+ "name": "benchmark/python/benchmarks/regression_benchmark.py::test_sddmm_sparse",
+ "value": 21.33905041385192,
+ "unit": "iter/sec",
+ "range": "stddev: 0.002583893132946909",
+ "extra": "mean: 46.862441421051486 msec\nrounds: 19"
+ }
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/dev/bench/index.html b/dev/bench/index.html
new file mode 100644
index 0000000..6c88780
--- /dev/null
+++ b/dev/bench/index.html
@@ -0,0 +1,281 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1, user-scalable=yes" />
+ <style>
+ html {
+ font-family: BlinkMacSystemFont,-apple-system,"Segoe UI",Roboto,Oxygen,Ubuntu,Cantarell,"Fira Sans","Droid Sans","Helvetica Neue",Helvetica,Arial,sans-serif;
+ -webkit-font-smoothing: antialiased;
+ background-color: #fff;
+ font-size: 16px;
+ }
+ body {
+ color: #4a4a4a;
+ margin: 8px;
+ font-size: 1em;
+ font-weight: 400;
+ }
+ header {
+ margin-bottom: 8px;
+ display: flex;
+ flex-direction: column;
+ }
+ main {
+ width: 100%;
+ display: flex;
+ flex-direction: column;
+ }
+ a {
+ color: #3273dc;
+ cursor: pointer;
+ text-decoration: none;
+ }
+ a:hover {
+ color: #000;
+ }
+ button {
+ color: #fff;
+ background-color: #3298dc;
+ border-color: transparent;
+ cursor: pointer;
+ text-align: center;
+ }
+ button:hover {
+ background-color: #2793da;
+ flex: none;
+ }
+ .spacer {
+ flex: auto;
+ }
+ .small {
+ font-size: 0.75rem;
+ }
+ footer {
+ margin-top: 16px;
+ display: flex;
+ align-items: center;
+ }
+ .header-label {
+ margin-right: 4px;
+ }
+ .benchmark-set {
+ margin: 8px 0;
+ width: 100%;
+ display: flex;
+ flex-direction: column;
+ }
+ .benchmark-title {
+ font-size: 3rem;
+ font-weight: 600;
+ word-break: break-word;
+ text-align: center;
+ }
+ .benchmark-graphs {
+ display: flex;
+ flex-direction: row;
+ justify-content: space-around;
+ align-items: center;
+ flex-wrap: wrap;
+ width: 100%;
+ }
+ .benchmark-chart {
+ max-width: 1000px;
+ }
+ </style>
+ <title>Benchmarks</title>
+ </head>
+
+ <body>
+ <header id="header">
+ <div class="header-item">
+ <strong class="header-label">Last Update:</strong>
+ <span id="last-update"></span>
+ </div>
+ <div class="header-item">
+ <strong class="header-label">Repository:</strong>
+ <a id="repository-link" rel="noopener"></a>
+ </div>
+ </header>
+ <main id="main"></main>
+ <footer>
+ <button id="dl-button">Download data as JSON</button>
+ <div class="spacer"></div>
+ <div class="small">Powered by <a rel="noopener" href="https://github.com/marketplace/actions/continuous-benchmark">github-action-benchmark</a></div>
+ </footer>
+
+ <script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.2/dist/Chart.min.js"></script>
+ <script src="data.js"></script>
+ <script id="main-script">
+ 'use strict';
+ (function() {
+ // Colors from https://github.com/github/linguist/blob/master/lib/linguist/languages.yml
+ const toolColors = {
+ cargo: '#dea584',
+ go: '#00add8',
+ benchmarkjs: '#f1e05a',
+ benchmarkluau: '#000080',
+ pytest: '#3572a5',
+ googlecpp: '#f34b7d',
+ catch2: '#f34b7d',
+ julia: '#a270ba',
+ jmh: '#b07219',
+ benchmarkdotnet: '#178600',
+ customBiggerIsBetter: '#38ff38',
+ customSmallerIsBetter: '#ff3838',
+ _: '#333333'
+ };
+
+ function init() {
+ function collectBenchesPerTestCase(entries) {
+ const map = new Map();
+ for (const entry of entries) {
+ const {commit, date, tool, benches} = entry;
+ for (const bench of benches) {
+ const result = { commit, date, tool, bench };
+ const arr = map.get(bench.name);
+ if (arr === undefined) {
+ map.set(bench.name, [result]);
+ } else {
+ arr.push(result);
+ }
+ }
+ }
+ return map;
+ }
+
+ const data = window.BENCHMARK_DATA;
+
+ // Render header
+ document.getElementById('last-update').textContent = new Date(data.lastUpdate).toString();
+ const repoLink = document.getElementById('repository-link');
+ repoLink.href = data.repoUrl;
+ repoLink.textContent = data.repoUrl;
+
+ // Render footer
+ document.getElementById('dl-button').onclick = () => {
+ const dataUrl = 'data:,' + JSON.stringify(data, null, 2);
+ const a = document.createElement('a');
+ a.href = dataUrl;
+ a.download = 'benchmark_data.json';
+ a.click();
+ };
+
+ // Prepare data points for charts
+ return Object.keys(data.entries).map(name => ({
+ name,
+ dataSet: collectBenchesPerTestCase(data.entries[name]),
+ }));
+ }
+
+ function renderAllChars(dataSets) {
+
+ function renderGraph(parent, name, dataset) {
+ const canvas = document.createElement('canvas');
+ canvas.className = 'benchmark-chart';
+ parent.appendChild(canvas);
+
+ const color = toolColors[dataset.length > 0 ? dataset[0].tool : '_'];
+ const data = {
+ labels: dataset.map(d => d.commit.id.slice(0, 7)),
+ datasets: [
+ {
+ label: name,
+ data: dataset.map(d => d.bench.value),
+ borderColor: color,
+ backgroundColor: color + '60', // Add alpha for #rrggbbaa
+ }
+ ],
+ };
+ const options = {
+ scales: {
+ xAxes: [
+ {
+ scaleLabel: {
+ display: true,
+ labelString: 'commit',
+ },
+ }
+ ],
+ yAxes: [
+ {
+ scaleLabel: {
+ display: true,
+ labelString: dataset.length > 0 ? dataset[0].bench.unit : '',
+ },
+ ticks: {
+ beginAtZero: true,
+ }
+ }
+ ],
+ },
+ tooltips: {
+ callbacks: {
+ afterTitle: items => {
+ const {index} = items[0];
+ const data = dataset[index];
+ return '\n' + data.commit.message + '\n\n' + data.commit.timestamp + ' committed by @' + data.commit.committer.username + '\n';
+ },
+ label: item => {
+ let label = item.value;
+ const { range, unit } = dataset[item.index].bench;
+ label += ' ' + unit;
+ if (range) {
+ label += ' (' + range + ')';
+ }
+ return label;
+ },
+ afterLabel: item => {
+ const { extra } = dataset[item.index].bench;
+ return extra ? '\n' + extra : '';
+ }
+ }
+ },
+ onClick: (_mouseEvent, activeElems) => {
+ if (activeElems.length === 0) {
+ return;
+ }
+ // XXX: Undocumented. How can we know the index?
+ const index = activeElems[0]._index;
+ const url = dataset[index].commit.url;
+ window.open(url, '_blank');
+ },
+ };
+
+ new Chart(canvas, {
+ type: 'line',
+ data,
+ options,
+ });
+ }
+
+ function renderBenchSet(name, benchSet, main) {
+ const setElem = document.createElement('div');
+ setElem.className = 'benchmark-set';
+ main.appendChild(setElem);
+
+ const nameElem = document.createElement('h1');
+ nameElem.className = 'benchmark-title';
+ nameElem.textContent = name;
+ setElem.appendChild(nameElem);
+
+ const graphsElem = document.createElement('div');
+ graphsElem.className = 'benchmark-graphs';
+ setElem.appendChild(graphsElem);
+
+ for (const [benchName, benches] of benchSet.entries()) {
+ renderGraph(graphsElem, benchName, benches)
+ }
+ }
+
+ const main = document.getElementById('main');
+ for (const {name, dataSet} of dataSets) {
+ renderBenchSet(name, dataSet, main);
+ }
+ }
+
+ renderAllChars(init()); // Start
+ })();
+ </script>
+ </body>
+</html>
