[mpact][benchmark] set up regression benchmark for each commit with graphs (#58)
diff --git a/.github/workflows/regression-benchmark.yml b/.github/workflows/regression-benchmark.yml
new file mode 100644
index 0000000..dd59a99
--- /dev/null
+++ b/.github/workflows/regression-benchmark.yml
@@ -0,0 +1,83 @@
+name: Regression benchmark
+
+on:
+ push:
+ branches: [ "main" ]
+
+permissions:
+ contents: write
+ deployments: write
+ pull-requests: write
+ repository-projects: write
+
+jobs:
+ benchmark:
+ name: Performance regression check
+ runs-on: ubuntu-latest
+ env:
+ CACHE_DIR: ${{ github.workspace }}/.ccache
+ PYTHONPATH: ${{ github.workspace }}/build/tools/mpact/python_packages/mpact
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ submodules: recursive
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Setup Python Version
+ uses: actions/setup-python@v5
+ with:
+ python-version: 3.11 # Install the python version needed
+
+ - name: Set up ccache
+ uses: hendrikmuhs/ccache-action@v1.2
+
+ - name: Install requirements
+ run: |
+ export CCACHE_DIR=${{ env.CACHE_DIR }}
+ python -m pip install --upgrade pip
+ python -m pip install --upgrade pip
+ python -m pip install pytest pytest-benchmark
+ python -m pip install -r externals/torch-mlir/requirements.txt
+ python -m pip install -r externals/torch-mlir/torchvision-requirements.txt
+
+ - name: Create build directory
+ run: mkdir build
+
+ - name: Configure CMake
+ run: >
+ cmake -GNinja -Bbuild
+ -DCMAKE_BUILD_TYPE=Release
+ -DLLVM_ENABLE_PROJECTS=mlir
+ -DLLVM_ENABLE_ASSERTIONS=ON
+ -DLLVM_EXTERNAL_PROJECTS="torch-mlir;mpact"
+ -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="${PWD}/externals/torch-mlir"
+ -DLLVM_EXTERNAL_MPACT_SOURCE_DIR="${PWD}"
+ -DLLVM_TARGETS_TO_BUILD=host
+ -DMLIR_ENABLE_BINDINGS_PYTHON=ON
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+ -DCMAKE_C_COMPILER=clang
+ -DCMAKE_CXX_COMPILER=clang++
+ "externals/torch-mlir/externals/llvm-project/llvm"
+
+ - name: Build
+ run: cmake --build build --target build-benchmark-mpact
+
+ - name: Run benchmark
+ run: pytest benchmark/python/benchmarks/regression_benchmark.py --benchmark-json output.json
+
+ - name: Store benchmark result
+ uses: benchmark-action/github-action-benchmark@v1
+ with:
+ tool: 'pytest'
+ output-file-path: output.json
+ fail-on-alert: true
+ # GitHub API token to make a commit comment
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ # Enable alert commit comment
+ comment-on-alert: true
+ # Mention @yinying-lisa-li in the commit comment
+ alert-comment-cc-users: '@yinying-lisa-li'
+ # Push and deploy GitHub pages branch automatically
+ auto-push: true
+ alert-threshold: 120%
diff --git a/benchmark/python/benchmarks/regression_benchmark.py b/benchmark/python/benchmarks/regression_benchmark.py
new file mode 100644
index 0000000..6b3d0e4
--- /dev/null
+++ b/benchmark/python/benchmarks/regression_benchmark.py
@@ -0,0 +1,63 @@
+import pytest
+from mpact.models.kernels import *
+from mpact_benchmark.utils.tensor_generator import generate_tensor
+
+SHAPE = (1024, 1024)
+SPARSITY = 0.8
+
+dense_tensor1 = generate_tensor(0, SHAPE, SPARSITY)
+dense_tensor2 = generate_tensor(1, SHAPE, SPARSITY)
+dense_tensor3 = generate_tensor(2, SHAPE, SPARSITY)
+dense_vector = generate_tensor(1, (SHAPE[0],), SPARSITY)
+
+sparse_tensor1 = dense_tensor1.to_sparse_csr()
+sparse_tensor2 = dense_tensor2.to_sparse_csr()
+sparse_tensor3 = dense_tensor3.to_sparse_csr()
+
+
+def test_mv_dense(benchmark):
+ benchmark(MVNet(), dense_tensor1, dense_vector)
+
+
+def test_mm_dense(benchmark):
+ benchmark(MMNet(), dense_tensor1, dense_tensor2)
+
+
+def test_add_dense(benchmark):
+ benchmark(AddNet(), dense_tensor1, dense_tensor2)
+
+
+def test_mul_dense(benchmark):
+ benchmark(MulNet(), dense_tensor1, dense_tensor2)
+
+
+def test_nop_dense(benchmark):
+ benchmark(SelfNet(), dense_tensor1)
+
+
+def test_sddmm_dense(benchmark):
+ benchmark(SDDMMNet(), dense_tensor1, dense_tensor2, dense_tensor3)
+
+
+def test_mv_sparse(benchmark):
+ benchmark(MVNet(), sparse_tensor1, dense_vector)
+
+
+def test_mm_sparse(benchmark):
+ benchmark(MMNet(), sparse_tensor1, sparse_tensor2)
+
+
+def test_add_sparse(benchmark):
+ benchmark(AddNet(), sparse_tensor1, sparse_tensor2)
+
+
+def test_mul_sparse(benchmark):
+ benchmark(MulNet(), sparse_tensor1, sparse_tensor2)
+
+
+def test_nop_sparse(benchmark):
+ benchmark(SelfNet(), sparse_tensor1)
+
+
+def test_sddmm_sparse(benchmark):
+ benchmark(SDDMMNet(), sparse_tensor1, dense_tensor2, dense_tensor3)
