Added benchmark code

.flake8

@@ -8,3 +8,4 @@ max-line-length = 119
 exclude =
     .venv/**
     build/**
+    benchmarks/**

benchmarks/src/README.md

@@ -0,0 +1,36 @@
+Benchmarks
+===========
+
+## Setting up environment
+
+```sh
+    python -m pip install -r requirements.txt
+```
+
+## Benchmark parameters
+
+The benchmark packages, rounds, array sizes, and numeric type may be specified on the constants at the top of [pytest_benchmark/common.py](pytest_benchmark/common.py).
+
+Alternatively, they may be specified individually at the top of each test file.
+
+
+## Running
+
+These are the steps to run the benchmarks, and produce the graphs 
+
+Run the benchmarks and store the results in `results.json`
+```sh
+    pytest .\pytest_benchmark --benchmark-json=results.json
+```
+
+To create graphs and store the timing results after creating the `results.json`, run:
+```sh
+    python graphs.py
+```
+
+To modify the tests being shown, modify the `TESTS` list at the top of the `graphs.py` file.
+To modify the labels shown, modify `PKG_LABELS`
+To modify the hardware display, modify `HARDWARE` 
+
+## Notes
+When running with `dpnp`, set the environment variable `DPNP_RAISE_EXCEPION_ON_NUMPY_FALLBACK` to 0.

benchmarks/src/graphs.py

@@ -0,0 +1,223 @@
+import json
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+BENCHMARKS_JSON = "results.json"
+
+# Hardware details shown in title
+HARDWARE = "AMD Ryzen 9 9900X 12-Core Processor 63032 MB (fp64 fp16)\noneAPI 2025.1.3 Intel(R) OpenCL Graphics: Intel(R) Arc(TM) B580 Graphics, 11873 MB (fp64 fp16)"
+
+# Show speedup in graph
+SHOW_NUMBERS = True
+
+# Round to digits after decimal
+ROUND_NUMBERS = 1
+
+# package list in graph order; arrayfire packages are added later
+PKG_NAMES = ["numpy", "dpnp", "cupy"]
+
+# color used in graphs
+PKG_COLOR = {
+    "numpy": "tab:blue",
+    "cupy": "tab:green",
+    "dpnp": "tab:red",
+    "afcpu": "tab:orange",
+    "afopencl": "tab:orange",
+    "afcuda": "tab:orange",
+    "afoneapi": "tab:orange",
+}
+
+# labels displayed in the graph
+PKG_LABELS = {
+    "numpy": "numpy[cpu]",
+    "dpnp": "dpnp[level_zero:gpu]",
+    "cupy": "cupy",
+    "afcpu": "afcpu",
+    "afcuda": "afcuda",
+    "afopencl": "afopencl[opencl:gpu]",
+    "afoneapi": "afoneapi[opencl:gpu]",
+}
+
+AFBACKENDS = ["afcpu", "afcuda", "afopencl", "afoneapi"]
+
+# Tests to be shown in graphs
+TESTS = [
+    "qr",
+    "neural_network",
+    "gemm",
+    "mandelbrot",
+    "nbody",
+    "pi",
+    "black_scholes",
+    "fft",
+    "normal",
+    "group_elementwise",
+    # Other tests
+    # 'svd
+    # 'cholesky',
+    # 'det',
+    # 'norm',
+    # 'uniform',
+    # 'inv'
+]
+
+
+def get_benchmark_data():
+    results = {}
+    descriptions = {}
+    with open(BENCHMARKS_JSON) as f:
+        js = json.load(f)
+        for bench in js["benchmarks"]:
+            test_name = bench["name"]
+            test_name = test_name[test_name.find("_") + 1 : test_name.find("[")]
+
+            key = bench["param"]
+            val = bench["stats"]["ops"]
+
+            if len(bench["extra_info"]) != 0 and (not test_name in descriptions):
+                descriptions[test_name] = bench["extra_info"]["description"]
+
+            if test_name not in results:
+                results[test_name] = {key: val}
+            else:
+                results[test_name][key] = val
+
+    return results, descriptions
+
+
+def create_graph(test_name, test_results):
+    names = []
+    values = []
+    for name in test_results:
+        names.append(name)
+        values.append(test_results[name])
+
+    bar = plt.bar(names, values)
+    plt.title(test_name)
+
+    plt.savefig("img/" + test_name + ".png")
+    plt.close()
+
+
+def generate_individual_graphs():
+    results, descriptions = get_benchmark_data()
+
+    for test in results:
+        create_graph(test, results[test])
+
+
+# Stores the timing results in a csv file
+def store_csv():
+    data_dict = {}
+    data_dict["Test(seconds)"] = []
+    results = {}
+    for pkg in PKG_LABELS.keys():
+        data_dict[pkg] = []
+        results[pkg] = {}
+
+    with open(BENCHMARKS_JSON) as f:
+        js = json.load(f)
+        for bench in js["benchmarks"]:
+            test_name = bench["name"]
+            test_name = test_name[test_name.find("_") + 1 : test_name.find("[")]
+
+            pkg = bench["param"]
+            time = bench["stats"]["mean"]
+
+            if not test_name in data_dict["Test(seconds)"]:
+                data_dict["Test(seconds)"].append(test_name)
+
+            results[pkg][test_name] = time
+
+    for test in data_dict["Test(seconds)"]:
+        for pkg in PKG_LABELS.keys():
+            if test in results[pkg]:
+                data_dict[pkg].append(results[pkg][test])
+            else:
+                data_dict[pkg].append(np.nan)
+
+    df = pd.DataFrame(data_dict)
+    df.to_csv("summary.csv")
+
+
+def generate_group_graph(test_list=None, show_numbers=False, filename="comparison"):
+    results, descriptions = get_benchmark_data()
+
+    width = 1 / (1 + len(PKG_NAMES))
+    multiplier = 0
+
+    tests = None
+    if test_list:
+        tests = test_list
+    else:
+        tests = results.keys()
+
+    tests_values = {}
+    x = np.arange(len(tests))
+
+    for name in PKG_NAMES:
+        tests_values[name] = []
+
+    max_val = 1
+    for test in tests:
+        for name in PKG_NAMES:
+            base_value = results[test]["numpy"]
+            if name in results[test]:
+                val = results[test][name] / base_value
+
+                if ROUND_NUMBERS:
+                    val = round(val, ROUND_NUMBERS)
+
+                if max_val < val:
+                    max_val = val
+
+                tests_values[name].append(val)
+            else:
+                tests_values[name].append(np.nan)
+
+    fig, ax = plt.subplots(layout="constrained")
+
+    for name in PKG_NAMES:
+        offset = width * multiplier
+        rects = ax.barh(x + offset, tests_values[name], width, label=PKG_LABELS[name], color=PKG_COLOR[name])
+
+        if show_numbers:
+            ax.bar_label(rects, padding=3, rotation=0)
+        multiplier += 1
+
+    xlabels = []
+    for test in tests:
+        xlabels.append(test + "\n" + descriptions[test])
+
+    ax.set_xlabel("Speedup")
+    ax.set_xscale("log")
+    ax.set_title(f"Runtime Comparison\n{HARDWARE}")
+    ax.set_yticks(x + width, xlabels, rotation=0)
+    xmin, xmax = ax.get_xlim()
+    ax.set_xlim(xmin, xmax * 2)
+
+    ax.legend(loc="lower right", ncols=len(PKG_NAMES))
+    fig.set_figheight(8)
+    fig.set_figwidth(13)
+    fig.savefig(f"img/{filename}.png")
+    plt.show()
+
+
+def main():
+    store_csv()
+    for backend in AFBACKENDS:
+        try:
+            filename = f"comparison_{backend}"
+            if not backend in PKG_NAMES:
+                PKG_NAMES.insert(1, backend)
+            generate_group_graph(TESTS, SHOW_NUMBERS, filename)
+            PKG_NAMES.remove(backend)
+        except Exception as e:
+            print(e)
+            print("No data for", backend)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/src/pytest_benchmark/common.py

@@ -0,0 +1,105 @@
+# cython: language_level=3
+# -*- coding: utf-8 -*-
+# *****************************************************************************
+# Copyright (c) 2016-2024, Intel Corporation
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# - Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# - Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+# THE POSSIBILITY OF SUCH DAMAGE.
+# *****************************************************************************
+
+import gc
+import math
+
+import cupy
+import dpctl
+import dpnp
+import numpy as np
+import pytest
+
+import arrayfire as af
+
+# modify parameters for most benchmarks
+ROUNDS = 30
+NSIZE = 2**13
+NNSIZE = NSIZE**2
+DTYPE = "float32"
+
+# comment a line to remove that package from testing
+PKGDICT = {
+    "dpnp": dpnp,
+    "numpy": np,
+    "cupy": cupy,
+    # "afcpu": af,
+    "afopencl": af,
+    "afcuda": af,
+    "afoneapi": af,
+}
+
+PKGS = []
+IDS = []
+
+for key, value in PKGDICT.items():
+    IDS.append(key)
+    PKGS.append(value)
+
+
+# Initialize packages and cleanup memory before each round
+def initialize_package(PKG_ID):
+    pkg = PKGDICT[PKG_ID]
+
+    try:
+        af.device_gc()
+        mempool = cupy.get_default_memory_pool()
+        mempool.free_all_blocks()
+    except:
+        pass
+
+    if PKG_ID == "afcpu":
+        af.set_backend(af.BackendType.cpu)
+        af.device_gc()
+        af.info()
+    elif PKG_ID == "afopencl":
+        af.set_backend(af.BackendType.opencl)
+        af.device_gc()
+        af.info()
+    elif PKG_ID == "afcuda":
+        af.set_backend(af.BackendType.cuda)
+        af.device_gc()
+        af.info()
+    elif PKG_ID == "afoneapi":
+        af.set_backend(af.BackendType.oneapi)
+        af.device_gc()
+        af.info()
+    elif PKG_ID == "numpy":
+        np.random.seed(0)
+    elif PKG_ID == "dpnp":
+        dpnp.random.seed(0)
+        print(dpctl.get_devices()[0])
+    elif PKG_ID == "cupy":
+        cupy.random.seed(0)
+        print(cupy.cuda.Device())
+        mempool = cupy.get_default_memory_pool()
+        mempool.free_all_blocks()
+    else:
+        raise NotImplementedError()
+
+    # Free all unused memory
+    gc.collect()