WIP: ENH: Add ITK-Wasm benchmark interfaces

Author: thewtex

.gitignore

@@ -42,3 +42,11 @@ cmake-build*/**
 doc/*.pyg
 doc/_minted-ITKPerformanceBenchmarking/*
 
+
+emscripten-build/
+micromamba/
+node_modules/
+test/pyodide-emscripten.tar.bz2
+wasi-build/
+wasm/typescript/
+wasm/python/

CMakeLists.txt

@@ -3,38 +3,42 @@ cmake_minimum_required(VERSION 3.16.3)
 if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD 17) # Supported values are ``14``, ``17`` and ``20``.
 endif()
+
 if(NOT CMAKE_CXX_STANDARD_REQUIRED)
   set(CMAKE_CXX_STANDARD_REQUIRED ON)
 endif()
+
 if(NOT CMAKE_CXX_EXTENSIONS)
   set(CMAKE_CXX_EXTENSIONS OFF)
 endif()
 
-## If building external to ITK, we need to capture
-## required build setting from ITK prior to calling
-## project()
+# # If building external to ITK, we need to capture
+# # required build setting from ITK prior to calling
+# # project()
 if(NOT ITK_SOURCE_DIR)
   find_package(ITK REQUIRED)
   list(APPEND CMAKE_MODULE_PATH ${ITK_CMAKE_DIR})
 
   foreach(p
-      ## Only policies introduced after the cmake_minimum_required
-      ## version need to explicitly be set to NEW.
-      # currently empty
-      )
+
+    # # Only policies introduced after the cmake_minimum_required
+    # # version need to explicitly be set to NEW.
+    # currently empty
+  )
     if(POLICY ${p})
       cmake_policy(SET ${p} NEW)
     endif()
   endforeach()
 
-
   # Set a default build type if none was specified
   if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
     message(STATUS "Setting build type to 'Release' as none was specified.")
     set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
+
     # Set the possible values of build type for cmake-gui
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
   endif()
+
   enable_language(C)
   enable_language(CXX)
 endif()
@@ -44,10 +48,16 @@ project(PerformanceBenchmarking)
 set(PerformanceBenchmarking_LIBRARIES PerformanceBenchmarking)
 
 include_directories(${CMAKE_BINARY_DIR})
+
 if(NOT ITK_SOURCE_DIR)
   include(ITKModuleExternal)
+
+  if(WASI OR EMSCRIPTEN)
+    set(BENCHMARK_USE_ITKWasm CACHE BOOL "Build with ITK-Wasm interface." ON)
+  endif()
 else()
   set(ITK_DIR ${CMAKE_BINARY_DIR})
   itk_module_impl()
 endif()
+
 add_subdirectory(examples)

environment.yml

@@ -0,0 +1,12 @@
+name: performance-benchmarking
+channels:
+  - conda-forge
+dependencies:
+  - pytest
+  - pytest-benchmark
+  - python=3.11
+  - pip
+  - pip:
+    - hatch
+    - itkwasm-mesh-io
+    - itkwasm-image-io

examples/CMakeLists.txt

@@ -1,13 +1,21 @@
-cmake_minimum_required(VERSION 3.10.2)
+cmake_minimum_required(VERSION 3.16.2)
 project(ITKBenchmarks NONE)
 
+option(BENCHMARK_USE_ITKWasm "Enable ITK-Wasm performance benchmarking interfaces" OFF)
+set(_itkwasm_deps)
+
+if(BENCHMARK_USE_ITKWasm)
+  list(APPEND _itkwasm_deps WebAssemblyInterface)
+  add_definitions(-DPERFORMANCE_BENCHMARKING_USE_ITKWASM)
+endif()
+
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/CMake ${CMAKE_MODULE_PATH})
 include(ITKBenchmarksExternalData)
 
 ExternalData_Expand_Arguments(ITKBenchmarksData
   BRAIN_IMAGE
   "DATA{Data/Input/brainweb165a10f17.mha}"
-  )
+)
 cmake_host_system_information(RESULT HOSTNAME QUERY HOSTNAME)
 string(TOLOWER "${HOSTNAME}" HOSTNAME_LOWER)
 set(TEST_OUTPUT_DIR "${PROJECT_BINARY_DIR}/Testing")
@@ -16,24 +24,29 @@ file(MAKE_DIRECTORY ${BENCHMARK_RESULTS_OUTPUT_DIR})
 
 include(CTest)
 
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
 option(BENCHMARK_ITK_CORE "Test the performance of ITK Core." ON)
+
 if(BENCHMARK_ITK_CORE)
   add_subdirectory(Core)
 endif()
 
-option(BENCHMARK_ITK_FILTERING "Test the performance of ITK Filters." ON)
-if(BENCHMARK_ITK_FILTERING)
-  add_subdirectory(Filtering)
-endif()
+# option(BENCHMARK_ITK_FILTERING "Test the performance of ITK Filters." ON)
 
-option(BENCHMARK_ITK_REGISTRATION "Test the performance of ITK Registration." ON)
-if(BENCHMARK_ITK_REGISTRATION)
-  add_subdirectory(Registration)
-endif()
+# if(BENCHMARK_ITK_FILTERING)
+# add_subdirectory(Filtering)
+# endif()
 
-option(BENCHMARK_ITK_SEGMENTATION "Test the performance of ITK Segmentation." ON)
-if(BENCHMARK_ITK_SEGMENTATION)
-  add_subdirectory(Segmentation)
-endif()
+# option(BENCHMARK_ITK_REGISTRATION "Test the performance of ITK Registration." ON)
+
+# if(BENCHMARK_ITK_REGISTRATION)
+# add_subdirectory(Registration)
+# endif()
+
+# option(BENCHMARK_ITK_SEGMENTATION "Test the performance of ITK Segmentation." ON)
 
+# if(BENCHMARK_ITK_SEGMENTATION)
+# add_subdirectory(Segmentation)
+# endif()
 ExternalData_Add_Target(ITKBenchmarksData)

examples/Core/CMakeLists.txt

@@ -2,19 +2,20 @@ project(ITKBenchmarkCore)
 
 find_package(ITK REQUIRED
   COMPONENTS
-    PerformanceBenchmarking
-    ITKCommon
-  )
+  PerformanceBenchmarking
+  ITKCommon
+  ${_itkwasm_deps}
+)
 include(${ITK_USE_FILE})
 
-
-add_executable(ThreadOverheadBenchmark ThreadOverhead.cxx )
-target_link_libraries(ThreadOverheadBenchmark ${ITK_LIBRARIES})
+add_executable(thread-overhead thread-overhead.cxx)
+target_link_libraries(thread-overhead PUBLIC ${ITK_LIBRARIES})
 add_test(
-  NAME ThreadOverheadBenchmark
-  COMMAND ThreadOverheadBenchmark
-  ${BENCHMARK_RESULTS_OUTPUT_DIR}/__DATESTAMP__ThreadOverheadBenchmark.json
-  1000 )
-set_property(TEST ThreadOverheadBenchmark APPEND PROPERTY LABELS Core)
-## performance tests should not be run in parallel
-set_tests_properties(ThreadOverheadBenchmark PROPERTIES RUN_SERIAL TRUE)
+  NAME thread-overhead-benchmark
+  COMMAND thread-overhead
+  ${BENCHMARK_RESULTS_OUTPUT_DIR}/__DATESTAMP__thread-overhead.json
+  --iterations 1000)
+set_property(TEST thread-overhead-benchmark APPEND PROPERTY LABELS Core)
+
+# # performance tests should not be run in parallel
+set_tests_properties(thread-overhead-benchmark PROPERTIES RUN_SERIAL TRUE)

examples/Core/ThreadOverhead.cxx renamed to examples/Core/thread-overhead.cxx

@@ -22,6 +22,11 @@
 #include "itkHighPriorityRealTimeProbe.h"
 #include "itkHighPriorityRealTimeProbesCollector.h"
 
+#if defined(PERFORMANCE_BENCHMARKING_USE_ITKWASM)
+#  include "itkPipeline.h"
+#  include "itkOutputTextStream.h"
+#endif
+
 #include <sstream>
 #include <fstream>
 #include "PerformanceBenchmarkingUtilities.h"
@@ -122,16 +127,46 @@ time_it(unsigned int threads, unsigned int iterations)
 int
 main(int argc, char * argv[])
 {
-  if (argc > 3)
+  bool          noPrintStdout = false;
+  bool          expandedReport = false;
+  bool          noPrintSystemInfo = false;
+  bool          noPrintReportHead = false;
+  constexpr int iterationsDefault = 500;
+  const int     threadsDefault = MultiThreaderName::GetGlobalDefaultNumberOfThreads();
+
+#if defined(PERFORMANCE_BENCHMARKING_USE_ITKWASM)
+  itk::wasm::Pipeline pipeline("thread-overhead", "Estimate the overhead cost per-thread.", argc, argv);
+
+  itk::wasm::OutputTextStream timingsJson;
+  pipeline.add_option("timings", timingsJson, "Internal timings")->type_name("OUTPUT_JSON");
+
+  pipeline.add_flag("--no-print-stdout", noPrintStdout, "Do not print to stdout.");
+  pipeline.add_flag("--expanded-report", expandedReport, "Print an expanded report.");
+  pipeline.add_flag("--no-print-system-info", noPrintSystemInfo, "Do not print system information.");
+  pipeline.add_flag("--no-print-report-head", noPrintReportHead, "Do not print the report header.");
+
+  int iterations = iterationsDefault;
+  pipeline.add_option("--iterations", iterations, "Number of iterations to run.");
+
+  int threads = threadsDefault;
+  pipeline.add_option("--threads", threads, "Number of threads to use.");
+
+  ITK_WASM_PARSE(pipeline);
+
+  std::ostream & timingsStream = timingsJson.Get();
+#else
+  if (argc > 5)
   {
     std::cerr << "Usage: " << std::endl;
-    std::cerr << argv[0] << " timingsFile [iterations [threads]]" << std::endl;
+    std::cerr << argv[0] << " timingsFile [--iterations iterations [--threads threads]]" << std::endl;
     return EXIT_FAILURE;
   }
 
   const std::string timingsFileName = ReplaceOccurrence(argv[1], "__DATESTAMP__", PerfDateStamp());
-  const int         iterations = (argc > 2) ? std::stoi(argv[2]) : 500;
-  const int         threads = (argc > 3) ? std::stoi(argv[3]) : MultiThreaderName::GetGlobalDefaultNumberOfThreads();
+  auto              timingsStream = std::ofstream(timingsFileName, std::ios_base::out);
+  const int         iterations = (argc > 3) ? std::stoi(argv[3]) : iterationsDefault;
+  const int         threads = (argc > 5) ? std::stoi(argv[5]) : threadsDefault;
+#endif
 
   if (threads == 1)
   {
@@ -142,7 +177,8 @@ main(int argc, char * argv[])
   ProbeType t1 = time_it(1, iterations);
   ProbeType t2 = time_it(threads, iterations);
 
-  WriteExpandedReport(timingsFileName, collector, true, true, false);
+  WriteExpandedReport(
+    timingsStream, collector, !noPrintStdout, expandedReport, !noPrintSystemInfo, !noPrintReportHead, false);
 
   double cost = (t2.GetMinimum() - t1.GetMinimum()) / (threads - 1.0);
 

include/PerformanceBenchmarkingUtilities.h

@@ -25,14 +25,16 @@ PerformanceBenchmarking_EXPORT std::string
                                PerfDateStamp();
 
 PerformanceBenchmarking_EXPORT std::string
-                               ReplaceOccurrence(std::string str, const std::string && findvalue, const std::string && replacevalue);
+ReplaceOccurrence(std::string str, const std::string && findvalue, const std::string && replacevalue);
 
 PerformanceBenchmarking_EXPORT std::string
                                DecorateWithBuildInformation(std::string inputJson);
 
 PerformanceBenchmarking_EXPORT void
-WriteExpandedReport(const std::string &                        timingsFileName,
+WriteExpandedReport(std::ostream &                             timingsStream,
                     itk::HighPriorityRealTimeProbesCollector & collector,
+                    bool                                       printStdout,
+                    bool                                       expandedReport,
                     bool                                       printSystemInfo,
                     bool                                       printReportHead,
                     bool                                       useTabs);

itk-module.cmake

@@ -16,9 +16,9 @@ For more information, see::
 itk_module(PerformanceBenchmarking
   ENABLE_SHARED
   PRIVATE_DEPENDS
-    ITKCommon
+  ITKCommon
   TEST_DEPENDS
-    ITKTestKernel
+  ITKTestKernel
   EXCLUDE_FROM_DEFAULT
   DESCRIPTION
   "${DOCUMENTATION}"

package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@itk-wasm/performance-benchmarking-build",
+  "version": "1.0.0",
+  "description": "npm scripts to generate itk-wasm artifacts.",
+  "private": true,
+  "packageManager": "pnpm@9.1.1",
+  "type": "module",
+  "itk-wasm": {
+    "emscripten-docker-image": "itkwasm/emscripten:20240517-32a1ffd9",
+    "wasi-docker-image": "itkwasm/wasi:20240517-32a1ffd9",
+    "typescript-package-name": "@itk-wasm/performance-benchmarking",
+    "typescript-output-dir": "wasm/typescript",
+    "python-package-name": "itkwasm-performance-benchmarking",
+    "python-output-dir": "wasm/python",
+    "package-description": "Benchmark ITK performance.",
+    "repository": "https://github.com/InsightSoftwareConsortium/ITKPerformanceBenchmarking"
+  },
+  "scripts": {
+    "build": "pnpm build:gen:typescript && pnpm -r build && pnpm build:gen:python",
+    "build:emscripten": "itk-wasm pnpm-script build:emscripten",
+    "build:emscripten:debug": "itk-wasm pnpm-script build:emscripten:debug",
+    "build:wasi": "itk-wasm pnpm-script build:wasi",
+    "build:wasi:debug": "itk-wasm pnpm-script build:wasi:debug",
+    "build:python:wasi": "itk-wasm pnpm-script build:python:wasi",
+    "bindgen:typescript": "itk-wasm pnpm-script bindgen:typescript",
+    "bindgen:python": "itk-wasm pnpm-script bindgen:python",
+    "build:gen:typescript": "itk-wasm pnpm-script build:gen:typescript",
+    "build:gen:python": "itk-wasm pnpm-script build:gen:python",
+    "build:micromamba": "itk-wasm pnpm-script build:micromamba",
+    "build:python:versionSync": "itk-wasm pnpm-script build:python:versionSync",
+    "publish:python": "itk-wasm pnpm-script publish:python",
+    "test": "pnpm test:data:download && pnpm -r test && pnpm test:python",
+    "test:data:download": "echo \"not needed\"",
+    "test:data:pack": "dam pack test/data test/data.tar.gz",
+    "test:python:wasi": "itk-wasm pnpm-script test:python:wasi",
+    "test:python:emscripten": "itk-wasm pnpm-script test:python:emscripten",
+    "test:python:dispatch": "itk-wasm pnpm-script test:python:dispatch",
+    "test:python": "itk-wasm pnpm-script test:python",
+    "test:wasi": "itk-wasm pnpm-script test:wasi -- -V",
+    "clean": "git clean -fdx -e node_modules"
+  },
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@itk-wasm/dam": "^1.1.1",
+    "@thewtex/setup-micromamba": "^1.9.7",
+    "itk-wasm": "^1.0.0-b.175"
+  }
+}