From 59ba73fadd9ebdcb02776d8583e92e57f4eec8d9 Mon Sep 17 00:00:00 2001 From: "Hans J. Johnson" Date: Sat, 16 Mar 2019 12:54:41 -0500 Subject: [PATCH 1/3] ENH: Replace --no-merges with --first-parent for comment When walking through the git histories, we want the parent versions of the previous commits rather than the --no-merges. The --no-merges results in failures for the merges that include (vnl, hdf5, or other Thirdparty updates). --- evaluate-itk-performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluate-itk-performance.py b/evaluate-itk-performance.py index f6a1b6b..fa53fbf 100755 --- a/evaluate-itk-performance.py +++ b/evaluate-itk-performance.py @@ -40,7 +40,7 @@ def __call__(self, parser, namespace, values, option_string=None): run_parser.add_argument('benchmark_bin', help='ITK performance benchmarks build directory', action = FullPaths) run_parser.add_argument('-r', '--rev-list', - help='Arguments for "git rev-list" to select the range of commits to benchmark, for example: "--no-merges v4.10.0..v5.0rc1"', + help='Arguments for "git rev-list" to select the range of commits to benchmark, for example: "--first-parent v4.10.0..v5.0rc1"', default='--no-merges HEAD~1..') upload_parser = subparsers.add_parser('upload', From 05a270b7a068b5c28e46e35e7044e7b8c47c8b3f Mon Sep 17 00:00:00 2001 From: "Hans J. Johnson" Date: Sat, 16 Mar 2019 13:01:49 -0500 Subject: [PATCH 2/3] ENH: Add ability to clean small parts of ITK build tree Clean the build tree of items from vcl and hdf5 builds that can be left over from other build locations. These old file artifacts can cause build failures due to the old file location taking precidence over the new file location (vcl_compiler_header.h), or imcompatible library bulding due to a glob expression including a file that does not belong to the source tree any longer (hdf5). --- evaluate-itk-performance.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/evaluate-itk-performance.py b/evaluate-itk-performance.py index fa53fbf..402b06e 100755 --- a/evaluate-itk-performance.py +++ b/evaluate-itk-performance.py @@ -7,6 +7,8 @@ import socket import json +import glob + def get_shell_output_as_string(commandlist): """ @@ -309,6 +311,18 @@ def has_sha(filepath): os.environ['ITKPERFORMANCEBENCHMARK_AUX_JSON'] = \ json.dumps(itk_information) + ## Remove vcl_compiler.h in build dir that takes precidence over older non-generated + ## vcl_compiler.h in the source tree (older source code). + if not os.path.exists( os.path.join( args.src, 'Modules','ThirdParty','VNL','src','vxl','vcl','vcl_compiler.h.in') ): + generated_vcl_headers = glob.glob(os.path.join( args.bin, 'Modules','ThirdParty','VNL','src','vxl','vcl',"*.h")) + for vcl_header_file in generated_vcl_headers: + os.remove( vcl_header_file ) + + ## HDF generates and tries to build .c files from other builds, clean these out + hdf5_generated_files = os.path.join( args.bin, 'Modules','ThirdParty','HDF5','src','itkhdf5',"*.c") + hdf5_bld_src_files = glob.glob( hdf5_generated_files ) + for hdf5_file in hdf5_bld_src_files: + os.remove( hdf5_file ) print('\nBuilding ITK...') build_itk(args.src, args.bin) From b08f3fb54a10fef7ffe51f4ef66d4f0d0a2eef82 Mon Sep 17 00:00:00 2001 From: "Hans J. Johnson" Date: Sat, 16 Mar 2019 13:00:37 -0500 Subject: [PATCH 3/3] ENH: Add ability to specify if NumberOfThreads is required. Select the patch that renames NumberOfThreads to NumberOfWorkUnits. Use a cmake driven option to indicate which code spelling should be used. --- CMakeLists.txt | 8 ++++++ evaluate-itk-performance.py | 28 ++++++++++++++++++--- include/PerformanceBenchmarkingUtilities.h | 16 ++++++------ src/PerformanceBenchmarkingInformation.h.in | 2 ++ 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 826eb50..8414385 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,6 +45,14 @@ if(NOT ITK_SOURCE_DIR) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() +project(PerformanceBenchmarking) + +option(ITK_USES_NUMBEROFTHREADS "If ITKv5 uses NumberOfThreads (not NumberOfWorkUnits" OFF) + +set(PerformanceBenchmarking_LIBRARIES PerformanceBenchmarking) + +include_directories(${CMAKE_BINARY_DIR}) +if(NOT ITK_SOURCE_DIR) include(ITKModuleExternal) else() set(ITK_DIR ${CMAKE_BINARY_DIR}) diff --git a/evaluate-itk-performance.py b/evaluate-itk-performance.py index 402b06e..b380375 100755 --- a/evaluate-itk-performance.py +++ b/evaluate-itk-performance.py @@ -43,7 +43,7 @@ def __call__(self, parser, namespace, values, option_string=None): help='ITK performance benchmarks build directory', action = FullPaths) run_parser.add_argument('-r', '--rev-list', help='Arguments for "git rev-list" to select the range of commits to benchmark, for example: "--first-parent v4.10.0..v5.0rc1"', - default='--no-merges HEAD~1..') + default='--first-parent HEAD~1..') upload_parser = subparsers.add_parser('upload', help='upload the benchmarks to data.kitware.com') @@ -154,20 +154,41 @@ def check_for_build_information(itk_src): has_itkbuildinformation = True return has_itkbuildinformation +# 2075084be6f9988b1ae2231bca607830fe6d772b is sha1 that rename NumberOfThreads into NumberOfWorkUnits in filters +# Author: Dzenan Zukic +# Date: Tue Jul 17 19:30:02 2018 +# so all ancestors need to prevent the benchmarking from using +def check_for_NumberOfThreads(itk_src): + os.chdir(itk_src) + try: + cmd = ['git', 'merge-base', + '--is-ancestor', 'HEAD', + '2075084be6f9988b1ae2231bca607830fe6d772b'] + has_itkNumberOfThreads = not bool(subprocess.check_call( cmd ) ) + except subprocess.CalledProcessError: + has_itkNumberOfThreads = False + return has_itkNumberOfThreads + def build_benchmarks(benchmark_src, benchmark_bin, itk_bin, - itk_has_buildinformation): + itk_has_buildinformation, + itk_has_NumberOfThreads): os.chdir(benchmark_bin) if itk_has_buildinformation: build_information_arg = '-DITK_HAS_INFORMATION_H:BOOL=ON' else: build_information_arg = '-DITK_HAS_INFORMATION_H:BOOL=OFF' + if itk_has_NumberOfThreads: + NumberOfThreads_arg = '-DITK_USES_NUMBEROFTHREADS:BOOL=ON' + else: + NumberOfThreads_arg = '-DITK_USES_NUMBEROFTHREADS:BOOL=OFF' subprocess.check_call(['cmake', '-G', 'Ninja', '-DCMAKE_BUILD_TYPE:STRING=Release', '-DCMAKE_CXX_STANDARD:STRING=17', '-DITK_DIR:PATH=' + itk_bin, build_information_arg, + NumberOfThreads_arg, benchmark_src]) subprocess.check_call(['ninja']) @@ -328,10 +349,11 @@ def has_sha(filepath): build_itk(args.src, args.bin) itk_has_buildinformation = check_for_build_information(args.src) + itk_has_NumberOfThreads = check_for_NumberOfThreads(args.src) print('\nBuilding benchmarks...') build_benchmarks(benchmark_src, args.benchmark_bin, args.bin, - itk_has_buildinformation) + itk_has_buildinformation, itk_has_NumberOfThreads) print('\nRunning benchmarks...') run_benchmarks(args.benchmark_bin, itk_information) diff --git a/include/PerformanceBenchmarkingUtilities.h b/include/PerformanceBenchmarkingUtilities.h index 480a870..33d579f 100644 --- a/include/PerformanceBenchmarkingUtilities.h +++ b/include/PerformanceBenchmarkingUtilities.h @@ -5,20 +5,20 @@ #ifndef PerformanceBenchmarkingUtilities_h #define PerformanceBenchmarkingUtilities_h -#include "PerformanceBenchmarkingExport.h" +#include "PerformanceBenchmarkingInformation.h" #include "jsonxx.h" #include //TODO: Move to utiliites #include "itkHighPriorityRealTimeProbesCollector.h" -#if ITK_VERSION_MAJOR >= 5 -# include "itkMultiThreaderBase.h" -using MultiThreaderName = itk::MultiThreaderBase; -# define SET_PARALLEL_UNITS(x) SetNumberOfWorkUnits(x) -#else -# include "itkMultiThreader.h" +#if ITK_VERSION_MAJOR < 5 || defined(ITK_USES_NUMBEROFTHREADS) +#include "itkMultiThreader.h" using MultiThreaderName = itk::MultiThreader; -# define SET_PARALLEL_UNITS(x) SetNumberOfThreads(x) +#define SET_PARALLEL_UNITS( x ) SetNumberOfThreads( x ) +#else +#include "itkMultiThreaderBase.h" +using MultiThreaderName = itk::MultiThreaderBase; +#define SET_PARALLEL_UNITS( x ) SetNumberOfWorkUnits( x ) #endif PerformanceBenchmarking_EXPORT std::string diff --git a/src/PerformanceBenchmarkingInformation.h.in b/src/PerformanceBenchmarkingInformation.h.in index af6f167..5c9905e 100644 --- a/src/PerformanceBenchmarkingInformation.h.in +++ b/src/PerformanceBenchmarkingInformation.h.in @@ -30,6 +30,8 @@ #include "PerformanceBenchmarkingExport.h" +#cmakedefine ITK_USES_NUMBEROFTHREADS + namespace itk {