Merge pull request #432 from ruby/rmf-add-tests

Add tests to the `run_benchmark` script and refactoring table formatting

Author: rafaelfranca

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
 
       - name: Run tests
-        run: ruby test/benchmarks_test.rb
+        run: rake test
 
       - name: Test run_benchmarks.rb
         run: ./run_benchmarks.rb

Rakefile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'rake/testtask'
+
+desc 'Run all tests'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+  t.warning = true
+end
+
+task default: :test

lib/benchmark_runner.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'json'
+require 'rbconfig'
+
+# Extracted helper methods from run_benchmarks.rb for testing
+module BenchmarkRunner
+  module_function
+
+  # Find the first available file number for output files
+  def free_file_no(directory)
+    (1..).each do |file_no|
+      out_path = File.join(directory, "output_%03d.csv" % file_no)
+      return file_no unless File.exist?(out_path)
+    end
+  end
+
+  # Get benchmark categories from metadata
+  def benchmark_categories(name, metadata)
+    benchmark_metadata = metadata[name] || {}
+    categories = [benchmark_metadata.fetch('category', 'other')]
+    categories << 'ractor' if benchmark_metadata['ractor']
+    categories
+  end
+
+  # Check if the name matches any of the names in a list of filters
+  def match_filter(entry, categories:, name_filters:, metadata:)
+    name_filters = process_name_filters(name_filters)
+    name = entry.sub(/\.rb\z/, '')
+    (categories.empty? || benchmark_categories(name, metadata).any? { |cat| categories.include?(cat) }) &&
+      (name_filters.empty? || name_filters.any? { |filter| filter === name })
+  end
+
+  # Process "/my_benchmark/i" into /my_benchmark/i
+  def process_name_filters(name_filters)
+    name_filters.map do |name_filter|
+      if name_filter[0] == "/"
+        regexp_str = name_filter[1..-1].reverse.sub(/\A(\w*)\//, "")
+        regexp_opts = ::Regexp.last_match(1).to_s
+        regexp_str.reverse!
+        r = /#{regexp_str}/
+        if !regexp_opts.empty?
+          # Convert option string to Regexp option flags
+          flags = 0
+          flags |= Regexp::IGNORECASE if regexp_opts.include?('i')
+          flags |= Regexp::MULTILINE if regexp_opts.include?('m')
+          flags |= Regexp::EXTENDED if regexp_opts.include?('x')
+          r = Regexp.new(regexp_str, flags)
+        end
+        r
+      else
+        name_filter
+      end
+    end
+  end
+
+  # Resolve the pre_init file path into a form that can be required
+  def expand_pre_init(path)
+    require 'pathname'
+
+    path = Pathname.new(path)
+
+    unless path.exist?
+      puts "--with-pre-init called with non-existent file!"
+      exit(-1)
+    end
+
+    if path.directory?
+      puts "--with-pre-init called with a directory, please pass a .rb file"
+      exit(-1)
+    end
+
+    library_name = path.basename(path.extname)
+    load_path = path.parent.expand_path
+
+    [
+      "-I", load_path,
+      "-r", library_name
+    ]
+  end
+
+  # Sort benchmarks with headlines first, then others, then micro
+  def sort_benchmarks(bench_names, metadata)
+    headline_benchmarks = metadata.select { |_, meta| meta['category'] == 'headline' }.keys
+    micro_benchmarks = metadata.select { |_, meta| meta['category'] == 'micro' }.keys
+
+    headline_names, bench_names = bench_names.partition { |name| headline_benchmarks.include?(name) }
+    micro_names, other_names = bench_names.partition { |name| micro_benchmarks.include?(name) }
+    headline_names.sort + other_names.sort + micro_names.sort
+  end
+
+  # Check which OS we are running
+  def os
+    @os ||= (
+      host_os = RbConfig::CONFIG['host_os']
+      case host_os
+      when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+        :windows
+      when /darwin|mac os/
+        :macosx
+      when /linux/
+        :linux
+      when /solaris|bsd/
+        :unix
+      else
+        raise "unknown os: #{host_os.inspect}"
+      end
+    )
+  end
+
+  # Generate setarch prefix for Linux
+  def setarch_prefix
+    # Disable address space randomization (for determinism)
+    prefix = ["setarch", `uname -m`.strip, "-R"]
+
+    # Abort if we don't have permission (perhaps in a docker container).
+    return [] unless system(*prefix, "true", out: File::NULL, err: File::NULL)
+
+    prefix
+  end
+end

lib/table_formatter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# Formats benchmark data as an ASCII table with aligned columns
+class TableFormatter
+  COLUMN_SEPARATOR = '  '
+  FAILURE_PLACEHOLDER = 'N/A'
+
+  def initialize(table_data, format, failures)
+    @header = table_data.first
+    @data_rows = table_data.drop(1)
+    @format = format
+    @failures = failures
+    @num_columns = @header.size
+  end
+
+  def to_s
+    rows = build_all_rows
+    col_widths = calculate_column_widths(rows)
+
+    format_table(rows, col_widths)
+  end
+
+  private
+
+  attr_reader :num_columns
+
+  def build_all_rows
+    [@header, *build_failure_rows, *build_formatted_data_rows]
+  end
+
+  def build_failure_rows
+    return [] if @failures.empty?
+
+    failed_benchmarks = extract_failed_benchmarks
+    failed_benchmarks.map { |name| build_failure_row(name) }
+  end
+
+  def extract_failed_benchmarks
+    @failures.flat_map { |_exe, data| data.keys }.uniq
+  end
+
+  def build_failure_row(benchmark_name)
+    [benchmark_name, *Array.new(num_columns - 1, FAILURE_PLACEHOLDER)]
+  end
+
+  def build_formatted_data_rows
+    @data_rows.map { |row| apply_format(row) }
+  end
+
+  def apply_format(row)
+    @format.zip(row).map { |fmt, data| fmt % data }
+  end
+
+  def calculate_column_widths(rows)
+    (0...num_columns).map do |col_index|
+      rows.map { |row| row[col_index].length }.max
+    end
+  end
+
+  def format_table(rows, col_widths)
+    separator = build_separator(col_widths)
+
+    formatted_rows = rows.map { |row| format_row(row, col_widths) }
+
+    [separator, *formatted_rows, separator].join("\n") + "\n"
+  end
+
+  def build_separator(col_widths)
+    col_widths.map { |width| '-' * width }.join(COLUMN_SEPARATOR)
+  end
+
+  def format_row(row, col_widths)
+    row.map.with_index { |cell, i| cell.ljust(col_widths[i]) }
+       .join(COLUMN_SEPARATOR)
+       .rstrip
+  end
+end