Extend runner and compare tool with extra precision metrics, platform-specific comparisons, and improved timeout handling (#305)

* Removed very large benchmarks from SPEC.

* Added benchmark, sha, and version metadata to the profile

* Removed extra imports

* Extending the compare and runner tools

* wip

* Revamp description and dataset uniqueness

* Added memtier_benchmark-playbook-session-storage

* Updated the memtier_benchmark-playbook-session-storage benchmark to match https://github.com/upstash/examples/tree/main/examples/ratelimit-with-redis

* Added caching and session-storage benchmarks

* Fixed SPEC validation.

* Fixed formatting

* fixed utils/tests/test_runner.py

* Fixed scc memtier benchmarks

* Added version label on by hash tags if version is present

* Fixed the timeseries naming

* Fixed ts prefix

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.274"
+version = "0.1.309"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

redis_benchmarks_specification/__cli__/stats.py

@@ -223,6 +223,9 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                         command = command.replace("'", "")
                         if "-key-pattern" in command:
                             continue
+                        # Skip command-ratio and other memtier arguments that start with -
+                        if command.startswith("-"):
+                            continue
                         command = command.lower()
                         if command not in tested_commands:
                             tested_commands.append(command)
@@ -238,14 +241,16 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                         if command not in tracked_commands_json:
                             tracked_commands_json[command] = command_json
 
-                        group = command_json["group"]
-                        if group not in tested_groups:
+                        # Only process if command_json has group information
+                        if "group" in command_json:
+                            group = command_json["group"]
+                            if group not in tested_groups:
 
-                            tested_groups.append(group)
-                        if group not in tracked_groups:
-                            tracked_groups.append(group)
-                            tracked_groups_hist[group] = 0
-                        tracked_groups_hist[group] = tracked_groups_hist[group] + 1
+                                tested_groups.append(group)
+                            if group not in tracked_groups:
+                                tracked_groups.append(group)
+                                tracked_groups_hist[group] = 0
+                            tracked_groups_hist[group] = tracked_groups_hist[group] + 1
 
                 # Calculate total connections
                 total_connections = clients * threads
@@ -262,12 +267,14 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                     data_sizes[data_size] = 0
                 data_sizes[data_size] = data_sizes[data_size] + 1
 
-                if tested_commands != origin_tested_commands:
+                if sorted(tested_commands) != sorted(origin_tested_commands):
                     requires_override = True
                     benchmark_config["tested-commands"] = tested_commands
                     logging.warn(
                         "there is a difference between specified test-commands in the yaml (name={}) and the ones we've detected {}!={}".format(
-                            test_name, origin_tested_commands, tested_commands
+                            test_name,
+                            sorted(origin_tested_commands),
+                            sorted(tested_commands),
                         )
                     )
 
@@ -323,12 +330,14 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                         )
                     )
 
-                if tested_groups != origin_tested_groups:
+                if sorted(tested_groups) != sorted(origin_tested_groups):
                     tested_groups_match_origin = False
                     benchmark_config["tested-groups"] = tested_groups
                     logging.warn(
                         "there is a difference between specified test-groups in the yaml (name={}) and the ones we've detected {}!={}".format(
-                            test_name, origin_tested_groups, tested_groups
+                            test_name,
+                            sorted(origin_tested_groups),
+                            sorted(tested_groups),
                         )
                     )