diff --git a/lib/benchpark/cmd/audit.py b/lib/benchpark/cmd/audit.py index 1115eb0a6..79264a2f8 100644 --- a/lib/benchpark/cmd/audit.py +++ b/lib/benchpark/cmd/audit.py @@ -8,14 +8,8 @@ import re import sys -import benchpark.paths import benchpark.repo -from benchpark.runtime import RuntimeResources - -bootstrapper = RuntimeResources(benchpark.paths.benchpark_home) # noqa -bootstrapper.bootstrap() # noqa - import ramble.config as cfg # noqa diff --git a/lib/benchpark/cmd/setup.py b/lib/benchpark/cmd/setup.py index 75b5e3400..2949d454f 100644 --- a/lib/benchpark/cmd/setup.py +++ b/lib/benchpark/cmd/setup.py @@ -15,7 +15,6 @@ import benchpark.paths from benchpark.debug import debug_print from benchpark.runtime import RuntimeResources -import benchpark.system # Note: it would be nice to vendor spack.llnl.util.link_tree, but that @@ -41,22 +40,6 @@ def symlink_tree(src, dst, include_fn=None): os.symlink(src_file, dst_symlink) -def setup_parser(root_parser): - root_parser.add_argument( - "experiment", - type=str, - help="The experiment (benchmark/ProgrammingModel) to run", - ) - root_parser.add_argument( - "system", type=str, help="The system on which to run the experiment" - ) - root_parser.add_argument( - "experiments_root", - type=str, - help="Where to install packages and store results for the experiments. Benchpark expects to manage this directory, and it should be empty/nonexistent the first time you run benchpark setup experiments.", - ) - - def command(args): """ experiments_root/ @@ -142,7 +125,9 @@ def include_fn(fname): initializer_script = experiments_root / "setup.sh" run_script = experiments_root / ".latest-experiment.sh" - per_workspace_setup = RuntimeResources(experiments_root) + per_workspace_setup = RuntimeResources( + experiments_root, upstream=RuntimeResources(benchpark.paths.benchpark_home) + ) # Parse experiment YAML for package_manager def find(d, tag): diff --git a/lib/benchpark/experiment.py b/lib/benchpark/experiment.py index 7a77f6253..673b1afaf 100644 --- a/lib/benchpark/experiment.py +++ b/lib/benchpark/experiment.py @@ -11,14 +11,8 @@ from benchpark.directives import ExperimentSystemBase from benchpark.directives import variant import benchpark.spec -import benchpark.paths -import benchpark.repo -import benchpark.runtime import benchpark.variant -bootstrapper = benchpark.runtime.RuntimeResources(benchpark.paths.benchpark_home) -bootstrapper.bootstrap() - import ramble.language.language_base # noqa import ramble.language.language_helpers # noqa diff --git a/lib/benchpark/paths.py b/lib/benchpark/paths.py index e58474ce6..6abe17fc2 100644 --- a/lib/benchpark/paths.py +++ b/lib/benchpark/paths.py @@ -20,3 +20,5 @@ def _source_location() -> pathlib.Path: global_ramble_path = benchpark_home / "ramble" global_spack_path = benchpark_home / "spack" hardware_descriptions = benchpark_root / "systems" / "all_hardware_descriptions" +checkout_versions = benchpark_root / "checkout-versions.yaml" +remote_urls = benchpark_root / "remote-urls.yaml" diff --git a/lib/benchpark/repo.py b/lib/benchpark/repo.py index 7b9243b98..ffff86b1f 100644 --- a/lib/benchpark/repo.py +++ b/lib/benchpark/repo.py @@ -10,15 +10,9 @@ from enum import Enum import benchpark.paths -import benchpark.runtime # isort: off -bootstrapper = benchpark.runtime.RuntimeResources( - benchpark.paths.benchpark_home -) # noqa -bootstrapper.bootstrap() # noqa - import llnl.util.lang # noqa import ramble.language.language_base # noqa import ramble.repository # noqa diff --git a/lib/benchpark/runtime.py b/lib/benchpark/runtime.py index 133b00f88..285da3f6f 100644 --- a/lib/benchpark/runtime.py +++ b/lib/benchpark/runtime.py @@ -46,7 +46,7 @@ def run_command(command_str, env=None, stdout=None, stderr=None): out, err = proc.communicate() if proc.returncode != 0: raise RuntimeError( - f"Failed command: {command_str}\nOutput: {stdout}\nError: {stderr}" + f"Failed command: {command_str}\nOutput: {out}\nError: {err}" ) return (out, err) @@ -64,22 +64,38 @@ def __call__(self, *args): class RuntimeResources: - def __init__(self, dest): - self.root = benchpark.paths.benchpark_root + def __init__(self, dest, upstream=None): self.dest = pathlib.Path(dest) + self.upstream = upstream - checkout_versions_location = self.root / "checkout-versions.yaml" - with open(checkout_versions_location, "r") as yaml_file: - data = yaml.safe_load(yaml_file) - self.ramble_commit = data["versions"]["ramble"] - self.spack_commit = data["versions"]["spack"] + self.ramble_location, self.spack_location = ( + self.dest / "ramble", + self.dest / "spack", + ) + + # Read pinned versions of ramble and spack + with open(benchpark.paths.checkout_versions, "r") as yaml_file: + data = yaml.safe_load(yaml_file)["versions"] + self.ramble_commit, self.spack_commit = data["ramble"], data["spack"] + + # Read remote urls for ramble and spack + with open(benchpark.paths.remote_urls, "r") as yaml_file: + data = yaml.safe_load(yaml_file)["urls"] + remote_ramble_url, remote_spack_url = data["ramble"], data["spack"] - self.ramble_location = self.dest / "ramble" - self.spack_location = self.dest / "spack" + # If this does not have an upstream, then we will be cloning from the URLs indicated in remote-urls.yaml + if self.upstream is None: + self.ramble_url, self.spack_url = remote_ramble_url, remote_spack_url + else: + # Clone from local "upstream" repository + self.ramble_url, self.spack_url = ( + self.upstream.ramble_location, + self.upstream.spack_location, + ) - def update_old_bootstrap(self, desired_commit, location): - # Store first 7 of hash in checkout-versions.yaml + def _check_and_update_bootstrap(self, desired_commit, location): with working_dir(location): + # length of hash is 7 in checkout-versions.yaml current_commit = run_command("git rev-parse HEAD")[0].strip()[:7] if current_commit != desired_commit: run_command("git fetch --all") @@ -92,7 +108,7 @@ def bootstrap(self): if not self.ramble_location.exists(): self._install_ramble() else: - self.update_old_bootstrap(self.ramble_commit, self.ramble_location) + self._check_and_update_bootstrap(self.ramble_commit, self.ramble_location) ramble_lib_path = self.ramble_location / "lib" / "ramble" externals = str(ramble_lib_path / "external") if externals not in sys.path: @@ -107,12 +123,12 @@ def bootstrap(self): if not self.spack_location.exists(): self._install_spack() else: - self.update_old_bootstrap(self.spack_commit, self.spack_location) + self._check_and_update_bootstrap(self.spack_commit, self.spack_location) def _install_ramble(self): print(f"Cloning Ramble to {self.ramble_location}") git_clone_commit( - "https://github.com/GoogleCloudPlatform/ramble.git", + self.ramble_url, self.ramble_commit, self.ramble_location, ) @@ -121,7 +137,9 @@ def _install_ramble(self): def _install_spack(self): print(f"Cloning Spack to {self.spack_location}") git_clone_commit( - "https://github.com/spack/spack.git", self.spack_commit, self.spack_location + self.spack_url, + self.spack_commit, + self.spack_location, ) debug_print(f"Done cloning Spack ({self.spack_location})") diff --git a/lib/benchpark/spec.py b/lib/benchpark/spec.py index 7c0c5f56b..f437f49fc 100644 --- a/lib/benchpark/spec.py +++ b/lib/benchpark/spec.py @@ -13,12 +13,7 @@ from typing import Iterable, Iterator, List, Match, Optional, Union from benchpark.error import BenchparkError -import benchpark.paths import benchpark.repo -import benchpark.runtime - -bootstrapper = benchpark.runtime.RuntimeResources(benchpark.paths.benchpark_home) -bootstrapper.bootstrap() import llnl.util.lang # noqa diff --git a/lib/benchpark/system.py b/lib/benchpark/system.py index 6469c3623..ccef44eff 100644 --- a/lib/benchpark/system.py +++ b/lib/benchpark/system.py @@ -7,21 +7,12 @@ import os import packaging.version import yaml +from typing import Dict, Tuple -import benchpark.paths from benchpark.directives import ExperimentSystemBase -import benchpark.repo -from benchpark.runtime import RuntimeResources - -from typing import Dict, Tuple import benchpark.spec import benchpark.variant -bootstrapper = RuntimeResources(benchpark.paths.benchpark_home) # noqa -bootstrapper.bootstrap() # noqa - -_repo_path = benchpark.repo.paths[benchpark.repo.ObjectTypes.systems] - def _hash_id(content_list): sha256_hash = hashlib.sha256() diff --git a/lib/main.py b/lib/main.py index 276ce4aa6..a65741c93 100755 --- a/lib/main.py +++ b/lib/main.py @@ -13,50 +13,35 @@ import yaml __version__ = "0.1.0" -if "-V" in sys.argv or "--version" in sys.argv: - print(__version__) - exit() -helpstr = """usage: main.py [-h] [-V] {tags,system,experiment,setup,unit-test,audit,info,list} ... - -Benchpark - -options: - -h, --help show this help message and exit - -V, --version show version number and exit - -Subcommands: - {tags,system,experiment,setup,unit-test,audit,info,list} - tags Tags in Benchpark experiments - system Initialize a system config - experiment Interact with experiments - setup Set up an experiment and prepare it to build/run - unit-test Run benchpark unit tests - audit Look for problems in System/Experiment repos - info Get information about Systems and Experiments - list List experiments, systems, benchmarks, and modifiers - analyze Perform canned analysis on the performance data (caliper files) after 'ramble on'""" -if "-h" == sys.argv[1] or "--help" == sys.argv[1]: - print(helpstr) - exit() - -import benchpark.cmd.audit # noqa: E402 -import benchpark.cmd.system # noqa: E402 -import benchpark.cmd.experiment # noqa: E402 -import benchpark.cmd.setup # noqa: E402 -import benchpark.cmd.show_build # noqa: E402 -import benchpark.cmd.unit_test # noqa: E402 -import benchpark.cmd.mirror # noqa: E402 -import benchpark.cmd.info # noqa: E402 -import benchpark.cmd.list # noqa: E402 -import benchpark.paths # noqa: E402 -from benchpark.accounting import benchpark_benchmarks # noqa: E402 - -try: - import benchpark.cmd.analyze # noqa: E402 - - analyze_installed = True -except ModuleNotFoundError: - analyze_installed = False +BOOTSTRAP = True +NOARGS = True if len(sys.argv) == 1 else False +if NOARGS or any(arg in sys.argv for arg in ["-V", "--version", "-h", "--help"]): + BOOTSTRAP = False + +if BOOTSTRAP: + import benchpark.paths # noqa: E402 + from benchpark.runtime import RuntimeResources + + bootstrapper = RuntimeResources(benchpark.paths.benchpark_home) # noqa + bootstrapper.bootstrap() # noqa + + import benchpark.cmd.audit # noqa: E402 + import benchpark.cmd.system # noqa: E402 + import benchpark.cmd.experiment # noqa: E402 + import benchpark.cmd.setup # noqa: E402 + import benchpark.cmd.show_build # noqa: E402 + import benchpark.cmd.unit_test # noqa: E402 + import benchpark.cmd.mirror # noqa: E402 + import benchpark.cmd.info # noqa: E402 + import benchpark.cmd.list # noqa: E402 + from benchpark.accounting import benchpark_benchmarks # noqa: E402 + + try: + import benchpark.cmd.analyze # noqa: E402 + + analyze_installed = True + except ModuleNotFoundError: + analyze_installed = False def main(): @@ -71,15 +56,77 @@ def main(): subparsers = parser.add_subparsers(title="Subcommands", dest="subcommand") actions = {} - benchpark_tags(subparsers, actions) - init_commands(subparsers, actions) + actions["system"] = subparsers.add_parser( + "system", help="Initialize a system config" + ) + actions["experiment"] = subparsers.add_parser( + "experiment", help="Interact with experiments" + ) + actions["setup"] = subparsers.add_parser( + "setup", help="Set up an experiment and prepare it to build/run" + ) + actions["setup"].add_argument( + "experiment", + type=str, + help="The experiment (benchmark/ProgrammingModel) to run", + ) + actions["setup"].add_argument( + "system", type=str, help="The system on which to run the experiment" + ) + actions["setup"].add_argument( + "experiments_root", + type=str, + help="Where to install packages and store results for the experiments. Benchpark expects to manage this directory, and it should be empty/nonexistent the first time you run benchpark setup experiments.", + ) + actions["unit-test"] = subparsers.add_parser( + "unit-test", help="Run benchpark unit tests" + ) + actions["audit"] = subparsers.add_parser( + "audit", help="Look for problems in System/Experiment repos" + ) + actions["mirror"] = subparsers.add_parser( + "mirror", help="Copy a benchpark workspace" + ) + actions["info"] = subparsers.add_parser( + "info", help="Get information about Systems and Experiments" + ) + actions["show-build"] = subparsers.add_parser( + "show-build", help="Show how spack built a benchmark" + ) + actions["list"] = subparsers.add_parser( + "list", help="List experiments, systems, benchmarks, and modifiers" + ) + actions["analyze"] = subparsers.add_parser( + "analyze", + help="Perform canned analysis on the performance data (caliper files) after 'ramble on'", + ) + actions["tags"] = subparsers.add_parser( + "tags", help="Tags in Benchpark experiments" + ) + actions["tags"].add_argument( + "-a", + "--application", + action="store", + help="The application for which to find Benchpark tags", + ) + actions["tags"].add_argument( + "-t", + "--tag", + action="store", + help="The tag for which to search in Benchpark experiments", + ) args, unknown_args = parser.parse_known_args() - no_args = True if len(sys.argv) == 1 else False - if no_args: + if args.version: + print(__version__) + return + elif not BOOTSTRAP: parser.print_help() - return 1 + return + + actions["tags"] = benchpark_tags_handler + init_commands(actions) exit_code = 0 @@ -158,57 +205,20 @@ def benchpark_check_tag(arg_str): return found -def init_commands(subparsers, actions_dict): +def init_commands(actions_dict): """This function is for initializing commands that are defined outside of this script. It is intended that all command setup will eventually be refactored in this way (e.g. `benchpark_setup` will be defined in another file. """ - system_parser = subparsers.add_parser("system", help="Initialize a system config") - benchpark.cmd.system.setup_parser(system_parser) - - experiment_parser = subparsers.add_parser( - "experiment", help="Interact with experiments" - ) - benchpark.cmd.experiment.setup_parser(experiment_parser) - - setup_parser = subparsers.add_parser( - "setup", help="Set up an experiment and prepare it to build/run" - ) - benchpark.cmd.setup.setup_parser(setup_parser) - - unit_test_parser = subparsers.add_parser( - "unit-test", help="Run benchpark unit tests" - ) - benchpark.cmd.unit_test.setup_parser(unit_test_parser) - - audit_parser = subparsers.add_parser( - "audit", help="Look for problems in System/Experiment repos" - ) - benchpark.cmd.audit.setup_parser(audit_parser) - - mirror_parser = subparsers.add_parser("mirror", help="Copy a benchpark workspace") - benchpark.cmd.mirror.setup_parser(mirror_parser) - - info_parser = subparsers.add_parser( - "info", help="Get information about Systems and Experiments" - ) - benchpark.cmd.info.setup_parser(info_parser) - - show_build_parser = subparsers.add_parser( - "show-build", help="Show how spack built a benchmark" - ) - benchpark.cmd.show_build.setup_parser(show_build_parser) - - list_parser = subparsers.add_parser( - "list", help="List experiments, systems, benchmarks, and modifiers" - ) - benchpark.cmd.list.setup_parser(list_parser) - - analyze_parser = subparsers.add_parser( - "analyze", - help="Perform canned analysis on the performance data (caliper files) after 'ramble on'", - ) + benchpark.cmd.system.setup_parser(actions_dict["system"]) + benchpark.cmd.experiment.setup_parser(actions_dict["experiment"]) + benchpark.cmd.unit_test.setup_parser(actions_dict["unit-test"]) + benchpark.cmd.audit.setup_parser(actions_dict["audit"]) + benchpark.cmd.mirror.setup_parser(actions_dict["mirror"]) + benchpark.cmd.info.setup_parser(actions_dict["info"]) + benchpark.cmd.show_build.setup_parser(actions_dict["show-build"]) + benchpark.cmd.list.setup_parser(actions_dict["list"]) actions_dict["system"] = benchpark.cmd.system.command actions_dict["experiment"] = benchpark.cmd.experiment.command @@ -220,7 +230,7 @@ def init_commands(subparsers, actions_dict): actions_dict["show-build"] = benchpark.cmd.show_build.command actions_dict["list"] = benchpark.cmd.list.command if analyze_installed: - benchpark.cmd.analyze.setup_parser(analyze_parser) + benchpark.cmd.analyze.setup_parser(actions_dict["analyze"]) actions_dict["analyze"] = benchpark.cmd.analyze.command else: @@ -249,23 +259,6 @@ def run_command(command_str, env=None): return (stdout, stderr) -def benchpark_tags(subparsers, actions_dict): - create_parser = subparsers.add_parser("tags", help="Tags in Benchpark experiments") - create_parser.add_argument( - "-a", - "--application", - action="store", - help="The application for which to find Benchpark tags", - ) - create_parser.add_argument( - "-t", - "--tag", - action="store", - help="The tag for which to search in Benchpark experiments", - ) - actions_dict["tags"] = benchpark_tags_handler - - def helper_experiments_tags(ramble_exe, benchmarks): # find all tags in Ramble applications (both in Ramble built-in and in Benchpark/repo) (tags_stdout, tags_stderr) = run_command(f"{ramble_exe} attributes --tags --all") diff --git a/remote-urls.yaml b/remote-urls.yaml new file mode 100644 index 000000000..2dbc1050c --- /dev/null +++ b/remote-urls.yaml @@ -0,0 +1,7 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 +urls: + ramble: https://github.com/GoogleCloudPlatform/ramble.git + spack: https://github.com/spack/spack.git