From 48d727b2344a571f2948865814dffc2f18fa4886 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 16 Feb 2022 10:40:40 +0100 Subject: [PATCH 1/4] improve compatibility with conda-smithy --- boa/cli/boa.py | 15 ++++++++++++--- boa/core/metadata.py | 39 ++++++++++++++++++++++++++++++++++----- boa/core/recipe_output.py | 2 ++ boa/core/run_build.py | 5 +++-- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/boa/cli/boa.py b/boa/cli/boa.py index 0d85f59a..ebb9dbc3 100644 --- a/boa/cli/boa.py +++ b/boa/cli/boa.py @@ -59,7 +59,7 @@ def main(config=None): "-i", "--interactive", action="store_true", - help="Use interactive mode if build fails", + help="Use (experimental) interactive mode if build fails", ) build_parser.add_argument( "--skip-existing", nargs="?", default="default", const="yes", @@ -75,11 +75,20 @@ def main(config=None): action="store_true", help="Continue building remaining recipes if a recipe fails.", ) - + build_parser.add_argument( + "--suppress-variables", + action="append", + help="CURRENTLY IGNORED! Do not display value of environment variables specified in build.script_env" + ) + build_parser.add_argument( + "--clobber-file", + action="append", + help="CURRENTLY IGNORED! Clobber data in meta.yaml with fields from this file. Jinja2 is not done on clobbered fields" + ) subparsers.add_parser( "build", parents=[parent_parser, build_parser, variant_parser], - help="build a recipe", + help="Build a recipe", ) transmute_parser = subparsers.add_parser( diff --git a/boa/core/metadata.py b/boa/core/metadata.py index 2a48fdeb..5f8a6d12 100644 --- a/boa/core/metadata.py +++ b/boa/core/metadata.py @@ -22,6 +22,8 @@ import json import copy +from boa.core.config import boa_config +console = boa_config.console def get_package_version_pin(specs, name): for s in specs: @@ -347,12 +349,11 @@ def get_hash_contents(self): """ # trim_build_only_deps(self, dependencies) - dependencies = ( + raw_dependencies = ( self.get_dependencies("build") + self.get_dependencies("host") - # self.output.requirements["build"] + self.output.requirements["host"] ) - dependencies = {x.name for x in dependencies} + dependencies = {x.name for x in raw_dependencies} # filter out ignored versions build_string_excludes = ["python", "r_base", "perl", "lua", "target_platform"] build_string_excludes.extend( @@ -380,12 +381,35 @@ def get_hash_contents(self): continue filtered_deps.append(req) - take_keys = set(self.config.variant.keys()) + take_keys = set(k for k in self.config.variant.keys() if k in dependencies) if "python" in take_keys and "python" not in dependencies: take_keys.remove("python") + # Add _compiler and _compiler_version if it was used + for dep in raw_dependencies: + if dep.is_compiler: + if f'{dep.splitted[1]}_compiler' in self.config.variant: + take_keys.add(f'{dep.splitted[1]}_compiler') + if f'{dep.splitted[1]}_compiler_version' in self.config.variant: + take_keys.add(f'{dep.splitted[1]}_compiler_version') + + if 'CONDA_BUILD_SYSROOT' in self.config.variant: + for dep in raw_dependencies: + if dep.is_compiler and dep.splitted[1] in ['c', 'cxx']: + take_keys.add('CONDA_BUILD_SYSROOT') + + # always add target_platform and channel_targets to hash + if ('target_platform' in self.config.variant) and not self.noarch: + take_keys.add('target_platform') + if 'channel_targets' in self.config.variant: + take_keys.add('channel_targets') + + console.print(self.config.variant) + # retrieve values - this dictionary is what makes up the hash. - return {key: self.config.variant[key] for key in take_keys} + hash_dict = {key: self.config.variant[key] for key in take_keys} + console.print("[bold]Hash dictionary:\n", hash_dict) + return hash_dict def info_index(self): arch = ( @@ -460,6 +484,11 @@ def copy(self): new = copy.deepcopy(self) return new + def get_used_vars(self, force_top_level=False): + return [] + def get_used_loop_vars(self, force_top_level=False): + return [] + def get_test_deps(self, py_files, pl_files, lua_files, r_files): specs = ["%s %s %s" % (self.name(), self.version(), self.build_id())] diff --git a/boa/core/recipe_output.py b/boa/core/recipe_output.py index 728f2c1e..1ec4dd59 100644 --- a/boa/core/recipe_output.py +++ b/boa/core/recipe_output.py @@ -278,6 +278,8 @@ def apply_variant(self, variant, differentiating_keys=()): copied = copy.deepcopy(self) copied.variant = variant + # copied.variants = [variant] + # copied.config.variants = [variant] for idx, r in enumerate(self.requirements["build"]): vname = r.name.replace("-", "_") if vname in variant: diff --git a/boa/core/run_build.py b/boa/core/run_build.py index 532a8825..b15ec40b 100644 --- a/boa/core/run_build.py +++ b/boa/core/run_build.py @@ -6,6 +6,7 @@ import itertools import json import pathlib +import copy from libmambapy import PrefixData from libmambapy import Context as MambaContext @@ -280,10 +281,9 @@ def to_build_tree(ydoc, variants, config, cbc, selected_features): # zip keys need to be contracted zipped_keys = cbc.get("zip_keys", []) - + print(f"Variant for {variant_name} ", variants.get(variant_name)) if variants.get(variant_name): v = variants[variant_name] - import copy vzipped = copy.copy(v) zippers = {} @@ -344,6 +344,7 @@ def to_build_tree(ydoc, variants, config, cbc, selected_features): unzipped_combinations.append(unz_combo) + print("Applying variant! ", c, differentiating_keys) for c in unzipped_combinations: x = output.apply_variant(c, differentiating_keys) final_outputs.append(x) From 60706624cacd1b367c850079f27ae646c1b600f5 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 16 Feb 2022 10:43:05 +0100 Subject: [PATCH 2/4] add boa api --- boa/api.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 boa/api.py diff --git a/boa/api.py b/boa/api.py new file mode 100644 index 00000000..04dd64a9 --- /dev/null +++ b/boa/api.py @@ -0,0 +1,87 @@ +from pathlib import Path + +from boa.core.render import render as core_render +from boa.core.utils import get_config +from boa.core.run_build import to_build_tree, get_dependency_variants +from boa.core.metadata import MetaData + + # os.path.join(forge_dir, forge_config["recipe_dir"]), + # platform=platform, + # arch=arch, + # ignore_system_variants=True, + # variants=migrated_combined_variant_spec, + # permit_undefined_jinja=True, + # finalize=False, + # bypass_env_check=True, + # channel_urls=forge_config.get("channels", {}).get( + # "sources", [] + # ), + +def render(recipe_dir, platform, arch, + ignore_system_variants=True, variants=None, permit_undefined_jinja=True, + finalize=False, bypass_env_check=True, channel_urls=None, selected_features=None): + + if not channel_urls: + channel_urls = [] + if not selected_features: + selected_features = dict() + + variant = {"target_platform": platform} + # cbc_file = Path(recipe_dir) / "conda_build_config.yaml" + cbc, config = get_config(recipe_dir, variant, []) + cbc["target_platform"] = [variant["target_platform"]] + + recipe_path = Path(recipe_dir) / "recipe.yaml" + ydoc = core_render(recipe_path, config) + + print("\n\n\nVARIANTS!!!\n\n", variants) + # this takes in all variants and outputs, builds a dependency tree and returns + # the final metadata + + print(config) + print(cbc) + + assembled_variants = {} + # if we have a outputs section, use that order the outputs + if ydoc.get("outputs"): + for o in ydoc["outputs"]: + # inherit from global package + pkg_meta = {} + pkg_meta.update(ydoc["package"]) + pkg_meta.update(o["package"]) + o["package"] = pkg_meta + + build_meta = {} + build_meta.update(ydoc.get("build")) + build_meta.update(o.get("build") or {}) + o["build"] = build_meta + + o["selected_features"] = selected_features + + assembled_variants[o["package"]["name"]] = get_dependency_variants( + o.get("requirements", {}), variants, config + ) + else: + # we only have one output + assembled_variants[ydoc["package"]["name"]] = get_dependency_variants( + ydoc.get("requirements", {}), variants, config + ) + + print("Selected variants: ", assembled_variants) + + sorted_outputs = to_build_tree(ydoc, + assembled_variants, + config, variants, + selected_features) + + metas = [] + for output in sorted_outputs: + meta = MetaData(recipe_path, output) + print(output) + meta.config.variants = {} + meta.config.input_variants = variants + # meta.config.variants = + metas.append((meta, None, None)) + print(metas) + return metas + # o.set_final_build_id(meta) From 0fdf9affab9b361273fa532fc0158fa712ad2366 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 16 Feb 2022 11:01:45 +0100 Subject: [PATCH 3/4] fix cli --- boa/cli/boa.py | 3 +-- boa/core/metadata.py | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/boa/cli/boa.py b/boa/cli/boa.py index ebb9dbc3..12576c53 100644 --- a/boa/cli/boa.py +++ b/boa/cli/boa.py @@ -77,12 +77,11 @@ def main(config=None): ) build_parser.add_argument( "--suppress-variables", - action="append", + action="store_true", help="CURRENTLY IGNORED! Do not display value of environment variables specified in build.script_env" ) build_parser.add_argument( "--clobber-file", - action="append", help="CURRENTLY IGNORED! Clobber data in meta.yaml with fields from this file. Jinja2 is not done on clobbered fields" ) subparsers.add_parser( diff --git a/boa/core/metadata.py b/boa/core/metadata.py index 5f8a6d12..6d58f995 100644 --- a/boa/core/metadata.py +++ b/boa/core/metadata.py @@ -485,9 +485,40 @@ def copy(self): return new def get_used_vars(self, force_top_level=False): - return [] + + raw_dependencies = ( + self.get_dependencies("build") + + self.get_dependencies("host") + ) + dependencies = {x.name for x in raw_dependencies} + + take_keys = set(k for k in self.config.variant.keys() if k in dependencies) + if "python" in take_keys and "python" not in dependencies: + take_keys.remove("python") + + # Add _compiler and _compiler_version if it was used + for dep in raw_dependencies: + if dep.is_compiler: + if f'{dep.splitted[1]}_compiler' in self.config.variant: + take_keys.add(f'{dep.splitted[1]}_compiler') + if f'{dep.splitted[1]}_compiler_version' in self.config.variant: + take_keys.add(f'{dep.splitted[1]}_compiler_version') + + if 'CONDA_BUILD_SYSROOT' in self.config.variant: + for dep in raw_dependencies: + if dep.is_compiler and dep.splitted[1] in ['c', 'cxx']: + take_keys.add('CONDA_BUILD_SYSROOT') + + # always add target_platform and channel_targets to hash + if ('target_platform' in self.config.variant) and not self.noarch: + take_keys.add('target_platform') + if 'channel_targets' in self.config.variant: + take_keys.add('channel_targets') + + return take_keys + def get_used_loop_vars(self, force_top_level=False): - return [] + return set() def get_test_deps(self, py_files, pl_files, lua_files, r_files): specs = ["%s %s %s" % (self.name(), self.version(), self.build_id())] From 5383d882cf9497be4988a66bb26a9f97c3c67a43 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 16 Feb 2022 13:02:11 +0100 Subject: [PATCH 4/4] add additional hack for smithy --- CHANGELOG.md | 2 +- boa/api.py | 53 +++++++++++++++++++----------------- boa/cli/boa.py | 4 +-- boa/core/metadata.py | 56 +++++++++++++++++++-------------------- boa/core/recipe_output.py | 7 +++++ 5 files changed, 67 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e633c14..06c8cd3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,4 +9,4 @@ - [boa] fix multi-output - [boa] fix keep run_export and existing spec when existing spec is not simple -- [mambabuild] allow testing multiple recipes (thanks @gabm) \ No newline at end of file +- [mambabuild] allow testing multiple recipes (thanks @gabm) diff --git a/boa/api.py b/boa/api.py index 04dd64a9..e2a03eb6 100644 --- a/boa/api.py +++ b/boa/api.py @@ -5,21 +5,31 @@ from boa.core.run_build import to_build_tree, get_dependency_variants from boa.core.metadata import MetaData - # os.path.join(forge_dir, forge_config["recipe_dir"]), - # platform=platform, - # arch=arch, - # ignore_system_variants=True, - # variants=migrated_combined_variant_spec, - # permit_undefined_jinja=True, - # finalize=False, - # bypass_env_check=True, - # channel_urls=forge_config.get("channels", {}).get( - # "sources", [] - # ), +# os.path.join(forge_dir, forge_config["recipe_dir"]), +# platform=platform, +# arch=arch, +# ignore_system_variants=True, +# variants=migrated_combined_variant_spec, +# permit_undefined_jinja=True, +# finalize=False, +# bypass_env_check=True, +# channel_urls=forge_config.get("channels", {}).get( +# "sources", [] +# ), -def render(recipe_dir, platform, arch, - ignore_system_variants=True, variants=None, permit_undefined_jinja=True, - finalize=False, bypass_env_check=True, channel_urls=None, selected_features=None): + +def render( + recipe_dir, + platform, + arch, + ignore_system_variants=True, + variants=None, + permit_undefined_jinja=True, + finalize=False, + bypass_env_check=True, + channel_urls=None, + selected_features=None, +): if not channel_urls: channel_urls = [] @@ -34,13 +44,9 @@ def render(recipe_dir, platform, arch, recipe_path = Path(recipe_dir) / "recipe.yaml" ydoc = core_render(recipe_path, config) - print("\n\n\nVARIANTS!!!\n\n", variants) # this takes in all variants and outputs, builds a dependency tree and returns # the final metadata - print(config) - print(cbc) - assembled_variants = {} # if we have a outputs section, use that order the outputs if ydoc.get("outputs"): @@ -69,10 +75,9 @@ def render(recipe_dir, platform, arch, print("Selected variants: ", assembled_variants) - sorted_outputs = to_build_tree(ydoc, - assembled_variants, - config, variants, - selected_features) + sorted_outputs = to_build_tree( + ydoc, assembled_variants, config, variants, selected_features + ) metas = [] for output in sorted_outputs: @@ -80,8 +85,8 @@ def render(recipe_dir, platform, arch, print(output) meta.config.variants = {} meta.config.input_variants = variants - # meta.config.variants = + # meta.config.variants = metas.append((meta, None, None)) print(metas) return metas - # o.set_final_build_id(meta) + # o.set_final_build_id(meta) diff --git a/boa/cli/boa.py b/boa/cli/boa.py index 12576c53..16422d09 100644 --- a/boa/cli/boa.py +++ b/boa/cli/boa.py @@ -78,11 +78,11 @@ def main(config=None): build_parser.add_argument( "--suppress-variables", action="store_true", - help="CURRENTLY IGNORED! Do not display value of environment variables specified in build.script_env" + help="CURRENTLY IGNORED! Do not display value of environment variables specified in build.script_env", ) build_parser.add_argument( "--clobber-file", - help="CURRENTLY IGNORED! Clobber data in meta.yaml with fields from this file. Jinja2 is not done on clobbered fields" + help="CURRENTLY IGNORED! Clobber data in meta.yaml with fields from this file. Jinja2 is not done on clobbered fields", ) subparsers.add_parser( "build", diff --git a/boa/core/metadata.py b/boa/core/metadata.py index 6d58f995..322563b8 100644 --- a/boa/core/metadata.py +++ b/boa/core/metadata.py @@ -23,8 +23,10 @@ import copy from boa.core.config import boa_config + console = boa_config.console + def get_package_version_pin(specs, name): for s in specs: x = s.split(" ") @@ -349,9 +351,8 @@ def get_hash_contents(self): """ # trim_build_only_deps(self, dependencies) - raw_dependencies = ( - self.get_dependencies("build") - + self.get_dependencies("host") + raw_dependencies = self.get_dependencies("build") + self.get_dependencies( + "host" ) dependencies = {x.name for x in raw_dependencies} # filter out ignored versions @@ -388,21 +389,21 @@ def get_hash_contents(self): # Add _compiler and _compiler_version if it was used for dep in raw_dependencies: if dep.is_compiler: - if f'{dep.splitted[1]}_compiler' in self.config.variant: - take_keys.add(f'{dep.splitted[1]}_compiler') - if f'{dep.splitted[1]}_compiler_version' in self.config.variant: - take_keys.add(f'{dep.splitted[1]}_compiler_version') + if f"{dep.splitted[1]}_compiler" in self.config.variant: + take_keys.add(f"{dep.splitted[1]}_compiler") + if f"{dep.splitted[1]}_compiler_version" in self.config.variant: + take_keys.add(f"{dep.splitted[1]}_compiler_version") - if 'CONDA_BUILD_SYSROOT' in self.config.variant: + if "CONDA_BUILD_SYSROOT" in self.config.variant: for dep in raw_dependencies: - if dep.is_compiler and dep.splitted[1] in ['c', 'cxx']: - take_keys.add('CONDA_BUILD_SYSROOT') + if dep.is_compiler and dep.splitted[1] in ["c", "cxx"]: + take_keys.add("CONDA_BUILD_SYSROOT") # always add target_platform and channel_targets to hash - if ('target_platform' in self.config.variant) and not self.noarch: - take_keys.add('target_platform') - if 'channel_targets' in self.config.variant: - take_keys.add('channel_targets') + if ("target_platform" in self.config.variant) and not self.noarch: + take_keys.add("target_platform") + if "channel_targets" in self.config.variant: + take_keys.add("channel_targets") console.print(self.config.variant) @@ -486,9 +487,8 @@ def copy(self): def get_used_vars(self, force_top_level=False): - raw_dependencies = ( - self.get_dependencies("build") - + self.get_dependencies("host") + raw_dependencies = self.get_dependencies("build") + self.get_dependencies( + "host" ) dependencies = {x.name for x in raw_dependencies} @@ -499,21 +499,21 @@ def get_used_vars(self, force_top_level=False): # Add _compiler and _compiler_version if it was used for dep in raw_dependencies: if dep.is_compiler: - if f'{dep.splitted[1]}_compiler' in self.config.variant: - take_keys.add(f'{dep.splitted[1]}_compiler') - if f'{dep.splitted[1]}_compiler_version' in self.config.variant: - take_keys.add(f'{dep.splitted[1]}_compiler_version') + if f"{dep.splitted[1]}_compiler" in self.config.variant: + take_keys.add(f"{dep.splitted[1]}_compiler") + if f"{dep.splitted[1]}_compiler_version" in self.config.variant: + take_keys.add(f"{dep.splitted[1]}_compiler_version") - if 'CONDA_BUILD_SYSROOT' in self.config.variant: + if "CONDA_BUILD_SYSROOT" in self.config.variant: for dep in raw_dependencies: - if dep.is_compiler and dep.splitted[1] in ['c', 'cxx']: - take_keys.add('CONDA_BUILD_SYSROOT') + if dep.is_compiler and dep.splitted[1] in ["c", "cxx"]: + take_keys.add("CONDA_BUILD_SYSROOT") # always add target_platform and channel_targets to hash - if ('target_platform' in self.config.variant) and not self.noarch: - take_keys.add('target_platform') - if 'channel_targets' in self.config.variant: - take_keys.add('channel_targets') + if ("target_platform" in self.config.variant) and not self.noarch: + take_keys.add("target_platform") + if "channel_targets" in self.config.variant: + take_keys.add("channel_targets") return take_keys diff --git a/boa/core/recipe_output.py b/boa/core/recipe_output.py index 1ec4dd59..cb3acbae 100644 --- a/boa/core/recipe_output.py +++ b/boa/core/recipe_output.py @@ -162,6 +162,7 @@ def __init__( parent = {} if selected_features is None: selected_features = {} + self.data = d self.data["source"] = d.get("source", parent.get("source", {})) self.config = config @@ -182,7 +183,13 @@ def set_section(sname): set_section("build") set_section("package") set_section("app") + + # TODO this is a hack ... set_section("extra") + set_section("about") + self.data["extra"] = self.sections["extra"] + self.data["about"] = self.sections["about"] + set_section("test") self.sections["files"] = d.get("files")