From be187515932492bb72e09e45f1c984c83eecd20f Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 10 Mar 2022 13:18:51 +0100 Subject: [PATCH 1/3] [skip ci] add jinja variant magics --- boa/core/recipe_output.py | 10 ++++++++ boa/core/render.py | 52 ++++++++++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/boa/core/recipe_output.py b/boa/core/recipe_output.py index ff5fba03..35a27d2b 100644 --- a/boa/core/recipe_output.py +++ b/boa/core/recipe_output.py @@ -117,6 +117,8 @@ def set_section(sname): self.parent = parent for section in ("build", "host", "run", "run_constrained"): + print(self.requirements.get(section)) + print([type(x) for x in self.requirements.get(section)]) self.requirements[section] = [ CondaBuildSpec(r) for r in (self.requirements.get(section) or []) ] @@ -178,6 +180,14 @@ def variant_keys(self): for s in self.sections["build"].get("skip", []): all_keys += ast_extract_syms(s) + for _, v in self.sections["build"].items(): + if "{{" in v: + print(v) + + for v in all_keys: + if "{{" in v: + print(v) + return [str(x) for x in all_keys] def all_requirements(self): diff --git a/boa/core/render.py b/boa/core/render.py index 72359ca3..5c4da82b 100644 --- a/boa/core/render.py +++ b/boa/core/render.py @@ -1,5 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause +import copy from ruamel.yaml import YAML import jinja2 @@ -12,13 +13,47 @@ console = boa_config.console +class TemplateStr(str): + def __new__(cls, template, value, missing_keys): + obj = str.__new__(cls, value) + obj.template = template + obj.missing_keys = missing_keys + return obj + + +class ContextDictAccessor(jinja2.runtime.Context): + + global_missing = [] + + def resolve_or_missing(self, key): + val = jinja2.runtime.Context.resolve_or_missing(self, key) + if val is jinja2.utils.missing: + ContextDictAccessor.global_missing.append(key) + return f"JINJA[{key}]" + return val + + +def render_jinja(value, context_dict, jenv): + if "{{" in value: + tmpl = jenv.from_string(value) + missing_vals = copy.copy(ContextDictAccessor.global_missing) + ContextDictAccessor.global_missing = [] + return TemplateStr(tmpl, tmpl.render(context_dict), missing_vals) + + return value + + def render_recursive(dict_or_array, context_dict, jenv): # check if it's a dict? if isinstance(dict_or_array, Mapping): for key, value in dict_or_array.items(): if isinstance(value, str): - tmpl = jenv.from_string(value) - dict_or_array[key] = tmpl.render(context_dict) + dict_or_array[key] = render_jinja(value, context_dict, jenv) + # tmpl = jenv.from_string(value) + # dict_or_array[key] = TemplateStr( + # tmpl, + # tmpl.render(context_dict) + # ) elif isinstance(value, Mapping): render_recursive(dict_or_array[key], context_dict, jenv) elif isinstance(value, Iterable): @@ -28,8 +63,7 @@ def render_recursive(dict_or_array, context_dict, jenv): for i in range(len(dict_or_array)): value = dict_or_array[i] if isinstance(value, str): - tmpl = jenv.from_string(value) - dict_or_array[i] = tmpl.render(context_dict) + dict_or_array[i] = render_jinja(value, context_dict, jenv) elif isinstance(value, Mapping): render_recursive(value, context_dict, jenv) elif isinstance(value, Iterable): @@ -174,8 +208,14 @@ def render(recipe_path, config=None): # step 2: fill out context dict context_dict = default_jinja_vars(config) + print(context_dict, type(context_dict)) + context_dict.update(ydoc.get("context", {})) + jenv = jinja2.Environment() + # use our special context dict accessor to register when we have missing variables + jenv.context_class = ContextDictAccessor + for key, value in context_dict.items(): if isinstance(value, str): tmpl = jenv.from_string(value) @@ -190,6 +230,6 @@ def render(recipe_path, config=None): # Normalize the entire recipe ydoc = normalize_recipe(ydoc) - # console.print("\n[yellow]Normalized recipe[/yellow]\n") - # console.print(ydoc) + console.print("\n[yellow]Normalized recipe[/yellow]\n") + console.print(ydoc) return ydoc From fa54795dba7639357b391d9093d990b943dc7391 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 10 Mar 2022 16:53:10 +0100 Subject: [PATCH 2/3] wip implement jinja variant expansion --- boa/core/conda_build_spec.py | 2 +- boa/core/recipe_output.py | 40 +++++++++++++++------ boa/core/render.py | 70 +++++++++++++++++++++++++++++------- boa/core/solver.py | 4 +-- 4 files changed, 89 insertions(+), 27 deletions(-) diff --git a/boa/core/conda_build_spec.py b/boa/core/conda_build_spec.py index 153d92bd..caa2a91c 100644 --- a/boa/core/conda_build_spec.py +++ b/boa/core/conda_build_spec.py @@ -138,7 +138,7 @@ def loosen_spec(self): def __repr__(self): self.loosen_spec() - return self.final + return str(self.final) def eval_pin_subpackage(self, all_outputs): pkg_name = self.name diff --git a/boa/core/recipe_output.py b/boa/core/recipe_output.py index 35a27d2b..ae6d978c 100644 --- a/boa/core/recipe_output.py +++ b/boa/core/recipe_output.py @@ -118,7 +118,6 @@ def set_section(sname): for section in ("build", "host", "run", "run_constrained"): print(self.requirements.get(section)) - print([type(x) for x in self.requirements.get(section)]) self.requirements[section] = [ CondaBuildSpec(r) for r in (self.requirements.get(section) or []) ] @@ -177,16 +176,17 @@ def variant_keys(self): "host", [] ) + for key in all_keys: + if isinstance(key, CondaBuildSpec) and hasattr(key.raw, "missing_keys"): + all_keys += key.raw.missing_keys + for s in self.sections["build"].get("skip", []): all_keys += ast_extract_syms(s) - for _, v in self.sections["build"].items(): - if "{{" in v: - print(v) - - for v in all_keys: - if "{{" in v: - print(v) + for k in all_keys: + print(type(k)) + print(k) + # print(all_keys) return [str(x) for x in all_keys] @@ -202,6 +202,27 @@ def apply_variant(self, variant, differentiating_keys=()): copied = copy.deepcopy(self) copied.variant = variant + + # TODO we need to do this recursively to also catch list items + for k in self.sections["build"]: + if hasattr(self.sections["build"][k], "missing_keys"): + print("replacing string in ", self.sections["build"][k]) + print("with: ", self.sections["build"][k].render(variant)) + copied.sections["build"][k] = copied.sections["build"][k].render( + variant + ) + + for idx, r in enumerate(self.requirements["build"]): + if hasattr(r.raw, "missing_keys"): + copied.requirements["build"][idx] = CondaBuildSpec( + r.raw.render(variant) + ) + + for idx, r in enumerate(self.requirements["host"]): + if hasattr(r.raw, "missing_keys"): + print("!!! --- RENDERING --- !!! ", r.raw.render(variant)) + copied.requirements["host"][idx] = CondaBuildSpec(r.raw.render(variant)) + for idx, r in enumerate(self.requirements["build"]): vname = r.name.replace("-", "_") if vname in variant: @@ -304,11 +325,10 @@ def specs_to_dict(specs): def __rich__(self): table = Table(box=rich.box.MINIMAL_DOUBLE_HEAD) - s = f"Output: {self.name} {self.version} BN: {self.build_number}\n" + s = f"Output: {self.name} {self.version} {self.final_build_id} BN: {self.build_number}\n" if hasattr(self, "differentiating_variant"): short_v = " ".join([val for val in self.differentiating_variant]) s += f"Variant: {short_v}\n" - s += "Build:\n" table.title = s table.add_column("Dependency") table.add_column("Version requirement") diff --git a/boa/core/render.py b/boa/core/render.py index 5c4da82b..9f0b870b 100644 --- a/boa/core/render.py +++ b/boa/core/render.py @@ -13,12 +13,56 @@ console = boa_config.console -class TemplateStr(str): - def __new__(cls, template, value, missing_keys): - obj = str.__new__(cls, value) - obj.template = template - obj.missing_keys = missing_keys - return obj +class TemplateStr: + def __init__(self, template, rendered, missing_keys, context_dict): + # obj = str.__new__(cls, rendered) + assert isinstance(rendered, str) + assert isinstance(template, str) + assert isinstance(missing_keys, list) + self.value = rendered + self.template = template + self.context_dict = context_dict + self.missing_keys = missing_keys + # return obj + + def split(self, sval=None): + if sval: + return self.value.split(sval) + return self.value.split() + + def render(self, variant_vars): + # execute render to record missing values! + jenv = jinja2.Environment() + cc = copy.copy(self.context_dict) + cc.update(variant_vars) + tmpl = jenv.from_string(self.template) + rendered = tmpl.render(cc) + return rendered + + def __str__(self): + # return str.__str__(self) + return str(self.value) + + def __repr__(self): + # return "TemplateStr{" + str.__str__(self) + "} (" + " ".join(self.missing_keys) + ")" + return "TS{" + self.value + "} (" + " ".join(self.missing_keys) + ")" + + def to_json(self): + return self.template + + # def __copy__(self): + # cls = self.__class__ + # result = cls.__new__(cls, self.template, str(self), self.missing_keys) + # return result + + # def __deepcopy__(self, memo): + # cls = self.__class__ + # template = copy.deepcopy(self.template) + # missing_keys = copy.deepcopy(self.missing_keys) + # # value = copy.deepcopy(str.__str__(self)) + # result = cls.__new__(cls, template, value, missing_keys) + # memo[id(self)] = result + # return result class ContextDictAccessor(jinja2.runtime.Context): @@ -35,10 +79,15 @@ def resolve_or_missing(self, key): def render_jinja(value, context_dict, jenv): if "{{" in value: + print(type(value), value) tmpl = jenv.from_string(value) - missing_vals = copy.copy(ContextDictAccessor.global_missing) + # execute render to record missing values! + rendered = tmpl.render(context_dict) + missing_vals = copy.deepcopy(ContextDictAccessor.global_missing) + if not missing_vals: + return rendered ContextDictAccessor.global_missing = [] - return TemplateStr(tmpl, tmpl.render(context_dict), missing_vals) + return TemplateStr(value, rendered, missing_vals, context_dict) return value @@ -49,11 +98,6 @@ def render_recursive(dict_or_array, context_dict, jenv): for key, value in dict_or_array.items(): if isinstance(value, str): dict_or_array[key] = render_jinja(value, context_dict, jenv) - # tmpl = jenv.from_string(value) - # dict_or_array[key] = TemplateStr( - # tmpl, - # tmpl.render(context_dict) - # ) elif isinstance(value, Mapping): render_recursive(dict_or_array[key], context_dict, jenv) elif isinstance(value, Iterable): diff --git a/boa/core/solver.py b/boa/core/solver.py index 0818d845..182fdf72 100644 --- a/boa/core/solver.py +++ b/boa/core/solver.py @@ -220,9 +220,7 @@ def solve(self, specs, pkg_cache_path=None): pkg_cache_path = pkgs_dirs package_cache = libmambapy.MultiPackageCache(pkg_cache_path) - return libmambapy.Transaction( - api_solver, package_cache, self.repos + list(self.local_repos.values()) - ) + return libmambapy.Transaction(api_solver, package_cache) def solve_for_action(self, specs, prefix): t = self.solve(specs) From ad2e2bb0450935af067daaa3507475b3c3e6e3d0 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Mon, 14 Mar 2022 10:58:11 +0100 Subject: [PATCH 3/3] simplify render recursive --- boa/core/render.py | 61 ++++++++++++++-------------------------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/boa/core/render.py b/boa/core/render.py index 9f0b870b..ad21388a 100644 --- a/boa/core/render.py +++ b/boa/core/render.py @@ -15,15 +15,10 @@ class TemplateStr: def __init__(self, template, rendered, missing_keys, context_dict): - # obj = str.__new__(cls, rendered) - assert isinstance(rendered, str) - assert isinstance(template, str) - assert isinstance(missing_keys, list) self.value = rendered self.template = template self.context_dict = context_dict self.missing_keys = missing_keys - # return obj def split(self, sval=None): if sval: @@ -40,30 +35,14 @@ def render(self, variant_vars): return rendered def __str__(self): - # return str.__str__(self) return str(self.value) def __repr__(self): - # return "TemplateStr{" + str.__str__(self) + "} (" + " ".join(self.missing_keys) + ")" return "TS{" + self.value + "} (" + " ".join(self.missing_keys) + ")" def to_json(self): return self.template - # def __copy__(self): - # cls = self.__class__ - # result = cls.__new__(cls, self.template, str(self), self.missing_keys) - # return result - - # def __deepcopy__(self, memo): - # cls = self.__class__ - # template = copy.deepcopy(self.template) - # missing_keys = copy.deepcopy(self.missing_keys) - # # value = copy.deepcopy(str.__str__(self)) - # result = cls.__new__(cls, template, value, missing_keys) - # memo[id(self)] = result - # return result - class ContextDictAccessor(jinja2.runtime.Context): @@ -92,26 +71,24 @@ def render_jinja(value, context_dict, jenv): return value -def render_recursive(dict_or_array, context_dict, jenv): - # check if it's a dict? - if isinstance(dict_or_array, Mapping): - for key, value in dict_or_array.items(): - if isinstance(value, str): - dict_or_array[key] = render_jinja(value, context_dict, jenv) - elif isinstance(value, Mapping): - render_recursive(dict_or_array[key], context_dict, jenv) - elif isinstance(value, Iterable): - render_recursive(dict_or_array[key], context_dict, jenv) - - elif isinstance(dict_or_array, Iterable): - for i in range(len(dict_or_array)): - value = dict_or_array[i] - if isinstance(value, str): - dict_or_array[i] = render_jinja(value, context_dict, jenv) - elif isinstance(value, Mapping): - render_recursive(value, context_dict, jenv) - elif isinstance(value, Iterable): - render_recursive(value, context_dict, jenv) +def render_recursive(dict_or_array, context_dict, jenv, key=None): + if key is not None: + das = dict_or_array[key] + if isinstance(das, str): + dict_or_array[key] = render_jinja(das, context_dict, jenv) + return + else: + das = dict_or_array + + if isinstance(das, Mapping): + for key in das.keys(): + render_recursive(das, context_dict, jenv, key=key) + return + + if isinstance(das, Iterable): + for i in range(len(das)): + render_recursive(das, context_dict, jenv, key=i) + return def flatten_selectors(ydoc, namespace): @@ -252,8 +229,6 @@ def render(recipe_path, config=None): # step 2: fill out context dict context_dict = default_jinja_vars(config) - print(context_dict, type(context_dict)) - context_dict.update(ydoc.get("context", {})) jenv = jinja2.Environment()