diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ac4345cc85..68423029ec 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -48,6 +48,7 @@ import re from collections import OrderedDict from contextlib import contextmanager +from typing import Optional import easybuild.tools.filetools as filetools from easybuild.base import fancylogger @@ -2234,6 +2235,40 @@ def resolve_template(value, tmpl_dict, expect_resolved=True): return value +def make_easyconfig_dict(ec: EasyConfig, original_spec: Optional[str] = None) -> dict: + """Embed the easyconfig into a dictionary with entries for name, dependencies etc.""" + result = { + 'ec': ec, + 'spec': ec.path, + 'short_mod_name': ec.short_mod_name, + 'full_mod_name': ec.full_mod_name, + 'dependencies': [], + 'builddependencies': [], + 'hiddendependencies': [], + 'hidden': ec.hidden, + } + if original_spec is not None: + result['original_spec'] = original_spec + + name = ec.name + # add build dependencies + for dep in ec['builddependencies']: + _log.debug("Adding build dependency %s for app %s." % (dep, name)) + result['builddependencies'].append(dep) + + # add dependencies (including build & hidden dependencies) + for dep in ec.dependencies(): + _log.debug("Adding dependency %s for app %s." % (dep, name)) + result['dependencies'].append(dep) + + # add toolchain as dependency too + if not is_system_toolchain(ec['toolchain']['name']): + tc = ec.toolchain.as_dict() + _log.debug("Adding toolchain %s as dependency for app %s." % (tc, name)) + result['dependencies'].append(tc) + return result + + def process_easyconfig(path, build_specs=None, validate=True, parse_only=False, hidden=None): """ Process easyconfig, returning some information for each block @@ -2270,43 +2305,12 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False, exit_code = EasyBuildExit.EASYCONFIG_ERROR raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=exit_code) - name = ec['name'] - - easyconfig = { - 'ec': ec, - } + if parse_only: + easyconfig = {'ec': ec} + else: + easyconfig = make_easyconfig_dict(ec, original_spec=path if len(blocks) > 1 else None) easyconfigs.append(easyconfig) - if not parse_only: - # also determine list of dependencies, module name (unless only parsed easyconfigs are requested) - easyconfig.update({ - 'spec': ec.path, - 'short_mod_name': ec.short_mod_name, - 'full_mod_name': ec.full_mod_name, - 'dependencies': [], - 'builddependencies': [], - 'hiddendependencies': [], - 'hidden': ec.hidden, - }) - if len(blocks) > 1: - easyconfig['original_spec'] = path - - # add build dependencies - for dep in ec['builddependencies']: - _log.debug("Adding build dependency %s for app %s." % (dep, name)) - easyconfig['builddependencies'].append(dep) - - # add dependencies (including build & hidden dependencies) - for dep in ec.dependencies(): - _log.debug("Adding dependency %s for app %s." % (dep, name)) - easyconfig['dependencies'].append(dep) - - # add toolchain as dependency too - if not is_system_toolchain(ec['toolchain']['name']): - tc = ec.toolchain.as_dict() - _log.debug("Adding toolchain %s as dependency for app %s." % (tc, name)) - easyconfig['dependencies'].append(tc) - if cache_key is not None: _easyconfigs_cache[cache_key] = [e.copy() for e in easyconfigs] diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 0711eef329..3c4ab1d666 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -77,7 +77,7 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, extra_ec_paths, auto_r return robot_path -def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True): +def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True, return_conflicts=False): """ Check for conflicts in dependency graphs for specified easyconfigs. @@ -85,15 +85,14 @@ def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True): :param modtool: ModulesTool instance to use :param check_inter_ec_conflicts: also check for conflicts between (dependencies of) listed easyconfigs :return: True if one or more conflicts were found, False otherwise + If return_conflicts is True, return list of conflicts instead of boolean and printing them to stderr """ ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) def mk_key(spec): """Create key for dictionary with all dependencies.""" - if 'ec' in spec: - spec = spec['ec'] - + spec = spec.get('ec', spec) return (spec['name'], det_full_ec_version(spec)) # determine whether any 'wrappers' are involved @@ -112,9 +111,7 @@ def mk_dep_keys(deps): if not dep.get('external_module', False): key = mk_key(dep) # replace 'wrapper' dependencies with the dependency they're wrapping - if key in wrapper_deps: - key = wrapper_deps[key] - res.append(key) + res.append(wrapper_deps.get(key, key)) return res # construct a dictionary: (name, installver) tuple to (build) dependencies @@ -201,15 +198,20 @@ def check_conflict(parent, dep1, dep2): vs_msg += "\n\t%s-%s as dep of: " % dep + ', '.join('%s-%s' % d for d in sorted(dep_of[dep])) if parent[0] is None: - sys.stderr.write("Conflict between (dependencies of) easyconfigs: %s\n" % vs_msg) + msg = "Conflict between (dependencies of) easyconfigs: " else: specname = '%s-%s' % parent - sys.stderr.write("Conflict found for dependencies of %s: %s\n" % (specname, vs_msg)) + msg = f"Conflict found for dependencies of {specname}: " + msg += vs_msg + if return_conflicts: + return msg + else: + print(msg, file=sys.stderr) return conflict # for each of the easyconfigs, check whether the dependencies (incl. build deps) contain any conflicts - res = False + res = [] if return_conflicts else False for (key, (build_deps, runtime_deps, multi_deps)) in deps_for.items(): # determine lists of runtime deps to iterate over @@ -225,8 +227,12 @@ def check_conflict(parent, dep1, dep2): for dep2 in (build_deps + runtime_deps)[i + 1:]: # don't worry about conflicts between module itself and any of its build deps if dep1 != key or dep2 not in build_deps: - res |= check_conflict(key, dep1, dep2) - + cur_res = check_conflict(key, dep1, dep2) + if return_conflicts: + if cur_res: + res.append(cur_res) + else: + res |= cur_res return res @@ -386,7 +392,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs] for ec in resolved_ecs: # only add easyconfig if it's not included yet (based on module name) - if not ec['full_mod_name'] in ordered_ec_mod_names: + if ec['full_mod_name'] not in ordered_ec_mod_names: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point @@ -409,7 +415,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] - candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] + candidates = [d for d in deps if EasyBuildMNS().det_full_module_name(d) not in being_installed] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any diff --git a/test/framework/robot.py b/test/framework/robot.py index 895c92f834..97d7cf5cef 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -40,7 +40,7 @@ import easybuild.framework.easyconfig.easyconfig as ecec import easybuild.tools.build_log import easybuild.tools.robot as robot -from easybuild.framework.easyconfig.easyconfig import process_easyconfig, EasyConfig +from easybuild.framework.easyconfig.easyconfig import process_easyconfig, EasyConfig, make_easyconfig_dict from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, find_resolved_modules, parse_easyconfigs from easybuild.framework.easyconfig.tweak import tweak from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy @@ -1415,11 +1415,11 @@ def test_check_conflicts(self): gzip_ec = os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb') gompi_ec = os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb') - ecs, _ = parse_easyconfigs([(gzip_ec, False), (gompi_ec, False)]) + non_conflict_ecs, _ = parse_easyconfigs([(gzip_ec, False), (gompi_ec, False)]) # no conflicts found, no output to stderr self.mock_stderr(True) - conflicts = check_conflicts(ecs, self.modtool) + conflicts = check_conflicts(non_conflict_ecs, self.modtool) stderr = self.get_stderr() self.mock_stderr(False) self.assertFalse(conflicts) @@ -1441,6 +1441,13 @@ def test_check_conflicts(self): self.assertTrue(conflicts) self.assertIn("Conflict found for dependencies of foss-2018a: GCC-4.6.4 vs GCC-6.4.0-2.28", stderr) + # Can also return the text + with self.mocked_stdout_stderr(mock_stdout=False) as mocked_stderr: + conflict_lst = check_conflicts(ecs, self.modtool, return_conflicts=True) + self.assertEqual('\n'.join(conflict_lst), stderr.strip()) + self.assertEqual(mocked_stderr.getvalue(), '') + self.assertEqual(check_conflicts(non_conflict_ecs, self.modtool, return_conflicts=True), []) + # conflicts between specified easyconfigs are also detected # direct conflict on software version @@ -1472,6 +1479,28 @@ def test_check_conflicts(self): # test use of check_inter_ec_conflicts self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False), "No conflicts found") + # Conflict in build dependencies is fine + hwloc_txt = read_file(os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb')) + gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')) + bzip_txt = read_file(os.path.join(test_easyconfigs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb')) + tc = re.search(r"toolchain *=.*", hwloc_txt)[0] + bzip_txt += f"\n{tc}" + gzip_txt += f"\n{tc}" + gzip_txt_2 = gzip_txt.replace('1.5', '2.0') + hwloc_txt += "\nbuilddependencies = [('gzip', '2.0')]" + bzip_txt += "\nbuilddependencies = [('gzip', '1.5')]" + ecs = [make_easyconfig_dict(EasyConfig(path=None, rawtxt=txt)) + for txt in (hwloc_txt, gzip_txt, gzip_txt_2, bzip_txt)] + self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False), + "No conflicts should be found") + # But fail if A depends on B which depends on C and A has C as a build dependency in another version + hwloc_txt += "\ndependencies = [('bzip2', '1.0.6')]" + bzip_txt = bzip_txt.replace("builddependencies", "dependencies") + ecs = [make_easyconfig_dict(EasyConfig(path=None, rawtxt=txt)) + for txt in (hwloc_txt, gzip_txt, gzip_txt_2, bzip_txt)] + self.assertTrue(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False, return_conflicts=True), + "Conflict should be found") + def test_check_conflicts_wrapper_deps(self): """Test check_conflicts when dependency 'wrappers' are involved.""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')