Skip to content

Add return_conflicts options to check_conflicts function and enhance tests #5000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

Add return_conflicts options to check_conflicts function and enhance tests #5000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ad705da
Add `return_conflicts` param to `check_conflicts`
Flamefire Sep 23, 2025
18ee4b8
Factor out `make_easyconfig_dict`
Flamefire Sep 23, 2025
9cd02c9
Add test of `check_conflicts` for build dependencies
Flamefire Sep 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 39 additions & 35 deletions easybuild/framework/easyconfig/easyconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import re
from collections import OrderedDict
from contextlib import contextmanager
from typing import Optional

import easybuild.tools.filetools as filetools
from easybuild.base import fancylogger
Expand Down Expand Up @@ -2234,6 +2235,40 @@ def resolve_template(value, tmpl_dict, expect_resolved=True):
return value


def make_easyconfig_dict(ec: EasyConfig, original_spec: Optional[str] = None) -> dict:
"""Embed the easyconfig into a dictionary with entries for name, dependencies etc."""
result = {
'ec': ec,
'spec': ec.path,
'short_mod_name': ec.short_mod_name,
'full_mod_name': ec.full_mod_name,
'dependencies': [],
'builddependencies': [],
'hiddendependencies': [],
'hidden': ec.hidden,
}
if original_spec is not None:
result['original_spec'] = original_spec

name = ec.name
# add build dependencies
for dep in ec['builddependencies']:
_log.debug("Adding build dependency %s for app %s." % (dep, name))
result['builddependencies'].append(dep)

# add dependencies (including build & hidden dependencies)
for dep in ec.dependencies():
_log.debug("Adding dependency %s for app %s." % (dep, name))
result['dependencies'].append(dep)

# add toolchain as dependency too
if not is_system_toolchain(ec['toolchain']['name']):
tc = ec.toolchain.as_dict()
_log.debug("Adding toolchain %s as dependency for app %s." % (tc, name))
result['dependencies'].append(tc)
return result


def process_easyconfig(path, build_specs=None, validate=True, parse_only=False, hidden=None):
"""
Process easyconfig, returning some information for each block
Expand Down Expand Up @@ -2270,43 +2305,12 @@ def process_easyconfig(path, build_specs=None, validate=True, parse_only=False,
exit_code = EasyBuildExit.EASYCONFIG_ERROR
raise EasyBuildError("Failed to process easyconfig %s: %s", spec, err.msg, exit_code=exit_code)

name = ec['name']

easyconfig = {
'ec': ec,
}
if parse_only:
easyconfig = {'ec': ec}
else:
easyconfig = make_easyconfig_dict(ec, original_spec=path if len(blocks) > 1 else None)
easyconfigs.append(easyconfig)

if not parse_only:
# also determine list of dependencies, module name (unless only parsed easyconfigs are requested)
easyconfig.update({
'spec': ec.path,
'short_mod_name': ec.short_mod_name,
'full_mod_name': ec.full_mod_name,
'dependencies': [],
'builddependencies': [],
'hiddendependencies': [],
'hidden': ec.hidden,
})
if len(blocks) > 1:
easyconfig['original_spec'] = path

# add build dependencies
for dep in ec['builddependencies']:
_log.debug("Adding build dependency %s for app %s." % (dep, name))
easyconfig['builddependencies'].append(dep)

# add dependencies (including build & hidden dependencies)
for dep in ec.dependencies():
_log.debug("Adding dependency %s for app %s." % (dep, name))
easyconfig['dependencies'].append(dep)

# add toolchain as dependency too
if not is_system_toolchain(ec['toolchain']['name']):
tc = ec.toolchain.as_dict()
_log.debug("Adding toolchain %s as dependency for app %s." % (tc, name))
easyconfig['dependencies'].append(tc)

if cache_key is not None:
_easyconfigs_cache[cache_key] = [e.copy() for e in easyconfigs]

Expand Down
34 changes: 20 additions & 14 deletions easybuild/tools/robot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,23 +77,22 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, extra_ec_paths, auto_r
return robot_path


def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True):
def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True, return_conflicts=False):
"""
Check for conflicts in dependency graphs for specified easyconfigs.

:param easyconfigs: list of easyconfig files (EasyConfig instances) to check for conflicts
:param modtool: ModulesTool instance to use
:param check_inter_ec_conflicts: also check for conflicts between (dependencies of) listed easyconfigs
:return: True if one or more conflicts were found, False otherwise
If return_conflicts is True, return list of conflicts instead of boolean and printing them to stderr
"""

ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True)

def mk_key(spec):
"""Create key for dictionary with all dependencies."""
if 'ec' in spec:
spec = spec['ec']

spec = spec.get('ec', spec)
return (spec['name'], det_full_ec_version(spec))

# determine whether any 'wrappers' are involved
Expand All @@ -112,9 +111,7 @@ def mk_dep_keys(deps):
if not dep.get('external_module', False):
key = mk_key(dep)
# replace 'wrapper' dependencies with the dependency they're wrapping
if key in wrapper_deps:
key = wrapper_deps[key]
res.append(key)
res.append(wrapper_deps.get(key, key))
return res

# construct a dictionary: (name, installver) tuple to (build) dependencies
Expand Down Expand Up @@ -201,15 +198,20 @@ def check_conflict(parent, dep1, dep2):
vs_msg += "\n\t%s-%s as dep of: " % dep + ', '.join('%s-%s' % d for d in sorted(dep_of[dep]))

if parent[0] is None:
sys.stderr.write("Conflict between (dependencies of) easyconfigs: %s\n" % vs_msg)
msg = "Conflict between (dependencies of) easyconfigs: "
else:
specname = '%s-%s' % parent
sys.stderr.write("Conflict found for dependencies of %s: %s\n" % (specname, vs_msg))
msg = f"Conflict found for dependencies of {specname}: "
msg += vs_msg
if return_conflicts:
return msg
else:
print(msg, file=sys.stderr)

return conflict

# for each of the easyconfigs, check whether the dependencies (incl. build deps) contain any conflicts
res = False
res = [] if return_conflicts else False
for (key, (build_deps, runtime_deps, multi_deps)) in deps_for.items():

# determine lists of runtime deps to iterate over
Expand All @@ -225,8 +227,12 @@ def check_conflict(parent, dep1, dep2):
for dep2 in (build_deps + runtime_deps)[i + 1:]:
# don't worry about conflicts between module itself and any of its build deps
if dep1 != key or dep2 not in build_deps:
res |= check_conflict(key, dep1, dep2)

cur_res = check_conflict(key, dep1, dep2)
if return_conflicts:
if cur_res:
res.append(cur_res)
else:
res |= cur_res
return res


Expand Down Expand Up @@ -386,7 +392,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro
ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs]
for ec in resolved_ecs:
# only add easyconfig if it's not included yet (based on module name)
if not ec['full_mod_name'] in ordered_ec_mod_names:
if ec['full_mod_name'] not in ordered_ec_mod_names:
ordered_ecs.append(ec)

# dependencies marked as external modules should be resolved via available modules at this point
Expand All @@ -409,7 +415,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
candidates = [d for d in deps if EasyBuildMNS().det_full_module_name(d) not in being_installed]
if candidates:
cand_dep = candidates[0]
# find easyconfig, might not find any
Expand Down
35 changes: 32 additions & 3 deletions test/framework/robot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
import easybuild.framework.easyconfig.easyconfig as ecec
import easybuild.tools.build_log
import easybuild.tools.robot as robot
from easybuild.framework.easyconfig.easyconfig import process_easyconfig, EasyConfig
from easybuild.framework.easyconfig.easyconfig import process_easyconfig, EasyConfig, make_easyconfig_dict
from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, find_resolved_modules, parse_easyconfigs
from easybuild.framework.easyconfig.tweak import tweak
from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy
Expand Down Expand Up @@ -1415,11 +1415,11 @@ def test_check_conflicts(self):

gzip_ec = os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb')
gompi_ec = os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb')
ecs, _ = parse_easyconfigs([(gzip_ec, False), (gompi_ec, False)])
non_conflict_ecs, _ = parse_easyconfigs([(gzip_ec, False), (gompi_ec, False)])

# no conflicts found, no output to stderr
self.mock_stderr(True)
conflicts = check_conflicts(ecs, self.modtool)
conflicts = check_conflicts(non_conflict_ecs, self.modtool)
stderr = self.get_stderr()
self.mock_stderr(False)
self.assertFalse(conflicts)
Expand All @@ -1441,6 +1441,13 @@ def test_check_conflicts(self):
self.assertTrue(conflicts)
self.assertIn("Conflict found for dependencies of foss-2018a: GCC-4.6.4 vs GCC-6.4.0-2.28", stderr)

# Can also return the text
with self.mocked_stdout_stderr(mock_stdout=False) as mocked_stderr:
conflict_lst = check_conflicts(ecs, self.modtool, return_conflicts=True)
self.assertEqual('\n'.join(conflict_lst), stderr.strip())
self.assertEqual(mocked_stderr.getvalue(), '')
self.assertEqual(check_conflicts(non_conflict_ecs, self.modtool, return_conflicts=True), [])

# conflicts between specified easyconfigs are also detected

# direct conflict on software version
Expand Down Expand Up @@ -1472,6 +1479,28 @@ def test_check_conflicts(self):
# test use of check_inter_ec_conflicts
self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False), "No conflicts found")

# Conflict in build dependencies is fine
hwloc_txt = read_file(os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'))
gzip_txt = read_file(os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.5-foss-2018a.eb'))
bzip_txt = read_file(os.path.join(test_easyconfigs, 'b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'))
tc = re.search(r"toolchain *=.*", hwloc_txt)[0]
bzip_txt += f"\n{tc}"
gzip_txt += f"\n{tc}"
gzip_txt_2 = gzip_txt.replace('1.5', '2.0')
hwloc_txt += "\nbuilddependencies = [('gzip', '2.0')]"
bzip_txt += "\nbuilddependencies = [('gzip', '1.5')]"
ecs = [make_easyconfig_dict(EasyConfig(path=None, rawtxt=txt))
for txt in (hwloc_txt, gzip_txt, gzip_txt_2, bzip_txt)]
self.assertFalse(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False),
"No conflicts should be found")
# But fail if A depends on B which depends on C and A has C as a build dependency in another version
hwloc_txt += "\ndependencies = [('bzip2', '1.0.6')]"
bzip_txt = bzip_txt.replace("builddependencies", "dependencies")
ecs = [make_easyconfig_dict(EasyConfig(path=None, rawtxt=txt))
for txt in (hwloc_txt, gzip_txt, gzip_txt_2, bzip_txt)]
self.assertTrue(check_conflicts(ecs, self.modtool, check_inter_ec_conflicts=False, return_conflicts=True),
"Conflict should be found")

def test_check_conflicts_wrapper_deps(self):
"""Test check_conflicts when dependency 'wrappers' are involved."""
test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
Expand Down