From c259c03531a98c07844f390d86f119a570b48905 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 27 Mar 2025 22:53:20 +0100 Subject: [PATCH 01/28] reverse logic that checks if filter matches context --- tools/filter.py | 137 +++++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 67 deletions(-) diff --git a/tools/filter.py b/tools/filter.py index 0caa2af8..b837df58 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -37,13 +37,13 @@ FILTER_COMPONENT_REPO ] -COMPONENT_TOO_SHORT = "component in filter spec '{component}:{pattern}' is too short; must be 3 characters or longer" -COMPONENT_UNKNOWN = "unknown component={component} in {component}:{pattern}" -FILTER_EMPTY_PATTERN = "pattern in filter string '{filter_string}' is empty" -FILTER_FORMAT_ERROR = "filter string '{filter_string}' does not conform to format 'component:pattern'" +COMPONENT_TOO_SHORT = "component in filter spec '{component}:{value}' is too short; must be 3 characters or longer" +COMPONENT_UNKNOWN = "unknown component={component} in {component}:{value}" +FILTER_EMPTY_VALUE = "value in filter string '{filter_string}' is empty" +FILTER_FORMAT_ERROR = "filter string '{filter_string}' does not conform to format 'component:value'" UNKNOWN_COMPONENT_CONST = "unknown component constant {component}" -Filter = namedtuple('Filter', ('component', 'pattern')) +Filter = namedtuple('Filter', ('component', 'value')) class EESSIBotActionFilterError(Exception): @@ -64,7 +64,7 @@ class EESSIBotActionFilter: Class for representing a filter that limits in which contexts bot commands are applied. A filter contains a list of key:value pairs where the key corresponds to a component (see FILTER_COMPONENTS) and the value is a - pattern used to filter commands based on the context a command is applied to. + string used to filter commands based on the context a command is applied to. """ def __init__(self, filter_string): """ @@ -101,15 +101,15 @@ def clear_all(self): """ self.action_filters = [] - def add_filter(self, component, pattern): + def add_filter(self, component, value): """ - Adds a filter given by a component and a pattern + Adds a filter given by a component and a value. Uses the full component + name even if only a prefix was used. Args: component (string): any prefix (min 3 characters long) of known filter components (see FILTER_COMPONENTS) - pattern (string): regular expression pattern that is applied to a - string representing the component + value (string): string that represents value of component Returns: None (implicitly) @@ -122,7 +122,7 @@ def add_filter(self, component, pattern): # - it is a 3+ character-long string, _and_ # - it is a prefix of one of the elements in FILTER_COMPONENTS if len(component) < 3: - msg = COMPONENT_TOO_SHORT.format(component=component, pattern=pattern) + msg = COMPONENT_TOO_SHORT.format(component=component, value=value) log(msg) raise EESSIBotActionFilterError(msg) full_component = None @@ -135,17 +135,17 @@ def add_filter(self, component, pattern): break if full_component: log(f"processing component {component}") - # replace '-' with '/' in pattern when using 'architecture' filter + # replace '-' with '/' in value when using 'architecture' filter # component (done to make sure that values are comparable) if full_component == FILTER_COMPONENT_ARCH: - pattern = pattern.replace('-', '/') - # replace '=' with '/' in pattern when using 'accelerator' filter + value = value.replace('-', '/') + # replace '=' with '/' in value when using 'accelerator' filter # component (done to make sure that values are comparable) if full_component == FILTER_COMPONENT_ACCEL: - pattern = pattern.replace('=', '/') - self.action_filters.append(Filter(full_component, pattern)) + value = value.replace('=', '/') + self.action_filters.append(Filter(full_component, value)) else: - msg = COMPONENT_UNKNOWN.format(component=component, pattern=pattern) + msg = COMPONENT_UNKNOWN.format(component=component, value=value) log(msg) raise EESSIBotActionFilterError(msg) @@ -154,14 +154,14 @@ def add_filter_from_string(self, filter_string): Adds a filter provided as a string Args: - filter_string (string): filter provided as component:pattern string + filter_string (string): filter provided as component:value string Returns: None (implicitly) Raises: EESSIBotActionFilterError: raised if filter_string does not conform - to 'component:pattern' format or pattern is empty + to 'component:value' format or value is empty """ _filter_split = filter_string.split(':') if len(_filter_split) != 2: @@ -169,48 +169,48 @@ def add_filter_from_string(self, filter_string): log(msg) raise EESSIBotActionFilterError(msg) if len(_filter_split[0]) < 3: - msg = COMPONENT_TOO_SHORT.format(component=_filter_split[0], pattern=_filter_split[1]) + msg = COMPONENT_TOO_SHORT.format(component=_filter_split[0], value=_filter_split[1]) log(msg) raise EESSIBotActionFilterError(msg) if len(_filter_split[1]) == 0: - msg = FILTER_EMPTY_PATTERN.format(filter_string=filter_string) + msg = FILTER_EMPTY_VALUE.format(filter_string=filter_string) log(msg) raise EESSIBotActionFilterError(msg) self.add_filter(_filter_split[0], _filter_split[1]) def get_filter_by_component(self, component): """ - Returns filter pattern for component. + Returns filter value for component. Args: component (string): one of FILTER_COMPONENTS Returns: - (list): list of pattern for filters whose component matches argument + (list): list of value for filters whose component matches argument """ if component not in FILTER_COMPONENTS: msg = UNKNOWN_COMPONENT_CONST.format(component=component) raise EESSIBotActionFilterError(msg) - pattern = [] + values = [] for _filter in self.action_filters: if component == _filter.component: - pattern.append(_filter.pattern) - return pattern + values.append(_filter.value) + return values - def remove_filter(self, component, pattern): + def remove_filter(self, component, value): """ - Removes all elements matching the filter given by (component, pattern) + Removes all elements matching the filter given by (component, value) Args: component (string): one of FILTER_COMPONENTS - pattern (string): regex that is applied to a string representing the component + value (string): value for the component Returns: None (implicitly) """ if len(component) < 3: - msg = COMPONENT_TOO_SHORT.format(component=component, pattern=pattern) + msg = COMPONENT_TOO_SHORT.format(component=component, value=value) log(msg) raise EESSIBotActionFilterError(msg) full_component = None @@ -223,7 +223,7 @@ def remove_filter(self, component, pattern): break if not full_component: # the component provided as argument is not in the list of FILTER_COMPONENTS - msg = COMPONENT_UNKNOWN.format(component=component, pattern=pattern) + msg = COMPONENT_UNKNOWN.format(component=component, value=value) log(msg) raise EESSIBotActionFilterError(msg) @@ -238,8 +238,8 @@ def remove_filter(self, component, pattern): # 3-character-long prefix (e.g., 'repository' and 'repeat' would # have the same prefix 'rep') _filter = self.action_filters[index] - if _filter.component.startswith(component) and pattern == _filter.pattern: - log(f"removing filter ({_filter.component}, {pattern})") + if _filter.component.startswith(component) and value == _filter.value: + log(f"removing filter ({_filter.component}, {value})") self.action_filters.pop(index) def to_string(self): @@ -255,52 +255,55 @@ def to_string(self): filter_str_list = [] for _filter in self.action_filters: cm = _filter.component - re = _filter.pattern + re = _filter.value filter_str_list.append(f"{cm}:{re}") return " ".join(filter_str_list) def check_filters(self, context): """ - Checks filters for a given context which is defined by one to five - components (accelerator, architecture, instance, job, repository) + Checks if a filter matches a given context which is defined by three + components (architecture, instance, repository) + A single component of a filter matches the corresponding component in a + context if the values for these components are exactly the same. Args: - context (dict) : dictionary that maps components to their value + context (dict) : dictionary that contents (component,value)-pairs + representing the context in which the method is called Returns: - True if no filters are defined or all defined filters match - their corresponding component in the given context - False if any defined filter does not match its corresponding - component in the given context + True if all defined components in a given context match their + corresponding component in a filter + False otherwise """ - # if no filters are defined we return True - if len(self.action_filters) == 0: - return True - - # we have at least one filter which has to match or we return False + # default result is False check = False # examples: # filter: 'arch:intel instance:AWS' --> evaluates to True if - # context['architecture'] matches 'intel' and if - # context['instance'] matches 'AWS' + # context['architecture'] is 'intel' and if context['instance'] is 'AWS' # filter: 'repository:eessi-2023.06' --> evaluates to True if - # context['repository'] matches 'eessi-2023.06' - - # we iterate over all defined filters - for af in self.action_filters: - if af.component in context: - value = context[af.component] - # replace - with / in architecture component - if af.component == FILTER_COMPONENT_ARCH: - value = value.replace('-', '/') - # replace = with / in accelerator component - if af.component == FILTER_COMPONENT_ACCEL: - value = value.replace('=', '/') - if re.search(af.pattern, value): - # if the pattern of the filter matches - check = True - else: - check = False - break - return check + # context['repository'] is 'eessi-2023.06' + + # we iterate over the three components 'architecture', 'instance' and + # 'repository' and verify if their value in the given context matches + # the corresponding value in the filter + context_components = [ FILTER_COMPONENT_ARCH, FILTER_COMPONENT_INST, FILTER_COMPONENT_REPO ] + for component in context_components: + filter_values = self.get_filter_by_component(component) + if len(filter_values) == 0: + # no filter for component provided --> at least one filter per + # component MUST be given + return False + if component in context: + # check if all values for the filter are identical to the value + # of the component in the context + context_value = context[component] + if component == FILTER_COMPONENT_ARCH: + context_value = context_value.replace('-', '/') + for filter_value in filter_values: + if filter_value != context_value: + return False + else: + # missing component in context, could lead to unexpected behavior + return False + return True From 6b60be3f04df8732b8d10748c10747256af1dacd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 27 Mar 2025 23:19:46 +0100 Subject: [PATCH 02/28] fix hound issues --- tools/filter.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/filter.py b/tools/filter.py index b837df58..38e38aa4 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -11,7 +11,6 @@ # Standard library imports from collections import namedtuple -import re # Third party imports (anything installed into the local Python environment) from pyghee.utils import log @@ -254,9 +253,9 @@ def to_string(self): """ filter_str_list = [] for _filter in self.action_filters: - cm = _filter.component - re = _filter.value - filter_str_list.append(f"{cm}:{re}") + component = _filter.component + value = _filter.value + filter_str_list.append(f"{component}:{value}") return " ".join(filter_str_list) def check_filters(self, context): @@ -287,7 +286,7 @@ def check_filters(self, context): # we iterate over the three components 'architecture', 'instance' and # 'repository' and verify if their value in the given context matches # the corresponding value in the filter - context_components = [ FILTER_COMPONENT_ARCH, FILTER_COMPONENT_INST, FILTER_COMPONENT_REPO ] + context_components = [FILTER_COMPONENT_ARCH, FILTER_COMPONENT_INST, FILTER_COMPONENT_REPO] for component in context_components: filter_values = self.get_filter_by_component(component) if len(filter_values) == 0: From f40589d02e57c3a3b8e2809571f3640d484890ff Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 27 Mar 2025 23:46:51 +0100 Subject: [PATCH 03/28] update tests for filters --- tests/test_tools_filter.py | 104 ++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/tests/test_tools_filter.py b/tests/test_tools_filter.py index b689aa60..880204fb 100644 --- a/tests/test_tools_filter.py +++ b/tests/test_tools_filter.py @@ -20,7 +20,7 @@ COMPONENT_UNKNOWN, EESSIBotActionFilter, EESSIBotActionFilterError, - FILTER_EMPTY_PATTERN, + FILTER_EMPTY_VALUE, FILTER_FORMAT_ERROR) @@ -34,9 +34,9 @@ def test_empty_action_filter(): def test_add_wellformed_filter_from_string(): af = EESSIBotActionFilter("") component = 'acc' - pattern = 'nvidia/cc80' - af.add_filter_from_string(f"{component}:{pattern}") - expected = f"accelerator:{pattern}" + value = 'nvidia/cc80' + af.add_filter_from_string(f"{component}:{value}") + expected = f"accelerator:{value}" actual = af.to_string() assert expected == actual @@ -52,29 +52,29 @@ def test_add_non_wellformed_filter_from_string(): assert str(err1.value) == expected_msg1 component2 = 'a' - pattern2 = 'zen4' - filter_string2 = f"{component2}:{pattern2}" + value2 = 'zen4' + filter_string2 = f"{component2}:{value2}" with pytest.raises(Exception) as err2: af.add_filter_from_string(filter_string2) assert err2.type == EESSIBotActionFilterError - expected_msg2 = COMPONENT_TOO_SHORT.format(component=component2, pattern=pattern2) + expected_msg2 = COMPONENT_TOO_SHORT.format(component=component2, value=value2) assert str(err2.value) == expected_msg2 component3 = 'arc' - pattern3 = '' - filter_string3 = f"{component3}:{pattern3}" + value3 = '' + filter_string3 = f"{component3}:{value3}" with pytest.raises(Exception) as err3: af.add_filter_from_string(filter_string3) assert err3.type == EESSIBotActionFilterError - expected_msg3 = FILTER_EMPTY_PATTERN.format(filter_string=filter_string3) + expected_msg3 = FILTER_EMPTY_VALUE.format(filter_string=filter_string3) assert str(err3.value) == expected_msg3 def test_add_single_action_filter(): af = EESSIBotActionFilter("") component = 'arch' - pattern = '.*intel.*' - af.add_filter(component, pattern) + value = '.*intel.*' + af.add_filter(component, value) expected = "architecture:.*intel.*" actual = af.to_string() assert expected == actual @@ -83,22 +83,22 @@ def test_add_single_action_filter(): def test_add_non_supported_component(): af = EESSIBotActionFilter("") component = 'machine' - pattern = '.*intel.*' + value = '.*intel.*' with pytest.raises(Exception) as err: - af.add_filter(component, pattern) + af.add_filter(component, value) assert err.type == EESSIBotActionFilterError - expected_msg = COMPONENT_UNKNOWN.format(component=component, pattern=pattern) + expected_msg = COMPONENT_UNKNOWN.format(component=component, value=value) assert str(err.value) == expected_msg def test_add_too_short_supported_component(): af = EESSIBotActionFilter("") component = 'a' - pattern = '.*intel.*' + value = '.*intel.*' with pytest.raises(Exception) as err: - af.add_filter(component, pattern) + af.add_filter(component, value) assert err.type == EESSIBotActionFilterError - expected_msg = COMPONENT_TOO_SHORT.format(component=component, pattern=pattern) + expected_msg = COMPONENT_TOO_SHORT.format(component=component, value=value) assert str(err.value) == expected_msg @@ -107,31 +107,31 @@ def test_add_too_short_supported_component(): def complex_filter(): af = EESSIBotActionFilter("") component1 = 'arch' - pattern1 = '.*intel.*' - af.add_filter(component1, pattern1) + value1 = '.*intel.*' + af.add_filter(component1, value1) component2 = 'repo' - pattern2 = 'nessi.no-2022.*' - af.add_filter(component2, pattern2) + value2 = 'nessi.no-2022.*' + af.add_filter(component2, value2) component3 = 'inst' - pattern3 = '[aA]' - af.add_filter(component3, pattern3) + value3 = '[aA]' + af.add_filter(component3, value3) yield af def test_remove_existing_filter(complex_filter): component1 = 'architecture' - pattern1 = '.*intel.*' - filter_string1 = f"{component1}:{pattern1}" + value1 = '.*intel.*' + filter_string1 = f"{component1}:{value1}" component2 = 'repository' - pattern2 = 'nessi.no-2022.*' - filter_string2 = f"{component2}:{pattern2}" + value2 = 'nessi.no-2022.*' + filter_string2 = f"{component2}:{value2}" component3 = 'instance' - pattern3 = '[aA]' - filter_string3 = f"{component3}:{pattern3}" + value3 = '[aA]' + filter_string3 = f"{component3}:{value3}" # remove last filter org_filter = copy.deepcopy(complex_filter) - org_filter.remove_filter(component3, pattern3) + org_filter.remove_filter(component3, value3) expected = filter_string1 expected += f" {filter_string2}" actual = org_filter.to_string() @@ -139,7 +139,7 @@ def test_remove_existing_filter(complex_filter): # remove second last filter org_filter = copy.deepcopy(complex_filter) - org_filter.remove_filter(component2, pattern2) + org_filter.remove_filter(component2, value2) expected = filter_string1 expected += f" {filter_string3}" actual = org_filter.to_string() @@ -147,7 +147,7 @@ def test_remove_existing_filter(complex_filter): # remove first filter org_filter = copy.deepcopy(complex_filter) - org_filter.remove_filter(component1, pattern1) + org_filter.remove_filter(component1, value1) expected = filter_string2 expected += f" {filter_string3}" actual = org_filter.to_string() @@ -156,11 +156,11 @@ def test_remove_existing_filter(complex_filter): def test_remove_non_existing_filter(complex_filter): component = 'accel' - pattern = 'amd/gfx90a' + value = 'amd/gfx90a' # remove non-existing filter org_filter = copy.deepcopy(complex_filter) - org_filter.remove_filter(component, pattern) + org_filter.remove_filter(component, value) org_filter_str = org_filter.to_string() complex_filter_str = complex_filter.to_string() assert org_filter_str == complex_filter_str @@ -168,24 +168,24 @@ def test_remove_non_existing_filter(complex_filter): def test_remove_filter_errors(complex_filter): component1 = 'ac' - pattern1 = 'amd/gfx90a' + value1 = 'amd/gfx90a' component2 = 'operating_system' - pattern2 = 'linux' + value2 = 'linux' # remove filter using too short component name org_filter = copy.deepcopy(complex_filter) with pytest.raises(Exception) as err1: - org_filter.remove_filter(component1, pattern1) + org_filter.remove_filter(component1, value1) assert err1.type == EESSIBotActionFilterError - expected_msg1 = COMPONENT_TOO_SHORT.format(component=component1, pattern=pattern1) + expected_msg1 = COMPONENT_TOO_SHORT.format(component=component1, value=value1) assert str(err1.value) == expected_msg1 # remove filter using unknown component name org_filter = copy.deepcopy(complex_filter) with pytest.raises(Exception) as err2: - org_filter.remove_filter(component2, pattern2) + org_filter.remove_filter(component2, value2) assert err2.type == EESSIBotActionFilterError - expected_msg2 = COMPONENT_UNKNOWN.format(component=component2, pattern=pattern2) + expected_msg2 = COMPONENT_UNKNOWN.format(component=component2, value=value2) assert str(err2.value) == expected_msg2 @@ -204,9 +204,9 @@ def test_check_matching_empty_filter(): def test_check_matching_simple_filter(): af = EESSIBotActionFilter("") component = 'arch' - pattern = '.*intel.*' - af.add_filter(component, pattern) - expected = f"architecture:{pattern}" + value = '.*intel.*' + af.add_filter(component, value) + expected = f"architecture:{value}" actual = af.to_string() assert expected == actual @@ -256,8 +256,8 @@ def test_non_match_architecture_repository_context(complex_filter): def arch_filter_slash_syntax(): af = EESSIBotActionFilter("") component = 'arch' - pattern = '.*/intel/.*' - af.add_filter(component, pattern) + value = '.*/intel/.*' + af.add_filter(component, value) yield af @@ -277,8 +277,8 @@ def test_match_architecture_syntax_slash(arch_filter_slash_syntax): def arch_filter_dash_syntax(): af = EESSIBotActionFilter("") component = 'arch' - pattern = '.*-intel-.*' - af.add_filter(component, pattern) + value = '.*-intel-.*' + af.add_filter(component, value) yield af @@ -298,8 +298,8 @@ def test_match_architecture_syntax_dash(arch_filter_dash_syntax): def accel_filter_slash_syntax(): af = EESSIBotActionFilter("") component = 'accel' - pattern = 'nvidia/.*' - af.add_filter(component, pattern) + value = 'nvidia/.*' + af.add_filter(component, value) yield af @@ -319,8 +319,8 @@ def test_match_accelerator_syntax_slash(accel_filter_slash_syntax): def accel_filter_equal_syntax(): af = EESSIBotActionFilter("") component = 'accel' - pattern = 'amd=gfx90a' - af.add_filter(component, pattern) + value = 'amd=gfx90a' + af.add_filter(component, value) yield af From c78feb0100251f07fb2ef5cb099aec65ba934a18 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 28 Mar 2025 00:20:39 +0100 Subject: [PATCH 04/28] update pytests --- tests/test_tools_filter.py | 59 +++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/tests/test_tools_filter.py b/tests/test_tools_filter.py index 880204fb..0c21d71b 100644 --- a/tests/test_tools_filter.py +++ b/tests/test_tools_filter.py @@ -195,8 +195,19 @@ def test_check_matching_empty_filter(): actual = af.to_string() assert expected == actual - context = {"arch": "foo"} + context = {"architecture": "foo"} actual = af.check_filters(context) + expected = False + assert expected == actual + + context = {"architecture": "foo", "instance": "me", "repository": "base"} + actual = af.check_filters(context) + expected = False + assert expected == actual + + af2 = EESSIBotActionFilter("arch:foo inst:me repo:base") + print("af2: %s" % af2.to_string()) + actual = af2.check_filters(context) expected = True assert expected == actual @@ -212,6 +223,12 @@ def test_check_matching_simple_filter(): context = {"architecture": "x86_64/intel/cascadelake"} actual = af.check_filters(context) + expected = False + assert expected == actual + + context = {"architecture": ".*intel.*", "instance": "[aA]", "repository": "nessi.no-2022.*"} + af2 = EESSIBotActionFilter("arch:.*intel.* inst:[aA] repo:nessi.no-2022.*") + actual = af2.check_filters(context) expected = True assert expected == actual @@ -233,14 +250,14 @@ def test_match_empty_context(complex_filter): def test_match_architecture_context(complex_filter): context = {"architecture": "x86_64/intel/cascadelake"} - expected = True + expected = False actual = complex_filter.check_filters(context) assert expected == actual def test_match_architecture_job_context(complex_filter): context = {"architecture": "x86_64/intel/cascadelake", "job": 1234} - expected = True + expected = False actual = complex_filter.check_filters(context) assert expected == actual @@ -254,20 +271,17 @@ def test_non_match_architecture_repository_context(complex_filter): @pytest.fixture def arch_filter_slash_syntax(): - af = EESSIBotActionFilter("") - component = 'arch' - value = '.*/intel/.*' - af.add_filter(component, value) + af = EESSIBotActionFilter("arch:.*/intel/.* repo:EESSI inst:aws") yield af def test_match_architecture_syntax_slash(arch_filter_slash_syntax): - context = {"architecture": "x86_64/intel/cascadelake", "repository": "EESSI"} + context = {"architecture": ".*/intel/.*", "repository": "EESSI", "instance": "aws"} expected = True actual = arch_filter_slash_syntax.check_filters(context) assert expected == actual - context = {"architecture": "x86_64-intel-cascadelake"} + context = {"architecture": ".*-intel-.*", "repository": "EESSI", "instance": "aws"} expected = True actual = arch_filter_slash_syntax.check_filters(context) assert expected == actual @@ -275,20 +289,17 @@ def test_match_architecture_syntax_slash(arch_filter_slash_syntax): @pytest.fixture def arch_filter_dash_syntax(): - af = EESSIBotActionFilter("") - component = 'arch' - value = '.*-intel-.*' - af.add_filter(component, value) + af = EESSIBotActionFilter("arch:.*-intel-.* repo:EESSI inst:azure") yield af def test_match_architecture_syntax_dash(arch_filter_dash_syntax): - context = {"architecture": "x86_64-intel-cascadelake", "repository": "EESSI"} + context = {"architecture": ".*-intel-.*", "repository": "EESSI", "instance": "azure"} expected = True actual = arch_filter_dash_syntax.check_filters(context) assert expected == actual - context = {"architecture": "x86_64/intel-cascadelake"} + context = {"architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} expected = True actual = arch_filter_dash_syntax.check_filters(context) assert expected == actual @@ -296,20 +307,17 @@ def test_match_architecture_syntax_dash(arch_filter_dash_syntax): @pytest.fixture def accel_filter_slash_syntax(): - af = EESSIBotActionFilter("") - component = 'accel' - value = 'nvidia/.*' - af.add_filter(component, value) + af = EESSIBotActionFilter("acc:nvidia/.* arch:.*-intel-.* repo:EESSI inst:azure") yield af def test_match_accelerator_syntax_slash(accel_filter_slash_syntax): - context = {"accelerator": "nvidia/cc70", "repository": "EESSI"} + context = {"accelerator": "nvidia/cc70", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} expected = True actual = accel_filter_slash_syntax.check_filters(context) assert expected == actual - context = {"accelerator": "nvidia=cc70"} + context = {"accelerator": "nvidia=cc70", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} expected = True actual = accel_filter_slash_syntax.check_filters(context) assert expected == actual @@ -317,20 +325,17 @@ def test_match_accelerator_syntax_slash(accel_filter_slash_syntax): @pytest.fixture def accel_filter_equal_syntax(): - af = EESSIBotActionFilter("") - component = 'accel' - value = 'amd=gfx90a' - af.add_filter(component, value) + af = EESSIBotActionFilter("acc:amd=gfx90a arch:.*-intel-.* repo:EESSI inst:azure") yield af def test_match_accelerator_syntax_equal(accel_filter_equal_syntax): - context = {"accelerator": "amd=gfx90a", "repository": "EESSI"} + context = {"accelerator": "amd=gfx90a", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} expected = True actual = accel_filter_equal_syntax.check_filters(context) assert expected == actual - context = {"accelerator": "amd/gfx90a"} + context = {"accelerator": "amd/gfx90a", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} expected = True actual = accel_filter_equal_syntax.check_filters(context) assert expected == actual From e5aca36e474638596592bc635f2c2bd122a5fe61 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 28 Mar 2025 00:22:33 +0100 Subject: [PATCH 05/28] remove unused variable --- tools/filter.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/filter.py b/tools/filter.py index 38e38aa4..ccf70da7 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -274,9 +274,6 @@ def check_filters(self, context): corresponding component in a filter False otherwise """ - # default result is False - check = False - # examples: # filter: 'arch:intel instance:AWS' --> evaluates to True if # context['architecture'] is 'intel' and if context['instance'] is 'AWS' From 0c01df6205cca6fad3847e87b1c51ead7c97d99b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 30 Mar 2025 08:40:07 +0200 Subject: [PATCH 06/28] Revert "update pytests" This reverts commit c78feb0100251f07fb2ef5cb099aec65ba934a18. We take a different approach aiming at only modifying the existing tests such that they continue to test what they originally were built to test, and add new tests that cover new or modified capabilities. --- tests/test_tools_filter.py | 59 +++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/tests/test_tools_filter.py b/tests/test_tools_filter.py index 0c21d71b..880204fb 100644 --- a/tests/test_tools_filter.py +++ b/tests/test_tools_filter.py @@ -195,19 +195,8 @@ def test_check_matching_empty_filter(): actual = af.to_string() assert expected == actual - context = {"architecture": "foo"} + context = {"arch": "foo"} actual = af.check_filters(context) - expected = False - assert expected == actual - - context = {"architecture": "foo", "instance": "me", "repository": "base"} - actual = af.check_filters(context) - expected = False - assert expected == actual - - af2 = EESSIBotActionFilter("arch:foo inst:me repo:base") - print("af2: %s" % af2.to_string()) - actual = af2.check_filters(context) expected = True assert expected == actual @@ -223,12 +212,6 @@ def test_check_matching_simple_filter(): context = {"architecture": "x86_64/intel/cascadelake"} actual = af.check_filters(context) - expected = False - assert expected == actual - - context = {"architecture": ".*intel.*", "instance": "[aA]", "repository": "nessi.no-2022.*"} - af2 = EESSIBotActionFilter("arch:.*intel.* inst:[aA] repo:nessi.no-2022.*") - actual = af2.check_filters(context) expected = True assert expected == actual @@ -250,14 +233,14 @@ def test_match_empty_context(complex_filter): def test_match_architecture_context(complex_filter): context = {"architecture": "x86_64/intel/cascadelake"} - expected = False + expected = True actual = complex_filter.check_filters(context) assert expected == actual def test_match_architecture_job_context(complex_filter): context = {"architecture": "x86_64/intel/cascadelake", "job": 1234} - expected = False + expected = True actual = complex_filter.check_filters(context) assert expected == actual @@ -271,17 +254,20 @@ def test_non_match_architecture_repository_context(complex_filter): @pytest.fixture def arch_filter_slash_syntax(): - af = EESSIBotActionFilter("arch:.*/intel/.* repo:EESSI inst:aws") + af = EESSIBotActionFilter("") + component = 'arch' + value = '.*/intel/.*' + af.add_filter(component, value) yield af def test_match_architecture_syntax_slash(arch_filter_slash_syntax): - context = {"architecture": ".*/intel/.*", "repository": "EESSI", "instance": "aws"} + context = {"architecture": "x86_64/intel/cascadelake", "repository": "EESSI"} expected = True actual = arch_filter_slash_syntax.check_filters(context) assert expected == actual - context = {"architecture": ".*-intel-.*", "repository": "EESSI", "instance": "aws"} + context = {"architecture": "x86_64-intel-cascadelake"} expected = True actual = arch_filter_slash_syntax.check_filters(context) assert expected == actual @@ -289,17 +275,20 @@ def test_match_architecture_syntax_slash(arch_filter_slash_syntax): @pytest.fixture def arch_filter_dash_syntax(): - af = EESSIBotActionFilter("arch:.*-intel-.* repo:EESSI inst:azure") + af = EESSIBotActionFilter("") + component = 'arch' + value = '.*-intel-.*' + af.add_filter(component, value) yield af def test_match_architecture_syntax_dash(arch_filter_dash_syntax): - context = {"architecture": ".*-intel-.*", "repository": "EESSI", "instance": "azure"} + context = {"architecture": "x86_64-intel-cascadelake", "repository": "EESSI"} expected = True actual = arch_filter_dash_syntax.check_filters(context) assert expected == actual - context = {"architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} + context = {"architecture": "x86_64/intel-cascadelake"} expected = True actual = arch_filter_dash_syntax.check_filters(context) assert expected == actual @@ -307,17 +296,20 @@ def test_match_architecture_syntax_dash(arch_filter_dash_syntax): @pytest.fixture def accel_filter_slash_syntax(): - af = EESSIBotActionFilter("acc:nvidia/.* arch:.*-intel-.* repo:EESSI inst:azure") + af = EESSIBotActionFilter("") + component = 'accel' + value = 'nvidia/.*' + af.add_filter(component, value) yield af def test_match_accelerator_syntax_slash(accel_filter_slash_syntax): - context = {"accelerator": "nvidia/cc70", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} + context = {"accelerator": "nvidia/cc70", "repository": "EESSI"} expected = True actual = accel_filter_slash_syntax.check_filters(context) assert expected == actual - context = {"accelerator": "nvidia=cc70", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} + context = {"accelerator": "nvidia=cc70"} expected = True actual = accel_filter_slash_syntax.check_filters(context) assert expected == actual @@ -325,17 +317,20 @@ def test_match_accelerator_syntax_slash(accel_filter_slash_syntax): @pytest.fixture def accel_filter_equal_syntax(): - af = EESSIBotActionFilter("acc:amd=gfx90a arch:.*-intel-.* repo:EESSI inst:azure") + af = EESSIBotActionFilter("") + component = 'accel' + value = 'amd=gfx90a' + af.add_filter(component, value) yield af def test_match_accelerator_syntax_equal(accel_filter_equal_syntax): - context = {"accelerator": "amd=gfx90a", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} + context = {"accelerator": "amd=gfx90a", "repository": "EESSI"} expected = True actual = accel_filter_equal_syntax.check_filters(context) assert expected == actual - context = {"accelerator": "amd/gfx90a", "architecture": ".*/intel-.*", "repository": "EESSI", "instance": "azure"} + context = {"accelerator": "amd/gfx90a"} expected = True actual = accel_filter_equal_syntax.check_filters(context) assert expected == actual From 41ef8041c054011187124fb78fcbe0af0043d616 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 30 Mar 2025 08:43:53 +0200 Subject: [PATCH 07/28] refactoring check_*filters functions --- tools/filter.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/tools/filter.py b/tools/filter.py index ccf70da7..799bed08 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -258,10 +258,11 @@ def to_string(self): filter_str_list.append(f"{component}:{value}") return " ".join(filter_str_list) - def check_filters(self, context): + def check_build_filters(self, context): """ - Checks if a filter matches a given context which is defined by three - components (architecture, instance, repository) + Checks if the filter in self matches a given context which is defined + by the three components FILTER_COMPONENT_ARCH, FILTER_COMPONENT_INST and + FILTER_COMPONENT_REPO for building. A single component of a filter matches the corresponding component in a context if the values for these components are exactly the same. @@ -270,7 +271,28 @@ def check_filters(self, context): representing the context in which the method is called Returns: - True if all defined components in a given context match their + True if all required components in a given context match their + corresponding component in a filter + False otherwise + """ + build_components = [FILTER_COMPONENT_ARCH, FILTER_COMPONENT_INST, FILTER_COMPONENT_REPO] + return self.check_filters(context, build_components) + + def check_filters(self, context, components): + """ + Checks if the filter in self matches a given context which is defined + by components. + A single component of a filter matches the corresponding component in a + context if the values for these components are exactly the same. + + Args: + context (dict) : dictionary that contents (component,value)-pairs + representing the context in which the method is called + components (list) : contains constants FILTER_COMPONENT_* that must + be checked + + Returns: + True if all required components in a given context match their corresponding component in a filter False otherwise """ @@ -280,11 +302,9 @@ def check_filters(self, context): # filter: 'repository:eessi-2023.06' --> evaluates to True if # context['repository'] is 'eessi-2023.06' - # we iterate over the three components 'architecture', 'instance' and - # 'repository' and verify if their value in the given context matches - # the corresponding value in the filter - context_components = [FILTER_COMPONENT_ARCH, FILTER_COMPONENT_INST, FILTER_COMPONENT_REPO] - for component in context_components: + # we iterate over the components and verify if their value in the given + # context matches the corresponding value in the filter + for component in components: filter_values = self.get_filter_by_component(component) if len(filter_values) == 0: # no filter for component provided --> at least one filter per From da091c9712535c085441774eee97bbf7080a5ea1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 1 Apr 2025 13:13:39 +0200 Subject: [PATCH 08/28] add string conversion for ACCEL filter - also improve docstring - add a potential TODO --- tools/filter.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/filter.py b/tools/filter.py index 799bed08..4fef7875 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -288,6 +288,9 @@ def check_filters(self, context, components): Args: context (dict) : dictionary that contents (component,value)-pairs representing the context in which the method is called + The context is typically defined by a combination of the build + target (CPU microarchitecture, CernVM-FS repository, optionally + accelerator architecture) and the build system. components (list) : contains constants FILTER_COMPONENT_* that must be checked @@ -302,6 +305,9 @@ def check_filters(self, context, components): # filter: 'repository:eessi-2023.06' --> evaluates to True if # context['repository'] is 'eessi-2023.06' + # TODO should we require a 'double'-match, that is, all components given + # have to match AND all filters given have to match? + # # we iterate over the components and verify if their value in the given # context matches the corresponding value in the filter for component in components: @@ -316,6 +322,8 @@ def check_filters(self, context, components): context_value = context[component] if component == FILTER_COMPONENT_ARCH: context_value = context_value.replace('-', '/') + if component == FILTER_COMPONENT_ACCEL: + context_value = context_value.replace('=', '/') for filter_value in filter_values: if filter_value != context_value: return False From 3d1899d6ec01f97f2f433bc203f8a72701a1ef4f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 1 Apr 2025 13:15:24 +0200 Subject: [PATCH 09/28] adjust tests for filters to use additional argument --- tests/test_tools_filter.py | 213 +++++++++++++++++++++++++++++++------ 1 file changed, 180 insertions(+), 33 deletions(-) diff --git a/tests/test_tools_filter.py b/tests/test_tools_filter.py index 880204fb..cde4f38e 100644 --- a/tests/test_tools_filter.py +++ b/tests/test_tools_filter.py @@ -20,8 +20,13 @@ COMPONENT_UNKNOWN, EESSIBotActionFilter, EESSIBotActionFilterError, + FILTER_COMPONENT_ACCEL, + FILTER_COMPONENT_ARCH, + FILTER_COMPONENT_INST, + FILTER_COMPONENT_REPO, FILTER_EMPTY_VALUE, - FILTER_FORMAT_ERROR) + FILTER_FORMAT_ERROR, + UNKNOWN_COMPONENT_CONST) def test_empty_action_filter(): @@ -189,19 +194,54 @@ def test_remove_filter_errors(complex_filter): assert str(err2.value) == expected_msg2 -def test_check_matching_empty_filter(): +def test_empty_filter_to_string(): af = EESSIBotActionFilter("") expected = '' actual = af.to_string() assert expected == actual + +def test_empty_filter_empty_context_no_component(): + af = EESSIBotActionFilter("") + context = {} + components = [] + actual = af.check_filters(context, components) + expected = True + assert expected == actual + + +def test_empty_filter_arch_context_no_component(): + af = EESSIBotActionFilter("") context = {"arch": "foo"} - actual = af.check_filters(context) + components = [] + actual = af.check_filters(context, components) expected = True assert expected == actual -def test_check_matching_simple_filter(): +def test_empty_filter_arch_context_arch_component(): + af = EESSIBotActionFilter("") + context = {"arch": "foo"} + components = [FILTER_COMPONENT_ARCH] + actual = af.check_filters(context, components) + expected = False + # if component to check is given, they need to be present in filter and + # context + assert expected == actual + + +def test_empty_filter_empty_context_arch_component(): + af = EESSIBotActionFilter("") + context = {} + components = [FILTER_COMPONENT_ARCH] + actual = af.check_filters(context, components) + expected = False + # if component to check is given, they need to be present in filter and + # context + assert expected == actual + + +def test_arch_filter_to_string(): af = EESSIBotActionFilter("") component = 'arch' value = '.*intel.*' @@ -210,13 +250,94 @@ def test_check_matching_simple_filter(): actual = af.to_string() assert expected == actual + +def test_arch_filter_no_context_other_component(): + af = EESSIBotActionFilter("") + component = 'arch' + value = 'x86_64/intel/cascadelake' + af.add_filter(component, value) + context = {"architecture": "x86_64/intel/cascadelake"} + components = ['OTHER'] + with pytest.raises(Exception) as err: + af.check_filters(context, components) + assert err.type == EESSIBotActionFilterError + expected_msg = UNKNOWN_COMPONENT_CONST.format(component=components[0]) + assert str(err.value) == expected_msg + + +def test_arch_filter_arch_context_arch_component(): + af = EESSIBotActionFilter("") + component = 'arch' + value = 'x86_64/intel/cascadelake' + af.add_filter(component, value) context = {"architecture": "x86_64/intel/cascadelake"} - actual = af.check_filters(context) + components = [FILTER_COMPONENT_ARCH] + actual = af.check_filters(context, components) expected = True assert expected == actual -def test_create_complex_filter(complex_filter): +def test_arch_filter_arch_context_no_component(): + af = EESSIBotActionFilter("") + component = 'arch' + value = 'x86_64/intel/cascadelake' + af.add_filter(component, value) + context = {"architecture": "x86_64/intel/cascadelake"} + components = [] + actual = af.check_filters(context, components) + expected = True + assert expected == actual + + +def test_arch_filter_no_context_no_component(): + af = EESSIBotActionFilter("") + component = 'arch' + value = 'x86_64/intel/cascadelake' + af.add_filter(component, value) + context = {} + components = [] + actual = af.check_filters(context, components) + expected = True + assert expected == actual + + +def test_arch_filter_no_context_arch_component(): + af = EESSIBotActionFilter("") + component = 'arch' + value = 'x86_64/intel/cascadelake' + af.add_filter(component, value) + context = {} + components = [FILTER_COMPONENT_ARCH] + actual = af.check_filters(context, components) + expected = False + assert expected == actual + + +def test_arch_filter_arch_context_inst_component(): + af = EESSIBotActionFilter("") + component = 'arch' + value = 'x86_64/intel/cascadelake' + af.add_filter(component, value) + context = {"architecture": "x86_64/intel/cascadelake"} + components = [FILTER_COMPONENT_INST] + actual = af.check_filters(context, components) + expected = False + assert expected == actual + + +def test_arch_filter_arch_context_two_components(): + af = EESSIBotActionFilter("") + component = 'arch' + value = 'x86_64/intel/cascadelake' + af.add_filter(component, value) + context = {"architecture": "x86_64/intel/cascadelake"} + components = [FILTER_COMPONENT_ARCH, FILTER_COMPONENT_INST] + actual = af.check_filters(context, components) + expected = False + assert expected == actual + + +def test_complex_filter_to_string(complex_filter): expected = "architecture:.*intel.*" expected += " repository:nessi.no-2022.*" expected += " instance:[aA]" @@ -224,31 +345,53 @@ def test_create_complex_filter(complex_filter): assert expected == actual -def test_match_empty_context(complex_filter): +def test_complex_filter_no_context_other_component(complex_filter): + context = {"architecture": ".*intel.*"} + components = ['OTHER'] + with pytest.raises(Exception) as err: + complex_filter.check_filters(context, components) + assert err.type == EESSIBotActionFilterError + expected_msg = UNKNOWN_COMPONENT_CONST.format(component=components[0]) + assert str(err.value) == expected_msg + + +def test_complex_filter_no_context_no_component(complex_filter): context = {} - expected = False - actual = complex_filter.check_filters(context) + components = [] + expected = True + actual = complex_filter.check_filters(context, components) assert expected == actual -def test_match_architecture_context(complex_filter): - context = {"architecture": "x86_64/intel/cascadelake"} +#def test_complex_filter_no_context_no_component(complex_filter): +# context = {} +# components = [] +# expected = True +# actual = complex_filter.check_filters(context, components) +# assert expected == actual + + +def test_complex_filter_arch_context_arch_component(complex_filter): + context = {"architecture": ".*intel.*"} + components = [FILTER_COMPONENT_ARCH] expected = True - actual = complex_filter.check_filters(context) + actual = complex_filter.check_filters(context, components) assert expected == actual -def test_match_architecture_job_context(complex_filter): - context = {"architecture": "x86_64/intel/cascadelake", "job": 1234} +def test_complex_filter_job_context_arch_component(complex_filter): + context = {"architecture": ".*intel.*", "job": 1234} + components = [FILTER_COMPONENT_ARCH] expected = True - actual = complex_filter.check_filters(context) + actual = complex_filter.check_filters(context, components) assert expected == actual -def test_non_match_architecture_repository_context(complex_filter): - context = {"architecture": "x86_64/intel/cascadelake", "repository": "EESSI"} +def test_complex_filter_repo_context_arch_and_repo_components(complex_filter): + context = {"architecture": ".*intel.*", "repository": "EESSI"} + components = [FILTER_COMPONENT_ARCH, FILTER_COMPONENT_REPO] expected = False - actual = complex_filter.check_filters(context) + actual = complex_filter.check_filters(context, components) assert expected == actual @@ -256,20 +399,21 @@ def test_non_match_architecture_repository_context(complex_filter): def arch_filter_slash_syntax(): af = EESSIBotActionFilter("") component = 'arch' - value = '.*/intel/.*' + value = 'x86_64/intel/cascadelake' af.add_filter(component, value) yield af def test_match_architecture_syntax_slash(arch_filter_slash_syntax): - context = {"architecture": "x86_64/intel/cascadelake", "repository": "EESSI"} + components = [FILTER_COMPONENT_ARCH] + context = {"architecture": "x86_64/intel/cascadelake"} expected = True - actual = arch_filter_slash_syntax.check_filters(context) + actual = arch_filter_slash_syntax.check_filters(context, components) assert expected == actual context = {"architecture": "x86_64-intel-cascadelake"} expected = True - actual = arch_filter_slash_syntax.check_filters(context) + actual = arch_filter_slash_syntax.check_filters(context, components) assert expected == actual @@ -277,20 +421,21 @@ def test_match_architecture_syntax_slash(arch_filter_slash_syntax): def arch_filter_dash_syntax(): af = EESSIBotActionFilter("") component = 'arch' - value = '.*-intel-.*' + value = 'x86_64-intel-cascadelake' af.add_filter(component, value) yield af def test_match_architecture_syntax_dash(arch_filter_dash_syntax): - context = {"architecture": "x86_64-intel-cascadelake", "repository": "EESSI"} + components = [FILTER_COMPONENT_ARCH] + context = {"architecture": "x86_64-intel-cascadelake"} expected = True - actual = arch_filter_dash_syntax.check_filters(context) + actual = arch_filter_dash_syntax.check_filters(context, components) assert expected == actual context = {"architecture": "x86_64/intel-cascadelake"} expected = True - actual = arch_filter_dash_syntax.check_filters(context) + actual = arch_filter_dash_syntax.check_filters(context, components) assert expected == actual @@ -298,20 +443,21 @@ def test_match_architecture_syntax_dash(arch_filter_dash_syntax): def accel_filter_slash_syntax(): af = EESSIBotActionFilter("") component = 'accel' - value = 'nvidia/.*' + value = 'nvidia/cc70' af.add_filter(component, value) yield af def test_match_accelerator_syntax_slash(accel_filter_slash_syntax): - context = {"accelerator": "nvidia/cc70", "repository": "EESSI"} + components = [FILTER_COMPONENT_ACCEL] + context = {"accelerator": "nvidia/cc70"} expected = True - actual = accel_filter_slash_syntax.check_filters(context) + actual = accel_filter_slash_syntax.check_filters(context, components) assert expected == actual context = {"accelerator": "nvidia=cc70"} expected = True - actual = accel_filter_slash_syntax.check_filters(context) + actual = accel_filter_slash_syntax.check_filters(context, components) assert expected == actual @@ -325,12 +471,13 @@ def accel_filter_equal_syntax(): def test_match_accelerator_syntax_equal(accel_filter_equal_syntax): - context = {"accelerator": "amd=gfx90a", "repository": "EESSI"} + components = [FILTER_COMPONENT_ACCEL] + context = {"accelerator": "amd=gfx90a"} expected = True - actual = accel_filter_equal_syntax.check_filters(context) + actual = accel_filter_equal_syntax.check_filters(context, components) assert expected == actual context = {"accelerator": "amd/gfx90a"} expected = True - actual = accel_filter_equal_syntax.check_filters(context) + actual = accel_filter_equal_syntax.check_filters(context, components) assert expected == actual From 43a2832dc0acd036c36c587bb16ee297fb768f82 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 10:51:19 +0200 Subject: [PATCH 10/28] ignore local .vscode directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 24e6f0f8..8f6b2b64 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ appbackup.cfg venv_bot_p37 *.swo events_log/ +.vscode/ From 2e260a78a6bb3eaf3b50ec3629b67663436b5cc4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 10:52:50 +0200 Subject: [PATCH 11/28] additional tests to increase test coverage + minor style improvements --- tests/test_tools_filter.py | 139 +++++++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 6 deletions(-) diff --git a/tests/test_tools_filter.py b/tests/test_tools_filter.py index cde4f38e..cb46cb49 100644 --- a/tests/test_tools_filter.py +++ b/tests/test_tools_filter.py @@ -13,6 +13,7 @@ import copy # Third party imports (anything installed into the local Python environment) +from unittest.mock import patch import pytest # Local application imports (anything from EESSI/eessi-bot-software-layer) @@ -363,12 +364,12 @@ def test_complex_filter_no_context_no_component(complex_filter): assert expected == actual -#def test_complex_filter_no_context_no_component(complex_filter): -# context = {} -# components = [] -# expected = True -# actual = complex_filter.check_filters(context, components) -# assert expected == actual +# def test_complex_filter_no_context_no_component(complex_filter): +# context = {} +# components = [] +# expected = True +# actual = complex_filter.check_filters(context, components) +# assert expected == actual def test_complex_filter_arch_context_arch_component(complex_filter): @@ -481,3 +482,129 @@ def test_match_accelerator_syntax_equal(accel_filter_equal_syntax): expected = True actual = accel_filter_equal_syntax.check_filters(context, components) assert expected == actual + +# All after this line is generated by Claude (via Cursor) +# import pytest +# from tools.filter import (EESSIBotActionFilter, +# EESSIBotActionFilterError, +# FILTER_COMPONENT_ARCH, +# FILTER_COMPONENT_INST, +# FILTER_COMPONENT_REPO) + + +def test_filter_creation(): + """Test basic filter creation from string""" + filter_str = "arch:x86_64 inst:AWS repo:eessi-2023.06" + action_filter = EESSIBotActionFilter(filter_str) + + # Check that filters were parsed correctly + assert len(action_filter.action_filters) == 3 + assert action_filter.get_filter_by_component(FILTER_COMPONENT_ARCH) == ['x86_64'] + assert action_filter.get_filter_by_component(FILTER_COMPONENT_INST) == ['AWS'] + assert action_filter.get_filter_by_component(FILTER_COMPONENT_REPO) == ['eessi-2023.06'] + + +def test_unexpected_exception_handling(): + """Test that unexpected exceptions in filter creation are logged and re-raised""" + # Create a mock exception + mock_exception = ValueError("Test unexpected error") + + # Mock the log function to verify it's called + with patch('tools.filter.log') as mock_log: + # Mock the add_filter_from_string method to raise our mock exception + with patch.object(EESSIBotActionFilter, 'add_filter_from_string', side_effect=mock_exception): + # Try to create a filter, which should raise our mock exception + with pytest.raises(ValueError) as exc_info: + EESSIBotActionFilter("arch:x86_64") + + # Verify the exception was re-raised + assert exc_info.value == mock_exception + + # Verify the log function was called with the expected message + mock_log.assert_called_once() + log_call_args = mock_log.call_args[0][0] + assert "Unexpected err=" in log_call_args + assert "Test unexpected error" in log_call_args + assert "type(err)=" in log_call_args + + +def test_filter_creation_invalid_format(): + """Test filter creation with invalid format""" + with pytest.raises(EESSIBotActionFilterError): + EESSIBotActionFilter("invalid_format") + + +def test_filter_creation_empty_value(): + """Test filter creation with empty value""" + with pytest.raises(EESSIBotActionFilterError): + EESSIBotActionFilter("arch:") + + +def test_filter_creation_short_component(): + """Test filter creation with component name too short""" + with pytest.raises(EESSIBotActionFilterError): + EESSIBotActionFilter("ar:x86_64") + + +def test_add_filter(): + """Test adding a filter manually""" + action_filter = EESSIBotActionFilter("") + action_filter.add_filter("arch", "x86_64") + assert action_filter.get_filter_by_component(FILTER_COMPONENT_ARCH) == ['x86_64'] + + +def test_remove_filter(): + """Test removing a filter""" + action_filter = EESSIBotActionFilter("arch:x86_64 inst:AWS") + action_filter.remove_filter("arch", "x86_64") + assert len(action_filter.get_filter_by_component(FILTER_COMPONENT_ARCH)) == 0 + assert len(action_filter.get_filter_by_component(FILTER_COMPONENT_INST)) == 1 + + +def test_check_build_filters(): + """Test checking build filters against a context""" + action_filter = EESSIBotActionFilter("arch:x86_64 inst:AWS repo:eessi-2023.06") + + # Matching context + matching_context = { + FILTER_COMPONENT_ARCH: "x86_64", + FILTER_COMPONENT_INST: "AWS", + FILTER_COMPONENT_REPO: "eessi-2023.06" + } + assert action_filter.check_build_filters(matching_context) is True + + # Non-matching context + non_matching_context = { + FILTER_COMPONENT_ARCH: "aarch64", + FILTER_COMPONENT_INST: "AWS", + FILTER_COMPONENT_REPO: "eessi-2023.06" + } + assert action_filter.check_build_filters(non_matching_context) is False + + +def test_to_string(): + """Test converting filters to string""" + filter_str = "arch:x86_64 inst:AWS repo:eessi-2023.06" + action_filter = EESSIBotActionFilter(filter_str) + # The to_string method returns the expanded component names + expected_str = "architecture:x86_64 instance:AWS repository:eessi-2023.06" + assert action_filter.to_string() == expected_str + + +def test_clear_all(): + """Test clearing all filters""" + action_filter = EESSIBotActionFilter("arch:x86_64 inst:AWS") + action_filter.clear_all() + assert len(action_filter.action_filters) == 0 + + +def test_architecture_filter_normalization(): + """Test that architecture filters are normalized (replacing all - with /)""" + action_filter = EESSIBotActionFilter("arch:x86_64-intel-haswell") + assert action_filter.get_filter_by_component(FILTER_COMPONENT_ARCH) == ['x86_64/intel/haswell'] + + # Test matching with normalized value - only check architecture component + context = { + FILTER_COMPONENT_ARCH: "x86_64/intel/haswell" + } + assert action_filter.check_filters(context, [FILTER_COMPONENT_ARCH]) is True From 715a8fac1d5659bec62bdd21f1880f1bef3747df Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 10:55:57 +0200 Subject: [PATCH 12/28] add coverage step to CI workflow --- .coveragerc | 8 ++++++++ .github/workflows/tests.yaml | 7 +++++++ 2 files changed, 15 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..e7731336 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +omit = + tests/* + */tests/* + */test/* + *test_*.py + */__pycache__/* + */venv/* \ No newline at end of file diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index dadec66f..a9ee4cf0 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -47,6 +47,13 @@ jobs: python -m pip install pytest-cov ./test.sh -q --cov=$PWD + - name: Run coverage and create report + run: | + python -m pip install coverage + coverage run -m pytest + coverage report -m + coverage html + - name: Run flake8 to verify PEP8-compliance of Python code run: | flake8 From f28e4126242fcd78c3e1e92531759e0cb30e2090 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 10:59:20 +0200 Subject: [PATCH 13/28] bump image to ubuntu 22.04 --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a9ee4cf0..5a01b78a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -17,7 +17,7 @@ on: [push, pull_request] permissions: read-all jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] From 24d4df7bcaab66699f9126129c49954b347f32e7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 11:00:44 +0200 Subject: [PATCH 14/28] remove trailing whitespace --- tests/test_tools_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tools_filter.py b/tests/test_tools_filter.py index cb46cb49..b223ccd6 100644 --- a/tests/test_tools_filter.py +++ b/tests/test_tools_filter.py @@ -525,7 +525,7 @@ def test_unexpected_exception_handling(): log_call_args = mock_log.call_args[0][0] assert "Unexpected err=" in log_call_args assert "Test unexpected error" in log_call_args - assert "type(err)=" in log_call_args + assert "type(err)=" in log_call_args def test_filter_creation_invalid_format(): From d497de7249713bc47da1445278f71af6f60e7525 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 11:09:44 +0200 Subject: [PATCH 15/28] drop 3.6, add 3.12; drop pytest-cov; include all *.py except tests/* --- .github/workflows/tests.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5a01b78a..f4b03f49 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] fail-fast: false steps: - name: checkout @@ -42,15 +42,11 @@ jobs: run: | ./test.sh --verbose - - name: Run test suite (with coverage) - run: | - python -m pip install pytest-cov - ./test.sh -q --cov=$PWD - - name: Run coverage and create report run: | python -m pip install coverage - coverage run -m pytest + PYTHON_SOURCES=$(find . -name "*.py" -not -path "tests/*") + coverage run --include="${PYTHON_SOURCES}" -m pytest coverage report -m coverage html From 6504e1504ad45fa232bab9feed847a80bad4dca5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 15:12:50 +0200 Subject: [PATCH 16/28] improve find --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f4b03f49..00dcbce1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -45,7 +45,7 @@ jobs: - name: Run coverage and create report run: | python -m pip install coverage - PYTHON_SOURCES=$(find . -name "*.py" -not -path "tests/*") + PYTHON_SOURCES=$(find . -name "*.py" -not -path "tests/*" -not -path ".github/*" | tr '\n' ',' | sed 's/,$//') coverage run --include="${PYTHON_SOURCES}" -m pytest coverage report -m coverage html From 8159572db77f834ba6a7cf9341af4c35598c48da Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 15:30:03 +0200 Subject: [PATCH 17/28] test that tries to import all python files and thus ensures that all appear in coverage report --- tests/test_coverage.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/test_coverage.py diff --git a/tests/test_coverage.py b/tests/test_coverage.py new file mode 100644 index 00000000..609efa0d --- /dev/null +++ b/tests/test_coverage.py @@ -0,0 +1,37 @@ +import os +import importlib +import pytest + +def test_import_all_python_files(): + """Test that imports all Python files to ensure they appear in coverage reports.""" + # Get the project root directory (where this test file is located) + project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # List of directories to scan for Python files + dirs_to_scan = ['connections', 'tasks', 'tools'] + + # Import all Python files + for dir_name in dirs_to_scan: + dir_path = os.path.join(project_root, dir_name) + if os.path.exists(dir_path): + for file_name in os.listdir(dir_path): + if file_name.endswith('.py') and not file_name.startswith('__'): + module_name = f"{dir_name}.{file_name[:-3]}" + try: + importlib.import_module(module_name) + except ImportError as e: + # Log the error but don't fail the test + print(f"Could not import {module_name}: {e}") + + # Import root Python files + for file_name in os.listdir(project_root): + if file_name.endswith('.py') and not file_name.startswith('__'): + module_name = file_name[:-3] + try: + importlib.import_module(module_name) + except ImportError as e: + # Log the error but don't fail the test + print(f"Could not import {module_name}: {e}") + + # The test always passes as its purpose is just to import files + assert True \ No newline at end of file From 34a46f26848c738ef2eff31fc16ad320b025aae2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 15:43:47 +0200 Subject: [PATCH 18/28] list files for which coverage report should be created --- .github/workflows/tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 00dcbce1..108efe49 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -46,6 +46,8 @@ jobs: run: | python -m pip install coverage PYTHON_SOURCES=$(find . -name "*.py" -not -path "tests/*" -not -path ".github/*" | tr '\n' ',' | sed 's/,$//') + echo "determine coverage of pytests for:" + echo "${PYTHON_SOURCES}" | tr ',' '\n' | sed 's/^\.\///' | awk '{print gsub(/\//, "/"), $0}' | sort -n | cut -d' ' -f2- | tree --fromfile -U coverage run --include="${PYTHON_SOURCES}" -m pytest coverage report -m coverage html From a6f253d76cc52acdce9852e50e2c8ab71e1b7c1b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 15:47:10 +0200 Subject: [PATCH 19/28] fix flake8 issues --- tests/test_coverage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_coverage.py b/tests/test_coverage.py index 609efa0d..dc5384a1 100644 --- a/tests/test_coverage.py +++ b/tests/test_coverage.py @@ -1,15 +1,15 @@ import os import importlib -import pytest + def test_import_all_python_files(): """Test that imports all Python files to ensure they appear in coverage reports.""" # Get the project root directory (where this test file is located) project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - + # List of directories to scan for Python files dirs_to_scan = ['connections', 'tasks', 'tools'] - + # Import all Python files for dir_name in dirs_to_scan: dir_path = os.path.join(project_root, dir_name) @@ -22,7 +22,7 @@ def test_import_all_python_files(): except ImportError as e: # Log the error but don't fail the test print(f"Could not import {module_name}: {e}") - + # Import root Python files for file_name in os.listdir(project_root): if file_name.endswith('.py') and not file_name.startswith('__'): @@ -32,6 +32,6 @@ def test_import_all_python_files(): except ImportError as e: # Log the error but don't fail the test print(f"Could not import {module_name}: {e}") - + # The test always passes as its purpose is just to import files - assert True \ No newline at end of file + assert True From 6ff027732d7ee315f4883a420d7c68d1ff9dd377 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 16:16:07 +0200 Subject: [PATCH 20/28] upload coverage reports as artifacts --- .github/workflows/tests.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 108efe49..fd9098fa 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -52,6 +52,15 @@ jobs: coverage report -m coverage html + - name: Upload coverage report + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v 4.6.2 + with: + name: coverage-report-python-${{ matrix.python }} + path: | + htmlcov + .coverage + retention-days: 7 + - name: Run flake8 to verify PEP8-compliance of Python code run: | flake8 From c53a696384290da665bb6e285e3d62edd06bf5f4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 16:22:34 +0200 Subject: [PATCH 21/28] comment upload step --- .github/workflows/tests.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index fd9098fa..dad916f2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -52,14 +52,14 @@ jobs: coverage report -m coverage html - - name: Upload coverage report - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v 4.6.2 - with: - name: coverage-report-python-${{ matrix.python }} - path: | - htmlcov - .coverage - retention-days: 7 +# - name: Upload coverage report +# uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v 4.6.2 +# with: +# name: coverage-report-python-${{ matrix.python }} +# path: | +# htmlcov +# .coverage +# retention-days: 7 - name: Run flake8 to verify PEP8-compliance of Python code run: | From 29487dbae0dad514a3948591605cfb6490c81be6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 16:26:48 +0200 Subject: [PATCH 22/28] update step that uploads artifacts --- .github/workflows/tests.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index dad916f2..1edc1854 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -52,14 +52,16 @@ jobs: coverage report -m coverage html -# - name: Upload coverage report -# uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v 4.6.2 -# with: -# name: coverage-report-python-${{ matrix.python }} -# path: | -# htmlcov -# .coverage -# retention-days: 7 + - name: Upload coverage report + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v 4.6.2 + with: + name: coverage-report-python-${{ matrix.python }} + path: htmlcov/ + retention-days: 7 + compression-level: 6 # default GNU Gzip level + if-no-files-found: warn + overwrite: false + include-hidden-files: false - name: Run flake8 to verify PEP8-compliance of Python code run: | From dbe15405cd5d3632149a369cf8b6792fe0ae0115 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 16:33:57 +0200 Subject: [PATCH 23/28] comment out upload of coverage report; postpone this to a subsequent PR that might use github pages --- .github/workflows/tests.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1edc1854..0cdab360 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -52,16 +52,16 @@ jobs: coverage report -m coverage html - - name: Upload coverage report - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v 4.6.2 - with: - name: coverage-report-python-${{ matrix.python }} - path: htmlcov/ - retention-days: 7 - compression-level: 6 # default GNU Gzip level - if-no-files-found: warn - overwrite: false - include-hidden-files: false +# - name: Upload coverage report +# uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v 4.6.2 +# with: +# name: coverage-report-python-${{ matrix.python }} +# path: htmlcov/ +# retention-days: 7 +# compression-level: 6 # default GNU Gzip level +# if-no-files-found: warn +# overwrite: false +# include-hidden-files: false - name: Run flake8 to verify PEP8-compliance of Python code run: | From 528f6dabf5dc558fc81d658a14316444cfd393b1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 17:08:32 +0200 Subject: [PATCH 24/28] polish .coveragerc --- .coveragerc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index e7731336..64e5de55 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,8 @@ [run] -omit = +omit = tests/* */tests/* */test/* *test_*.py */__pycache__/* - */venv/* \ No newline at end of file + */venv/* From a76ae338ded5cd462d492d50582a5400eec8790f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 17:13:11 +0200 Subject: [PATCH 25/28] cleanup naming of test step --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0cdab360..d49a619d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -38,7 +38,7 @@ jobs: python -m pip install pytest python -m pip install --upgrade flake8 - - name: Run test suite (without coverage) + - name: Run test suite run: | ./test.sh --verbose From 8f870fd8eb5f5265118ffb014c2b6b8231e7a8c9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 18:17:50 +0200 Subject: [PATCH 26/28] use check_build_filters --- tasks/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/build.py b/tasks/build.py index 7a0b1c83..97183bb5 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -621,7 +621,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter): log(f"{fn}(): checking filter {action_filter.to_string()}") context = {"architecture": arch, "repository": repo_id, "instance": app_name} log(f"{fn}(): context is '{json.dumps(context, indent=4)}'") - if not action_filter.check_filters(context): + if not action_filter.check_build_filters(context): log(f"{fn}(): context does NOT satisfy filter(s), skipping") continue else: From 71b9995c31ad6522ff6b2ba454b54ed560295d56 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 19:10:02 +0200 Subject: [PATCH 27/28] add log messages to check_filters --- tools/filter.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/filter.py b/tools/filter.py index 4fef7875..976017c2 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -315,6 +315,7 @@ def check_filters(self, context, components): if len(filter_values) == 0: # no filter for component provided --> at least one filter per # component MUST be given + log(f"no filter for component '{component}' provided; check_filters returns False") return False if component in context: # check if all values for the filter are identical to the value @@ -326,8 +327,13 @@ def check_filters(self, context, components): context_value = context_value.replace('=', '/') for filter_value in filter_values: if filter_value != context_value: + log_msg = "filter value '%s' does not match context value '%s' for component '%s';" + log_msg += " check_filters returns False" + log(log_msg % (filter_value, context_value, component)) return False else: - # missing component in context, could lead to unexpected behavior + # missing required component in context, could lead to unexpected behavior + log_msg = "missing required component '%s' in context; check_filters returns False" + log(log_msg % component) return False return True From 46a4f75c6410325375f63f19d573d6c5b842ee92 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 21 Apr 2025 19:37:18 +0200 Subject: [PATCH 28/28] strip leading OS name from architecture value if it starts with linux --- tools/filter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/filter.py b/tools/filter.py index 976017c2..d34b1d07 100644 --- a/tools/filter.py +++ b/tools/filter.py @@ -322,6 +322,10 @@ def check_filters(self, context, components): # of the component in the context context_value = context[component] if component == FILTER_COMPONENT_ARCH: + if context_value.startswith('linux'): + # strip leading OS name from architecture string + context_value = '/'.join(context_value.split('/')[1:]) + # replace '-' with '/' in architecture string context_value = context_value.replace('-', '/') if component == FILTER_COMPONENT_ACCEL: context_value = context_value.replace('=', '/')