From d05686fbf916abf3460beb14b220958199e4be92 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Sat, 10 Jan 2026 12:46:41 +0100 Subject: [PATCH 1/5] make tracestates copy-able --- robotmbt/tracestate.py | 8 ++++++++ utest/test_tracestate.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/robotmbt/tracestate.py b/robotmbt/tracestate.py index 959db914..2426ec28 100644 --- a/robotmbt/tracestate.py +++ b/robotmbt/tracestate.py @@ -39,6 +39,14 @@ def __init__(self, scenario_indexes: list[int]): self._snapshots = [] # Keeps details for elements in trace self._open_refinements = [] + def copy(self): + cp = TraceState(self.c_pool.keys()) + cp.c_pool.update(self.c_pool) + cp._tried = [triedlist[:] for triedlist in self._tried] + cp._snapshots = self._snapshots[:] + cp._open_refinements = self._open_refinements[:] + return cp + @property def model(self): """returns the model as it is at the end of the current trace""" diff --git a/utest/test_tracestate.py b/utest/test_tracestate.py index 77c1462a..57e0a1b4 100644 --- a/utest/test_tracestate.py +++ b/utest/test_tracestate.py @@ -512,6 +512,29 @@ def test_only_completed_scenarios_affect_drought(self): ts.confirm_full_scenario(2, 'two remainder', {}) self.assertEqual(ts.coverage_drought, 0) + def test_tracestates_can_be_copied(self): + ts = TraceState([1, 2, 3]) + ts.confirm_full_scenario(1, 'one', {}) + ts.confirm_full_scenario(2, 'two', {}) + cp = ts.copy() + self.assertEqual(ts.c_pool, cp.c_pool) + self.assertEqual(ts.tried, cp.tried) + self.assertEqual(ts.active_refinements, cp.active_refinements) + self.assertEqual(ts[-1], cp[-1]) + + def test_tracestate_copies_are_independent(self): + ts = TraceState([1, 2, 3]) + ts.confirm_full_scenario(1, 'one', {}) + ts.confirm_full_scenario(2, 'two', {}) + cp = ts.copy() + cp.push_partial_scenario(3, 'three', {}) + self.assertNotEqual(ts.active_refinements, cp.active_refinements) + cp.confirm_full_scenario(3, 'two', {}) + self.assertEqual(len(cp), len(ts)+2) + self.assertNotEqual(ts.c_pool, cp.c_pool) + cp.rewind() + self.assertIn(3, cp.tried) + self.assertNotIn(3, ts.tried) if __name__ == '__main__': unittest.main() From 03666b55998d12f5c6dfc816b2a43b07c12c12f3 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Sat, 10 Jan 2026 13:05:34 +0100 Subject: [PATCH 2/5] pep8 newline --- utest/test_tracestate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utest/test_tracestate.py b/utest/test_tracestate.py index 57e0a1b4..f41d9cc1 100644 --- a/utest/test_tracestate.py +++ b/utest/test_tracestate.py @@ -536,5 +536,6 @@ def test_tracestate_copies_are_independent(self): self.assertIn(3, cp.tried) self.assertNotIn(3, ts.tried) + if __name__ == '__main__': unittest.main() From 787b33669a8360a53d89592a2677753bfb759a27 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Sat, 10 Jan 2026 13:21:52 +0100 Subject: [PATCH 3/5] first pre-exploration experiment --- robotmbt/suiteprocessors.py | 74 ++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index aa6b7f9d..4c9d8adb 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -103,8 +103,80 @@ def process_test_suite(self, in_suite, *, seed='new'): self._report_tracestate_wrapup(tracestate) return self.out_suite + def _explore_paths(self): + tracestates = [] + ids = [s.src_id for s in self.scenarios] + for scenario in self.scenarios: + scenarios = ids[:] + this_scenario = scenarios.pop(scenarios.index(scenario.src_id)) + random.shuffle(scenarios) + scenarios.insert(0, this_scenario) + logger.debug(f"Exploring with prio order {scenarios}") + tracestates.append(TraceState(scenarios)) + count = 0 + candidate_id = tracestates[-1].next_candidate(retry=False) + while candidate_id is not None: + count += 1 + candidate = self._select_scenario_variant(candidate_id, tracestates[-1]) + if candidate: # No valid variant available in the current state + modeller.try_to_fit_in_scenario(candidate, tracestates[-1]) + else: + tracestates[-1].reject_scenario(candidate_id) + candidate_id = tracestates[-1].next_candidate(retry=False) + logger.debug(f"Result trace: {tracestates[-1].id_trace} " + f"of length {len(tracestates[-1])} after exploring {count} options.") + + map(self._report_tracestate_to_user, tracestates) + + # suggested prio-order: + # - all ids from src_id list of longest trace + # - but all never reached scenarios (all tracestates) are moved to the front + # - all unreached in this trace move in second place + longest = self._longest_trace(tracestates) + unreached = self._unreached_scenarios(tracestates) + suggested_order = [] + suggested_order += [id for id in longest.c_pool if id in unreached] + suggested_order += [id for id in longest.c_pool if id not in suggested_order and longest.count(id) == 0] + suggested_order += [id for id in longest.c_pool if longest.count(id) > 0] + logger.debug(f"1st suggestion: {suggested_order}") + + latest_new = self._last_new_coverage(tracestates) + second_suggestion = [] + second_suggestion += [id for id in latest_new.c_pool if id in unreached] + second_suggestion += [id for id in latest_new.c_pool + if id not in second_suggestion and latest_new.count(id) == 0] + second_suggestion += [id for id in latest_new.c_pool if latest_new.count(id) > 0] + logger.debug(f"2nd suggestion: {second_suggestion}") + if second_suggestion != suggested_order: + logger.warn(f"Found one: {suggested_order} vs {second_suggestion}") + return suggested_order + + @staticmethod + def _longest_trace(tracestate_list): + lengths = [len(ts) for ts in tracestate_list] + return tracestate_list[lengths.index(max(lengths))] + + @staticmethod + def _last_new_coverage(tracestate_list): + ids = [] + last_trace = None + for tracestate in tracestate_list: + for id in [int(long_id.split('.')[0]) for long_id in tracestate.id_trace]: + if id not in ids: + ids.append(id) + last_trace = tracestate + return last_trace + + @staticmethod + def _unreached_scenarios(tracestate_list): + total_coverage = dict().fromkeys(tracestate_list[0].c_pool, 0) + for ts in tracestate_list: + total_coverage = {k: total_coverage[k]+v for k, v in ts.c_pool.items()} + return [id for id in total_coverage if total_coverage[id] == 0] + def _try_to_reach_full_coverage(self, allow_duplicate_scenarios): - tracestate = TraceState(self.shuffled) + suggested_prio_order = self._explore_paths() + tracestate = TraceState(suggested_prio_order) while not tracestate.coverage_reached(): candidate_id = tracestate.next_candidate(retry=allow_duplicate_scenarios) if candidate_id is None: # No more candidates remaining for this level From 940967071e0c027d405537c0728f3843f4b6cbab Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Sun, 11 Jan 2026 17:30:33 +0100 Subject: [PATCH 4/5] cleanup --- .../02__reusing_seed_reproduces_trace.robot | 2 +- .../03__retrace_with_refinement.robot | 2 +- .../04__retrace_with_step_modifiers.robot | 2 +- .../random_seeds/05__retrace_combined.robot | 4 +- robotmbt/suiteprocessors.py | 79 ++++++++++--------- 5 files changed, 47 insertions(+), 42 deletions(-) diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/02__reusing_seed_reproduces_trace.robot b/atest/robotMBT tests/07__processor_options/random_seeds/02__reusing_seed_reproduces_trace.robot index 28e5e180..d0e2b737 100644 --- a/atest/robotMBT tests/07__processor_options/random_seeds/02__reusing_seed_reproduces_trace.robot +++ b/atest/robotMBT tests/07__processor_options/random_seeds/02__reusing_seed_reproduces_trace.robot @@ -5,7 +5,7 @@ Documentation This suite uses the `seed` argument to reproduce a previously ... indicated by the number string. Suite Setup Run keywords Set suite variable ${trace} ${empty} ... AND Treat this test suite Model-based seed=aqmou-eelcuu-sniu-ugsyek-jyhoor -Suite Teardown Should be equal ${trace} 6930142758 +Suite Teardown Should be equal ${trace} 6879523014 Library robotmbt *** Test Cases *** diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot b/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot index fc39b74a..c9b2e81e 100644 --- a/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot +++ b/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot @@ -9,7 +9,7 @@ Documentation This suite uses the `seed` argument to reproduce a previously ... traces possible of varying length. Suite Setup Run keywords Set suite variable ${trace} ${empty} ... AND Treat this test suite Model-based seed=xou-pumj-ihj-oibiyc-surer -Suite Teardown Should be equal ${trace} B1A5BY3B4BX6B2 +Suite Teardown Should be equal ${trace} B2A6B5BY3BX1B4 Library robotmbt *** Test Cases *** diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/04__retrace_with_step_modifiers.robot b/atest/robotMBT tests/07__processor_options/random_seeds/04__retrace_with_step_modifiers.robot index 3ffdfe43..d3a4a52c 100644 --- a/atest/robotMBT tests/07__processor_options/random_seeds/04__retrace_with_step_modifiers.robot +++ b/atest/robotMBT tests/07__processor_options/random_seeds/04__retrace_with_step_modifiers.robot @@ -6,7 +6,7 @@ Documentation This suite uses the `seed` argument to reproduce a previously ... included, the modifers will number the scenarios at random. The reproduced trace ... must match for both the scenario order and the data order. Suite Setup Treat this test suite Model-based seed=iulr-vih-esycu-eyl-yfa -Suite Teardown Should be equal ${trace} H6G3E5I4D9F8J2B1A7C0 +Suite Teardown Should be equal ${trace} H2G3C1E7I5B9F0A4D6J8 Library robotmbt *** Test Cases *** diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot b/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot index cfbca3b2..3fe638ea 100644 --- a/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot +++ b/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot @@ -15,8 +15,8 @@ Documentation This suite uses the `seed` argument to reproduce a previously ... a data choice by using step modifiers. The lower level includes a path choice. ... Both low-level scenarios are equally valid, the only difference is that the data ... choice is included either once or twice. -Suite Setup Treat this test suite Model-based seed=gujuqt-iakm-oexo-xnu-huba -Suite Teardown Should be equal ${trace} ATQQPRSPRPXY +Suite Setup Treat this test suite Model-based seed=kece-zwu-eihho-yli-rbixx +Suite Teardown Should be equal ${trace} APSSTQTQPXY Library robotmbt *** Test Cases *** diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index 4c9d8adb..bbcec6e0 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -87,8 +87,7 @@ def process_test_suite(self, in_suite, *, seed='new'): "\n\t".join([f"{s.src_id}: {s.name}" for s in self.scenarios])) self._init_randomiser(seed) - self.shuffled = [s.src_id for s in self.scenarios] - random.shuffle(self.shuffled) # Keep a single shuffle for all TraceStates (non-essential) + self._prio_order = self._pre_explore_paths()[0] # a short trace without the need for repeating scenarios is preferred tracestate = self._try_to_reach_full_coverage(allow_duplicate_scenarios=False) @@ -103,53 +102,60 @@ def process_test_suite(self, in_suite, *, seed='new'): self._report_tracestate_wrapup(tracestate) return self.out_suite - def _explore_paths(self): + def _pre_explore_paths(self): + id_list = [s.src_id for s in self.scenarios] + random.shuffle(id_list) # pre-shuffle to prevent scenario 1 from always getting first prio tracestates = [] - ids = [s.src_id for s in self.scenarios] - for scenario in self.scenarios: - scenarios = ids[:] - this_scenario = scenarios.pop(scenarios.index(scenario.src_id)) + for prio_id in id_list: + # For each available id, give it pole position in the prio order and randomise the rest. + # The randomised bit will give some good variation and the prio_id will always be tried + # first giving more (not all) insight in its dependencies. + scenarios = id_list[:] + scenarios.remove(prio_id) random.shuffle(scenarios) - scenarios.insert(0, this_scenario) + scenarios.insert(0, prio_id) logger.debug(f"Exploring with prio order {scenarios}") - tracestates.append(TraceState(scenarios)) + tracestate = TraceState(scenarios) count = 0 - candidate_id = tracestates[-1].next_candidate(retry=False) + candidate_id = tracestate.next_candidate(retry=False) while candidate_id is not None: count += 1 - candidate = self._select_scenario_variant(candidate_id, tracestates[-1]) + candidate = self._select_scenario_variant(candidate_id, tracestate) if candidate: # No valid variant available in the current state - modeller.try_to_fit_in_scenario(candidate, tracestates[-1]) + modeller.try_to_fit_in_scenario(candidate, tracestate) else: - tracestates[-1].reject_scenario(candidate_id) - candidate_id = tracestates[-1].next_candidate(retry=False) - logger.debug(f"Result trace: {tracestates[-1].id_trace} " - f"of length {len(tracestates[-1])} after exploring {count} options.") - - map(self._report_tracestate_to_user, tracestates) + tracestate.reject_scenario(candidate_id) + candidate_id = tracestate.next_candidate(retry=False) + tracestates.append(tracestate) + logger.debug(f"Result trace: {tracestate.id_trace} " + f"of length {len(tracestate)} after exploring {count} options.") - # suggested prio-order: + # first suggested prio-order: # - all ids from src_id list of longest trace - # - but all never reached scenarios (all tracestates) are moved to the front - # - all unreached in this trace move in second place - longest = self._longest_trace(tracestates) + # - but all unreached scenarios (over all traces) are moved to the front + # - and all unreached in this trace move into second place unreached = self._unreached_scenarios(tracestates) - suggested_order = [] - suggested_order += [id for id in longest.c_pool if id in unreached] - suggested_order += [id for id in longest.c_pool if id not in suggested_order and longest.count(id) == 0] - suggested_order += [id for id in longest.c_pool if longest.count(id) > 0] - logger.debug(f"1st suggestion: {suggested_order}") + longest = self._longest_trace(tracestates) + first_suggestion = [id for id in longest.c_pool if id in unreached] + first_suggestion += [id for id in longest.c_pool if id not in first_suggestion and longest.count(id) == 0] + first_suggestion += [id for id in longest.c_pool if longest.count(id) > 0] + logger.debug(f"Prio order suggestion: {first_suggestion}") + # second suggested prio-order: + # - Take the last trace that saw a scenario inserted that had not been reached in any of the prior + # traces. There may be a unique sequence in here to reach that scenario. + # - reordering to insert unreached sceanrio is the same as with the first suggested prio-order latest_new = self._last_new_coverage(tracestates) - second_suggestion = [] - second_suggestion += [id for id in latest_new.c_pool if id in unreached] + second_suggestion = [id for id in latest_new.c_pool if id in unreached] second_suggestion += [id for id in latest_new.c_pool if id not in second_suggestion and latest_new.count(id) == 0] second_suggestion += [id for id in latest_new.c_pool if latest_new.count(id) > 0] - logger.debug(f"2nd suggestion: {second_suggestion}") - if second_suggestion != suggested_order: - logger.warn(f"Found one: {suggested_order} vs {second_suggestion}") - return suggested_order + + suggestions = [first_suggestion] + if first_suggestion != second_suggestion: + logger.debug(f"2nd suggestion: {second_suggestion}") + suggestions.append(second_suggestion) + return suggestions @staticmethod def _longest_trace(tracestate_list): @@ -159,9 +165,9 @@ def _longest_trace(tracestate_list): @staticmethod def _last_new_coverage(tracestate_list): ids = [] - last_trace = None + last_trace = tracestate_list[0] for tracestate in tracestate_list: - for id in [int(long_id.split('.')[0]) for long_id in tracestate.id_trace]: + for id in [int(float(long_id)) for long_id in tracestate.id_trace]: if id not in ids: ids.append(id) last_trace = tracestate @@ -175,8 +181,7 @@ def _unreached_scenarios(tracestate_list): return [id for id in total_coverage if total_coverage[id] == 0] def _try_to_reach_full_coverage(self, allow_duplicate_scenarios): - suggested_prio_order = self._explore_paths() - tracestate = TraceState(suggested_prio_order) + tracestate = TraceState(self._prio_order) while not tracestate.coverage_reached(): candidate_id = tracestate.next_candidate(retry=allow_duplicate_scenarios) if candidate_id is None: # No more candidates remaining for this level From e33dbd5ebbe1e8e21fb4a9ab6325f2377d994202 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 16 Jan 2026 15:52:53 +0100 Subject: [PATCH 5/5] apply random walk when including repetition --- .../03__retrace_with_refinement.robot | 2 +- .../random_seeds/05__retrace_combined.robot | 2 +- robotmbt/suiteprocessors.py | 10 ++++----- robotmbt/tracestate.py | 21 +++++++++++-------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot b/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot index c9b2e81e..5461c2d8 100644 --- a/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot +++ b/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot @@ -9,7 +9,7 @@ Documentation This suite uses the `seed` argument to reproduce a previously ... traces possible of varying length. Suite Setup Run keywords Set suite variable ${trace} ${empty} ... AND Treat this test suite Model-based seed=xou-pumj-ihj-oibiyc-surer -Suite Teardown Should be equal ${trace} B2A6B5BY3BX1B4 +Suite Teardown Should be equal ${trace} B3AY2A5A6B4BX1 Library robotmbt *** Test Cases *** diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot b/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot index 3fe638ea..5b23e5f9 100644 --- a/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot +++ b/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot @@ -16,7 +16,7 @@ Documentation This suite uses the `seed` argument to reproduce a previously ... Both low-level scenarios are equally valid, the only difference is that the data ... choice is included either once or twice. Suite Setup Treat this test suite Model-based seed=kece-zwu-eihho-yli-rbixx -Suite Teardown Should be equal ${trace} APSSTQTQPXY +Suite Teardown Should be equal ${trace} APPSRTTPPXY Library robotmbt *** Test Cases *** diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index bbcec6e0..52380e41 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -90,11 +90,11 @@ def process_test_suite(self, in_suite, *, seed='new'): self._prio_order = self._pre_explore_paths()[0] # a short trace without the need for repeating scenarios is preferred - tracestate = self._try_to_reach_full_coverage(allow_duplicate_scenarios=False) + tracestate = self._try_to_reach_full_coverage(allow_duplicate_scenarios=False, randomise=False) if not tracestate.coverage_reached(): logger.debug("Direct trace not available. Allowing repetition of scenarios") - tracestate = self._try_to_reach_full_coverage(allow_duplicate_scenarios=True) + tracestate = self._try_to_reach_full_coverage(allow_duplicate_scenarios=True, randomise=True) if not tracestate.coverage_reached(): raise Exception("Unable to compose a consistent suite") @@ -117,7 +117,7 @@ def _pre_explore_paths(self): logger.debug(f"Exploring with prio order {scenarios}") tracestate = TraceState(scenarios) count = 0 - candidate_id = tracestate.next_candidate(retry=False) + candidate_id = tracestate.next_candidate(retry=False, randomise=False) while candidate_id is not None: count += 1 candidate = self._select_scenario_variant(candidate_id, tracestate) @@ -180,10 +180,10 @@ def _unreached_scenarios(tracestate_list): total_coverage = {k: total_coverage[k]+v for k, v in ts.c_pool.items()} return [id for id in total_coverage if total_coverage[id] == 0] - def _try_to_reach_full_coverage(self, allow_duplicate_scenarios): + def _try_to_reach_full_coverage(self, allow_duplicate_scenarios, randomise=False): tracestate = TraceState(self._prio_order) while not tracestate.coverage_reached(): - candidate_id = tracestate.next_candidate(retry=allow_duplicate_scenarios) + candidate_id = tracestate.next_candidate(retry=allow_duplicate_scenarios, randomise=randomise) if candidate_id is None: # No more candidates remaining for this level if not tracestate.can_rewind(): break diff --git a/robotmbt/tracestate.py b/robotmbt/tracestate.py index 2426ec28..fbf82a69 100644 --- a/robotmbt/tracestate.py +++ b/robotmbt/tracestate.py @@ -30,6 +30,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import random + class TraceState: def __init__(self, scenario_indexes: list[int]): self.c_pool = {index: 0 for index in scenario_indexes} @@ -76,16 +78,17 @@ def coverage_reached(self): def get_trace(self): return [snap.scenario for snap in self._snapshots] - def next_candidate(self, retry=False): - for i in self.c_pool: - if i not in self._tried[-1] and not self.is_refinement_active(i) and self.count(i) == 0: - return i - if not retry: + def next_candidate(self, retry: bool=False, randomise=False): + untried_candidates = [i for i in self.c_pool if i not in self._tried[-1] + and not self.is_refinement_active(i)] + uncovered_candidates = [i for i in untried_candidates if self.count(i) == 0] + + if uncovered_candidates: + return random.choice(uncovered_candidates) if randomise else uncovered_candidates[0] + elif not retry or not untried_candidates: return None - for i in self.c_pool: - if i not in self._tried[-1] and not self.is_refinement_active(i): - return i - return None + else: + return random.choice(untried_candidates) if randomise else untried_candidates[0] def count(self, index): """