From 3585abae92329a664d4c9626b05365f74f9e4b1e Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 4 Mar 2021 10:53:36 +0100 Subject: [PATCH 1/9] embunit: Use Outputter for all embunit tests This allows for use of OUTPUT=XML in all tests using embunit, not just `tests/unittests`. The other output formats work without the outputter being explicitly set. --- sys/include/embUnit.h | 2 +- tests/unittests/main.c | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sys/include/embUnit.h b/sys/include/embUnit.h index 6a17f1bcd55e..0e57ef1045fc 100644 --- a/sys/include/embUnit.h +++ b/sys/include/embUnit.h @@ -50,7 +50,7 @@ # include "embUnit/TextUIRunner.h" -# define TESTS_START() TextUIRunner_start() +# define TESTS_START() TextUIRunner_startWithOutputter(OUTPUTTER) # define TESTS_RUN(t) TextUIRunner_runTest(t) # define TESTS_END() TextUIRunner_end() #else diff --git a/tests/unittests/main.c b/tests/unittests/main.c index 8128bd5dd17d..809af0c8f584 100644 --- a/tests/unittests/main.c +++ b/tests/unittests/main.c @@ -30,10 +30,6 @@ int main(void) xtimer_init(); #endif -#ifdef OUTPUT - TextUIRunner_setOutputter(OUTPUTTER); -#endif - TESTS_START(); #ifndef NO_TEST_SUITES UNCURRY(RUN_TEST_SUITES, TEST_SUITES) From 45ca1dfb96fa6c1d9a3fce85adf9e975cc541966 Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 4 Mar 2021 10:55:50 +0100 Subject: [PATCH 2/9] embunit/ColorOutputter: properly reset colors and styles after tests --- sys/embunit/ColorOutputter.c | 2 +- sys/embunit/ColorTextColors.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/embunit/ColorOutputter.c b/sys/embunit/ColorOutputter.c index 3c489e1562dd..e7cce64cf64e 100644 --- a/sys/embunit/ColorOutputter.c +++ b/sys/embunit/ColorOutputter.c @@ -53,7 +53,7 @@ void ColorOutputter_printStatistics(OutputterRef self, TestResultRef result) else { printf("\n" BGGREEN SBOLD "OK" SDEFAULT " (%d tests)", result->runCount); } - printf(LINEFILL BGDEFAULT "\n"); + printf(LINEFILL BGDEFAULT "\n" ANSI_RESET); } static const OutputterImplement ColorOutputterImplement = { diff --git a/sys/embunit/ColorTextColors.h b/sys/embunit/ColorTextColors.h index f615a9982a5f..1e7bd30ca2b2 100644 --- a/sys/embunit/ColorTextColors.h +++ b/sys/embunit/ColorTextColors.h @@ -44,6 +44,7 @@ extern "C" { #define SBOLD "\033[1m" #define SDEFAULT "\033[21m" #define LINEFILL "\033[K" +#define ANSI_RESET "\033[0m" /** * @} From fa5798c9001f07a2c484e9a8a38648f99703301f Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 4 Mar 2021 11:15:00 +0100 Subject: [PATCH 3/9] testrunner: try to guess OUTPUT format of unittests and act accordingly --- dist/pythonlibs/testrunner/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/dist/pythonlibs/testrunner/__init__.py b/dist/pythonlibs/testrunner/__init__.py index 8182de518904..4aafce2443b0 100755 --- a/dist/pythonlibs/testrunner/__init__.py +++ b/dist/pythonlibs/testrunner/__init__.py @@ -54,10 +54,24 @@ def check_unittests(child, timeout=TIMEOUT, nb_tests=None): to perform an exact match against that number. """ if nb_tests is None: - child.expect(r'OK \((\d+) tests\)', timeout=timeout) - return int(child.match.group(1)) - _tests = int(nb_tests) - child.expect_exact('OK ({} tests)'.format(_tests), timeout=timeout) + # need escape sequence so don't use raw string + res = child.expect(['OK(\033\\[21m)? \\((\\d+) tests\\)', + r'(\d+)'], timeout=timeout) + if res == 0: + _tests = int(child.match.group(2)) + else: + _tests = int(child.match.group(1)) + else: + _tests = int(nb_tests) + # need escape sequence so don't use raw string + res = child.expect(['OK(\033\\[21m)? \\({} tests\\)'.format(_tests), + r'{}'.format(_tests)], + timeout=timeout) + if res == 1: + res = child.expect([r'\d+', ''], + timeout=timeout) + if res == 0: + raise AssertionError("Unittests failed") return _tests From f47b1252fc1c03d6ed273aadd5754095b79464e3 Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 4 Mar 2021 11:34:59 +0100 Subject: [PATCH 4/9] tests: use run_check_unittests() where appropriate --- tests/driver_ds1307/tests-with-config/01-run.py | 13 ++++++++++--- tests/gnrc_sixlowpan_frag_minfwd/tests/01-run.py | 8 ++------ tests/gnrc_sixlowpan_frag_sfr/tests/01-run.py | 8 ++------ tests/pkg_libbase58/tests/01-run.py | 8 ++------ tests/suit_manifest/tests/01-run.py | 4 ++-- 5 files changed, 18 insertions(+), 23 deletions(-) diff --git a/tests/driver_ds1307/tests-with-config/01-run.py b/tests/driver_ds1307/tests-with-config/01-run.py index 87ecdb87ccd5..ccd5ab24b020 100755 --- a/tests/driver_ds1307/tests-with-config/01-run.py +++ b/tests/driver_ds1307/tests-with-config/01-run.py @@ -7,12 +7,19 @@ # directory for more details. import sys -from testrunner import run + +import pexpect + +from testrunner import run, run_check_unittests def testfunc(child): - child.expect([r"OK \([0-9]+ tests\)", - r"error: unable to initialize RTC \[I2C initialization error\]"]) + res = child.expect([ + r"error: unable to initialize RTC \[I2C initialization error\]", + pexpect.TIMEOUT, + ]) + if res == 1: + run_check_unittests() if __name__ == "__main__": diff --git a/tests/gnrc_sixlowpan_frag_minfwd/tests/01-run.py b/tests/gnrc_sixlowpan_frag_minfwd/tests/01-run.py index 6d8b056e2d59..2eac21932f00 100755 --- a/tests/gnrc_sixlowpan_frag_minfwd/tests/01-run.py +++ b/tests/gnrc_sixlowpan_frag_minfwd/tests/01-run.py @@ -8,12 +8,8 @@ # directory for more details. import sys -from testrunner import run - - -def testfunc(child): - child.expect(r"OK \(\d+ tests\)") +from testrunner import run_check_unittests if __name__ == "__main__": - sys.exit(run(testfunc)) + sys.exit(run_check_unittests()) diff --git a/tests/gnrc_sixlowpan_frag_sfr/tests/01-run.py b/tests/gnrc_sixlowpan_frag_sfr/tests/01-run.py index 6d8b056e2d59..2eac21932f00 100755 --- a/tests/gnrc_sixlowpan_frag_sfr/tests/01-run.py +++ b/tests/gnrc_sixlowpan_frag_sfr/tests/01-run.py @@ -8,12 +8,8 @@ # directory for more details. import sys -from testrunner import run - - -def testfunc(child): - child.expect(r"OK \(\d+ tests\)") +from testrunner import run_check_unittests if __name__ == "__main__": - sys.exit(run(testfunc)) + sys.exit(run_check_unittests()) diff --git a/tests/pkg_libbase58/tests/01-run.py b/tests/pkg_libbase58/tests/01-run.py index 5ba9b468a288..0f594d325692 100755 --- a/tests/pkg_libbase58/tests/01-run.py +++ b/tests/pkg_libbase58/tests/01-run.py @@ -7,12 +7,8 @@ # directory for more details. import sys -from testrunner import run - - -def testfunc(child): - child.expect(u"OK \\([0-9]+ tests\\)") +from testrunner import run_check_unittests if __name__ == "__main__": - sys.exit(run(testfunc, timeout=120)) + sys.exit(run_check_unittests(timeout=120)) diff --git a/tests/suit_manifest/tests/01-run.py b/tests/suit_manifest/tests/01-run.py index 388f894852c2..22dd0aa9e08f 100755 --- a/tests/suit_manifest/tests/01-run.py +++ b/tests/suit_manifest/tests/01-run.py @@ -8,7 +8,7 @@ import os import sys -from testrunner import run +from testrunner import run, run_check_unittests def testfunc(child): @@ -17,7 +17,7 @@ def testfunc(child): # 16 seconds on `samr21-xpro` # >50 seconds on `nrf51dk` timeout = 60 if board != 'native' else -1 - child.expect(r"OK \(\d+ tests\)", timeout=timeout) + run_check_unittests(timeout=timeout) if __name__ == "__main__": From d4c624e7f21d4d9630b3eb425ac26944e32cc322 Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Fri, 3 Sep 2021 15:22:44 +0200 Subject: [PATCH 5/9] tests/gnrc_lorawan: fix output definition for test --- tests/gnrc_lorawan/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/gnrc_lorawan/Makefile b/tests/gnrc_lorawan/Makefile index 9532a2510ed3..b3e15f897b2c 100644 --- a/tests/gnrc_lorawan/Makefile +++ b/tests/gnrc_lorawan/Makefile @@ -1,7 +1,8 @@ # name of your application include ../Makefile.tests_common -CFLAGS += -DOUTPUT=TEXT +# set embunit output to test +OUTPUT = TEXT # Add unittest framework USEMODULE += embunit From 3621508f10d658743744d0387ba5d5df44bf2994 Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 9 Sep 2021 17:35:32 +0200 Subject: [PATCH 6/9] tests/gnrc_ipv6_ext_frag: fix output definition for test --- tests/gnrc_ipv6_ext_frag/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/gnrc_ipv6_ext_frag/Makefile b/tests/gnrc_ipv6_ext_frag/Makefile index c7ba5f083fed..9afc46c5b0ed 100644 --- a/tests/gnrc_ipv6_ext_frag/Makefile +++ b/tests/gnrc_ipv6_ext_frag/Makefile @@ -4,7 +4,8 @@ include ../Makefile.tests_common export TAP ?= tap0 -CFLAGS += -DOUTPUT=TEXT +OUTPUT = TEXT + CFLAGS += -DTEST_SUITES="gnrc_ipv6_ext_frag" ifeq (native,$(BOARD)) From aa3638a80b7289c4506b0530b3d8b024f944a7b2 Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 9 Sep 2021 17:35:45 +0200 Subject: [PATCH 7/9] tests/gnrc_rpl_srh: fix output definition for test --- tests/gnrc_rpl_srh/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gnrc_rpl_srh/Makefile b/tests/gnrc_rpl_srh/Makefile index c7e3112c4ba8..26ab1a364740 100644 --- a/tests/gnrc_rpl_srh/Makefile +++ b/tests/gnrc_rpl_srh/Makefile @@ -4,7 +4,7 @@ include ../Makefile.tests_common export TAP ?= tap0 -CFLAGS += -DOUTPUT=TEXT +OUTPUT = TEXT # use Ethernet as link-layer protocol ifeq (native,$(BOARD)) From 2e2b91f2afcccda6ea57e35f41898e5cd8c74414 Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 9 Sep 2021 17:42:20 +0200 Subject: [PATCH 8/9] fixup! tests: use run_check_unittests() where appropriate --- tests/suit_manifest/tests/01-run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/suit_manifest/tests/01-run.py b/tests/suit_manifest/tests/01-run.py index 22dd0aa9e08f..160ff09733f7 100755 --- a/tests/suit_manifest/tests/01-run.py +++ b/tests/suit_manifest/tests/01-run.py @@ -8,7 +8,7 @@ import os import sys -from testrunner import run, run_check_unittests +from testrunner import run, check_unittests def testfunc(child): @@ -17,7 +17,7 @@ def testfunc(child): # 16 seconds on `samr21-xpro` # >50 seconds on `nrf51dk` timeout = 60 if board != 'native' else -1 - run_check_unittests(timeout=timeout) + check_unittests(child, timeout=timeout) if __name__ == "__main__": From e682a648d21e6237a1c2a94056d26b88cbfc2d9b Mon Sep 17 00:00:00 2001 From: Martine Lenders Date: Thu, 9 Sep 2021 17:35:45 +0200 Subject: [PATCH 9/9] fixup! tests: use run_check_unittests() where appropriate --- tests/suit_manifest/tests/01-run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suit_manifest/tests/01-run.py b/tests/suit_manifest/tests/01-run.py index 160ff09733f7..960e38c4d9b3 100755 --- a/tests/suit_manifest/tests/01-run.py +++ b/tests/suit_manifest/tests/01-run.py @@ -17,7 +17,7 @@ def testfunc(child): # 16 seconds on `samr21-xpro` # >50 seconds on `nrf51dk` timeout = 60 if board != 'native' else -1 - check_unittests(child, timeout=timeout) + assert check_unittests(child, timeout=timeout) > 0 if __name__ == "__main__":