tests/test_tools: add a test for the testing tools environment

tests/test_tools/Makefile

@@ -0,0 +1,11 @@
+DEVELHELP = 0
+include ../Makefile.tests_common
+
+USEMODULE += shell
+
+TEST_ON_CI_WHITELIST += all
+
+# Disable shell echo and prompt to not have them in the way for testing
+CFLAGS += -DSHELL_NO_ECHO=1 -DSHELL_NO_PROMPT=1
+
+include $(RIOTBASE)/Makefile.include

tests/test_tools/README.md

@@ -0,0 +1,8 @@
+`test_tools`
+============
+
+This test is here to verify the test tools integration with your board and test
+setup.
+
+It verify the assumptions required for testing on the board behaviour
+through make term.

tests/test_tools/main.c

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2019 Freie Universität Berlin
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+/**
+ * @file
+ * @brief       Specific shell implementation for testing the testing tools.
+ *
+ * @author      Gaëtan Harter <gaetan.harter@fu-berlin.de>
+ *
+ */
+
+#include <stdio.h>
+
+#include "shell_commands.h"
+#include "shell.h"
+
+#if !defined(SHELL_NO_ECHO) || !defined(SHELL_NO_PROMPT)
+#error This test assumes no shell echo or shell prompt
+#endif
+
+
+/**
+ * @brief true - do nothing, successfully
+ *
+ * true [ignored command line arguments]
+ *
+ * Description taken from `man true` in coreutils.
+ *
+ * @param[in] argc  Number of arguments
+ * @param[in] argv  Array of arguments
+ *
+ * @return  0
+ *
+ */
+static int cmd_true(int argc, char **argv)
+{
+    (void)argc;
+    (void)argv;
+    return 0;
+}
+
+
+/**
+ * @brief shellping, replies shellpong
+ *
+ * Test if the shell is ready to take commands
+ *
+ * @param[in] argc  Number of arguments
+ * @param[in] argv  Array of arguments
+ *
+ * @return  0
+ *
+ */
+static int cmd_shellping(int argc, char **argv)
+{
+    (void)argc;
+    (void)argv;
+    puts("shellpong");
+    return 0;
+}
+
+
+static const shell_command_t shell_commands[] = {
+    { "shellping", "Just print 'shellpong'", cmd_shellping },
+    { "true", "do nothing, successfully", cmd_true },
+    { NULL, NULL, NULL }
+};
+
+int main(void)
+{
+    puts("Running 'tests_tools' application");
+
+    char line_buf[SHELL_DEFAULT_BUFSIZE];
+    shell_run(shell_commands, line_buf, SHELL_DEFAULT_BUFSIZE);
+
+    return 0;
+}

tests/test_tools/tests/01-run.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Test behaviour of the test running and the term program interaction."""
+
+import sys
+import pexpect
+from testrunner import run
+
+
+def _shellping(child, timeout=1):
+    """Issue a 'shellping' command.
+
+    Raises a pexpect exception on failure.
+    :param timeout: timeout for the answer
+    """
+    child.sendline('shellping')
+    child.expect_exact('shellpong\r\n', timeout=timeout)
+
+
+def _wait_shell_ready(child, numtries=5):
+    """Wait until the shell is ready by using 'shellping'."""
+    for _ in range(numtries - 1):
+        try:
+            _shellping(child)
+        except pexpect.TIMEOUT:
+            pass
+        else:
+            break
+    else:
+        # This one should fail
+        _shellping(child)
+
+
+def _test_no_local_echo(child):
+    """Verify that there is not local echo while testing."""
+    msg = 'true this should not be echoed'
+    child.sendline(msg)
+    res = child.expect_exact([pexpect.TIMEOUT, msg], timeout=1)
+    assert res == 0, "There should have been a timeout and not match stdin"
+
+
+def testfunc(child):
+    """Run some tests to verify the board under test behaves correctly.
+
+    It currently tests:
+
+    * local echo
+    """
+    child.expect_exact("Running 'tests_tools' application")
+
+    _wait_shell_ready(child)
+
+    # Verify there is no local and remote echo as it is disabled
+    _test_no_local_echo(child)
+
+    # The node should still answer after the previous one
+    _shellping(child)
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc))