diff --git a/dist/pythonlibs/riotnode/.coveragerc b/dist/pythonlibs/riotnode/.coveragerc new file mode 100644 index 000000000000..3a2d4abfb5a2 --- /dev/null +++ b/dist/pythonlibs/riotnode/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = riotnode/tests/* diff --git a/dist/pythonlibs/riotnode/.gitignore b/dist/pythonlibs/riotnode/.gitignore new file mode 100644 index 000000000000..6091f4ddf75f --- /dev/null +++ b/dist/pythonlibs/riotnode/.gitignore @@ -0,0 +1,112 @@ +# Manually added: +# All xml reports +*.xml + +#### joe made this: http://goel.io/joe + +#####=== Python ===##### + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/dist/pythonlibs/riotnode/README.rst b/dist/pythonlibs/riotnode/README.rst new file mode 100644 index 000000000000..983f65e3c2ee --- /dev/null +++ b/dist/pythonlibs/riotnode/README.rst @@ -0,0 +1,28 @@ +RIOT Node abstraction +===================== + +This provides python object abstraction of a node. +The first goal is to be the starting point for the serial abstraction and +build on top of that to provide higher level abstraction like over the shell. + +It could provide an RPC interface to a node in Python over the serial port +and maybe also over network. + +The goal is here to be test environment agnostic and be usable in any test +framework and also without it. + + +Testing +------- + +Run `tox` to run the whole test suite: + +:: + + tox + ... + ________________________________ summary ________________________________ + test: commands succeeded + lint: commands succeeded + flake8: commands succeeded + congratulations :) diff --git a/dist/pythonlibs/riotnode/TODO.rst b/dist/pythonlibs/riotnode/TODO.rst new file mode 100644 index 000000000000..a5b775d39e68 --- /dev/null +++ b/dist/pythonlibs/riotnode/TODO.rst @@ -0,0 +1,24 @@ +TODO list +========= + +Some list of things I would like to do but not for first publication. + + +Legacy handling +--------------- + +Some handling was directly taken from ``testrunner``, without a justified/tested +reason. I just used it to not break existing setup for nothing. +I have more details in the code. + +* Ignoring reset return value and error message +* Use killpg(SIGKILL) to kill terminal + + +Testing +------- + +The current 'node' implementation is an ideal node where all output is captured +and reset directly resets. Having wilder implementations with output loss (maybe +as a deamon with a ``flash`` pre-requisite and sometime no ``reset`` would be +interesting. diff --git a/dist/pythonlibs/riotnode/out.pdf b/dist/pythonlibs/riotnode/out.pdf new file mode 100644 index 000000000000..f75d9a10ba12 Binary files /dev/null and b/dist/pythonlibs/riotnode/out.pdf differ diff --git a/dist/pythonlibs/riotnode/requirements.txt b/dist/pythonlibs/riotnode/requirements.txt new file mode 100644 index 000000000000..479e00d0a874 --- /dev/null +++ b/dist/pythonlibs/riotnode/requirements.txt @@ -0,0 +1,2 @@ +# Use the current setup.py for requirements +. diff --git a/dist/pythonlibs/riotnode/riotnode/__init__.py b/dist/pythonlibs/riotnode/riotnode/__init__.py new file mode 100644 index 000000000000..5df249a04756 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/__init__.py @@ -0,0 +1,11 @@ +"""RIOT Node abstraction. + +This prodives python object abstraction of a node. +The first goal is to be the starting point for the serial abstraction and +build on top of that to provide higher level abstraction like over the shell. + +It could provide an RPC interface to a node in Python over the serial port +and maybe also over network. +""" + +__version__ = '0.1.0' diff --git a/dist/pythonlibs/riotnode/riotnode/node.py b/dist/pythonlibs/riotnode/riotnode/node.py new file mode 100644 index 000000000000..1274f45399fb --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/node.py @@ -0,0 +1,205 @@ +"""RIOTNode abstraction. + +Define class to abstract a node over the RIOT build system. +""" + +import os +import time +import logging +import subprocess +import contextlib + +import pexpect + +from . import utils + +DEVNULL = open(os.devnull, 'w') + + +class TermSpawn(pexpect.spawn): + """Subclass to adapt the behaviour to our need. + + * change default `__init__` values + * disable local 'echo' to not match send messages + * 'utf-8/replace' by default + * default timeout + * tweak exception: + * replace the value with the called pattern + * remove exception context from inside pexpect implementation + """ + + def __init__(self, # pylint:disable=too-many-arguments + command, timeout=10, echo=False, + encoding='utf-8', codec_errors='replace', **kwargs): + super().__init__(command, timeout=timeout, echo=echo, + encoding=encoding, codec_errors=codec_errors, + **kwargs) + + def expect(self, pattern, *args, **kwargs): + # pylint:disable=arguments-differ + try: + return super().expect(pattern, *args, **kwargs) + except (pexpect.TIMEOUT, pexpect.EOF) as exc: + raise self._pexpect_exception(exc, pattern) + + def expect_exact(self, pattern, *args, **kwargs): + # pylint:disable=arguments-differ + try: + return super().expect_exact(pattern, *args, **kwargs) + except (pexpect.TIMEOUT, pexpect.EOF) as exc: + raise self._pexpect_exception(exc, pattern) + + @staticmethod + def _pexpect_exception(exc, pattern): + """Tweak pexpect exception. + + * Put the calling 'pattern' as value + * Remove exception context + """ + exc.pexpect_value = exc.value + exc.value = pattern + + # Remove exception context + exc.__cause__ = None + exc.__traceback__ = None + return exc + + +class RIOTNode(): + """Class abstracting a RIOTNode in an application. + + This should abstract the build system integration. + + :param application_directory: relative directory to the application. + :param env: dictionary of environment variables that should be used. + These overwrites values coming from `os.environ` and can help + define factories where environment comes from a file or if the + script is not executed from the build system context. + + Environment variable configuration + + :environment BOARD: current RIOT board type. + :environment RIOT_TERM_START_DELAY: delay before `make term` is said to be + ready after calling. + """ + + TERM_SPAWN_CLASS = TermSpawn + TERM_STARTED_DELAY = int(os.environ.get('RIOT_TERM_START_DELAY') or 3) + + MAKE_ARGS = () + RESET_TARGETS = ('reset',) + + def __init__(self, application_directory='.', env=None): + self._application_directory = application_directory + + # TODO I am not satisfied by this, but would require changing all the + # environment handling, just put a note until I can fix it. + # I still want to show a PR before this + # I would prefer getting either no environment == os.environ or the + # full environment to use. + # It should also change the `TERM_STARTED_DELAY` thing. + self.env = os.environ.copy() + self.env.update(env or {}) + + self.term = None # type: pexpect.spawn + + self.logger = logging.getLogger(__name__) + + @property + def application_directory(self): + """Absolute path to the current directory.""" + return os.path.abspath(self._application_directory) + + def board(self): + """Return board type.""" + return self.env['BOARD'] + + def reset(self): + """Reset current node.""" + # Ignoring 'reset' return value was taken from `testrunner`. + # For me it should not be done for all boards as it should be an error. + # I would rather fix it in the build system or have a per board + # configuration. + + # Make reset yields error on some boards even if successful + # Ignore printed errors and returncode + self.make_run(self.RESET_TARGETS, stdout=DEVNULL, stderr=DEVNULL) + + @contextlib.contextmanager + def run_term(self, reset=True, **startkwargs): + """Terminal context manager.""" + try: + self.start_term(**startkwargs) + if reset: + self.reset() + yield self.term + finally: + self.stop_term() + + def start_term(self, **spawnkwargs): + """Start the terminal. + + The function is blocking until it is ready. + It waits some time until the terminal is ready and resets the node. + """ + self.stop_term() + + term_cmd = self.make_command(['term']) + self.term = self.TERM_SPAWN_CLASS(term_cmd[0], args=term_cmd[1:], + env=self.env, **spawnkwargs) + + # on many platforms, the termprog needs a short while to be ready + time.sleep(self.TERM_STARTED_DELAY) + + def _term_pid(self): + """Terminal pid or None.""" + return getattr(self.term, 'pid', None) + + def stop_term(self): + """Stop the terminal.""" + with utils.ensure_all_subprocesses_stopped(self._term_pid(), + self.logger): + self._safe_term_close() + + def _safe_term_close(self): + """Safe 'term.close'. + + Handles possible exceptions. + """ + try: + self.term.close() + except AttributeError: + # Not initialized + pass + except ProcessLookupError: + self.logger.warning('Process already stopped') + except pexpect.ExceptionPexpect: + # Not sure how to cover this in a test + # 'make term' is not killed by 'term.close()' + self.logger.critical('Could not close make term') + + def make_run(self, targets, *runargs, **runkwargs): + """Call make `targets` for current RIOTNode context. + + It is using `subprocess.run` internally. + + :param targets: make targets + :param *runargs: args passed to subprocess.run + :param *runkwargs: kwargs passed to subprocess.run + :return: subprocess.CompletedProcess object + """ + command = self.make_command(targets) + return subprocess.run(command, env=self.env, *runargs, **runkwargs) + + def make_command(self, targets): + """Make command for current RIOTNode context. + + :return: list of command arguments (for example for subprocess) + """ + command = ['make'] + command.extend(self.MAKE_ARGS) + if self._application_directory != '.': + dir_cmd = '--no-print-directory', '-C', self.application_directory + command.extend(dir_cmd) + command.extend(targets) + return command diff --git a/dist/pythonlibs/riotnode/riotnode/tests/__init__.py b/dist/pythonlibs/riotnode/riotnode/tests/__init__.py new file mode 100644 index 000000000000..b32d7cf7e2d9 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/__init__.py @@ -0,0 +1 @@ +"""riotnode.tests directory.""" diff --git a/dist/pythonlibs/riotnode/riotnode/tests/node_test.py b/dist/pythonlibs/riotnode/riotnode/tests/node_test.py new file mode 100644 index 000000000000..812a737967db --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/node_test.py @@ -0,0 +1,234 @@ +"""riotnode.node test module.""" + +import os +import sys +import signal +import tempfile + +import pytest +import pexpect + +import riotnode.node + +CURDIR = os.path.dirname(__file__) +APPLICATIONS_DIR = os.path.join(CURDIR, 'utils', 'application') + + +def test_riotnode_application_dir(): + """Test the creation of a riotnode with an `application_dir`.""" + riotbase = os.path.abspath(os.environ['RIOTBASE']) + application = os.path.join(riotbase, 'examples/hello-world') + board = 'native' + + env = {'BOARD': board} + node = riotnode.node.RIOTNode(application, env) + + assert node.application_directory == application + assert node.board() == board + + clean_cmd = ['make', '--no-print-directory', '-C', application, 'clean'] + assert node.make_command(['clean']) == clean_cmd + + +def test_riotnode_curdir(): + """Test the creation of a riotnode with current directory.""" + riotbase = os.path.abspath(os.environ['RIOTBASE']) + application = os.path.join(riotbase, 'examples/hello-world') + board = 'native' + + _curdir = os.getcwd() + _environ = os.environ.copy() + try: + os.environ['BOARD'] = board + os.chdir(application) + + node = riotnode.node.RIOTNode() + + assert node.application_directory == application + assert node.board() == board + assert node.make_command(['clean']) == ['make', 'clean'] + finally: + os.chdir(_curdir) + os.environ.clear() + os.environ.update(_environ) + + +@pytest.fixture(name='app_pidfile_env') +def fixture_app_pidfile_env(): + """Environment to use application pidfile""" + with tempfile.NamedTemporaryFile() as tmpfile: + yield {'PIDFILE': tmpfile.name} + + +def test_running_echo_application(app_pidfile_env): + """Test basic functionnalities with the 'echo' application.""" + env = {'BOARD': 'board', 'APPLICATION': './echo.py'} + env.update(app_pidfile_env) + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + + with node.run_term(logfile=sys.stdout) as child: + child.expect_exact('Starting RIOT node') + + # Test multiple echo + for i in range(16): + child.sendline('Hello Test {}'.format(i)) + child.expect(r'Hello Test (\d+)', timeout=1) + num = int(child.match.group(1)) + assert i == num + + +def test_running_term_with_reset(app_pidfile_env): + """Test that node resets on run_term.""" + env = {'BOARD': 'board'} + env.update(app_pidfile_env) + env['APPLICATION'] = './hello.py' + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + + with node.run_term(logfile=sys.stdout) as child: + # Firmware should have started twice + child.expect_exact('Starting RIOT node') + child.expect_exact('Hello World') + child.expect_exact('Starting RIOT node') + child.expect_exact('Hello World') + + +def test_running_term_without_reset(app_pidfile_env): + """Test not resetting node on run_term.""" + env = {'BOARD': 'board'} + env.update(app_pidfile_env) + env['APPLICATION'] = './hello.py' + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + + with node.run_term(reset=False, logfile=sys.stdout) as child: + child.expect_exact('Starting RIOT node') + child.expect_exact('Hello World') + # Firmware should start only once + with pytest.raises(pexpect.exceptions.TIMEOUT): + child.expect_exact('Starting RIOT node', timeout=1) + + +def test_running_error_cases(app_pidfile_env): + """Test basic functionnalities with the 'echo' application. + + This tests: + * stopping already stopped child + """ + # Use only 'echo' as process to exit directly + env = {'BOARD': 'board', + 'NODE_WRAPPER': 'echo', 'APPLICATION': 'Starting RIOT node'} + env.update(app_pidfile_env) + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + + with node.run_term(logfile=sys.stdout) as child: + child.expect_exact('Starting RIOT node') + + # Term is already finished and expect should trigger EOF + with pytest.raises(pexpect.EOF): + child.expect('this should eof') + + # Exiting the context manager should not crash when node is killed + + +def test_expect_not_matching_stdin(app_pidfile_env): + """Test that expect does not match stdin.""" + env = {'BOARD': 'board', 'APPLICATION': './hello.py'} + env.update(app_pidfile_env) + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + + with node.run_term(logfile=sys.stdout) as child: + child.expect_exact('Starting RIOT node') + child.expect_exact('Hello World') + + msg = "This should not be matched as it is on stdin" + child.sendline(msg) + matched = child.expect_exact([pexpect.TIMEOUT, msg], timeout=1) + assert matched == 0 + # This would have matched with `node.run_term(echo=True)` + + +def test_expect_value(app_pidfile_env): + """Test that expect value is being changed to the pattern.""" + env = {'BOARD': 'board', 'APPLICATION': './echo.py'} + env.update(app_pidfile_env) + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + + with node.run_term(logfile=sys.stdout) as child: + child.expect_exact('Starting RIOT node') + + # Exception is 'exc_info.value' and pattern is in 'exc.value' + child.sendline('lowercase') + with pytest.raises(pexpect.TIMEOUT) as exc_info: + child.expect('UPPERCASE', timeout=0.5) + assert str(exc_info.value) == 'UPPERCASE' + + # value updated and old value saved + assert exc_info.value.value == 'UPPERCASE' + assert exc_info.value.pexpect_value.startswith('Timeout exceeded.') + + # check the context is removed (should be only 2 levels) + assert len(exc_info.traceback) == 2 + + child.sendline('lowercase') + with pytest.raises(pexpect.TIMEOUT) as exc_info: + child.expect_exact('UPPERCASE', timeout=0.5) + assert str(exc_info.value) == 'UPPERCASE' + + +def test_term_cleanup(app_pidfile_env): + """Test a terminal that does a cleanup after kill. + + The term process should be able to run its cleanup. + """ + # Always run as 'deleted=True' to deleted even on early exception + # File must exist at the end of the context manager + with tempfile.NamedTemporaryFile(delete=True) as tmpfile: + env = {'BOARD': 'board'} + env.update(app_pidfile_env) + env['APPLICATION'] = './create_file.py %s' % tmpfile.name + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + with node.run_term(logfile=sys.stdout) as child: + child.expect_exact('Running') + # Ensure script is started correctly + content = open(tmpfile.name, 'r', encoding='utf-8').read() + assert content == 'Running\n' + + # File should not exist anymore so no error to create one + # File must exist to be cleaned by tempfile + open(tmpfile.name, 'x') + + +def test_killing_a_broken_term(app_pidfile_env): + """Test killing a terminal that can only be killed with SIGKILL.""" + env = {'BOARD': 'board', 'APPLICATION': './sigkill_script.py'} + env.update(app_pidfile_env) + + node = riotnode.node.RIOTNode(APPLICATIONS_DIR, env) + node.TERM_STARTED_DELAY = 1 + + with node.run_term(logfile=sys.stdout) as child: + child.expect_exact('Kill me with SIGKILL!') + child.expect(r'My PID: (\d+)') + term_pid = int(child.match.group(1)) + + msg = 'Some term process where not stopped' + with pytest.raises(RuntimeError, match=msg): + node.stop_term() + + # Send a SIGKILL to the process, it should raise an error as it is stopped + # And if it was running, it will be cleaned + with pytest.raises(ProcessLookupError): + os.kill(term_pid, signal.SIGKILL) diff --git a/dist/pythonlibs/riotnode/riotnode/tests/riotnode_test.py b/dist/pythonlibs/riotnode/riotnode/tests/riotnode_test.py new file mode 100644 index 000000000000..5ef2b2d13bb1 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/riotnode_test.py @@ -0,0 +1,10 @@ +"""riotnode.__init__ tests""" +import riotnode + + +def test_version(): + """Test there is an `__version__` attriubte. + + Goal is to have a test to run the test environment. + """ + assert getattr(riotnode, '__version__', None) is not None diff --git a/dist/pythonlibs/riotnode/riotnode/tests/utils/application/Makefile b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/Makefile new file mode 100644 index 000000000000..64050f0b10ba --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/Makefile @@ -0,0 +1,16 @@ +.PHONY: all flash reset term + +PIDFILE ?= /tmp/riotnode_test_pid +NODEPID = $(shell cat $(firstword $(wildcard $(PIDFILE)) /dev/null)) + +NODE_WRAPPER ?= ./node.py +APPLICATION ?= ./echo.py + +all: +flash: + +reset: + kill -USR1 $(NODEPID) 2>/dev/null || true + +term: + sh -c 'echo $$$$ > $(PIDFILE); exec $(NODE_WRAPPER) $(APPLICATION)' diff --git a/dist/pythonlibs/riotnode/riotnode/tests/utils/application/create_file.py b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/create_file.py new file mode 100755 index 000000000000..bd2e63cae5c5 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/create_file.py @@ -0,0 +1,36 @@ +#! /usr/bin/env python3 +"""Implement creating a file and deleting at exit + +This should show a terminal program doing a cleanup. +""" + +import os +import sys +import atexit +import signal +import argparse + +PARSER = argparse.ArgumentParser() +PARSER.add_argument('running_file') + + +def main(): + """Create a file and delete it after program exits.""" + args = PARSER.parse_args() + + # Trigger atexit on SIGHUP + signal.signal(signal.SIGHUP, (lambda *_: sys.exit(0))) + + # Delete file after program closes + # This should be the case if normally terminated + atexit.register(os.remove, args.running_file) + + with open(args.running_file, 'w', encoding='utf-8') as rfile: + rfile.write('Running\n') + print('Running') + while True: + signal.pause() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dist/pythonlibs/riotnode/riotnode/tests/utils/application/echo.py b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/echo.py new file mode 100755 index 000000000000..6e716f3b8f91 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/echo.py @@ -0,0 +1,16 @@ +#! /usr/bin/env python3 +"""Firmware implementing echoing line inputs.""" + +import sys + + +def main(): + """Print some header and echo the output.""" + print('Starting RIOT node') + print('This example will echo') + while True: + print(input()) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dist/pythonlibs/riotnode/riotnode/tests/utils/application/hello.py b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/hello.py new file mode 100755 index 000000000000..5c6d09cabb2c --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/hello.py @@ -0,0 +1,17 @@ +#! /usr/bin/env python3 +"""Firmware implementing a simple hello-world.""" + +import sys +import signal + + +def main(): + """Print some header and do nothing.""" + print('Starting RIOT node') + print('Hello World') + while True: + signal.pause() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dist/pythonlibs/riotnode/riotnode/tests/utils/application/node.py b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/node.py new file mode 100755 index 000000000000..a3a9abed5551 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/node.py @@ -0,0 +1,74 @@ +#! /usr/bin/env python3 +"""Wrap an application to behave like a board firmware. + ++ Start a command given as argument ++ Handle 'reset' the firmware when receiving `SIGUSR1` + + +Ideas for extensions: + +* resetting or not on reset +* See how to implement loosing some of the output on first startup +""" + + +import sys +import signal +import threading +import argparse +import subprocess + +PARSER = argparse.ArgumentParser() +PARSER.add_argument('argument', nargs='+', default=[]) + +# Signals sent by 'pexpect' + SIGTERM +FORWARDED_SIGNALS = (signal.SIGHUP, signal.SIGCONT, signal.SIGINT, + signal.SIGTERM) + + +def forward_signal(signum, proc): + """Forward signal to child.""" + if not proc.poll(): + proc.send_signal(signum) + + +def _run_cmd(args, termonsig=signal.SIGUSR1, **popenkwargs): + """Run a subprocess of `args`. + + It will be terminated on `termonsig` signal. + + :param args: command arguments + :param termonsig: terminate the process on `termonsig` signal + :param **popenkwargs: Popen kwargs + :return: True if process should be restarted + """ + restart_process = threading.Event() + proc = subprocess.Popen(args, **popenkwargs) + + # Forward cleanup processes to child + for sig in FORWARDED_SIGNALS: + signal.signal(sig, lambda signum, _: forward_signal(signum, proc)) + + # set 'termonsig' handler for reset + def _reset(*_): + """Terminate process and set the 'restart_process' flag.""" + restart_process.set() + proc.terminate() + signal.signal(termonsig, _reset) + + proc.wait() + return restart_process.is_set() + + +def main(): + """Run an application in a loop. + + On 'SIGUSR1' the application will be reset. + """ + args = PARSER.parse_args() + while _run_cmd(args.argument): + pass + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dist/pythonlibs/riotnode/riotnode/tests/utils/application/sigkill_script.py b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/sigkill_script.py new file mode 100755 index 000000000000..5a2c036386da --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/utils/application/sigkill_script.py @@ -0,0 +1,18 @@ +#! /usr/bin/env python3 +"""Ignore SIGINT/SIGTERM/SIGHUP, kill with SIGKILL.""" +import os +import signal + + +def main(): + """Only kill this program with SIGKILL.""" + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGHUP, signal.SIG_IGN) + print('Kill me with SIGKILL!') + print('My PID: %u' % os.getpid()) + signal.pause() + + +if __name__ == '__main__': + main() diff --git a/dist/pythonlibs/riotnode/riotnode/tests/utils_test.py b/dist/pythonlibs/riotnode/riotnode/tests/utils_test.py new file mode 100644 index 000000000000..9e627a3a6fe2 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/tests/utils_test.py @@ -0,0 +1,123 @@ +"""rionode.utils test module.""" + +import time +import multiprocessing +import logging + +import psutil +import pytest + +import riotnode.utils + + +def _process_spawn(num, queue): + """Function for a process which spawns `num` number of processes and reports + their PIDs back. + """ + procs = [] + procs_pids = [] + + for _ in range(num): + proc = multiprocessing.Process(target=_process_wait) + proc.start() + procs.append(proc) + procs_pids.append(proc.pid) + + queue.put(procs_pids) + + # wait for all + for proc in procs: + proc.join() + + +def _process_wait(): + """Function for a dummy process which does nothing.""" + while True: + time.sleep(1) + + +def test_ensuring_all_subproc_are_stopped(): + """Tests that all subprocesses stopped.""" + queue = multiprocessing.Queue() + spawner = multiprocessing.Process(target=_process_spawn, args=(2, queue)) + spawner.start() + procs_pids = queue.get() + + procs = [] + for pid in procs_pids: + procs.append(psutil.Process(pid)) + + # should raise an exception as processes are not stopped + with pytest.raises(RuntimeError): + with riotnode.utils.ensure_all_subprocesses_stopped(spawner.pid, + logging): + pass + + # now all processes should be stopped + running = False + for proc in procs: + if proc.is_running(): + running = True + proc.terminate() + + try: + parent = psutil.Process(spawner.pid) + if parent.is_running(): + running = True + parent.terminate() + except psutil.NoSuchProcess: + pass + + assert not running + + +def test_getting_pid_subprocesses(): + """Tests getting all subprocesses PIDs.""" + queue = multiprocessing.Queue() + spawner = multiprocessing.Process(target=_process_spawn, args=(2, queue)) + spawner.start() + procs_pids = queue.get() + + subprocesses = riotnode.utils.pid_subprocesses(spawner.pid) + subprocesses_pids = (s.pid for s in subprocesses) + + try: + # returns all subprocesses and the parent + assert len(subprocesses) == len(procs_pids) + 1 + assert spawner.pid in subprocesses_pids + for pid in procs_pids: + assert pid in subprocesses_pids + + assert riotnode.utils.pid_subprocesses(None) == [] + finally: + for proc in subprocesses: + proc.terminate() + + +def test_ensuring_all_procs_are_stopped(): + """Tests that all processes are stopped.""" + queue = multiprocessing.Queue() + spawner = multiprocessing.Process(target=_process_spawn, args=(2, queue)) + spawner.start() + procs_pids = queue.get() + + procs = [] + for pid in procs_pids: + procs.append(psutil.Process(pid)) + + # Should raise exception as processes are running + with pytest.raises(RuntimeError): + riotnode.utils.ensure_processes_stopped(procs, logging) + + # All processes should be stopped + running = False + for proc in procs: + if proc.is_running(): + running = True + proc.terminate() + + spawner.terminate() + assert not running + + # Now it should not raise exceptions + riotnode.utils.ensure_processes_stopped(procs, logging) diff --git a/dist/pythonlibs/riotnode/riotnode/utils.py b/dist/pythonlibs/riotnode/riotnode/utils.py new file mode 100644 index 000000000000..1e621a3feb62 --- /dev/null +++ b/dist/pythonlibs/riotnode/riotnode/utils.py @@ -0,0 +1,69 @@ +"""Some utilities functions""" + +import contextlib + +import psutil + + +@contextlib.contextmanager +def ensure_all_subprocesses_stopped(pid, logger, timeout=3): + """Ensure all subprocesses for 'pid' are correctly stopped. + + context manager. + """ + try: + processes = pid_subprocesses(pid) + yield + finally: + ensure_processes_stopped(processes, logger, timeout=timeout) + + +def pid_subprocesses(pid): + """Return subprocesses for pid including itself. + + If pid is None return nothing. + """ + if pid is None: + return [] + + try: + proc = psutil.Process(pid) + processes = [proc] + processes.extend(proc.children(recursive=True)) + return processes + except psutil.NoSuchProcess: + return [] + + +def ensure_processes_stopped(processes, logger, timeout=3): + """Ensure processes are correctly stopped and kill them if not. + + Raise a RuntimeError if it is not the case. + """ + # Use a list to call all of them + stopped_processes = [_ensure_process_stopped(p, logger, timeout) + for p in processes] + if not all(stopped_processes): + raise RuntimeError('Some term process where not stopped') + + +def _ensure_process_stopped(proc, logger, timeout=3): + """Ensure the given process has been stopped. + + If it has not, do a critical logging message and kill it. + :type proc: psutil.Process + :return: True if process was stopped + """ + if not proc.is_running(): + return True + logger.critical('process %u:%s was not stopped', proc.pid, proc.status()) + + proc.kill() + try: + proc.wait(timeout) + except psutil.TimeoutExpired: # pragma: no cover + # No sure how to cover this… + logger.critical( + 'process %u:%s is not killable', proc.pid, proc.status()) + + return False diff --git a/dist/pythonlibs/riotnode/setup.cfg b/dist/pythonlibs/riotnode/setup.cfg new file mode 100644 index 000000000000..16d2ea34817e --- /dev/null +++ b/dist/pythonlibs/riotnode/setup.cfg @@ -0,0 +1,15 @@ +[tool:pytest] +addopts = -v --junit-xml=test-report.xml + --doctest-modules + --cov=riotnode --cov-branch + --cov-report=term --cov-report=xml --cov-report=html +testpaths = riotnode + +[lint] +lint-reports = no +lint-disable = locally-disabled,star-args +lint-msg-template = {path}:{line}: [{msg_id}({symbol}), {obj}] {msg} + +[flake8] +exclude = .tox,dist,doc,build,*.egg +max-complexity = 10 diff --git a/dist/pythonlibs/riotnode/setup.py b/dist/pythonlibs/riotnode/setup.py new file mode 100644 index 000000000000..7f5b5591ff9e --- /dev/null +++ b/dist/pythonlibs/riotnode/setup.py @@ -0,0 +1,48 @@ +#! /usr/bin/env python3 + +import os +from setuptools import setup, find_packages + +PACKAGE = 'riotnode' +LICENSE = 'LGPLv2.1' +URL = 'https://github.com/RIOT-OS/RIOT' + + +def get_version(package): + """ Extract package version without importing file + Importing cause issues with coverage, + (modules can be removed from sys.modules to prevent this) + Importing __init__.py triggers importing rest and then requests too + + Inspired from pep8 setup.py + """ + with open(os.path.join(package, '__init__.py')) as init_fd: + for line in init_fd: + if line.startswith('__version__'): + return eval(line.split('=')[-1]) # pylint:disable=eval-used + return None + + +setup( + name=PACKAGE, + version=get_version(PACKAGE), + description='RIOTNode python abstraction', + long_description=open('README.rst').read(), + author='Gaëtan Harter', + author_email='gaetan.harter@fu-berlin.de', + url=URL, + license=LICENSE, + download_url=URL, + packages=find_packages(), + classifiers=['Development Status :: 2 - Pre-Alpha', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Intended Audience :: End Users/Desktop', + 'Environment :: Console', + 'Topic :: Utilities', ], + install_requires=['pexpect', 'psutil'], + python_requires='>=3.5', +) diff --git a/dist/pythonlibs/riotnode/tox.ini b/dist/pythonlibs/riotnode/tox.ini new file mode 100644 index 000000000000..37670ebe1c11 --- /dev/null +++ b/dist/pythonlibs/riotnode/tox.ini @@ -0,0 +1,33 @@ +[tox] +envlist = test,lint,flake8 + +[testenv] +basepython = python3 +passenv = RIOTBASE +setenv = + RIOTBASE = {toxinidir}/../../.. + package = riotnode +commands = + test: {[testenv:test]commands} + lint: {[testenv:lint]commands} + flake8: {[testenv:flake8]commands} + +[testenv:test] +deps = + pytest + pytest-cov +commands = + pytest + +[testenv:lint] +deps = + pylint + pytest +commands = + pylint {envsitepackagesdir}/{env:package} + # This does not check files in 'tests/utils/application' + +[testenv:flake8] +deps = flake8 +commands = + flake8 diff --git a/dist/pythonlibs/sitecustomize.py b/dist/pythonlibs/sitecustomize.py new file mode 100644 index 000000000000..cb020d758b47 --- /dev/null +++ b/dist/pythonlibs/sitecustomize.py @@ -0,0 +1,7 @@ +import os +import sys + +# Allow importing packages implemented in sub directories +# Prepend as the directory has the same name as the package + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'riotnode')) diff --git a/dist/pythonlibs/testrunner/__init__.py b/dist/pythonlibs/testrunner/__init__.py index 31fa260eb469..a6e80e5b9e9b 100755 --- a/dist/pythonlibs/testrunner/__init__.py +++ b/dist/pythonlibs/testrunner/__init__.py @@ -28,13 +28,13 @@ def run(testfunc, timeout=TIMEOUT, echo=True, traceback=False): try: testfunc(child) except pexpect.TIMEOUT: - trace = find_exc_origin(sys.exc_info()[2]) + trace = find_exc_origin(sys.exc_info()[2], pexpect_path=pexpect_path) print("Timeout in expect script at \"%s\" (%s:%d)" % trace) if traceback: print_tb(sys.exc_info()[2]) return 1 except pexpect.EOF: - trace = find_exc_origin(sys.exc_info()[2]) + trace = find_exc_origin(sys.exc_info()[2], pexpect_path=pexpect_path) print("Unexpected end of file in expect script at \"%s\" (%s:%d)" % trace) if traceback: