From 9679d0548a42af831f37edabe1d63ab6e7746db4 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Thu, 28 Feb 2019 16:57:23 +0100 Subject: [PATCH 1/9] makefiles: introduce rawterm recipe and variables. `make rawterm` is the new interface for automated access to the serial port. It will be used in testing. In contrast with `make term`, which is intended for human-machine-interaction, rawterm is meant to comminicate the node with automated scripts running on the host machine. rawterm must: - Not have any input or output buffering. - Not modify input or output in any way (no extra prompts, logging artifacts, translations etc.) - Have local echo disabled. - Not print anything to stdout that was not generated by the node (i.e. no startup banners.) With this definition it is very easy to implement a generic "nice" terminal (that is `make term`) as a wrapper around rawterm. --- Makefile.include | 4 +++- makefiles/info.inc.mk | 2 ++ makefiles/tools/serial.inc.mk | 3 +++ makefiles/vars.inc.mk | 18 +++++++++++++++++- 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Makefile.include b/Makefile.include index d0468a86124b..eb79038f3315 100644 --- a/Makefile.include +++ b/Makefile.include @@ -543,8 +543,10 @@ flash-only: $(FLASHDEPS) preflash: $(BUILD_BEFORE_FLASH) $(PREFLASHER) $(PREFFLAGS) +rawterm: $(filter flash, $(MAKECMDGOALS)) $(TERMDEPS) + $(RAWTERMPROG) $(RAWTERMFLAGS) + term: $(filter flash, $(MAKECMDGOALS)) $(TERMDEPS) - $(call check_cmd,$(TERMPROG),Terminal program) $(TERMPROG) $(TERMFLAGS) list-ttys: diff --git a/makefiles/info.inc.mk b/makefiles/info.inc.mk index 239b819c114b..b14fc6869768 100644 --- a/makefiles/info.inc.mk +++ b/makefiles/info.inc.mk @@ -80,6 +80,8 @@ info-build: @echo '' @echo 'TERMPROG: $(TERMPROG)' @echo 'TERMFLAGS: $(TERMFLAGS)' + @echo 'RAWTERMPROG: $(RAWTERMPROG)' + @echo 'RAWTERMFLAGS: $(RAWTERMFLAGS)' @echo 'PORT: $(PORT)' @echo '' @echo 'DEBUGGER: $(DEBUGGER)' diff --git a/makefiles/tools/serial.inc.mk b/makefiles/tools/serial.inc.mk index 8abcf7fe1d1f..7a9d7105d02a 100644 --- a/makefiles/tools/serial.inc.mk +++ b/makefiles/tools/serial.inc.mk @@ -23,4 +23,7 @@ else ifeq ($(RIOT_TERMINAL),socat) else ifeq ($(RIOT_TERMINAL),picocom) TERMPROG ?= picocom TERMFLAGS ?= --nolock --imap lfcrlf --baud "$(BAUD)" "$(PORT)" +else ifeq ($(RIOT_TERMINAL),custom) + # Do nothing. This name is reserved so that TERMFLAGS does not get + # set if the user wants to override TERMPROG. endif diff --git a/makefiles/vars.inc.mk b/makefiles/vars.inc.mk index bff4d74f52a1..5ef69db158a8 100644 --- a/makefiles/vars.inc.mk +++ b/makefiles/vars.inc.mk @@ -72,8 +72,24 @@ export GIT_CACHE_DIR # path to git-cache cache directory export FLASHER # The command to call on "make flash". export FFLAGS # The parameters to supply to FLASHER. export FLASH_ADDR # Define an offset to flash code into ROM memory. -# TERMPROG # The command to call on "make term". + +# --- Terminal Access --- # + +# RIOT_TERMINAL # Preset to use for the "user friendly" terminal. + # TERMPROG and TERMFLAGS will be set according to + # RIOT_TERMINAL. The special name "custom" can be + # used to leave these variables undefined. +# TERMPROG # The command to call on "make term". This should be + # "user friendly" terminal, with line editing-history, etc. + # By default, rlwrap is used around the RAWTERM program. # TERMFLAGS # Additional parameters to supply to TERMPROG. + +# RIOT_RAWTERMINAL # Like RIOT_TERMINAL but for the raw/testing terminal. +# RAWTERMPROG # The command to call on "make rawterm". This terminal + # is used for testing. It should have no buffering, + # and no translation or modification of input or output. +# RAWTERMFLAGS # Additional parameters to supply to RAWTERMPROG. + export PORT # The port to connect the TERMPROG to. export ELFFILE # The unstripped result of the compilation. export HEXFILE # The stripped result of the compilation. From 53800ba824c1577c665a8a19acd333998b7f27aa Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Thu, 28 Feb 2019 17:32:35 +0100 Subject: [PATCH 2/9] makefiles/serial: Add raw terminal programs Add socat, picocom and jlink (for rtt) as terminals, and remove socat from the cooked terminal list. --- boards/ruuvitag/Makefile.include | 3 +-- boards/thingy52/Makefile.include | 3 +-- makefiles/tools/jlink.inc.mk | 5 +++++ makefiles/tools/serial.inc.mk | 18 ++++++++++++++---- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/boards/ruuvitag/Makefile.include b/boards/ruuvitag/Makefile.include index eb284c27fc33..2444cc71952e 100644 --- a/boards/ruuvitag/Makefile.include +++ b/boards/ruuvitag/Makefile.include @@ -3,8 +3,7 @@ CPU_MODEL = nrf52832xxaa # for this board, we are using Segger's RTT as default terminal interface USEMODULE += stdio_rtt -TERMPROG = $(RIOTTOOLS)/jlink/jlink.sh -TERMFLAGS = term_rtt +RIOT_RAWTERMINAL = jlink # use shared Makefile.include include $(RIOTBOARD)/common/nrf52xxxdk/Makefile.include diff --git a/boards/thingy52/Makefile.include b/boards/thingy52/Makefile.include index bf6e40dba2c8..5cd5b0bd3027 100644 --- a/boards/thingy52/Makefile.include +++ b/boards/thingy52/Makefile.include @@ -3,8 +3,7 @@ CPU_MODEL = nrf52832xxaa # for this board, we are using Segger's RTT as default terminal interface USEMODULE += stdio_rtt -TERMPROG = $(RIOTTOOLS)/jlink/jlink.sh -TERMFLAGS = term_rtt +RIOT_RAWTERMINAL = jlink # use shared Makefile.include include $(RIOTBOARD)/common/nrf52/Makefile.include diff --git a/makefiles/tools/jlink.inc.mk b/makefiles/tools/jlink.inc.mk index 61abdd129065..9a73b2407330 100644 --- a/makefiles/tools/jlink.inc.mk +++ b/makefiles/tools/jlink.inc.mk @@ -9,3 +9,8 @@ export FFLAGS ?= flash $(FLASHFILE) export DEBUGGER_FLAGS ?= debug $(ELFFILE) export DEBUGSERVER_FLAGS ?= debug-server export RESET_FLAGS ?= reset + +ifeq ($(RIOT_RAWTERMINAL),jlink) + RAWTERMPROG ?= $(RIOTTOOLS)/jlink/jlink.sh + RAWTERMFLAGS ?= term_rtt +endif diff --git a/makefiles/tools/serial.inc.mk b/makefiles/tools/serial.inc.mk index 7a9d7105d02a..6a11eb41e0de 100644 --- a/makefiles/tools/serial.inc.mk +++ b/makefiles/tools/serial.inc.mk @@ -16,10 +16,6 @@ RIOT_TERMINAL ?= pyterm ifeq ($(RIOT_TERMINAL),pyterm) TERMPROG ?= $(RIOTTOOLS)/pyterm/pyterm TERMFLAGS ?= -p "$(PORT)" -b "$(BAUD)" -else ifeq ($(RIOT_TERMINAL),socat) - SOCAT_OUTPUT ?= - - TERMPROG ?= $(RIOT_TERMINAL) - TERMFLAGS ?= $(SOCAT_OUTPUT) open:$(PORT),b$(BAUD),echo=0,raw else ifeq ($(RIOT_TERMINAL),picocom) TERMPROG ?= picocom TERMFLAGS ?= --nolock --imap lfcrlf --baud "$(BAUD)" "$(PORT)" @@ -27,3 +23,17 @@ else ifeq ($(RIOT_TERMINAL),custom) # Do nothing. This name is reserved so that TERMFLAGS does not get # set if the user wants to override TERMPROG. endif + +RIOT_RAWTERMINAL ?= socat +ifeq ($(RIOT_RAWTERMINAL),socat) + SOCAT_LOCAL ?= STDIO,icanon=0,echo=0,escape=0x04 + SOCAT_REMOTE ?= open:$(PORT),b$(BAUD),echo=0,raw + RAWTERMPROG ?= $(RIOT_RAWTERMINAL) + RAWTERMFLAGS ?= $(SOCAT_LOCAL) $(SOCAT_REMOTE) +else ifeq ($(RIOT_RAWTERMINAL),picocom) + RAWTERMPROG ?= picocom + RAWTERMFLAGS ?= --nolock -q --imap lfcrlf --baud "$(BAUD)" "$(PORT)" +else ifeq ($(RIOT_RAWTERMINAL),custom) + # Do nothing. This name is reserved so that TERMFLAGS does not get + # set if the user wants to override TERMPROG. +endif From 3849f8a680f7179f2b5e1fe95e7934827556afb9 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Thu, 28 Feb 2019 17:33:42 +0100 Subject: [PATCH 3/9] makefiles/serial: add rlwrap support. Rlwrap allows using any raw terminal (if it is really raw) and adding readline support around it, with history that is even preserved thoughout sessions. The -C option is used to create a different history file for each board. --- makefiles/tools/serial.inc.mk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/makefiles/tools/serial.inc.mk b/makefiles/tools/serial.inc.mk index 6a11eb41e0de..054149afab23 100644 --- a/makefiles/tools/serial.inc.mk +++ b/makefiles/tools/serial.inc.mk @@ -16,6 +16,11 @@ RIOT_TERMINAL ?= pyterm ifeq ($(RIOT_TERMINAL),pyterm) TERMPROG ?= $(RIOTTOOLS)/pyterm/pyterm TERMFLAGS ?= -p "$(PORT)" -b "$(BAUD)" +else ifeq ($(RIOT_TERMINAL),rlwrap) + TERMPROG ?= $(RIOT_TERMINAL) + RLWRAP_PROMPT ?= -pPurple -S 'RIOT $$ ' + RLWRAP_FLAGS ?= -a -C RIOT-$(BOARD) + TERMFLAGS ?= $(RLWRAP_PROMPT) $(RLWRAP_FLAGS) $(RAWTERMPROG) $(RAWTERMFLAGS) else ifeq ($(RIOT_TERMINAL),picocom) TERMPROG ?= picocom TERMFLAGS ?= --nolock --imap lfcrlf --baud "$(BAUD)" "$(PORT)" From 990020e995288d0fd1f67dd50911cfc5a0c3378c Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Fri, 1 Mar 2019 16:01:42 +0100 Subject: [PATCH 4/9] boards/lobaro-lorabox: force RIOT_TERMINAL=pyterm The lobaro-lorabox sets some pyterm-specific TERMFLAGS, so we must make sure that pyterm is selected. --- boards/lobaro-lorabox/Makefile.include | 2 ++ 1 file changed, 2 insertions(+) diff --git a/boards/lobaro-lorabox/Makefile.include b/boards/lobaro-lorabox/Makefile.include index 45b63d68dd32..b955686ebbdf 100644 --- a/boards/lobaro-lorabox/Makefile.include +++ b/boards/lobaro-lorabox/Makefile.include @@ -10,6 +10,8 @@ PORT_LINUX ?= /dev/ttyUSB0 PORT_DARWIN ?= $(firstword $(sort $(wildcard /dev/tty.usbmodem*))) # setup serial terminal +# The --set-rts option is specific to pyterm so we are forced to use it. +RIOT_TERMINAL = pyterm include $(RIOTMAKE)/tools/serial.inc.mk FLASHER = $(RIOTTOOLS)/stm32loader/stm32loader.py From 7a2e825c57528b591e3d8e41b5f503d1e6ac71c0 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Fri, 1 Mar 2019 16:05:23 +0100 Subject: [PATCH 5/9] boards/msba2: force RIOT_TERMINAL=pyterm The msba2 sets some pyterm-specific TERMFLAGS, so we must make sure that pyterm is selected. --- boards/common/msba2/Makefile.include | 2 ++ 1 file changed, 2 insertions(+) diff --git a/boards/common/msba2/Makefile.include b/boards/common/msba2/Makefile.include index bcd3499f8e9c..4505a7a578b6 100644 --- a/boards/common/msba2/Makefile.include +++ b/boards/common/msba2/Makefile.include @@ -19,6 +19,8 @@ PORT_LINUX ?= /dev/ttyUSB0 # This does not make a lot of sense, but it has the same value as the previous code PORT_DARWIN ?= /dev/tty.usbserial-ARM +# The -tg option is specific to pyterm so we are forced to use it. +RIOT_TERMINAL = pyterm TERMFLAGS += -tg -p "$(PORT)" include $(RIOTMAKE)/tools/serial.inc.mk From 086480aa94ef2e7d128c99925e92f040ac7931b7 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Mon, 4 Mar 2019 14:37:05 +0100 Subject: [PATCH 6/9] boards/native: Use socat as a termprog. The riot executable is NOT a terminal program. Therefore, to mantain consistency with the rest of the targets and avoid hacky-hacks the TERMPROG should not be ELFFILE. This commit creates a new variables specially for the native platform, NATIVE_EXEC, NATIVE_FLAGS. The (raw)terminal program is socat, which wraps NATIVE_EXEC. This also has the benefit of disabling buffering and echo in the terminal so that narive behaves more similar to bare-metal RIOT. Doing this terminal reconfiguration in RIOT is possible, but dangerous as the settings must be reset after exit or the users terminal emulator will be left broken. Using socat as RIOT_RAWTERMINAL means rlwrap can be used as RIOT_TERMINAL, further unifying the behaviour of native and bare-metal. --- boards/native/Makefile.include | 35 ++++++++++++++++++++++++---------- makefiles/vars.inc.mk | 4 ++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/boards/native/Makefile.include b/boards/native/Makefile.include index 91b4c84b1fd2..bea7669f4096 100644 --- a/boards/native/Makefile.include +++ b/boards/native/Makefile.include @@ -40,7 +40,30 @@ else export DEBUGGER ?= gdb endif -TERMPROG ?= $(ELFFILE) +# set the tap interface for term/valgrind +# FIXME: this is not a port. It should be renames as "NET_INTERFACE" or +# something else that is more honest. +ifneq (,$(filter netdev_default gnrc_netdev_default,$(USEMODULE))) + export PORT ?= tap0 +else + export PORT = +endif + +RIOT_TERMINAL ?= rlwrap +# Silence the "Warning: no PORT set!" message +# PORT ?= not-applicable +# Wrapping native in socat disables line buffering so the behavior of native +# is more similar to that of bare-metal RIOT. +RIOT_RAWTERMINAL ?= socat +NATIVE_EXEC ?= $(ELFFILE) +NATIVE_FLAGS += $(PORT) # Flags to the native executable + +# checkout the "sigint" flags. Without it ctrl-c just exits socat, which breaks +# the pipe on the RIOT executable causing SIGPIPE and a dirty exit. +SOCAT_REMOTE = exec:"$(NATIVE_EXEC) $(NATIVE_FLAGS)",sigint $(PORT) + +include $(RIOTMAKE)/tools/serial.inc.mk + export FLASHER = true export VALGRIND ?= valgrind export CGANNOTATE ?= cg_annotate @@ -93,14 +116,6 @@ else endif export LINKFLAGS += -ffunction-sections -# set the tap interface for term/valgrind -ifneq (,$(filter netdev_default gnrc_netdev_default,$(USEMODULE))) - export PORT ?= tap0 -else - export PORT = -endif - -TERMFLAGS := $(PORT) $(TERMFLAGS) export ASFLAGS = ifeq ($(shell basename $(DEBUGGER)),lldb) @@ -117,7 +132,7 @@ debug-valgrind-server: export VALGRIND_FLAGS ?= --vgdb=yes --vgdb-error=0 -v \ --leak-check=full --track-origins=yes --fullpath-after=$(RIOTBASE) \ --read-var-info=yes term-cachegrind: export CACHEGRIND_FLAGS += --tool=cachegrind -term-gprof: TERMPROG = GMON_OUT_PREFIX=gmon.out $(ELFFILE) +term-gprof: export GMON_OUT_PREFIX=gmon.out all-valgrind: export CFLAGS += -DHAVE_VALGRIND_H -g3 all-valgrind: export NATIVEINCLUDES += $(shell pkg-config valgrind --cflags) all-debug: export CFLAGS += -g3 diff --git a/makefiles/vars.inc.mk b/makefiles/vars.inc.mk index 5ef69db158a8..e0277acc685c 100644 --- a/makefiles/vars.inc.mk +++ b/makefiles/vars.inc.mk @@ -90,6 +90,10 @@ export FLASH_ADDR # Define an offset to flash code into ROM memory. # and no translation or modification of input or output. # RAWTERMFLAGS # Additional parameters to supply to RAWTERMPROG. +# --- Native-specific --- # +# NATIVE_EXEC # Executable for native builds. +# NATIVE_FLAGS # Command line arguments for the native executable. + export PORT # The port to connect the TERMPROG to. export ELFFILE # The unstripped result of the compilation. export HEXFILE # The stripped result of the compilation. From 0ad9540ae2dece9a0cc0d4b3f80e8a6a75f1ace9 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Mon, 4 Mar 2019 15:04:03 +0100 Subject: [PATCH 7/9] examples,tests: replace TERMFLAGS by NATIVE_FLAGS for native. The native executable is no longer the TERMPROG. Therefore, the flags are not given by TERMFLAGS, but by NATIVE_FLAGS. --- examples/gnrc_border_router/Makefile | 2 +- tests/gnrc_ipv6_ext/Makefile | 2 +- tests/gnrc_rpl_srh/Makefile | 2 +- tests/socket_zep/Makefile | 2 +- tests/socket_zep/tests/01-run.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/gnrc_border_router/Makefile b/examples/gnrc_border_router/Makefile index 82f8f7ff945a..ba1277c8d0b2 100644 --- a/examples/gnrc_border_router/Makefile +++ b/examples/gnrc_border_router/Makefile @@ -35,7 +35,7 @@ ifeq (,$(filter native,$(BOARD))) CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE) -DUSE_ETHOS_FOR_STDIO else GNRC_NETIF_NUMOF := 2 - TERMFLAGS += -z [::1]:17754 + NATIVE_FLAGS += -z [::1]:17754 USEMODULE += socket_zep endif diff --git a/tests/gnrc_ipv6_ext/Makefile b/tests/gnrc_ipv6_ext/Makefile index 009a309074b1..fe5c5efb1d82 100644 --- a/tests/gnrc_ipv6_ext/Makefile +++ b/tests/gnrc_ipv6_ext/Makefile @@ -19,7 +19,7 @@ export TAP ?= tap0 ifeq (native,$(BOARD)) USEMODULE += netdev_tap - TERMFLAGS ?= $(TAP) + NATIVE_FLAGS ?= $(TAP) else USEMODULE += ethos diff --git a/tests/gnrc_rpl_srh/Makefile b/tests/gnrc_rpl_srh/Makefile index 098caf6be454..5f794762110c 100644 --- a/tests/gnrc_rpl_srh/Makefile +++ b/tests/gnrc_rpl_srh/Makefile @@ -22,7 +22,7 @@ CFLAGS += -DOUTPUT=TEXT ifeq (native,$(BOARD)) USEMODULE += netdev_tap - TERMFLAGS ?= $(TAP) + NATIVE_FLAGS ?= $(TAP) else USEMODULE += ethos diff --git a/tests/socket_zep/Makefile b/tests/socket_zep/Makefile index dbf82761e22c..c6c54cff85e5 100644 --- a/tests/socket_zep/Makefile +++ b/tests/socket_zep/Makefile @@ -10,6 +10,6 @@ USEMODULE += socket_zep CFLAGS += -DDEVELHELP -TERMFLAGS ?= -z [::1]:17754 +NATIVE_FLAGS ?= -z [::1]:17754 include $(RIOTBASE)/Makefile.include diff --git a/tests/socket_zep/tests/01-run.py b/tests/socket_zep/tests/01-run.py index 843f3421b74e..57cd92a7ce4e 100755 --- a/tests/socket_zep/tests/01-run.py +++ b/tests/socket_zep/tests/01-run.py @@ -54,7 +54,7 @@ def testfunc(child): if __name__ == "__main__": - os.environ['TERMFLAGS'] = "-z [%s]:%d,[%s]:%d" % ( + os.environ['NATIVE_FLAGS'] = "-z [%s]:%d,[%s]:%d" % ( zep_params['local_addr'], zep_params['local_port'], zep_params['remote_addr'], zep_params['remote_port']) s = socket.socket(family=socket.AF_INET6, type=socket.SOCK_DGRAM) From 88e14f61d6de8c37a5dc069e11d11f38dbf37869 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Wed, 6 Mar 2019 16:12:33 +0100 Subject: [PATCH 8/9] testrunner: Use `make rawterm` Use the raw terminal to run the tests. This is supposed to simplify the test scripts as it it no longer possible for the output/decorations of the terminal program to be confused with the output of the node. In addition, the raw terminal is more likely to be more lightweight. The lack of buffering in the raw terminal makes it possible to test low level uart functions. Decoupling the UI terminal from the testing terminal also gives us more freedom to improve or change the UI without worrying about breaking the tests. --- dist/pythonlibs/testrunner/spawn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/pythonlibs/testrunner/spawn.py b/dist/pythonlibs/testrunner/spawn.py index 84164d1a1d0e..de4974bbb592 100644 --- a/dist/pythonlibs/testrunner/spawn.py +++ b/dist/pythonlibs/testrunner/spawn.py @@ -36,7 +36,7 @@ def find_exc_origin(exc_info): def setup_child(timeout=10, spawnclass=pexpect.spawnu, env=None, logfile=None): - child = spawnclass("make term", env=env, timeout=timeout, + child = spawnclass("make rawterm", env=env, timeout=timeout, codec_errors='replace', echo=False) # on many platforms, the termprog needs a short while to be ready... From 8d74a1bc54b2fa45eededde88491450cb956a940 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Tue, 28 May 2019 15:55:08 +0200 Subject: [PATCH 9/9] fixup! connect native to a pty in socat. This is needed in order to set the options correctly (otherwise one gets an "inappropriate ioctl") --- boards/native/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boards/native/Makefile.include b/boards/native/Makefile.include index bea7669f4096..75598427b68b 100644 --- a/boards/native/Makefile.include +++ b/boards/native/Makefile.include @@ -60,7 +60,7 @@ NATIVE_FLAGS += $(PORT) # Flags to the native executable # checkout the "sigint" flags. Without it ctrl-c just exits socat, which breaks # the pipe on the RIOT executable causing SIGPIPE and a dirty exit. -SOCAT_REMOTE = exec:"$(NATIVE_EXEC) $(NATIVE_FLAGS)",sigint $(PORT) +SOCAT_REMOTE = exec:"$(NATIVE_EXEC) $(NATIVE_FLAGS)",sigint,pty $(PORT) include $(RIOTMAKE)/tools/serial.inc.mk