diff --git a/test_tracetools_launch/test/test_tracetools_launch/test_trace_action.py b/test_tracetools_launch/test/test_tracetools_launch/test_trace_action.py index c95589b4..19b8f565 100644 --- a/test_tracetools_launch/test/test_tracetools_launch/test_trace_action.py +++ b/test_tracetools_launch/test/test_tracetools_launch/test_trace_action.py @@ -19,6 +19,7 @@ import shutil import tempfile import textwrap +from typing import Iterable from typing import List from typing import Optional from typing import TextIO @@ -116,8 +117,14 @@ def _check_trace_action( perform_typed_substitution(context, action.append_trace, bool) ) self.assertEqual(0, len(action.events_kernel)) + # Covert events_ust to list of lists if needed + if all(not isinstance(x, Iterable) or isinstance(x, str) for x in events_ust): + events_ust = [events_ust] self.assertEqual( - events_ust, [perform_substitutions(context, x) for x in action.events_ust]) + events_ust, + [[perform_substitutions(context, x) for x in channel_events] + for channel_events in action.events_ust], + ) self.assertEqual( subbuffer_size_ust, perform_typed_substitution(context, action.subbuffer_size_ust, int) @@ -617,6 +624,26 @@ def test_append_trace(self) -> None: shutil.rmtree(tmpdir) + def test_multiple_channels(self) -> None: + tmpdir = tempfile.mkdtemp(prefix='TestTraceAction__test_multiple_channels') + + action = Trace( + session_name='my-session-name', + base_path=tmpdir, + events_kernel=[], + syscalls=[], + events_ust=[ + ['ros2:*'], + ['*'], + ], + subbuffer_size_ust=524288, + subbuffer_size_kernel=1048576, + ) + context = self._assert_launch_no_errors([action]) + self._check_trace_action(action, context, tmpdir, events_ust=[['ros2:*'], ['*']]) + + shutil.rmtree(tmpdir) + if __name__ == '__main__': unittest.main() diff --git a/tracetools_launch/tracetools_launch/action.py b/tracetools_launch/tracetools_launch/action.py index 9eef60d1..be3f5ff2 100644 --- a/tracetools_launch/tracetools_launch/action.py +++ b/tracetools_launch/tracetools_launch/action.py @@ -19,6 +19,7 @@ import re import shlex from typing import cast +from typing import Generator from typing import Iterable from typing import List from typing import Mapping @@ -154,8 +155,11 @@ def __init__( :param events_ust: the list of ROS UST events to enable; if it's `None`, the default ROS events are used for a normal session, and the default ROS initialization events are used for the snapshot session in case of a dual session; if it's an empty list, no UST - events are enabled - :param events_kernel: the list of kernel events to enable + events are enabled; if a list of lists is provided, one channel per list is created + with the corresponding UST events enabled in each channel + :param events_kernel: the list of kernel events to enable; if a list of lists is provided, + one channel per list is created with the corresponding kernel events enabled in each + channel :param syscalls: the list of syscalls to enable :param context_fields: the names of context fields to enable if it's a list or a set, the context fields are enabled for both kernel and userspace; @@ -194,8 +198,23 @@ def __init__( if events_ust is None: events_ust = names.DEFAULT_EVENTS_ROS if not self._dual_session \ else names.DEFAULT_INIT_EVENTS_ROS - self._events_ust = [normalize_to_list_of_substitutions(x) for x in events_ust] - self._events_kernel = [normalize_to_list_of_substitutions(x) for x in events_kernel] + # Convert to list of lists if a single list is provided + if events_ust and all( + not isinstance(x, Iterable) or isinstance(x, (str, Generator)) for x in events_ust + ): + events_ust = [events_ust] + if events_kernel and all( + not isinstance(x, Iterable) or isinstance(x, (str, Generator)) for x in events_kernel + ): + events_kernel = [events_kernel] + self._events_ust = [ + [normalize_to_list_of_substitutions(x) for x in channel_events_ust] + for channel_events_ust in events_ust + ] + self._events_kernel = [ + [normalize_to_list_of_substitutions(x) for x in channel_events_kernel] + for channel_events_kernel in events_kernel + ] self._syscalls = [normalize_to_list_of_substitutions(x) for x in syscalls] self._context_fields: Union[ Mapping[str, List[List[Substitution]]], List[List[Substitution]] @@ -236,11 +255,11 @@ def trace_directory(self) -> Optional[str]: return self._trace_directory @property - def events_ust(self) -> List[List[Substitution]]: + def events_ust(self) -> List[List[List[Substitution]]]: return self._events_ust @property - def events_kernel(self) -> List[List[Substitution]]: + def events_kernel(self) -> List[List[List[Substitution]]]: return self._events_kernel @property @@ -320,6 +339,9 @@ def _append_arg(): arg.append(sub) if arg: result_args.append(arg) + + # Convert list of lists of substitutions to list of substitutions + result_args = [(arg for arg in sublist) for sublist in result_args] return result_args @classmethod @@ -446,8 +468,14 @@ def execute(self, context: LaunchContext) -> List[Action]: dual_session = perform_typed_substitution(context, self._dual_session, bool) base_path = perform_substitutions(context, self._base_path) append_trace = perform_typed_substitution(context, self._append_trace, bool) - events_ust = [perform_substitutions(context, x) for x in self._events_ust] - events_kernel = [perform_substitutions(context, x) for x in self._events_kernel] + events_ust = [ + [perform_substitutions(context, x) for x in channel_events_ust] + for channel_events_ust in self._events_ust + ] + events_kernel = [ + [perform_substitutions(context, x) for x in channel_events_kernel] + for channel_events_kernel in self._events_kernel + ] syscalls = [perform_substitutions(context, x) for x in self._syscalls] context_fields = ( { @@ -516,18 +544,23 @@ def destroy(event: Event, context: LaunchContext) -> None: context.register_event_handler(OnShutdown(on_shutdown=destroy)) return self._ld_preload_actions - def _get_ld_preload_actions(self, events_ust: List[str]) -> List[Action]: + def _get_ld_preload_actions(self, events_ust: List[List[str]]) -> List[Action]: ld_preload_actions: List[Action] = [] # Add LD_PRELOAD actions if corresponding events are enabled - if self.has_libc_wrapper_events(events_ust): + if any(self.has_libc_wrapper_events(channel_events_ust) + for channel_events_ust in events_ust): ld_preload_actions.append(LdPreload(self.LIB_LIBC_WRAPPER)) - if self.has_pthread_wrapper_events(events_ust): + if any(self.has_pthread_wrapper_events(channel_events_ust) + for channel_events_ust in events_ust): ld_preload_actions.append(LdPreload(self.LIB_PTHREAD_WRAPPER)) - if self.has_dl_events(events_ust): + if any(self.has_dl_events(channel_events_ust) + for channel_events_ust in events_ust): ld_preload_actions.append(LdPreload(self.LIB_DL)) # Warn if events match both normal AND fast profiling libs - has_fast_profiling_events = self.has_profiling_events(events_ust, True) - has_normal_profiling_events = self.has_profiling_events(events_ust, False) + has_fast_profiling_events = any(self.has_profiling_events(channel_events_ust, True) + for channel_events_ust in events_ust) + has_normal_profiling_events = any(self.has_profiling_events(channel_events_ust, False) + for channel_events_ust in events_ust) # In practice, the first lib in the LD_PRELOAD list will be used, so the fast one here if has_fast_profiling_events: ld_preload_actions.append(LdPreload(self.LIB_PROFILE_FAST)) diff --git a/tracetools_trace/tracetools_trace/tools/lttng_impl.py b/tracetools_trace/tracetools_trace/tools/lttng_impl.py index 0ac6f9d5..d0ba9c84 100644 --- a/tracetools_trace/tracetools_trace/tools/lttng_impl.py +++ b/tracetools_trace/tracetools_trace/tools/lttng_impl.py @@ -201,12 +201,12 @@ def setup( dual_session: bool = False, base_path: str, append_trace: bool = False, - ros_events: Union[List[str], Set[str]] = DEFAULT_EVENTS_ROS, - kernel_events: Union[List[str], Set[str]] = [], + ros_events: Union[List[List[str]], List[Set[str]]] = DEFAULT_EVENTS_ROS, + kernel_events: Union[List[List[str]], List[Set[str]]] = [], syscalls: Union[List[str], Set[str]] = [], context_fields: Union[List[str], Set[str], Dict[str, List[str]]] = DEFAULT_CONTEXT, - channel_name_ust: str = 'ros2', - channel_name_kernel: str = 'kchan', + channel_name_ust_prefix: str = 'ros2', + channel_name_kernel_prefix: str = 'kchan', subbuffer_size_ust: int = 8 * 4096, subbuffer_size_kernel: int = 32 * 4096, ) -> Optional[str]: @@ -228,8 +228,8 @@ def setup( which will be created if needed :param append_trace: whether to append to the trace directory if it already exists, otherwise an error is reported - :param ros_events: list of ROS events to enable - :param kernel_events: list of kernel events to enable + :param ros_events: list of lists of ROS events to enable in each UST channel + :param kernel_events: list of lists of kernel events to enable in each kernel channel :param syscalls: list of syscalls to enable these will be part of the kernel channel :param context_fields: the names of context fields to enable @@ -237,8 +237,8 @@ def setup( if it's a dictionary: { domain type string -> context fields list } with the domain type string being either `names.DOMAIN_TYPE_KERNEL` or `names.DOMAIN_TYPE_USERSPACE` - :param channel_name_ust: the UST channel name - :param channel_name_kernel: the kernel channel name + :param channel_name_ust_prefix: prefix of the UST channel name + :param channel_name_kernel_prefix: prefix of the kernel channel name :param subbuffer_size_ust: the size of the subbuffers for userspace events (defaults to 8 times the usual page size) :param subbuffer_size_kernel: the size of the subbuffers for kernel events (defaults to 32 @@ -294,11 +294,16 @@ def setup( ' see: https://github.com/ros2/ros2_tracing#building' ) + num_channels_ust = len(ros_events) + num_channels_kernel = len(kernel_events) + # Convert lists to sets - if not isinstance(ros_events, set): - ros_events = set(ros_events) - if not isinstance(kernel_events, set): - kernel_events = set(kernel_events) + for i in range(num_channels_ust): + if not isinstance(ros_events[i], set): + ros_events[i] = set(ros_events[i]) + for i in range(num_channels_kernel): + if not isinstance(kernel_events[i], set): + kernel_events[i] = set(kernel_events[i]) if not isinstance(syscalls, set): syscalls = set(syscalls) if isinstance(context_fields, list): @@ -331,92 +336,96 @@ def setup( full_path=full_path, ) - # Enable channel, events, and contexts for each domain + # Enable channels, events, and contexts for each domain if ust_enabled: domain = DOMAIN_TYPE_USERSPACE domain_type = lttngpy.LTTNG_DOMAIN_UST - channel_name = channel_name_ust - _enable_channel( - session_name=session_name, - domain_type=domain_type, - # Per-user buffer - buffer_type=lttngpy.LTTNG_BUFFER_PER_UID, - channel_name=channel_name, - # Overwrite if snapshot mode, otherwise discard - overwrite=int(snapshot_mode), - # We use 2 sub-buffers in normal mode because the number of sub-buffers is pointless in - # discard mode, and switching between sub-buffers introduces noticeable CPU overhead. - # In snapshot mode, we use 4 sub-buffers to lose less data when sub-buffers are over- - # written, because when all sub-buffers are full the oldest one is discarded entirely. - # See: https://lttng.org/docs/v2.13/#doc-channel-subbuf-size-vs-subbuf-count - subbuf_size=subbuffer_size_ust, - num_subbuf=4 if snapshot_mode else 2, - # Ignore switch timer interval and use read timer instead - switch_timer_interval=0, - read_timer_interval=200, - # mmap channel output (only option for UST) - output=lttngpy.LTTNG_EVENT_MMAP, - ) - _enable_events( - session_name=session_name, - domain_type=domain_type, - event_type=lttngpy.LTTNG_EVENT_TRACEPOINT, - channel_name=channel_name, - events=ros_events, - ) - _add_contexts( - session_name=session_name, - domain_type=domain_type, - channel_name=channel_name, - context_fields=contexts_dict.get(domain), - ) + for i in range(num_channels_ust): + channel_name_ust = channel_name_ust_prefix+(f'_{i}') + _enable_channel( + session_name=session_name, + domain_type=domain_type, + # Per-user buffer + buffer_type=lttngpy.LTTNG_BUFFER_PER_UID, + channel_name=channel_name_ust, + # Overwrite if snapshot mode, otherwise discard + overwrite=int(snapshot_mode), + # We use 2 sub-buffers in normal mode because the number of sub-buffers is + # pointless in discard mode, and switching between sub-buffers introduces + # noticeable CPU overhead. In snapshot mode, we use 4 sub-buffers to lose less data + # when sub-buffers are over-written, because when all sub-buffers are full the + # oldest one is discarded entirely. + # See: https://lttng.org/docs/v2.13/#doc-channel-subbuf-size-vs-subbuf-count + subbuf_size=subbuffer_size_ust, + num_subbuf=4 if snapshot_mode else 2, + # Ignore switch timer interval and use read timer instead + switch_timer_interval=0, + read_timer_interval=200, + # mmap channel output (only option for UST) + output=lttngpy.LTTNG_EVENT_MMAP, + ) + _enable_events( + session_name=session_name, + domain_type=domain_type, + event_type=lttngpy.LTTNG_EVENT_TRACEPOINT, + channel_name=channel_name_ust, + events=ros_events[i], + ) + _add_contexts( + session_name=session_name, + domain_type=domain_type, + channel_name=channel_name_ust, + context_fields=contexts_dict.get(domain), + ) if kernel_enabled: domain = DOMAIN_TYPE_KERNEL domain_type = lttngpy.LTTNG_DOMAIN_KERNEL - channel_name = channel_name_kernel - _enable_channel( - session_name=session_name, - domain_type=domain_type, - # Global buffer (only option for kernel domain) - buffer_type=lttngpy.LTTNG_BUFFER_GLOBAL, - channel_name=channel_name, - # Overwrite if snapshot mode, otherwise discard - overwrite=int(snapshot_mode), - # We use 2 sub-buffers in normal mode because the number of sub-buffers is pointless in - # discard mode, and switching between sub-buffers introduces noticeable CPU overhead. - # In snapshot mode, we use 4 sub-buffers to lose less data when sub-buffers are over- - # written, because when all sub-buffers are full the oldest one is discarded entirely. - # See: https://lttng.org/docs/v2.13/#doc-channel-subbuf-size-vs-subbuf-count - subbuf_size=subbuffer_size_kernel, - num_subbuf=4 if snapshot_mode else 2, - # Ignore switch timer interval and use read timer instead - switch_timer_interval=0, - read_timer_interval=200, - # mmap channel output instead of splice - output=lttngpy.LTTNG_EVENT_MMAP, - ) - if kernel_events: - _enable_events( + for i in range(num_channels_kernel): + channel_name_kernel = channel_name_kernel_prefix+(f'_{i}') + _enable_channel( session_name=session_name, domain_type=domain_type, - event_type=lttngpy.LTTNG_EVENT_TRACEPOINT, - channel_name=channel_name, - events=kernel_events, + # Global buffer (only option for kernel domain) + buffer_type=lttngpy.LTTNG_BUFFER_GLOBAL, + channel_name=channel_name_kernel, + # Overwrite if snapshot mode, otherwise discard + overwrite=int(snapshot_mode), + # We use 2 sub-buffers in normal mode because the number of sub-buffers is + # pointless in discard mode, and switching between sub-buffers introduces + # noticeable CPU overhead. In snapshot mode, we use 4 sub-buffers to lose less data + # when sub-buffers are over-written, because when all sub-buffers are full the + # oldest one is discarded entirely. + # See: https://lttng.org/docs/v2.13/#doc-channel-subbuf-size-vs-subbuf-count + subbuf_size=subbuffer_size_ust, + num_subbuf=4 if snapshot_mode else 2, + # Ignore switch timer interval and use read timer instead + switch_timer_interval=0, + read_timer_interval=200, + # mmap channel output instead of splice + output=lttngpy.LTTNG_EVENT_MMAP, ) - if syscalls: - _enable_events( + if kernel_events: + _enable_events( + session_name=session_name, + domain_type=domain_type, + event_type=lttngpy.LTTNG_EVENT_TRACEPOINT, + channel_name=channel_name_kernel, + events=kernel_events[i], + ) + if syscalls: + _enable_events( + session_name=session_name, + domain_type=domain_type, + event_type=lttngpy.LTTNG_EVENT_SYSCALL, + channel_name=channel_name_kernel, + events=syscalls, + ) + _add_contexts( session_name=session_name, domain_type=domain_type, - event_type=lttngpy.LTTNG_EVENT_SYSCALL, - channel_name=channel_name, - events=syscalls, + channel_name=channel_name_kernel, + context_fields=contexts_dict.get(domain), ) - _add_contexts( - session_name=session_name, - domain_type=domain_type, - channel_name=channel_name, - context_fields=contexts_dict.get(domain), - ) return full_path diff --git a/tracetools_trace/tracetools_trace/trace.py b/tracetools_trace/tracetools_trace/trace.py index c46d32d2..5140e8b1 100644 --- a/tracetools_trace/tracetools_trace/trace.py +++ b/tracetools_trace/tracetools_trace/trace.py @@ -23,6 +23,7 @@ from typing import List from typing import Optional from typing import Tuple +from typing import Union from tracetools_trace.tools import args from tracetools_trace.tools import lttng @@ -114,8 +115,8 @@ def init( dual_session: bool, base_path: Optional[str], append_trace: bool, - ros_events: List[str], - kernel_events: List[str], + ros_events: Union[List[str], List[List[str]]], + kernel_events: Union[List[str], List[List[str]]], syscalls: List[str], context_fields: List[str], display_list: bool, @@ -136,8 +137,12 @@ def init( or `None` for default :param append_trace: whether to append to the trace directory if it already exists, otherwise an error is reported - :param ros_events: list of ROS events to enable - :param kernel_events: list of kernel events to enable + :param ros_events: list of ROS events to enable; if a list of lists is provided, + one channel per list is created with the corresponding ROS events enabled in each + channel + :param kernel_events: list of kernel events to enable; if a list of lists is provided, + one channel per list is created with the corresponding kernel events enabled in each + channel :param syscalls: list of syscalls to enable :param context_fields: list of context fields to enable :param display_list: whether to display list(s) of enabled events and context names @@ -178,6 +183,12 @@ def init( if interactive: input('press enter to start...') + # Convert to list of lists if needed + if ros_events and all(isinstance(e, str) for e in ros_events): + ros_events = [ros_events] + if kernel_events and all(isinstance(e, str) for e in kernel_events): + kernel_events = [kernel_events] + trace_directory = lttng.lttng_init( session_name=session_name, snapshot_mode=snapshot_mode,