From 66de9a5695b1288175860ec5516ca4fff593420a Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Sat, 29 Apr 2023 18:15:36 -0400 Subject: [PATCH 01/42] initial commit --- pipit/plotting/core.py | 109 ++++++++++++++++++++++++++++ pipit/plotting/theme.py | 69 ++++++++++++++++++ pipit/plotting/util.py | 154 ++++++++++++++++++++++++++++++++++++++++ pipit/trace.py | 9 +++ 4 files changed, 341 insertions(+) create mode 100644 pipit/plotting/core.py create mode 100644 pipit/plotting/theme.py create mode 100644 pipit/plotting/util.py diff --git a/pipit/plotting/core.py b/pipit/plotting/core.py new file mode 100644 index 00000000..cc6543e9 --- /dev/null +++ b/pipit/plotting/core.py @@ -0,0 +1,109 @@ +import numpy as np +from bokeh.models import ( + ColorBar, + HoverTool, + LinearColorMapper, + LogColorMapper, + NumeralTickFormatter, +) +from bokeh.plotting import figure + +from .util import ( + clamp, + get_process_ticker, + get_size_hover_formatter, + get_size_tick_formatter, + get_tooltips, + show, +) + + +def comm_matrix( + data, output="size", cmap="log", palette="Viridis256", return_figure=False +): + """ + Plot the Trace's communication matrix. + + Args: + data (numpy.ndarray): a 2D numpy array of shape (N, N) containing the + communication matrix between N processes. + output (str, optional): "size" or "count" + cmap (str, optional): Specifies the color mapping. Options are "log", + "linear", and "any" + palette (str, optional): Name of Bokeh color palette to use. Defaults to + "Viridis256". + return_figure (bool, optional): Whether to return the Bokeh figure + object. Defaults to False, which displays the result and returns nothing. + + Returns: + None or Bokeh figure object + """ + + N = data.shape[0] + + # Define color mapper + if cmap == "linear": + color_mapper = LinearColorMapper(palette=palette, low=0, high=np.amax(data)) + elif cmap == "log": + color_mapper = LogColorMapper( + palette=palette, low=max(np.amin(data), 1), high=np.amax(data) + ) + elif cmap == "any": + color_mapper = LinearColorMapper(palette=palette, low=1, high=1) + + # Create bokeh plot + p = figure( + x_axis_label="Sender", + y_axis_label="Receiver", + x_range=(-0.5, N - 0.5), + y_range=(N - 0.5, -0.5), + x_axis_location="above", + tools="hover,pan,reset,wheel_zoom,save", + width=90 + clamp(N * 30, 200, 500), + height=10 + clamp(N * 30, 200, 500), + toolbar_location="below", + ) + + # Add glyphs and layouts + p.image( + image=[np.flipud(data)], + x=-0.5, + y=N - 0.5, + dw=N, + dh=N, + color_mapper=color_mapper, + ) + + color_bar = ColorBar( + color_mapper=color_mapper, + formatter=get_size_tick_formatter(ignore_range=cmap == "log") + if output == "size" + else NumeralTickFormatter(), + width=15, + ) + p.add_layout(color_bar, "right") + + # Customize plot + p.axis.ticker = get_process_ticker(N=N) + # p.axis.major_tick_line_color = None + p.grid.visible = False + + # Configure hover + hover = p.select(HoverTool) + hover.tooltips = get_tooltips( + { + "Sender": "Process $x{0.}", + "Receiver": "Process $y{0.}", + "Bytes": "@image{custom}", + } + if output == "size" + else { + "Sender": "Process $x{0.}", + "Receiver": "Process $y{0.}", + "Count": "@image", + } + ) + hover.formatters = {"@image": get_size_hover_formatter()} + + # Return plot + return show(p, return_figure=return_figure) diff --git a/pipit/plotting/theme.py b/pipit/plotting/theme.py new file mode 100644 index 00000000..7c63d2a9 --- /dev/null +++ b/pipit/plotting/theme.py @@ -0,0 +1,69 @@ +DEFAULT = """ + attrs: + Plot: + height: 350 + width: 700 + Axis: + axis_label_text_font_style: "bold" + minor_tick_line_color: null + Toolbar: + autohide: true + logo: null + HoverTool: + point_policy: "follow_mouse" + Legend: + label_text_font_size: "8.5pt" + spacing: 10 + border_line_color: null + glyph_width: 16 + glyph_height: 16 + Scatter: + size: 9 + DataRange1d: + range_padding: 0.05 +""" + +PAPER = """ + attrs: + Plot: + height: 350 + width: 700 + toolbar_location: "above" + outline_line_width: 0 + Title: + text_font_size: "0pt" + text_font: "Gill Sans" + Axis: + axis_label_text_font_style: "bold" + axis_label_text_font_size: "18pt" + axis_label_text_font: "Gill Sans" + major_label_text_font_size: "16pt" + major_label_text_font: "Gill Sans" + minor_tick_line_color: null + ColorBar: + major_label_text_font_size: "16pt" + major_label_text_font: "Gill Sans" + Toolbar: + autohide: true + logo: null + HoverTool: + point_policy: "follow_mouse" + Legend: + label_text_font_size: "15pt" + label_text_font: "Gill Sans" + spacing: -1 + padding: 0 + border_line_color: null + glyph_width: 16 + glyph_height: 16 + margin: 5 + Scatter: + size: 12 + DataRange1d: + range_padding: 0.05 +""" + +themes = { + "default": DEFAULT, + "paper": PAPER, +} diff --git a/pipit/plotting/util.py b/pipit/plotting/util.py new file mode 100644 index 00000000..13d5637b --- /dev/null +++ b/pipit/plotting/util.py @@ -0,0 +1,154 @@ +import yaml +from bokeh.models import BasicTicker, CustomJSHover, FuncTickFormatter +from bokeh.plotting import output_notebook +from bokeh.plotting import show as bk_show +from bokeh.themes import Theme + +from .theme import themes + +# Constants +notebook_url = "http://localhost:8888" +theme = "default" + +# JS expression to convert bytes to human-readable string +# "x" is the value (in bytes) being compared +# "y" is the value (in bytes) being formatted +JS_FORMAT_SIZE = """ + if(x < 1e3) + return (y).toFixed(2) + " B"; + if(x < 1e6) + return (y / 1e3).toFixed(2) + " kB"; + if(x < 1e9) + return (y / 1e6).toFixed(2) + " MB"; + if(x < 1e12) + return (y / 1e9).toFixed(2) + " GB"; + if(x < 1e15) + return (y / 1e12).toFixed(2) + " TB"; + else + return (y / 1e15).toFixed(2) + " PB"; +""" + + +# Utility functions +def in_notebook(): + """Returns True if we are in notebook environment, False otherwise""" + try: + from IPython import get_ipython + + if "IPKernelApp" not in get_ipython().config: # pragma: no cover + return False + except ImportError: + return False + except AttributeError: + return False + return True + + +def show(p, return_figure=False): + """Used to wrap return values of plotting functions. + + If return_figure is True, then just returns the figure object, otherwise starts a + Bokeh server containing the figure. If we are in a notebook, displays the + figure in the output cell, otherwise shows figure in new browser tab. + + See https://docs.bokeh.org/en/latest/docs/user_guide/output/jupyter.html#bokeh-server-applications, # noqa E501 + https://docs.bokeh.org/en/latest/docs/user_guide/server/library.html. + """ + if return_figure: + return p + + # Load Gill Sans font from CDN + if in_notebook() and theme == "paper": + from IPython.display import HTML, display_html + + display_html( + HTML( + """ + + """ + ) + ) + + # Create a Bokeh app containing the figure + def bkapp(doc): + doc.add_root(p) + doc.theme = Theme( + json=yaml.load( + themes[theme], + Loader=yaml.FullLoader, + ) + ) + + if in_notebook(): + # If notebook, show it in output cell + output_notebook(hide_banner=True) + bk_show(bkapp, notebook_url=notebook_url) + else: + # If standalone, start HTTP server and show in browser + from bokeh.server.server import Server + + server = Server({"/": bkapp}, port=0, allow_websocket_origin=["*"]) + server.start() + server.io_loop.add_callback(server.show, "/") + server.io_loop.start() + + +def get_tooltips(tooltips_dict): + """Returns nicely formatted HTML tooltips from a dict""" + + html = "" + for k, v in tooltips_dict.items(): + html += f""" +
+ {k}:  + {v} +
+ """ + html += """ + + """ + return html + + +def clamp(value, min_val, max_val): + """Clamps value to min and max bounds""" + + if value < min_val: + return min_val + if value > max_val: + return max_val + return value + + +# Custom tickers and formatters +def get_process_ticker(N): + return BasicTicker( + base=2, desired_num_ticks=min(N, 16), min_interval=1, num_minor_ticks=0 + ) + + +def get_size_hover_formatter(): + return CustomJSHover( + code=f""" + let x = value; + let y = value; + {JS_FORMAT_SIZE} + """ + ) + + +def get_size_tick_formatter(ignore_range=False): + x = "tick" if ignore_range else "Math.max(...ticks) - Math.min(...ticks);" + return FuncTickFormatter( + code=f""" + let x = {x} + let y = tick; + {JS_FORMAT_SIZE} + """ + ) diff --git a/pipit/trace.py b/pipit/trace.py index 6f3838b6..ab5158db 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -681,3 +681,12 @@ def calc_exc_time_in_bin(events): df.insert(0, "bin_end", edges[1:]) return df + + def plot_comm_matrix(self, output="size", *args, **kwargs): + from .plotting import core + + # Generate the data + data = self.comm_matrix(output=output) + + # Return the Bokeh plot + return core.comm_matrix(data, output=output, *args, **kwargs) From 39655d717229b04abdd36ac6a357c5c811684d12 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Sat, 29 Apr 2023 18:28:23 -0400 Subject: [PATCH 02/42] clean up --- pipit/plotting/core.py | 17 ++++++++--------- pipit/plotting/util.py | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pipit/plotting/core.py b/pipit/plotting/core.py index cc6543e9..33e8a5fb 100644 --- a/pipit/plotting/core.py +++ b/pipit/plotting/core.py @@ -19,26 +19,26 @@ def comm_matrix( - data, output="size", cmap="log", palette="Viridis256", return_figure=False + data, output="size", cmap="log", palette="Viridis256", return_fig=False ): """ - Plot the Trace's communication matrix. + Plot the trace's communication matrix. Args: data (numpy.ndarray): a 2D numpy array of shape (N, N) containing the communication matrix between N processes. - output (str, optional): "size" or "count" + output (str, optional): Specifies whether the matrix contains "size" + or "count" values. Defaults to "size". cmap (str, optional): Specifies the color mapping. Options are "log", - "linear", and "any" + "linear", and "any". Defaults to "log". palette (str, optional): Name of Bokeh color palette to use. Defaults to "Viridis256". - return_figure (bool, optional): Whether to return the Bokeh figure + return_fig (bool, optional): Specifies whether to return the Bokeh figure object. Defaults to False, which displays the result and returns nothing. Returns: - None or Bokeh figure object + Bokeh figure object if return_fig, None otherwise """ - N = data.shape[0] # Define color mapper @@ -85,7 +85,6 @@ def comm_matrix( # Customize plot p.axis.ticker = get_process_ticker(N=N) - # p.axis.major_tick_line_color = None p.grid.visible = False # Configure hover @@ -106,4 +105,4 @@ def comm_matrix( hover.formatters = {"@image": get_size_hover_formatter()} # Return plot - return show(p, return_figure=return_figure) + return show(p, return_fig=return_fig) diff --git a/pipit/plotting/util.py b/pipit/plotting/util.py index 13d5637b..5a962dc9 100644 --- a/pipit/plotting/util.py +++ b/pipit/plotting/util.py @@ -44,7 +44,7 @@ def in_notebook(): return True -def show(p, return_figure=False): +def show(p, return_fig=False): """Used to wrap return values of plotting functions. If return_figure is True, then just returns the figure object, otherwise starts a @@ -54,7 +54,7 @@ def show(p, return_figure=False): See https://docs.bokeh.org/en/latest/docs/user_guide/output/jupyter.html#bokeh-server-applications, # noqa E501 https://docs.bokeh.org/en/latest/docs/user_guide/server/library.html. """ - if return_figure: + if return_fig: return p # Load Gill Sans font from CDN From 4fc2d10776d8898885ceabaec04d2c01ffe92105 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Sat, 29 Apr 2023 19:14:26 -0400 Subject: [PATCH 03/42] add generate_palette --- pipit/plotting/util.py | 132 ++++++++++++++++++++++++++++++++++------- pipit/trace.py | 4 ++ 2 files changed, 116 insertions(+), 20 deletions(-) diff --git a/pipit/plotting/util.py b/pipit/plotting/util.py index 5a962dc9..9d76ad5f 100644 --- a/pipit/plotting/util.py +++ b/pipit/plotting/util.py @@ -6,30 +6,12 @@ from .theme import themes -# Constants +# Global constants notebook_url = "http://localhost:8888" theme = "default" -# JS expression to convert bytes to human-readable string -# "x" is the value (in bytes) being compared -# "y" is the value (in bytes) being formatted -JS_FORMAT_SIZE = """ - if(x < 1e3) - return (y).toFixed(2) + " B"; - if(x < 1e6) - return (y / 1e3).toFixed(2) + " kB"; - if(x < 1e9) - return (y / 1e6).toFixed(2) + " MB"; - if(x < 1e12) - return (y / 1e9).toFixed(2) + " GB"; - if(x < 1e15) - return (y / 1e12).toFixed(2) + " TB"; - else - return (y / 1e15).toFixed(2) + " PB"; -""" - -# Utility functions +# Helper functions def in_notebook(): """Returns True if we are in notebook environment, False otherwise""" try: @@ -127,6 +109,26 @@ def clamp(value, min_val, max_val): # Custom tickers and formatters + +# JS expression to convert bytes to human-readable string +# "x" is the value (in bytes) being compared +# "y" is the value (in bytes) being formatted +JS_FORMAT_SIZE = """ + if(x < 1e3) + return (y).toFixed(2) + " B"; + if(x < 1e6) + return (y / 1e3).toFixed(2) + " kB"; + if(x < 1e9) + return (y / 1e6).toFixed(2) + " MB"; + if(x < 1e12) + return (y / 1e9).toFixed(2) + " GB"; + if(x < 1e15) + return (y / 1e12).toFixed(2) + " TB"; + else + return (y / 1e15).toFixed(2) + " PB"; +""" + + def get_process_ticker(N): return BasicTicker( base=2, desired_num_ticks=min(N, 16), min_interval=1, num_minor_ticks=0 @@ -152,3 +154,93 @@ def get_size_tick_formatter(ignore_range=False): {JS_FORMAT_SIZE} """ ) + + +# Color palette + +# Default color palette is based on bokeh.palettes.Category20_20 +# See https://docs.bokeh.org/en/latest/docs/reference/palettes.html#d3-palettes + +DEFAULT_RESERVED = { + "MPI_Send": "#1f77b4", + "MPI_Isend": "#1f77b4", + "MPI_Recv": "#d62728", + "MPI_Irecv": "#d62728", + "MPI_Wait": "#c7c7c7", + "MPI_Waitany": "#c7c7c7", + "MPI_Waitall": "#c7c7c7", + "Idle": "#c7c7c7", +} + +DEFAULT_LIGHT = [ + "#aec7e8", + "#ffbb78", + "#98df8a", + "#ff9896", + "#c5b0d5", + "#c49c94", + "#f7b6d2", + "#dbdb8d", + "#9edae5", +] + +DEFAULT_DARK = [ + "#ff7f0e", + "#2ca02c", + "#9467bd", + "#8c564b", + "#e377c2", + "#bcbd22", + "#17becf", +] + + +def generate_palette( + trace, + reserved=DEFAULT_RESERVED, + light=DEFAULT_LIGHT, + dark=DEFAULT_DARK, +): + """Generates color palette for a trace. + + Assigns light colors for even depths, and dark colors for odd depths to maximize + contrast in nested functions. + + Returns: + dict: Dictionary mapping function name to CSS color value. + """ + + # Calculate inc time and depth + trace.calc_inc_metrics(["Timestamp (ns)"]) + trace._match_caller_callee() + + # Get all function names from trace + func = trace.events[trace.events["Event Type"] == "Enter"] + names = reversed(trace.flat_profile(["time.inc"]).index.tolist()) + + # Get the depth of each function + depths = ( + func.groupby("Name")["_depth"] + .agg(lambda x: x.value_counts().index[0]) + .to_dict() + ) + + # Start with palette being a copy of reserved colors + palette = reserved.copy() + + # Initialize indices for light and dark colors + dark_index = 0 + light_index = 0 + + # Iterate over function names and assign colors to each + for i, f in enumerate(names): + if f not in palette: + # Assign light color for even-depth, and dark color for odd-depth + if depths[f] % 2 == 0: + palette[f] = light[light_index % len(light)] + light_index += 1 + else: + palette[f] = dark[dark_index % len(dark)] + dark_index += 1 + + return palette diff --git a/pipit/trace.py b/pipit/trace.py index ab5158db..f679c5f9 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -26,6 +26,10 @@ def __init__(self, definitions, events): self.inc_metrics = [] self.exc_metrics = [] + from .plotting.util import generate_palette + + self.palette = generate_palette(self) + @staticmethod def from_otf2(dirname, num_processes=None): """Read an OTF2 trace into a new Trace object.""" From aed28e92741e843a36e1a89ec902cb911c56fd24 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Sun, 30 Apr 2023 13:44:47 -0400 Subject: [PATCH 04/42] add message_histogram --- pipit/plotting/core.py | 48 ++++++++++++++++++++++++++++++++++++++++-- pipit/trace.py | 9 ++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/pipit/plotting/core.py b/pipit/plotting/core.py index 33e8a5fb..385311fd 100644 --- a/pipit/plotting/core.py +++ b/pipit/plotting/core.py @@ -21,8 +21,7 @@ def comm_matrix( data, output="size", cmap="log", palette="Viridis256", return_fig=False ): - """ - Plot the trace's communication matrix. + """Plots the trace's communication matrix. Args: data (numpy.ndarray): a 2D numpy array of shape (N, N) containing the @@ -106,3 +105,48 @@ def comm_matrix( # Return plot return show(p, return_fig=return_fig) + + +def message_histogram( + data, + return_fig=False, +): + """Plots the trace's message size histogram. + + Args: + data (hist, edges): Histogram and edges + return_fig (bool, optional): Specifies whether to return the Bokeh figure + object. Defaults to False, which displays the result and returns nothing. + + Returns: + Bokeh figure object if return_fig, None otherwise + """ + hist, edges = data + + # Create bokeh plot + p = figure( + x_axis_label="Message size", + y_axis_label="Number of messages", + tools="hover,save", + ) + p.y_range.start = 0 + + # Add glyphs and layouts + p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:]) + + # Customize plot + p.xaxis.formatter = get_size_tick_formatter() + p.yaxis.formatter = NumeralTickFormatter() + + # Configure hover + hover = p.select(HoverTool) + hover.tooltips = get_tooltips( + {"Bin": "@left{custom} - @right{custom}", "Number of messages": "@top"} + ) + hover.formatters = { + "@left": get_size_hover_formatter(), + "@right": get_size_hover_formatter(), + } + + # Return plot + return show(p, return_fig=return_fig) diff --git a/pipit/trace.py b/pipit/trace.py index f679c5f9..aeb8392c 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -694,3 +694,12 @@ def plot_comm_matrix(self, output="size", *args, **kwargs): # Return the Bokeh plot return core.comm_matrix(data, output=output, *args, **kwargs) + + def plot_message_histogram(self, bins=20, *args, **kwargs): + from .plotting import core + + # Generate the data + data = self.message_histogram(bins=bins) + + # Return the Bokeh plot + return core.message_histogram(data, *args, **kwargs) From e852e85751df07b60c606e88dc8ea2e7e336ad8e Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Sun, 30 Apr 2023 14:45:30 -0400 Subject: [PATCH 05/42] update theme --- pipit/plotting/theme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipit/plotting/theme.py b/pipit/plotting/theme.py index 7c63d2a9..4a2c821f 100644 --- a/pipit/plotting/theme.py +++ b/pipit/plotting/theme.py @@ -3,6 +3,7 @@ Plot: height: 350 width: 700 + background_fill_color: "#fafafa" Axis: axis_label_text_font_style: "bold" minor_tick_line_color: null From 80079e0d9807c563907e7e09c41437be3ecfa4bd Mon Sep 17 00:00:00 2001 From: hsirkar Date: Fri, 15 Sep 2023 07:52:10 -0400 Subject: [PATCH 06/42] rename plotting to vis --- pipit/trace.py | 2 +- pipit/{plotting => vis}/core.py | 0 pipit/{plotting => vis}/theme.py | 0 pipit/{plotting => vis}/util.py | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename pipit/{plotting => vis}/core.py (100%) rename pipit/{plotting => vis}/theme.py (100%) rename pipit/{plotting => vis}/util.py (100%) diff --git a/pipit/trace.py b/pipit/trace.py index aeb8392c..a3855f85 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -26,7 +26,7 @@ def __init__(self, definitions, events): self.inc_metrics = [] self.exc_metrics = [] - from .plotting.util import generate_palette + from .vis.util import generate_palette self.palette = generate_palette(self) diff --git a/pipit/plotting/core.py b/pipit/vis/core.py similarity index 100% rename from pipit/plotting/core.py rename to pipit/vis/core.py diff --git a/pipit/plotting/theme.py b/pipit/vis/theme.py similarity index 100% rename from pipit/plotting/theme.py rename to pipit/vis/theme.py diff --git a/pipit/plotting/util.py b/pipit/vis/util.py similarity index 100% rename from pipit/plotting/util.py rename to pipit/vis/util.py From 66836ba07900f983f6a705af7fc1805073063886 Mon Sep 17 00:00:00 2001 From: hsirkar Date: Fri, 15 Sep 2023 08:11:47 -0400 Subject: [PATCH 07/42] cleanup --- docs/examples/vis.ipynb | 718 ++++++++++++++++++++++++++++++++++++++++ pipit/trace.py | 4 +- pipit/vis/theme.py | 70 ---- pipit/vis/util.py | 49 +-- 4 files changed, 748 insertions(+), 93 deletions(-) create mode 100644 docs/examples/vis.ipynb delete mode 100644 pipit/vis/theme.py diff --git a/docs/examples/vis.ipynb b/docs/examples/vis.ipynb new file mode 100644 index 00000000..be315daa --- /dev/null +++ b/docs/examples/vis.ipynb @@ -0,0 +1,718 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "99c9dcc2", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "import sys\n", + "sys.path.append('../../')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "09adcb68", + "metadata": {}, + "outputs": [], + "source": [ + "import pipit as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "aaef6dc5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ping_pong = pp.Trace.from_otf2(\"../../pipit/tests/data/ping-pong-otf2\")\n", + "ping_pong" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0c82808a", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " const force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "const JS_MIME_TYPE = 'application/javascript';\n", + " const HTML_MIME_TYPE = 'text/html';\n", + " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", + " const CLASS_NAME = 'output_bokeh rendered_html';\n", + "\n", + " /**\n", + " * Render data to the DOM node\n", + " */\n", + " function render(props, node) {\n", + " const script = document.createElement(\"script\");\n", + " node.appendChild(script);\n", + " }\n", + "\n", + " /**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + " function handleClearOutput(event, handle) {\n", + " const cell = handle.cell;\n", + "\n", + " const id = cell.output_area._bokeh_element_id;\n", + " const server_id = cell.output_area._bokeh_server_id;\n", + " // Clean up Bokeh references\n", + " if (id != null && id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + "\n", + " if (server_id !== undefined) {\n", + " // Clean up Bokeh references\n", + " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", + " cell.notebook.kernel.execute(cmd_clean, {\n", + " iopub: {\n", + " output: function(msg) {\n", + " const id = msg.content.text.trim();\n", + " if (id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + " }\n", + " }\n", + " });\n", + " // Destroy server and session\n", + " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", + " cell.notebook.kernel.execute(cmd_destroy);\n", + " }\n", + " }\n", + "\n", + " /**\n", + " * Handle when a new output is added\n", + " */\n", + " function handleAddOutput(event, handle) {\n", + " const output_area = handle.output_area;\n", + " const output = handle.output;\n", + "\n", + " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", + " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + "\n", + " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + "\n", + " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", + " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", + " // store reference to embed id on output_area\n", + " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " }\n", + " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " const bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " const script_attrs = bk_div.children[0].attributes;\n", + " for (let i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + " }\n", + "\n", + " function register_renderer(events, OutputArea) {\n", + "\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " const toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[toinsert.length - 1]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " /* Handle when an output is cleared or removed */\n", + " events.on('clear_output.CodeCell', handleClearOutput);\n", + " events.on('delete.Cell', handleClearOutput);\n", + "\n", + " /* Handle when a new output is added */\n", + " events.on('output_added.OutputArea', handleAddOutput);\n", + "\n", + " /**\n", + " * Register the mime type and append_mime function with output_area\n", + " */\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " /* Is output safe? */\n", + " safe: true,\n", + " /* Index of renderer in `output_area.display_order` */\n", + " index: 0\n", + " });\n", + " }\n", + "\n", + " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", + " if (root.Jupyter !== undefined) {\n", + " const events = require('base/js/events');\n", + " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", + "\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " }\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " const NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"
    \\n\"+\n", + " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", + " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", + " \"
\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " const el = document.getElementById(null);\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS is loading...\";\n", + " }\n", + " if (root.Bokeh !== undefined) {\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error(url) {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (let i = 0; i < css_urls.length; i++) {\n", + " const url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " for (let i = 0; i < js_urls.length; i++) {\n", + " const url = js_urls[i];\n", + " const element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n", + " const css_urls = [];\n", + "\n", + " const inline_js = [ function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if (root.Bokeh !== undefined || force === true) {\n", + " for (let i = 0; i < inline_js.length; i++) {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " }\n", + "} else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + " }\n", + "\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(css_urls, js_urls, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(window));" + ], + "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.bokehjs_exec.v0+json": "", + "text/html": [ + "" + ] + }, + "metadata": { + "application/vnd.bokehjs_exec.v0+json": { + "server_id": "0b0fda93376e4b0487c5c480a0fa92f3" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "ping_pong.plot_comm_matrix()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1fb7433e", + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " const force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "const JS_MIME_TYPE = 'application/javascript';\n", + " const HTML_MIME_TYPE = 'text/html';\n", + " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", + " const CLASS_NAME = 'output_bokeh rendered_html';\n", + "\n", + " /**\n", + " * Render data to the DOM node\n", + " */\n", + " function render(props, node) {\n", + " const script = document.createElement(\"script\");\n", + " node.appendChild(script);\n", + " }\n", + "\n", + " /**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + " function handleClearOutput(event, handle) {\n", + " const cell = handle.cell;\n", + "\n", + " const id = cell.output_area._bokeh_element_id;\n", + " const server_id = cell.output_area._bokeh_server_id;\n", + " // Clean up Bokeh references\n", + " if (id != null && id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + "\n", + " if (server_id !== undefined) {\n", + " // Clean up Bokeh references\n", + " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", + " cell.notebook.kernel.execute(cmd_clean, {\n", + " iopub: {\n", + " output: function(msg) {\n", + " const id = msg.content.text.trim();\n", + " if (id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + " }\n", + " }\n", + " });\n", + " // Destroy server and session\n", + " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", + " cell.notebook.kernel.execute(cmd_destroy);\n", + " }\n", + " }\n", + "\n", + " /**\n", + " * Handle when a new output is added\n", + " */\n", + " function handleAddOutput(event, handle) {\n", + " const output_area = handle.output_area;\n", + " const output = handle.output;\n", + "\n", + " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", + " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + "\n", + " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + "\n", + " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", + " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", + " // store reference to embed id on output_area\n", + " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " }\n", + " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " const bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " const script_attrs = bk_div.children[0].attributes;\n", + " for (let i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + " }\n", + "\n", + " function register_renderer(events, OutputArea) {\n", + "\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " const toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[toinsert.length - 1]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " /* Handle when an output is cleared or removed */\n", + " events.on('clear_output.CodeCell', handleClearOutput);\n", + " events.on('delete.Cell', handleClearOutput);\n", + "\n", + " /* Handle when a new output is added */\n", + " events.on('output_added.OutputArea', handleAddOutput);\n", + "\n", + " /**\n", + " * Register the mime type and append_mime function with output_area\n", + " */\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " /* Is output safe? */\n", + " safe: true,\n", + " /* Index of renderer in `output_area.display_order` */\n", + " index: 0\n", + " });\n", + " }\n", + "\n", + " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", + " if (root.Jupyter !== undefined) {\n", + " const events = require('base/js/events');\n", + " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", + "\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " }\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " const NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"
    \\n\"+\n", + " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", + " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", + " \"
\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " const el = document.getElementById(null);\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS is loading...\";\n", + " }\n", + " if (root.Bokeh !== undefined) {\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error(url) {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (let i = 0; i < css_urls.length; i++) {\n", + " const url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " for (let i = 0; i < js_urls.length; i++) {\n", + " const url = js_urls[i];\n", + " const element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n", + " const css_urls = [];\n", + "\n", + " const inline_js = [ function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if (root.Bokeh !== undefined || force === true) {\n", + " for (let i = 0; i < inline_js.length; i++) {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " }\n", + "} else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + " }\n", + "\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(css_urls, js_urls, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(window));" + ], + "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.bokehjs_exec.v0+json": "", + "text/html": [ + "" + ] + }, + "metadata": { + "application/vnd.bokehjs_exec.v0+json": { + "server_id": "5a58b28e0dd7442294d0ad41562d3a57" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "ping_pong.plot_message_histogram()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pipit/trace.py b/pipit/trace.py index a3855f85..33f1b669 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -687,7 +687,7 @@ def calc_exc_time_in_bin(events): return df def plot_comm_matrix(self, output="size", *args, **kwargs): - from .plotting import core + from .vis import core # Generate the data data = self.comm_matrix(output=output) @@ -696,7 +696,7 @@ def plot_comm_matrix(self, output="size", *args, **kwargs): return core.comm_matrix(data, output=output, *args, **kwargs) def plot_message_histogram(self, bins=20, *args, **kwargs): - from .plotting import core + from .vis import core # Generate the data data = self.message_histogram(bins=bins) diff --git a/pipit/vis/theme.py b/pipit/vis/theme.py deleted file mode 100644 index 4a2c821f..00000000 --- a/pipit/vis/theme.py +++ /dev/null @@ -1,70 +0,0 @@ -DEFAULT = """ - attrs: - Plot: - height: 350 - width: 700 - background_fill_color: "#fafafa" - Axis: - axis_label_text_font_style: "bold" - minor_tick_line_color: null - Toolbar: - autohide: true - logo: null - HoverTool: - point_policy: "follow_mouse" - Legend: - label_text_font_size: "8.5pt" - spacing: 10 - border_line_color: null - glyph_width: 16 - glyph_height: 16 - Scatter: - size: 9 - DataRange1d: - range_padding: 0.05 -""" - -PAPER = """ - attrs: - Plot: - height: 350 - width: 700 - toolbar_location: "above" - outline_line_width: 0 - Title: - text_font_size: "0pt" - text_font: "Gill Sans" - Axis: - axis_label_text_font_style: "bold" - axis_label_text_font_size: "18pt" - axis_label_text_font: "Gill Sans" - major_label_text_font_size: "16pt" - major_label_text_font: "Gill Sans" - minor_tick_line_color: null - ColorBar: - major_label_text_font_size: "16pt" - major_label_text_font: "Gill Sans" - Toolbar: - autohide: true - logo: null - HoverTool: - point_policy: "follow_mouse" - Legend: - label_text_font_size: "15pt" - label_text_font: "Gill Sans" - spacing: -1 - padding: 0 - border_line_color: null - glyph_width: 16 - glyph_height: 16 - margin: 5 - Scatter: - size: 12 - DataRange1d: - range_padding: 0.05 -""" - -themes = { - "default": DEFAULT, - "paper": PAPER, -} diff --git a/pipit/vis/util.py b/pipit/vis/util.py index 9d76ad5f..daa2954c 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -4,12 +4,33 @@ from bokeh.plotting import show as bk_show from bokeh.themes import Theme -from .theme import themes - # Global constants -notebook_url = "http://localhost:8888" -theme = "default" - +NOTEBOOK_URL = "http://localhost:8888" +THEME = """ + attrs: + Plot: + height: 350 + width: 700 + background_fill_color: "#fafafa" + Axis: + axis_label_text_font_style: "bold" + minor_tick_line_color: null + Toolbar: + autohide: true + logo: null + HoverTool: + point_policy: "follow_mouse" + Legend: + label_text_font_size: "8.5pt" + spacing: 10 + border_line_color: null + glyph_width: 16 + glyph_height: 16 + Scatter: + size: 9 + DataRange1d: + range_padding: 0.05 +""" # Helper functions def in_notebook(): @@ -39,26 +60,12 @@ def show(p, return_fig=False): if return_fig: return p - # Load Gill Sans font from CDN - if in_notebook() and theme == "paper": - from IPython.display import HTML, display_html - - display_html( - HTML( - """ - - """ - ) - ) - # Create a Bokeh app containing the figure def bkapp(doc): doc.add_root(p) doc.theme = Theme( json=yaml.load( - themes[theme], + THEME, Loader=yaml.FullLoader, ) ) @@ -66,7 +73,7 @@ def bkapp(doc): if in_notebook(): # If notebook, show it in output cell output_notebook(hide_banner=True) - bk_show(bkapp, notebook_url=notebook_url) + bk_show(bkapp, notebook_url=NOTEBOOK_URL) else: # If standalone, start HTTP server and show in browser from bokeh.server.server import Server From 980d33b5f3bb2e451ea7ee27adf71e363d58503e Mon Sep 17 00:00:00 2001 From: hsirkar Date: Fri, 15 Sep 2023 08:24:09 -0400 Subject: [PATCH 08/42] add bokeh and datashader to CI --- .github/workflows/unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index a59b3664..28d94a6f 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -26,7 +26,7 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip - pip install --upgrade numpy pandas pytest otf2 + pip install --upgrade numpy pandas pytest otf2 bokeh==2.4.3 datashader - name: Lint and format check with flake8 and black if: ${{ matrix.python-version == 3.9 }} @@ -54,7 +54,7 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip - pip install --upgrade numpy pandas pytest otf2 + pip install --upgrade numpy pandas pytest otf2 bokeh==2.4.3 datashader - name: Basic test with pytest run: | From 0fc638fa0be622fc15a8eea44771bcb17d914887 Mon Sep 17 00:00:00 2001 From: hsirkar Date: Fri, 15 Sep 2023 08:30:06 -0400 Subject: [PATCH 09/42] remove python 3.6 support --- .github/workflows/unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 28d94a6f..01194ecf 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -12,9 +12,9 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] include: - - python-version: "3.6" + - python-version: "3.7" os: ubuntu-20.04 steps: From aa9bb134c6cc2a0da8b9f408f0038e289ab7c081 Mon Sep 17 00:00:00 2001 From: hsirkar Date: Fri, 15 Sep 2023 08:32:10 -0400 Subject: [PATCH 10/42] formatting --- docs/examples/vis.ipynb | 3 ++- pipit/vis/util.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/examples/vis.ipynb b/docs/examples/vis.ipynb index be315daa..29002ca6 100644 --- a/docs/examples/vis.ipynb +++ b/docs/examples/vis.ipynb @@ -10,7 +10,8 @@ "%load_ext autoreload\n", "%autoreload 2\n", "import sys\n", - "sys.path.append('../../')" + "\n", + "sys.path.append(\"../../\")" ] }, { diff --git a/pipit/vis/util.py b/pipit/vis/util.py index daa2954c..2d361182 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -32,6 +32,7 @@ range_padding: 0.05 """ + # Helper functions def in_notebook(): """Returns True if we are in notebook environment, False otherwise""" From d2d0edec054a3a9fb6c10158306ae3e67550e23a Mon Sep 17 00:00:00 2001 From: hsirkar Date: Fri, 15 Sep 2023 08:51:09 -0400 Subject: [PATCH 11/42] fix black issues --- pipit/graph.py | 2 +- pipit/readers/otf2_reader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipit/graph.py b/pipit/graph.py index 5903bea6..32db90c7 100644 --- a/pipit/graph.py +++ b/pipit/graph.py @@ -79,7 +79,7 @@ def _calculate_level(self): return 1 + self.parent._calculate_level() def __eq__(self, obj) -> bool: - if type(obj) != Node: + if not isinstance(obj, Node): return False else: return self._pipit_nid == obj._pipit_nid diff --git a/pipit/readers/otf2_reader.py b/pipit/readers/otf2_reader.py index 1cd5cd54..d23d2cf1 100644 --- a/pipit/readers/otf2_reader.py +++ b/pipit/readers/otf2_reader.py @@ -244,7 +244,7 @@ def events_reader(self, rank_size): if str(loc.type)[13:] == "CPU_THREAD": # don't add metric events as a separate row, # and add their values into columns instead - if type(event) == otf2.events.Metric: + if isinstance(event, otf2.events.Metric): # Since the location is a cpu thread, we know # that the metric event is of type MetricClass, # which has a list of MetricMembers. From 587d39361237448c58eb4a7b08633666859b2228 Mon Sep 17 00:00:00 2001 From: hsirkar Date: Wed, 4 Oct 2023 10:04:47 -0400 Subject: [PATCH 12/42] add global config object --- docs/examples/vis.ipynb | 677 ++-------------------------------------- pipit/__init__.py | 1 + pipit/config.py | 28 ++ pipit/vis/util.py | 33 +- 4 files changed, 58 insertions(+), 681 deletions(-) create mode 100644 pipit/config.py diff --git a/docs/examples/vis.ipynb b/docs/examples/vis.ipynb index 29002ca6..62bc0b12 100644 --- a/docs/examples/vis.ipynb +++ b/docs/examples/vis.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "99c9dcc2", + "execution_count": null, + "id": "e5841460", "metadata": {}, "outputs": [], "source": [ @@ -16,8 +16,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "09adcb68", + "execution_count": null, + "id": "da5ebec2", "metadata": {}, "outputs": [], "source": [ @@ -26,21 +26,12 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "aaef6dc5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "id": "7070057d", + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "ping_pong = pp.Trace.from_otf2(\"../../pipit/tests/data/ping-pong-otf2\")\n", "ping_pong" @@ -48,648 +39,32 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "0c82808a", + "execution_count": null, + "id": "5906905a", + "metadata": {}, + "outputs": [], + "source": [ + "pp.config[\"notebook_url\"] = \"http://localhost:8889\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93113475", "metadata": { "scrolled": true }, - "outputs": [ - { - "data": { - "application/javascript": [ - "(function(root) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " const force = true;\n", - "\n", - " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", - " root._bokeh_onload_callbacks = [];\n", - " root._bokeh_is_loading = undefined;\n", - " }\n", - "\n", - "const JS_MIME_TYPE = 'application/javascript';\n", - " const HTML_MIME_TYPE = 'text/html';\n", - " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", - " const CLASS_NAME = 'output_bokeh rendered_html';\n", - "\n", - " /**\n", - " * Render data to the DOM node\n", - " */\n", - " function render(props, node) {\n", - " const script = document.createElement(\"script\");\n", - " node.appendChild(script);\n", - " }\n", - "\n", - " /**\n", - " * Handle when an output is cleared or removed\n", - " */\n", - " function handleClearOutput(event, handle) {\n", - " const cell = handle.cell;\n", - "\n", - " const id = cell.output_area._bokeh_element_id;\n", - " const server_id = cell.output_area._bokeh_server_id;\n", - " // Clean up Bokeh references\n", - " if (id != null && id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - "\n", - " if (server_id !== undefined) {\n", - " // Clean up Bokeh references\n", - " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", - " cell.notebook.kernel.execute(cmd_clean, {\n", - " iopub: {\n", - " output: function(msg) {\n", - " const id = msg.content.text.trim();\n", - " if (id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - " }\n", - " }\n", - " });\n", - " // Destroy server and session\n", - " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", - " cell.notebook.kernel.execute(cmd_destroy);\n", - " }\n", - " }\n", - "\n", - " /**\n", - " * Handle when a new output is added\n", - " */\n", - " function handleAddOutput(event, handle) {\n", - " const output_area = handle.output_area;\n", - " const output = handle.output;\n", - "\n", - " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", - " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", - " return\n", - " }\n", - "\n", - " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", - "\n", - " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", - " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", - " // store reference to embed id on output_area\n", - " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", - " }\n", - " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", - " const bk_div = document.createElement(\"div\");\n", - " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", - " const script_attrs = bk_div.children[0].attributes;\n", - " for (let i = 0; i < script_attrs.length; i++) {\n", - " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", - " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", - " }\n", - " // store reference to server id on output_area\n", - " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", - " }\n", - " }\n", - "\n", - " function register_renderer(events, OutputArea) {\n", - "\n", - " function append_mime(data, metadata, element) {\n", - " // create a DOM node to render to\n", - " const toinsert = this.create_output_subarea(\n", - " metadata,\n", - " CLASS_NAME,\n", - " EXEC_MIME_TYPE\n", - " );\n", - " this.keyboard_manager.register_events(toinsert);\n", - " // Render to node\n", - " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", - " render(props, toinsert[toinsert.length - 1]);\n", - " element.append(toinsert);\n", - " return toinsert\n", - " }\n", - "\n", - " /* Handle when an output is cleared or removed */\n", - " events.on('clear_output.CodeCell', handleClearOutput);\n", - " events.on('delete.Cell', handleClearOutput);\n", - "\n", - " /* Handle when a new output is added */\n", - " events.on('output_added.OutputArea', handleAddOutput);\n", - "\n", - " /**\n", - " * Register the mime type and append_mime function with output_area\n", - " */\n", - " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", - " /* Is output safe? */\n", - " safe: true,\n", - " /* Index of renderer in `output_area.display_order` */\n", - " index: 0\n", - " });\n", - " }\n", - "\n", - " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", - " if (root.Jupyter !== undefined) {\n", - " const events = require('base/js/events');\n", - " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", - "\n", - " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", - " register_renderer(events, OutputArea);\n", - " }\n", - " }\n", - " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", - " root._bokeh_timeout = Date.now() + 5000;\n", - " root._bokeh_failed_load = false;\n", - " }\n", - "\n", - " const NB_LOAD_WARNING = {'data': {'text/html':\n", - " \"
\\n\"+\n", - " \"

\\n\"+\n", - " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", - " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", - " \"

\\n\"+\n", - " \"
    \\n\"+\n", - " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", - " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", - " \"
\\n\"+\n", - " \"\\n\"+\n", - " \"from bokeh.resources import INLINE\\n\"+\n", - " \"output_notebook(resources=INLINE)\\n\"+\n", - " \"\\n\"+\n", - " \"
\"}};\n", - "\n", - " function display_loaded() {\n", - " const el = document.getElementById(null);\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS is loading...\";\n", - " }\n", - " if (root.Bokeh !== undefined) {\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", - " }\n", - " } else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(display_loaded, 100)\n", - " }\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " try {\n", - " root._bokeh_onload_callbacks.forEach(function(callback) {\n", - " if (callback != null)\n", - " callback();\n", - " });\n", - " } finally {\n", - " delete root._bokeh_onload_callbacks\n", - " }\n", - " console.debug(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(css_urls, js_urls, callback) {\n", - " if (css_urls == null) css_urls = [];\n", - " if (js_urls == null) js_urls = [];\n", - "\n", - " root._bokeh_onload_callbacks.push(callback);\n", - " if (root._bokeh_is_loading > 0) {\n", - " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", - "\n", - " function on_load() {\n", - " root._bokeh_is_loading--;\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", - " run_callbacks()\n", - " }\n", - " }\n", - "\n", - " function on_error(url) {\n", - " console.error(\"failed to load \" + url);\n", - " }\n", - "\n", - " for (let i = 0; i < css_urls.length; i++) {\n", - " const url = css_urls[i];\n", - " const element = document.createElement(\"link\");\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.rel = \"stylesheet\";\n", - " element.type = \"text/css\";\n", - " element.href = url;\n", - " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " for (let i = 0; i < js_urls.length; i++) {\n", - " const url = js_urls[i];\n", - " const element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.async = false;\n", - " element.src = url;\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " };\n", - "\n", - " function inject_raw_css(css) {\n", - " const element = document.createElement(\"style\");\n", - " element.appendChild(document.createTextNode(css));\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n", - " const css_urls = [];\n", - "\n", - " const inline_js = [ function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - "function(Bokeh) {\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " if (root.Bokeh !== undefined || force === true) {\n", - " for (let i = 0; i < inline_js.length; i++) {\n", - " inline_js[i].call(root, root.Bokeh);\n", - " }\n", - "} else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(run_inline_js, 100);\n", - " } else if (!root._bokeh_failed_load) {\n", - " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", - " root._bokeh_failed_load = true;\n", - " } else if (force !== true) {\n", - " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", - " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", - " }\n", - " }\n", - "\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(css_urls, js_urls, function() {\n", - " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(window));" - ], - "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.bokehjs_exec.v0+json": "", - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.bokehjs_exec.v0+json": { - "server_id": "0b0fda93376e4b0487c5c480a0fa92f3" - } - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "ping_pong.plot_comm_matrix()" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "1fb7433e", + "execution_count": null, + "id": "a4aff933", "metadata": {}, - "outputs": [ - { - "data": { - "application/javascript": [ - "(function(root) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " const force = true;\n", - "\n", - " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", - " root._bokeh_onload_callbacks = [];\n", - " root._bokeh_is_loading = undefined;\n", - " }\n", - "\n", - "const JS_MIME_TYPE = 'application/javascript';\n", - " const HTML_MIME_TYPE = 'text/html';\n", - " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", - " const CLASS_NAME = 'output_bokeh rendered_html';\n", - "\n", - " /**\n", - " * Render data to the DOM node\n", - " */\n", - " function render(props, node) {\n", - " const script = document.createElement(\"script\");\n", - " node.appendChild(script);\n", - " }\n", - "\n", - " /**\n", - " * Handle when an output is cleared or removed\n", - " */\n", - " function handleClearOutput(event, handle) {\n", - " const cell = handle.cell;\n", - "\n", - " const id = cell.output_area._bokeh_element_id;\n", - " const server_id = cell.output_area._bokeh_server_id;\n", - " // Clean up Bokeh references\n", - " if (id != null && id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - "\n", - " if (server_id !== undefined) {\n", - " // Clean up Bokeh references\n", - " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", - " cell.notebook.kernel.execute(cmd_clean, {\n", - " iopub: {\n", - " output: function(msg) {\n", - " const id = msg.content.text.trim();\n", - " if (id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - " }\n", - " }\n", - " });\n", - " // Destroy server and session\n", - " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", - " cell.notebook.kernel.execute(cmd_destroy);\n", - " }\n", - " }\n", - "\n", - " /**\n", - " * Handle when a new output is added\n", - " */\n", - " function handleAddOutput(event, handle) {\n", - " const output_area = handle.output_area;\n", - " const output = handle.output;\n", - "\n", - " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", - " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", - " return\n", - " }\n", - "\n", - " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", - "\n", - " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", - " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", - " // store reference to embed id on output_area\n", - " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", - " }\n", - " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", - " const bk_div = document.createElement(\"div\");\n", - " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", - " const script_attrs = bk_div.children[0].attributes;\n", - " for (let i = 0; i < script_attrs.length; i++) {\n", - " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", - " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", - " }\n", - " // store reference to server id on output_area\n", - " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", - " }\n", - " }\n", - "\n", - " function register_renderer(events, OutputArea) {\n", - "\n", - " function append_mime(data, metadata, element) {\n", - " // create a DOM node to render to\n", - " const toinsert = this.create_output_subarea(\n", - " metadata,\n", - " CLASS_NAME,\n", - " EXEC_MIME_TYPE\n", - " );\n", - " this.keyboard_manager.register_events(toinsert);\n", - " // Render to node\n", - " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", - " render(props, toinsert[toinsert.length - 1]);\n", - " element.append(toinsert);\n", - " return toinsert\n", - " }\n", - "\n", - " /* Handle when an output is cleared or removed */\n", - " events.on('clear_output.CodeCell', handleClearOutput);\n", - " events.on('delete.Cell', handleClearOutput);\n", - "\n", - " /* Handle when a new output is added */\n", - " events.on('output_added.OutputArea', handleAddOutput);\n", - "\n", - " /**\n", - " * Register the mime type and append_mime function with output_area\n", - " */\n", - " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", - " /* Is output safe? */\n", - " safe: true,\n", - " /* Index of renderer in `output_area.display_order` */\n", - " index: 0\n", - " });\n", - " }\n", - "\n", - " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", - " if (root.Jupyter !== undefined) {\n", - " const events = require('base/js/events');\n", - " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", - "\n", - " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", - " register_renderer(events, OutputArea);\n", - " }\n", - " }\n", - " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", - " root._bokeh_timeout = Date.now() + 5000;\n", - " root._bokeh_failed_load = false;\n", - " }\n", - "\n", - " const NB_LOAD_WARNING = {'data': {'text/html':\n", - " \"
\\n\"+\n", - " \"

\\n\"+\n", - " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", - " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", - " \"

\\n\"+\n", - " \"
    \\n\"+\n", - " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", - " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", - " \"
\\n\"+\n", - " \"\\n\"+\n", - " \"from bokeh.resources import INLINE\\n\"+\n", - " \"output_notebook(resources=INLINE)\\n\"+\n", - " \"\\n\"+\n", - " \"
\"}};\n", - "\n", - " function display_loaded() {\n", - " const el = document.getElementById(null);\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS is loading...\";\n", - " }\n", - " if (root.Bokeh !== undefined) {\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", - " }\n", - " } else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(display_loaded, 100)\n", - " }\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " try {\n", - " root._bokeh_onload_callbacks.forEach(function(callback) {\n", - " if (callback != null)\n", - " callback();\n", - " });\n", - " } finally {\n", - " delete root._bokeh_onload_callbacks\n", - " }\n", - " console.debug(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(css_urls, js_urls, callback) {\n", - " if (css_urls == null) css_urls = [];\n", - " if (js_urls == null) js_urls = [];\n", - "\n", - " root._bokeh_onload_callbacks.push(callback);\n", - " if (root._bokeh_is_loading > 0) {\n", - " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", - "\n", - " function on_load() {\n", - " root._bokeh_is_loading--;\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", - " run_callbacks()\n", - " }\n", - " }\n", - "\n", - " function on_error(url) {\n", - " console.error(\"failed to load \" + url);\n", - " }\n", - "\n", - " for (let i = 0; i < css_urls.length; i++) {\n", - " const url = css_urls[i];\n", - " const element = document.createElement(\"link\");\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.rel = \"stylesheet\";\n", - " element.type = \"text/css\";\n", - " element.href = url;\n", - " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " for (let i = 0; i < js_urls.length; i++) {\n", - " const url = js_urls[i];\n", - " const element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.async = false;\n", - " element.src = url;\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " };\n", - "\n", - " function inject_raw_css(css) {\n", - " const element = document.createElement(\"style\");\n", - " element.appendChild(document.createTextNode(css));\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n", - " const css_urls = [];\n", - "\n", - " const inline_js = [ function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - "function(Bokeh) {\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " if (root.Bokeh !== undefined || force === true) {\n", - " for (let i = 0; i < inline_js.length; i++) {\n", - " inline_js[i].call(root, root.Bokeh);\n", - " }\n", - "} else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(run_inline_js, 100);\n", - " } else if (!root._bokeh_failed_load) {\n", - " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", - " root._bokeh_failed_load = true;\n", - " } else if (force !== true) {\n", - " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", - " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", - " }\n", - " }\n", - "\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(css_urls, js_urls, function() {\n", - " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(window));" - ], - "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.3.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.3.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.bokehjs_exec.v0+json": "", - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.bokehjs_exec.v0+json": { - "server_id": "5a58b28e0dd7442294d0ad41562d3a57" - } - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "ping_pong.plot_message_histogram()" ] diff --git a/pipit/__init__.py b/pipit/__init__.py index 1a014093..dc64218f 100644 --- a/pipit/__init__.py +++ b/pipit/__init__.py @@ -4,3 +4,4 @@ # SPDX-License-Identifier: MIT from .trace import Trace # noqa: F401 +from .config import config # noqa: F401 diff --git a/pipit/config.py b/pipit/config.py new file mode 100644 index 00000000..ff575453 --- /dev/null +++ b/pipit/config.py @@ -0,0 +1,28 @@ +config = { + "notebook_url": "http://localhost:8888", + "theme": """ + attrs: + Plot: + height: 350 + width: 700 + background_fill_color: "#fafafa" + Axis: + axis_label_text_font_style: "bold" + minor_tick_line_color: null + Toolbar: + autohide: true + logo: null + HoverTool: + point_policy: "follow_mouse" + Legend: + label_text_font_size: "8.5pt" + spacing: 10 + border_line_color: null + glyph_width: 16 + glyph_height: 16 + Scatter: + size: 9 + DataRange1d: + range_padding: 0.05 + """, +} diff --git a/pipit/vis/util.py b/pipit/vis/util.py index 2d361182..7fae4832 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -3,34 +3,7 @@ from bokeh.plotting import output_notebook from bokeh.plotting import show as bk_show from bokeh.themes import Theme - -# Global constants -NOTEBOOK_URL = "http://localhost:8888" -THEME = """ - attrs: - Plot: - height: 350 - width: 700 - background_fill_color: "#fafafa" - Axis: - axis_label_text_font_style: "bold" - minor_tick_line_color: null - Toolbar: - autohide: true - logo: null - HoverTool: - point_policy: "follow_mouse" - Legend: - label_text_font_size: "8.5pt" - spacing: 10 - border_line_color: null - glyph_width: 16 - glyph_height: 16 - Scatter: - size: 9 - DataRange1d: - range_padding: 0.05 -""" +from pipit import config # Helper functions @@ -66,7 +39,7 @@ def bkapp(doc): doc.add_root(p) doc.theme = Theme( json=yaml.load( - THEME, + config["theme"], Loader=yaml.FullLoader, ) ) @@ -74,7 +47,7 @@ def bkapp(doc): if in_notebook(): # If notebook, show it in output cell output_notebook(hide_banner=True) - bk_show(bkapp, notebook_url=NOTEBOOK_URL) + bk_show(bkapp, notebook_url=config["notebook_url"]) else: # If standalone, start HTTP server and show in browser from bokeh.server.server import Server From ec3430bc3c3ebb7bb4db8864ed3c74386a27bb4d Mon Sep 17 00:00:00 2001 From: hsirkar Date: Wed, 4 Oct 2023 10:11:06 -0400 Subject: [PATCH 13/42] Fix comm matrix bug :) --- pipit/vis/core.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pipit/vis/core.py b/pipit/vis/core.py index 385311fd..543bc548 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -52,8 +52,8 @@ def comm_matrix( # Create bokeh plot p = figure( - x_axis_label="Sender", - y_axis_label="Receiver", + x_axis_label="Receiver", + y_axis_label="Sender", x_range=(-0.5, N - 0.5), y_range=(N - 0.5, -0.5), x_axis_location="above", @@ -90,14 +90,14 @@ def comm_matrix( hover = p.select(HoverTool) hover.tooltips = get_tooltips( { - "Sender": "Process $x{0.}", - "Receiver": "Process $y{0.}", + "Sender": "Process $y{0.}", + "Receiver": "Process $x{0.}", "Bytes": "@image{custom}", } if output == "size" else { - "Sender": "Process $x{0.}", - "Receiver": "Process $y{0.}", + "Sender": "Process $y{0.}", + "Receiver": "Process $x{0.}", "Count": "@image", } ) From 0dd89ae47a00e3239a902242a07d6cc31fba36f4 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Wed, 28 Feb 2024 17:53:00 -0500 Subject: [PATCH 14/42] update config --- docs/examples/dsl2-Copy2.ipynb | 822 +++++++++++++++++++++++++++++++++ pipit/readers/otf2_reader.py | 6 +- pipit/util/cct.py | 8 +- pipit/util/config.py | 45 ++ pipit/vis/core.py | 26 +- pipit/vis/util.py | 10 +- 6 files changed, 894 insertions(+), 23 deletions(-) create mode 100644 docs/examples/dsl2-Copy2.ipynb diff --git a/docs/examples/dsl2-Copy2.ipynb b/docs/examples/dsl2-Copy2.ipynb new file mode 100644 index 00000000..7c47de9e --- /dev/null +++ b/docs/examples/dsl2-Copy2.ipynb @@ -0,0 +1,822 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "8227bf9b-a386-4cb0-9eba-a7e258bf1d00", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload \n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "7c95d344-9406-4c66-978a-e7032df0967b", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append(\"../..\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a8ea89f2-3d70-486c-9901-aeb4900169cb", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import pipit as pp" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "589e87fd-d8d1-4ddf-a426-b911410379c4", + "metadata": {}, + "outputs": [], + "source": [ + "trace = pp.Trace.from_otf2(\"../../pipit/tests/data/ping-pong-otf2/\")\n", + "pandas_df = trace.events[[\"Timestamp (ns)\", \"Name\", \"Event Type\", \"Process\", \"Thread\"]].copy()\n", + "pandas_df[\"Process\"] = pandas_df[\"Process\"].astype(int)\n", + "idx = pandas_df.groupby(['Process']).cumcount()\n", + "pandas_df[\"idx\"] = idx" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3867d2a2-ff3e-4c8c-9c09-46a167a0fde4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'pandas'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pp.set_option(\"backend\", \"pandas\")\n", + "pp.get_option(\"backend\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "156df562-4431-40b8-b6b0-b7cfbe6206d0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TraceDataset (0 traces, 0 events)\n" + ] + } + ], + "source": [ + "ds = pp.dsl2.TraceDataset()\n", + "ds.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "7fa7fe70-0c61-4503-9b27-3bae52d2f1c3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Event (rank=1, idx=0, thread=0, timestamp=0.0, event_type='Instant', name='ProgramBegin')" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# simulating a reader\n", + "row = pandas_df.iloc[0]\n", + "evt = pp.dsl2.Event(\n", + " rank=row[\"Process\"],\n", + " idx=row[\"idx\"],\n", + " thread=row[\"Thread\"],\n", + " timestamp=row[\"Timestamp (ns)\"],\n", + " event_type=row[\"Event Type\"],\n", + " name=row[\"Name\"],\n", + ")\n", + "evt" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "5abbffd5-3246-4331-9a40-86b5a6eda0d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TraceDataset (1 trace, 0 events)\n" + ] + } + ], + "source": [ + "ds.push_event(evt)\n", + "ds.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "bca1c384-0239-46b6-94d0-86439635eb80", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+-------------+--------------+--------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+-------------+--------------+--------------|\n", + "| 1 | 0 | 0 | 0 | Instant | ProgramBegin |\n", + "+--------+-------+----------+-------------+--------------+--------------+\n", + "TraceDataset (1 trace, 1 event)\n" + ] + } + ], + "source": [ + "ds.flush()\n", + "ds.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "2910c4d8-e40d-4683-b0d4-7943a6be34a6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+--------------------+--------------+-----------------------|\n", + "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin |\n", + "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) |\n", + "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init |\n", + "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", + "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", + "| ... | ... | ... | ... | ... | ... |\n", + "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", + "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", + "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", + "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", + "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "TraceDataset (2 traces, 120 events)\n" + ] + } + ], + "source": [ + "for i in range(1, len(pandas_df)):\n", + " row = pandas_df.iloc[i]\n", + " evt = pp.dsl2.Event(\n", + " rank=row[\"Process\"],\n", + " idx=row[\"idx\"],\n", + " thread=row[\"Thread\"],\n", + " timestamp=row[\"Timestamp (ns)\"],\n", + " event_type=row[\"Event Type\"],\n", + " name=row[\"Name\"],\n", + " )\n", + " ds.push_event(evt)\n", + "\n", + "ds.flush()\n", + "ds.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "72fac6a1-0486-4ab5-b95d-af594afe0996", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+-------------+--------------+---------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+-------------+--------------+---------------|\n", + "| 1 | 0 | 0 | 0 | Instant | ProgramBegin |\n", + "| 1 | 1 | 0 | 30083.1 | Enter | int main(int, |\n", + "| | | | | | char**) |\n", + "| 1 | 2 | 0 | 40288.3 | Enter | MPI_Init |\n", + "| 0 | 0 | 0 | 307731 | Instant | ProgramBegin |\n", + "| 0 | 1 | 0 | 336980 | Enter | int main(int, |\n", + "| | | | | | char**) |\n", + "+--------+-------+----------+-------------+--------------+---------------+\n", + "TraceDataset (2 traces, 5 events)\n" + ] + } + ], + "source": [ + "ds.head().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "3cba6972-052d-41df-a448-43f383fdf04e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+-------------+--------------+---------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+-------------+--------------+---------------|\n", + "| 1 | 57 | 0 | 1.99575e+08 | Leave | MPI_Finalize |\n", + "| 0 | 58 | 0 | 1.99575e+08 | Leave | int main(int, |\n", + "| | | | | | char**) |\n", + "| 1 | 58 | 0 | 1.99577e+08 | Leave | int main(int, |\n", + "| | | | | | char**) |\n", + "| 0 | 59 | 0 | 1.99603e+08 | Instant | ProgramEnd |\n", + "| 1 | 59 | 0 | 1.99604e+08 | Instant | ProgramEnd |\n", + "+--------+-------+----------+-------------+--------------+---------------+\n", + "TraceDataset (2 traces, 5 events)\n" + ] + } + ], + "source": [ + "ds.tail().show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "c8a7099f-c446-4800-b55a-c263b742a814", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+--------------------+--------------+-----------------------|\n", + "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", + "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", + "| 0 | 2 | 0 | 346054.77444467926 | Enter | MPI_Init |\n", + "| 0 | 3 | 0 | 193643138.1741584 | Leave | MPI_Init |\n", + "| 1 | 3 | 0 | 193643835.4832178 | Leave | MPI_Init |\n", + "| ... | ... | ... | ... | ... | ... |\n", + "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", + "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", + "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", + "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", + "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "TraceDataset (2 traces, 117 events)\n" + ] + } + ], + "source": [ + "ds.filter(\"timestamp > 1e5\").show()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "ebf8f71a-6a30-4c90-a9b6-ecb2215192f7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+----------+--------------------+--------------+-----------------------+\n", + "| idx | thread | timestamp | event_type | name |\n", + "|-------+----------+--------------------+--------------+-----------------------|\n", + "| 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", + "| 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", + "| 2 | 0 | 346054.77444467926 | Enter | MPI_Init |\n", + "| 3 | 0 | 193643138.1741584 | Leave | MPI_Init |\n", + "| 4 | 0 | 193651646.20379105 | Enter | MPI_Comm_size |\n", + "| ... | ... | ... | ... | ... |\n", + "| 55 | 0 | 199320512.07918367 | Leave | MPI_Recv |\n", + "| 56 | 0 | 199514778.2785141 | Enter | MPI_Finalize |\n", + "| 57 | 0 | 199573648.15437022 | Leave | MPI_Finalize |\n", + "| 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", + "| 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", + "+-------+----------+--------------------+--------------+-----------------------+\n", + "_Trace (rank=0, 60 events)\n" + ] + } + ], + "source": [ + "ds.traces[0].show()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "bde23e54-6f1b-4620-a90a-903ba8294c27", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Event (rank=0, idx=54, thread=0, timestamp=199319973.70504335, event_type='Instant', name='MpiRecv')" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.traces[0].loc[54]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "d1562ea5-8048-4c3c-8813-139ab83fcd33", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+----------+-------------+--------------+----------+\n", + "| idx | thread | timestamp | event_type | name |\n", + "|-------+----------+-------------+--------------+----------|\n", + "| 40 | 0 | 1.94911e+08 | Leave | MPI_Send |\n", + "| 41 | 0 | 1.94912e+08 | Enter | MPI_Recv |\n", + "| 42 | 0 | 1.95132e+08 | Instant | MpiRecv |\n", + "| 43 | 0 | 1.95132e+08 | Leave | MPI_Recv |\n", + "| 44 | 0 | 1.95718e+08 | Enter | MPI_Send |\n", + "| 45 | 0 | 1.95719e+08 | Instant | MpiSend |\n", + "| 46 | 0 | 1.9614e+08 | Leave | MPI_Send |\n", + "| 47 | 0 | 1.9614e+08 | Enter | MPI_Recv |\n", + "| 48 | 0 | 1.96584e+08 | Instant | MpiRecv |\n", + "| 49 | 0 | 1.96584e+08 | Leave | MPI_Recv |\n", + "| 50 | 0 | 1.97613e+08 | Enter | MPI_Send |\n", + "| 51 | 0 | 1.97614e+08 | Instant | MpiSend |\n", + "| 52 | 0 | 1.98506e+08 | Leave | MPI_Send |\n", + "| 53 | 0 | 1.98507e+08 | Enter | MPI_Recv |\n", + "| 54 | 0 | 1.9932e+08 | Instant | MpiRecv |\n", + "+-------+----------+-------------+--------------+----------+\n", + "_Trace (rank=0, 15 events)\n" + ] + } + ], + "source": [ + "ds.traces[0].loc[40:54].show()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "857acd69-9a15-454c-96c3-bdc4ba8ef955", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+--------------------+--------------+-----------------------|\n", + "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", + "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", + "| 0 | 2 | 0 | 346054.77444467926 | Enter | MPI_Init |\n", + "| 0 | 3 | 0 | 193643138.1741584 | Leave | MPI_Init |\n", + "| 0 | 4 | 0 | 193651646.20379105 | Enter | MPI_Comm_size |\n", + "| ... | ... | ... | ... | ... | ... |\n", + "| 0 | 55 | 0 | 199320512.07918367 | Leave | MPI_Recv |\n", + "| 0 | 56 | 0 | 199514778.2785141 | Enter | MPI_Finalize |\n", + "| 0 | 57 | 0 | 199573648.15437022 | Leave | MPI_Finalize |\n", + "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", + "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "TraceDataset (1 trace, 60 events)\n" + ] + } + ], + "source": [ + "ds.loc[0].show()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "2eeb2fae-2dca-4742-90de-725d45d1e5bb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+--------------------+--------------+-----------------------|\n", + "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin |\n", + "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) |\n", + "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init |\n", + "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", + "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", + "| ... | ... | ... | ... | ... | ... |\n", + "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", + "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", + "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", + "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", + "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "TraceDataset (2 traces, 120 events)\n" + ] + } + ], + "source": [ + "ds.loc[0:2].show()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "5e843a24-ff1a-44af-975a-15f83df89b07", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Event (rank=0, idx=59, thread=0, timestamp=199603304.5511645, event_type='Instant', name='ProgramEnd')" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.loc[0, 59]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2223f5fc-1433-443e-bd5f-1ae1c23faf28", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "120" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.map_traces(lambda trace: trace.count()).reduce(\"sum\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "de60e9ad-5aff-4f08-9a83-9d6cd086bc06", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+--------------------+\n", + "| idx | result |\n", + "|-------+--------------------|\n", + "| 0 | 307730.9358165928 |\n", + "| 1 | 336979.73374932166 |\n", + "| 2 | 346054.77444467926 |\n", + "| 3 | 193643138.1741584 |\n", + "| 4 | 193651646.20379105 |\n", + "| ... | ... |\n", + "| 55 | 199320512.07918367 |\n", + "| 56 | 199514778.2785141 |\n", + "| 57 | 199573648.15437022 |\n", + "| 58 | 199575243.23094556 |\n", + "| 59 | 199603304.5511645 |\n", + "+-------+--------------------+\n", + "DictLike (idx -> result, 60 items)\n" + ] + } + ], + "source": [ + "ds.traces[0].map_events(lambda event: event.timestamp).show()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "ac27a0b1-26df-4d6d-b9ff-35b77bcfb7c3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------+--------------------+\n", + "| idx | result |\n", + "|-------+--------------------|\n", + "| 0 | 307730.9358165928 |\n", + "| 1 | 336979.73374932166 |\n", + "| 2 | 346054.77444467926 |\n", + "| 3 | 193643138.1741584 |\n", + "| 4 | 193651646.20379105 |\n", + "| ... | ... |\n", + "| 55 | 199320512.07918367 |\n", + "| 56 | 199514778.2785141 |\n", + "| 57 | 199573648.15437022 |\n", + "| 58 | 199575243.23094556 |\n", + "| 59 | 199603304.5511645 |\n", + "+-------+--------------------+\n", + "DictLike (idx -> result, 60 items)\n" + ] + } + ], + "source": [ + "ds.traces[0].map_events(lambda event: event.timestamp).show()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "7afbf369-7fe4-40c2-97c4-b80d339fcbcb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "185373520.9350972" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.traces[0].map_events(lambda event: event.timestamp).reduce(\"mean\")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "44bd35aa-8763-4d11-808b-b42e10a95e81", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "11122246941.50224" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds.map_traces(lambda trace: trace.map_events(lambda event: event.timestamp).reduce(\"sum\")).reduce(\"mean\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "8821997b-1154-4e61-93c9-4cca21b948eb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+----------+\n", + "| rank | result |\n", + "|--------+----------|\n", + "| 1 | 60 |\n", + "| 0 | 60 |\n", + "+--------+----------+\n", + "DictLike (rank -> result, 2 items)\n" + ] + } + ], + "source": [ + "ds.map_traces(lambda trace: trace.count()).show()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "8aa72310-728d-42c9-9e6a-dcc13f9a0299", + "metadata": {}, + "outputs": [], + "source": [ + "# matching_evt = np.full(ds.traces[0].count(), np.nan)\n", + "# matching_ts = np.full(ds.traces[0].count(), np.nan)\n", + "\n", + "# stack = []\n", + "\n", + "# for event in ds.traces[0].iter_events():\n", + "# if event.event_type == \"Enter\":\n", + "# stack.append((event.idx, event.timestamp))\n", + "# elif event.event_type == \"Leave\":\n", + "# enter_idx, enter_ts = stack.pop()\n", + "# matching_evt[enter_idx] = event.idx\n", + "# matching_ts[enter_idx] = event.timestamp\n", + "# matching_evt[event.idx] = enter_idx\n", + "# matching_ts[event.idx] = enter_ts\n", + "\n", + "# ds.traces[0].add_column(\"matching_evt\", matching_evt)\n", + "# ds.traces[0].add_column(\"matching_ts\", matching_ts)\n", + "\n", + "# del matching_evt\n", + "# del matching_ts\n", + "\n", + "# ds.traces[0].show()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "06005b18-cc7e-4363-9264-ea0c50142575", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "| rank | idx | thread | timestamp | event_type | name |\n", + "|--------+-------+----------+--------------------+--------------+-----------------------|\n", + "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin |\n", + "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) |\n", + "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init |\n", + "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", + "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", + "| ... | ... | ... | ... | ... | ... |\n", + "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", + "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", + "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", + "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", + "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", + "+--------+-------+----------+--------------------+--------------+-----------------------+\n", + "TraceDataset (2 traces, 120 events)\n" + ] + } + ], + "source": [ + "ds.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "3312389e-71c4-411d-b8e0-9d8c14367359", + "metadata": {}, + "outputs": [], + "source": [ + "analyzer = pp.dsl2.Analyzer(ds)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "008cdda4-7d70-4501-9f43-55c6b465e1f2", + "metadata": {}, + "outputs": [], + "source": [ + "analyzer._match_events()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "c5d1a502-d728-4831-a608-9577c2c843b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+\n", + "| rank | idx | thread | timestamp | event_type | name | matching_evt | matching_ts |\n", + "|--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------|\n", + "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin | -1 | nan |\n", + "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) | 58 | 199576798.2158296 |\n", + "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init | 3 | 193643835.4832178 |\n", + "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin | -1 | nan |\n", + "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) | 58 | 199575243.23094556 |\n", + "| ... | ... | ... | ... | ... | ... | ... | ... |\n", + "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize | 56 | 199529686.66029385 |\n", + "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) | 1 | 336979.73374932166 |\n", + "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) | 1 | 30083.086937435106 |\n", + "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd | -1 | nan |\n", + "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd | -1 | nan |\n", + "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+\n", + "TraceDataset (2 traces, 120 events)\n" + ] + } + ], + "source": [ + "analyzer.ds.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "ada9f092-fcbc-4f90-bb36-c22e8353d6fe", + "metadata": {}, + "outputs": [], + "source": [ + "analyzer._match_caller_callee()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6c856475-0ea0-4acb-8c2b-2b1bfcdb5391", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+---------+-------+--------------------------+\n", + "| rank | idx | thread | timestamp | event_type | name | matching_evt | matching_ts | depth | par | children |\n", + "|--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+---------+-------+--------------------------|\n", + "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin | -1 | nan | 0 | -1 | [] |\n", + "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) | 58 | 199576798.2158296 | 0 | -1 | [2, 4, 6, 8, 11, 14, 17, |\n", + "| | | | | | | | | | | 20, 23, 26, 29, 32, 35, |\n", + "| | | | | | | | | | | 38, 41, 44, 47, 50, 53, |\n", + "| | | | | | | | | | | 56] |\n", + "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init | 3 | 193643835.4832178 | 1 | 1 | [] |\n", + "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin | -1 | nan | 0 | -1 | [] |\n", + "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) | 58 | 199575243.23094556 | 0 | -1 | [2, 4, 6, 8, 11, 14, 17, |\n", + "| | | | | | | | | | | 20, 23, 26, 29, 32, 35, |\n", + "| | | | | | | | | | | 38, 41, 44, 47, 50, 53, |\n", + "| | | | | | | | | | | 56] |\n", + "| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |\n", + "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize | 56 | 199529686.66029385 | 0 | -1 | [] |\n", + "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) | 1 | 336979.73374932166 | 0 | -1 | [] |\n", + "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) | 1 | 30083.086937435106 | 0 | -1 | [] |\n", + "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd | -1 | nan | 0 | -1 | [] |\n", + "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd | -1 | nan | 0 | -1 | [] |\n", + "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+---------+-------+--------------------------+\n", + "TraceDataset (2 traces, 120 events)\n" + ] + } + ], + "source": [ + "analyzer.ds.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pipit/readers/otf2_reader.py b/pipit/readers/otf2_reader.py index b1fae889..b685c3a7 100644 --- a/pipit/readers/otf2_reader.py +++ b/pipit/readers/otf2_reader.py @@ -316,9 +316,9 @@ def events_reader(self, rank_size): if value is not None and key != "time": # uses field_to_val to convert all data types # and ensure that there are no pickling errors - attributes_dict[ - self.field_to_val(key) - ] = self.handle_data(value) + attributes_dict[self.field_to_val(key)] = ( + self.handle_data(value) + ) event_attributes.append(attributes_dict) else: # nan attributes for leave rows diff --git a/pipit/util/cct.py b/pipit/util/cct.py index 6557a588..b4992271 100644 --- a/pipit/util/cct.py +++ b/pipit/util/cct.py @@ -86,9 +86,11 @@ def create_cct(events): # add node as root or child of its # parent depending on current depth - graph.add_root( - curr_node - ) if curr_depth == 0 else parent_node.add_child(curr_node) + ( + graph.add_root(curr_node) + if curr_depth == 0 + else parent_node.add_child(curr_node) + ) # Update nodes stack, column, and current depth nodes_stack.append(curr_node) diff --git a/pipit/util/config.py b/pipit/util/config.py index f71bb53f..8dcfa857 100644 --- a/pipit/util/config.py +++ b/pipit/util/config.py @@ -83,6 +83,23 @@ def url_validator(key, value): ) +# Validator to check if theme is valid YAML +def theme_validator(key, value): + import yaml + + try: + yaml.safe_load(value) + except yaml.YAMLError: + raise ValueError( + ( + 'Error loading configuration: The Value "{}" for Configuration "{}"' + + "must be a valid YAML" + ).format(value, key) + ) + else: + return True + + registered_options = { "log_level": { "default": "INFO", @@ -92,6 +109,34 @@ def url_validator(key, value): "default": "http://localhost:8888", "validator": url_validator, }, + "theme": { + "default": """ + attrs: + Plot: + height: 350 + width: 700 + background_fill_color: "#fafafa" + Axis: + axis_label_text_font_style: "bold" + minor_tick_line_color: null + Toolbar: + autohide: true + logo: null + HoverTool: + point_policy: "follow_mouse" + Legend: + label_text_font_size: "8.5pt" + spacing: 10 + border_line_color: null + glyph_width: 16 + glyph_height: 16 + Scatter: + size: 9 + DataRange1d: + range_padding: 0.05 + """, + "validator": theme_validator, + }, } global_config = {key: registered_options[key]["default"] for key in registered_options} diff --git a/pipit/vis/core.py b/pipit/vis/core.py index 543bc548..59ee5654 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -38,7 +38,7 @@ def comm_matrix( Returns: Bokeh figure object if return_fig, None otherwise """ - N = data.shape[0] + num_ranks = data.shape[0] # Define color mapper if cmap == "linear": @@ -54,12 +54,12 @@ def comm_matrix( p = figure( x_axis_label="Receiver", y_axis_label="Sender", - x_range=(-0.5, N - 0.5), - y_range=(N - 0.5, -0.5), + x_range=(-0.5, num_ranks - 0.5), + y_range=(num_ranks - 0.5, -0.5), x_axis_location="above", tools="hover,pan,reset,wheel_zoom,save", - width=90 + clamp(N * 30, 200, 500), - height=10 + clamp(N * 30, 200, 500), + width=90 + clamp(num_ranks * 30, 200, 500), + height=10 + clamp(num_ranks * 30, 200, 500), toolbar_location="below", ) @@ -67,23 +67,25 @@ def comm_matrix( p.image( image=[np.flipud(data)], x=-0.5, - y=N - 0.5, - dw=N, - dh=N, + y=num_ranks - 0.5, + dw=num_ranks, + dh=num_ranks, color_mapper=color_mapper, ) color_bar = ColorBar( color_mapper=color_mapper, - formatter=get_size_tick_formatter(ignore_range=cmap == "log") - if output == "size" - else NumeralTickFormatter(), + formatter=( + get_size_tick_formatter(ignore_range=cmap == "log") + if output == "size" + else NumeralTickFormatter() + ), width=15, ) p.add_layout(color_bar, "right") # Customize plot - p.axis.ticker = get_process_ticker(N=N) + p.axis.ticker = get_process_ticker(num_ranks=num_ranks) p.grid.visible = False # Configure hover diff --git a/pipit/vis/util.py b/pipit/vis/util.py index 7fae4832..d36b79d1 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -3,7 +3,7 @@ from bokeh.plotting import output_notebook from bokeh.plotting import show as bk_show from bokeh.themes import Theme -from pipit import config +import pipit as pp # Helper functions @@ -39,7 +39,7 @@ def bkapp(doc): doc.add_root(p) doc.theme = Theme( json=yaml.load( - config["theme"], + pp.get_option("theme"), Loader=yaml.FullLoader, ) ) @@ -47,7 +47,7 @@ def bkapp(doc): if in_notebook(): # If notebook, show it in output cell output_notebook(hide_banner=True) - bk_show(bkapp, notebook_url=config["notebook_url"]) + bk_show(bkapp, notebook_url=pp.get_option("notebook_url")) else: # If standalone, start HTTP server and show in browser from bokeh.server.server import Server @@ -110,9 +110,9 @@ def clamp(value, min_val, max_val): """ -def get_process_ticker(N): +def get_process_ticker(num_ranks): return BasicTicker( - base=2, desired_num_ticks=min(N, 16), min_interval=1, num_minor_ticks=0 + base=2, desired_num_ticks=min(num_ranks, 16), min_interval=1, num_minor_ticks=0 ) From 50315e0aa8ac6230337d737005fb7ad272bd98fc Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Wed, 28 Feb 2024 19:16:09 -0500 Subject: [PATCH 15/42] upgrade to bokeh 3 --- docs/examples/vis.ipynb | 692 ++++++++++++++++++++++++++++++++++++++-- pipit/vis/core.py | 45 ++- pipit/vis/util.py | 30 +- requirements.txt | 1 + 4 files changed, 698 insertions(+), 70 deletions(-) diff --git a/docs/examples/vis.ipynb b/docs/examples/vis.ipynb index 62bc0b12..e4541df6 100644 --- a/docs/examples/vis.ipynb +++ b/docs/examples/vis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "e5841460", "metadata": {}, "outputs": [], @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "da5ebec2", "metadata": {}, "outputs": [], @@ -26,12 +26,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "7070057d", - "metadata": { - "scrolled": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "ping_pong = pp.Trace.from_otf2(\"../../pipit/tests/data/ping-pong-otf2\")\n", "ping_pong" @@ -39,35 +48,680 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "5906905a", + "execution_count": 4, + "id": "07605e08-385c-4918-8405-f56ace459268", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[ 0., 4177920.],\n", + " [4177920., 0.]])" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "pp.config[\"notebook_url\"] = \"http://localhost:8889\"" + "data = ping_pong.comm_matrix()\n", + "data" ] }, { "cell_type": "code", - "execution_count": null, - "id": "93113475", - "metadata": { - "scrolled": true - }, - "outputs": [], + "execution_count": 5, + "id": "2476f1f3-a6cf-49f2-8e3e-0e004f088504", + "metadata": {}, + "outputs": [ + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " const force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "const JS_MIME_TYPE = 'application/javascript';\n", + " const HTML_MIME_TYPE = 'text/html';\n", + " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", + " const CLASS_NAME = 'output_bokeh rendered_html';\n", + "\n", + " /**\n", + " * Render data to the DOM node\n", + " */\n", + " function render(props, node) {\n", + " const script = document.createElement(\"script\");\n", + " node.appendChild(script);\n", + " }\n", + "\n", + " /**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + " function handleClearOutput(event, handle) {\n", + " const cell = handle.cell;\n", + "\n", + " const id = cell.output_area._bokeh_element_id;\n", + " const server_id = cell.output_area._bokeh_server_id;\n", + " // Clean up Bokeh references\n", + " if (id != null && id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + "\n", + " if (server_id !== undefined) {\n", + " // Clean up Bokeh references\n", + " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", + " cell.notebook.kernel.execute(cmd_clean, {\n", + " iopub: {\n", + " output: function(msg) {\n", + " const id = msg.content.text.trim();\n", + " if (id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + " }\n", + " }\n", + " });\n", + " // Destroy server and session\n", + " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", + " cell.notebook.kernel.execute(cmd_destroy);\n", + " }\n", + " }\n", + "\n", + " /**\n", + " * Handle when a new output is added\n", + " */\n", + " function handleAddOutput(event, handle) {\n", + " const output_area = handle.output_area;\n", + " const output = handle.output;\n", + "\n", + " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", + " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + "\n", + " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + "\n", + " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", + " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", + " // store reference to embed id on output_area\n", + " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " }\n", + " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " const bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " const script_attrs = bk_div.children[0].attributes;\n", + " for (let i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + " }\n", + "\n", + " function register_renderer(events, OutputArea) {\n", + "\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " const toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[toinsert.length - 1]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " /* Handle when an output is cleared or removed */\n", + " events.on('clear_output.CodeCell', handleClearOutput);\n", + " events.on('delete.Cell', handleClearOutput);\n", + "\n", + " /* Handle when a new output is added */\n", + " events.on('output_added.OutputArea', handleAddOutput);\n", + "\n", + " /**\n", + " * Register the mime type and append_mime function with output_area\n", + " */\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " /* Is output safe? */\n", + " safe: true,\n", + " /* Index of renderer in `output_area.display_order` */\n", + " index: 0\n", + " });\n", + " }\n", + "\n", + " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", + " if (root.Jupyter !== undefined) {\n", + " const events = require('base/js/events');\n", + " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", + "\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " }\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " const NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"
    \\n\"+\n", + " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", + " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", + " \"
\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " const el = document.getElementById(null);\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS is loading...\";\n", + " }\n", + " if (root.Bokeh !== undefined) {\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error(url) {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (let i = 0; i < css_urls.length; i++) {\n", + " const url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " for (let i = 0; i < js_urls.length; i++) {\n", + " const url = js_urls[i];\n", + " const element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n", + " const css_urls = [];\n", + "\n", + " const inline_js = [ function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if (root.Bokeh !== undefined || force === true) {\n", + " for (let i = 0; i < inline_js.length; i++) {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " }\n", + "} else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + " }\n", + "\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(css_urls, js_urls, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(window));" + ], + "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.bokehjs_exec.v0+json": "", + "text/html": [ + "" + ] + }, + "metadata": { + "application/vnd.bokehjs_exec.v0+json": { + "server_id": "7a1c6a300f4241deaba4ca613e4d18a5" + } + }, + "output_type": "display_data" + } + ], "source": [ "ping_pong.plot_comm_matrix()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "a4aff933", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/javascript": [ + "(function(root) {\n", + " function now() {\n", + " return new Date();\n", + " }\n", + "\n", + " const force = true;\n", + "\n", + " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", + " root._bokeh_onload_callbacks = [];\n", + " root._bokeh_is_loading = undefined;\n", + " }\n", + "\n", + "const JS_MIME_TYPE = 'application/javascript';\n", + " const HTML_MIME_TYPE = 'text/html';\n", + " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", + " const CLASS_NAME = 'output_bokeh rendered_html';\n", + "\n", + " /**\n", + " * Render data to the DOM node\n", + " */\n", + " function render(props, node) {\n", + " const script = document.createElement(\"script\");\n", + " node.appendChild(script);\n", + " }\n", + "\n", + " /**\n", + " * Handle when an output is cleared or removed\n", + " */\n", + " function handleClearOutput(event, handle) {\n", + " const cell = handle.cell;\n", + "\n", + " const id = cell.output_area._bokeh_element_id;\n", + " const server_id = cell.output_area._bokeh_server_id;\n", + " // Clean up Bokeh references\n", + " if (id != null && id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + "\n", + " if (server_id !== undefined) {\n", + " // Clean up Bokeh references\n", + " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", + " cell.notebook.kernel.execute(cmd_clean, {\n", + " iopub: {\n", + " output: function(msg) {\n", + " const id = msg.content.text.trim();\n", + " if (id in Bokeh.index) {\n", + " Bokeh.index[id].model.document.clear();\n", + " delete Bokeh.index[id];\n", + " }\n", + " }\n", + " }\n", + " });\n", + " // Destroy server and session\n", + " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", + " cell.notebook.kernel.execute(cmd_destroy);\n", + " }\n", + " }\n", + "\n", + " /**\n", + " * Handle when a new output is added\n", + " */\n", + " function handleAddOutput(event, handle) {\n", + " const output_area = handle.output_area;\n", + " const output = handle.output;\n", + "\n", + " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", + " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", + " return\n", + " }\n", + "\n", + " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", + "\n", + " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", + " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", + " // store reference to embed id on output_area\n", + " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", + " }\n", + " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", + " const bk_div = document.createElement(\"div\");\n", + " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", + " const script_attrs = bk_div.children[0].attributes;\n", + " for (let i = 0; i < script_attrs.length; i++) {\n", + " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", + " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", + " }\n", + " // store reference to server id on output_area\n", + " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", + " }\n", + " }\n", + "\n", + " function register_renderer(events, OutputArea) {\n", + "\n", + " function append_mime(data, metadata, element) {\n", + " // create a DOM node to render to\n", + " const toinsert = this.create_output_subarea(\n", + " metadata,\n", + " CLASS_NAME,\n", + " EXEC_MIME_TYPE\n", + " );\n", + " this.keyboard_manager.register_events(toinsert);\n", + " // Render to node\n", + " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", + " render(props, toinsert[toinsert.length - 1]);\n", + " element.append(toinsert);\n", + " return toinsert\n", + " }\n", + "\n", + " /* Handle when an output is cleared or removed */\n", + " events.on('clear_output.CodeCell', handleClearOutput);\n", + " events.on('delete.Cell', handleClearOutput);\n", + "\n", + " /* Handle when a new output is added */\n", + " events.on('output_added.OutputArea', handleAddOutput);\n", + "\n", + " /**\n", + " * Register the mime type and append_mime function with output_area\n", + " */\n", + " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", + " /* Is output safe? */\n", + " safe: true,\n", + " /* Index of renderer in `output_area.display_order` */\n", + " index: 0\n", + " });\n", + " }\n", + "\n", + " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", + " if (root.Jupyter !== undefined) {\n", + " const events = require('base/js/events');\n", + " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", + "\n", + " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", + " register_renderer(events, OutputArea);\n", + " }\n", + " }\n", + " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", + " root._bokeh_timeout = Date.now() + 5000;\n", + " root._bokeh_failed_load = false;\n", + " }\n", + "\n", + " const NB_LOAD_WARNING = {'data': {'text/html':\n", + " \"
\\n\"+\n", + " \"

\\n\"+\n", + " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", + " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", + " \"

\\n\"+\n", + " \"
    \\n\"+\n", + " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", + " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", + " \"
\\n\"+\n", + " \"\\n\"+\n", + " \"from bokeh.resources import INLINE\\n\"+\n", + " \"output_notebook(resources=INLINE)\\n\"+\n", + " \"\\n\"+\n", + " \"
\"}};\n", + "\n", + " function display_loaded() {\n", + " const el = document.getElementById(null);\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS is loading...\";\n", + " }\n", + " if (root.Bokeh !== undefined) {\n", + " if (el != null) {\n", + " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", + " }\n", + " } else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(display_loaded, 100)\n", + " }\n", + " }\n", + "\n", + " function run_callbacks() {\n", + " try {\n", + " root._bokeh_onload_callbacks.forEach(function(callback) {\n", + " if (callback != null)\n", + " callback();\n", + " });\n", + " } finally {\n", + " delete root._bokeh_onload_callbacks\n", + " }\n", + " console.debug(\"Bokeh: all callbacks have finished\");\n", + " }\n", + "\n", + " function load_libs(css_urls, js_urls, callback) {\n", + " if (css_urls == null) css_urls = [];\n", + " if (js_urls == null) js_urls = [];\n", + "\n", + " root._bokeh_onload_callbacks.push(callback);\n", + " if (root._bokeh_is_loading > 0) {\n", + " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", + " return null;\n", + " }\n", + " if (js_urls == null || js_urls.length === 0) {\n", + " run_callbacks();\n", + " return null;\n", + " }\n", + " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", + " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", + "\n", + " function on_load() {\n", + " root._bokeh_is_loading--;\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", + " run_callbacks()\n", + " }\n", + " }\n", + "\n", + " function on_error(url) {\n", + " console.error(\"failed to load \" + url);\n", + " }\n", + "\n", + " for (let i = 0; i < css_urls.length; i++) {\n", + " const url = css_urls[i];\n", + " const element = document.createElement(\"link\");\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.rel = \"stylesheet\";\n", + " element.type = \"text/css\";\n", + " element.href = url;\n", + " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " for (let i = 0; i < js_urls.length; i++) {\n", + " const url = js_urls[i];\n", + " const element = document.createElement('script');\n", + " element.onload = on_load;\n", + " element.onerror = on_error.bind(null, url);\n", + " element.async = false;\n", + " element.src = url;\n", + " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", + " document.head.appendChild(element);\n", + " }\n", + " };\n", + "\n", + " function inject_raw_css(css) {\n", + " const element = document.createElement(\"style\");\n", + " element.appendChild(document.createTextNode(css));\n", + " document.body.appendChild(element);\n", + " }\n", + "\n", + " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n", + " const css_urls = [];\n", + "\n", + " const inline_js = [ function(Bokeh) {\n", + " Bokeh.set_log_level(\"info\");\n", + " },\n", + "function(Bokeh) {\n", + " }\n", + " ];\n", + "\n", + " function run_inline_js() {\n", + " if (root.Bokeh !== undefined || force === true) {\n", + " for (let i = 0; i < inline_js.length; i++) {\n", + " inline_js[i].call(root, root.Bokeh);\n", + " }\n", + "} else if (Date.now() < root._bokeh_timeout) {\n", + " setTimeout(run_inline_js, 100);\n", + " } else if (!root._bokeh_failed_load) {\n", + " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", + " root._bokeh_failed_load = true;\n", + " } else if (force !== true) {\n", + " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", + " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", + " }\n", + " }\n", + "\n", + " if (root._bokeh_is_loading === 0) {\n", + " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", + " run_inline_js();\n", + " } else {\n", + " load_libs(css_urls, js_urls, function() {\n", + " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", + " run_inline_js();\n", + " });\n", + " }\n", + "}(window));" + ], + "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.bokehjs_exec.v0+json": "", + "text/html": [ + "" + ] + }, + "metadata": { + "application/vnd.bokehjs_exec.v0+json": { + "server_id": "c29ef1105a8a4690aa88355a3b32c0e1" + } + }, + "output_type": "display_data" + } + ], "source": [ "ping_pong.plot_message_histogram()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40bfed1c-5cce-48e0-b220-288b7ac16d26", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/pipit/vis/core.py b/pipit/vis/core.py index 59ee5654..19acc8ca 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -13,7 +13,6 @@ get_process_ticker, get_size_hover_formatter, get_size_tick_formatter, - get_tooltips, show, ) @@ -38,7 +37,7 @@ def comm_matrix( Returns: Bokeh figure object if return_fig, None otherwise """ - num_ranks = data.shape[0] + nranks = data.shape[0] # Define color mapper if cmap == "linear": @@ -54,12 +53,12 @@ def comm_matrix( p = figure( x_axis_label="Receiver", y_axis_label="Sender", - x_range=(-0.5, num_ranks - 0.5), - y_range=(num_ranks - 0.5, -0.5), + x_range=(-0.5, nranks - 0.5), + y_range=(nranks - 0.5, -0.5), x_axis_location="above", tools="hover,pan,reset,wheel_zoom,save", - width=90 + clamp(num_ranks * 30, 200, 500), - height=10 + clamp(num_ranks * 30, 200, 500), + width=90 + clamp(nranks * 30, 200, 500), + height=10 + clamp(nranks * 30, 200, 500), toolbar_location="below", ) @@ -67,9 +66,9 @@ def comm_matrix( p.image( image=[np.flipud(data)], x=-0.5, - y=num_ranks - 0.5, - dw=num_ranks, - dh=num_ranks, + y=-0.5, + dw=nranks, + dh=nranks, color_mapper=color_mapper, ) @@ -85,24 +84,16 @@ def comm_matrix( p.add_layout(color_bar, "right") # Customize plot - p.axis.ticker = get_process_ticker(num_ranks=num_ranks) + p.axis.ticker = get_process_ticker(nranks=nranks) p.grid.visible = False # Configure hover hover = p.select(HoverTool) - hover.tooltips = get_tooltips( - { - "Sender": "Process $y{0.}", - "Receiver": "Process $x{0.}", - "Bytes": "@image{custom}", - } - if output == "size" - else { - "Sender": "Process $y{0.}", - "Receiver": "Process $x{0.}", - "Count": "@image", - } - ) + hover.tooltips = [ + ("sender", "$y{0.}"), + ("receiver", "$x{0.}"), + ("value", "@image") if output == "count" else ("value", "@image{custom}"), + ] hover.formatters = {"@image": get_size_hover_formatter()} # Return plot @@ -139,12 +130,14 @@ def message_histogram( # Customize plot p.xaxis.formatter = get_size_tick_formatter() p.yaxis.formatter = NumeralTickFormatter() + p.xgrid.visible = False # Configure hover hover = p.select(HoverTool) - hover.tooltips = get_tooltips( - {"Bin": "@left{custom} - @right{custom}", "Number of messages": "@top"} - ) + hover.tooltips = [ + ("Bin", "@left{custom} - @right{custom}"), + ("Count", "@top"), + ] hover.formatters = { "@left": get_size_hover_formatter(), "@right": get_size_hover_formatter(), diff --git a/pipit/vis/util.py b/pipit/vis/util.py index d36b79d1..ce250692 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -1,5 +1,5 @@ import yaml -from bokeh.models import BasicTicker, CustomJSHover, FuncTickFormatter +from bokeh.models import BasicTicker, CustomJSHover, CustomJSTickFormatter from bokeh.plotting import output_notebook from bokeh.plotting import show as bk_show from bokeh.themes import Theme @@ -36,6 +36,7 @@ def show(p, return_fig=False): # Create a Bokeh app containing the figure def bkapp(doc): + doc.clear() doc.add_root(p) doc.theme = Theme( json=yaml.load( @@ -58,27 +59,6 @@ def bkapp(doc): server.io_loop.start() -def get_tooltips(tooltips_dict): - """Returns nicely formatted HTML tooltips from a dict""" - - html = "" - for k, v in tooltips_dict.items(): - html += f""" -
- {k}:  - {v} -
- """ - html += """ - - """ - return html - - def clamp(value, min_val, max_val): """Clamps value to min and max bounds""" @@ -110,9 +90,9 @@ def clamp(value, min_val, max_val): """ -def get_process_ticker(num_ranks): +def get_process_ticker(nranks): return BasicTicker( - base=2, desired_num_ticks=min(num_ranks, 16), min_interval=1, num_minor_ticks=0 + base=2, desired_num_ticks=min(nranks, 16), min_interval=1, num_minor_ticks=0 ) @@ -128,7 +108,7 @@ def get_size_hover_formatter(): def get_size_tick_formatter(ignore_range=False): x = "tick" if ignore_range else "Math.max(...ticks) - Math.min(...ticks);" - return FuncTickFormatter( + return CustomJSTickFormatter( code=f""" let x = {x} let y = tick; diff --git a/requirements.txt b/requirements.txt index 1f317793..7ad07110 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ numpy otf2 pandas +bokeh \ No newline at end of file From 7b4a8ce4fd6769809c070b6a35906804df4b2078 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Wed, 28 Feb 2024 19:17:12 -0500 Subject: [PATCH 16/42] unpin 2.4.3 --- .github/workflows/unit-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 01194ecf..d395f0ca 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -26,7 +26,7 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip - pip install --upgrade numpy pandas pytest otf2 bokeh==2.4.3 datashader + pip install --upgrade numpy pandas pytest otf2 bokeh datashader - name: Lint and format check with flake8 and black if: ${{ matrix.python-version == 3.9 }} @@ -54,7 +54,7 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip - pip install --upgrade numpy pandas pytest otf2 bokeh==2.4.3 datashader + pip install --upgrade numpy pandas pytest otf2 bokeh datashader - name: Basic test with pytest run: | From a50078799793f4b7c9e61ac766c306d50838bfb8 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Wed, 28 Feb 2024 19:19:40 -0500 Subject: [PATCH 17/42] fix flake8 and black --- pipit/readers/otf2_reader.py | 6 +++--- pipit/readers/projections_reader.py | 12 ++++++------ pipit/trace.py | 8 ++++---- pipit/util/cct.py | 8 +++++--- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pipit/readers/otf2_reader.py b/pipit/readers/otf2_reader.py index b1fae889..b685c3a7 100644 --- a/pipit/readers/otf2_reader.py +++ b/pipit/readers/otf2_reader.py @@ -316,9 +316,9 @@ def events_reader(self, rank_size): if value is not None and key != "time": # uses field_to_val to convert all data types # and ensure that there are no pickling errors - attributes_dict[ - self.field_to_val(key) - ] = self.handle_data(value) + attributes_dict[self.field_to_val(key)] = ( + self.handle_data(value) + ) event_attributes.append(attributes_dict) else: # nan attributes for leave rows diff --git a/pipit/readers/projections_reader.py b/pipit/readers/projections_reader.py index 38153e8f..dd59ffe2 100644 --- a/pipit/readers/projections_reader.py +++ b/pipit/readers/projections_reader.py @@ -513,10 +513,10 @@ def _read_log_file(self, rank_size) -> pd.DataFrame: pe = int(line_arr[5]) msglen = int(line_arr[6]) send_time = int(line_arr[7]) * 1000 - numPEs = int(line_arr[8]) - destPEs = [] - for i in (0, numPEs): - destPEs.append(int(line_arr[9 + i])) + num_procs = int(line_arr[8]) + dest_procs = [] + for i in (0, num_procs): + dest_procs.append(int(line_arr[9 + i])) details = { "From PE": pe, @@ -525,7 +525,7 @@ def _read_log_file(self, rank_size) -> pd.DataFrame: "Message Length": msglen, "Event ID": event, "Send Time": send_time, - "Destinatopn PEs": destPEs, + "Destinatopn PEs": dest_procs, } _add_to_trace_dict( @@ -534,7 +534,7 @@ def _read_log_file(self, rank_size) -> pd.DataFrame: "Instant", time, pe_num, - "To " + str(numPEs) + "processors", + "To " + str(num_procs) + "processors", ) # Processing of chare (i.e. execution) ? diff --git a/pipit/trace.py b/pipit/trace.py index 2b4f111a..fcc9f75d 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -597,7 +597,7 @@ def load_imbalance(self, metric="time.exc", num_processes=1): return imbalance_df - def idle_time(self, idle_functions=["Idle"], MPI_events=False): + def idle_time(self, idle_functions=["Idle"], mpi_events=False): # dict for creating a new dataframe idle_times = {"Process": [], "Idle Time": []} @@ -605,19 +605,19 @@ def idle_time(self, idle_functions=["Idle"], MPI_events=False): idle_times["Process"].append(process) idle_times["Idle Time"].append( self._calculate_idle_time_for_process( - process, idle_functions, MPI_events + process, idle_functions, mpi_events ) ) return pd.DataFrame(idle_times) def _calculate_idle_time_for_process( - self, process, idle_functions=["Idle"], MPI_events=False + self, process, idle_functions=["Idle"], mpi_events=False ): # calculate inclusive metrics if "time.inc" not in self.events.columns: self.calc_inc_metrics() - if MPI_events: + if mpi_events: idle_functions += ["MPI_Wait", "MPI_Waitall", "MPI_Recv"] # filter the dataframe to include only 'Enter' events within the specified # process with the specified function names diff --git a/pipit/util/cct.py b/pipit/util/cct.py index 6557a588..b4992271 100644 --- a/pipit/util/cct.py +++ b/pipit/util/cct.py @@ -86,9 +86,11 @@ def create_cct(events): # add node as root or child of its # parent depending on current depth - graph.add_root( - curr_node - ) if curr_depth == 0 else parent_node.add_child(curr_node) + ( + graph.add_root(curr_node) + if curr_depth == 0 + else parent_node.add_child(curr_node) + ) # Update nodes stack, column, and current depth nodes_stack.append(curr_node) From 6ada6e0f70aa17d820eb6a9632c14b63f490d919 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Wed, 28 Feb 2024 19:50:45 -0500 Subject: [PATCH 18/42] clean up --- docs/examples/vis.ipynb | 684 +--------------------------------------- pipit/trace.py | 4 - pipit/vis/core.py | 7 +- pipit/vis/util.py | 90 ------ 4 files changed, 12 insertions(+), 773 deletions(-) diff --git a/docs/examples/vis.ipynb b/docs/examples/vis.ipynb index e4541df6..2109ff99 100644 --- a/docs/examples/vis.ipynb +++ b/docs/examples/vis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "e5841460", "metadata": {}, "outputs": [], @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "da5ebec2", "metadata": {}, "outputs": [], @@ -26,21 +26,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "7070057d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "ping_pong = pp.Trace.from_otf2(\"../../pipit/tests/data/ping-pong-otf2\")\n", "ping_pong" @@ -48,680 +37,23 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "07605e08-385c-4918-8405-f56ace459268", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 0., 4177920.],\n", - " [4177920., 0.]])" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "data = ping_pong.comm_matrix()\n", - "data" - ] - }, - { - "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "2476f1f3-a6cf-49f2-8e3e-0e004f088504", "metadata": {}, - "outputs": [ - { - "data": { - "application/javascript": [ - "(function(root) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " const force = true;\n", - "\n", - " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", - " root._bokeh_onload_callbacks = [];\n", - " root._bokeh_is_loading = undefined;\n", - " }\n", - "\n", - "const JS_MIME_TYPE = 'application/javascript';\n", - " const HTML_MIME_TYPE = 'text/html';\n", - " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", - " const CLASS_NAME = 'output_bokeh rendered_html';\n", - "\n", - " /**\n", - " * Render data to the DOM node\n", - " */\n", - " function render(props, node) {\n", - " const script = document.createElement(\"script\");\n", - " node.appendChild(script);\n", - " }\n", - "\n", - " /**\n", - " * Handle when an output is cleared or removed\n", - " */\n", - " function handleClearOutput(event, handle) {\n", - " const cell = handle.cell;\n", - "\n", - " const id = cell.output_area._bokeh_element_id;\n", - " const server_id = cell.output_area._bokeh_server_id;\n", - " // Clean up Bokeh references\n", - " if (id != null && id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - "\n", - " if (server_id !== undefined) {\n", - " // Clean up Bokeh references\n", - " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", - " cell.notebook.kernel.execute(cmd_clean, {\n", - " iopub: {\n", - " output: function(msg) {\n", - " const id = msg.content.text.trim();\n", - " if (id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - " }\n", - " }\n", - " });\n", - " // Destroy server and session\n", - " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", - " cell.notebook.kernel.execute(cmd_destroy);\n", - " }\n", - " }\n", - "\n", - " /**\n", - " * Handle when a new output is added\n", - " */\n", - " function handleAddOutput(event, handle) {\n", - " const output_area = handle.output_area;\n", - " const output = handle.output;\n", - "\n", - " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", - " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", - " return\n", - " }\n", - "\n", - " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", - "\n", - " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", - " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", - " // store reference to embed id on output_area\n", - " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", - " }\n", - " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", - " const bk_div = document.createElement(\"div\");\n", - " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", - " const script_attrs = bk_div.children[0].attributes;\n", - " for (let i = 0; i < script_attrs.length; i++) {\n", - " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", - " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", - " }\n", - " // store reference to server id on output_area\n", - " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", - " }\n", - " }\n", - "\n", - " function register_renderer(events, OutputArea) {\n", - "\n", - " function append_mime(data, metadata, element) {\n", - " // create a DOM node to render to\n", - " const toinsert = this.create_output_subarea(\n", - " metadata,\n", - " CLASS_NAME,\n", - " EXEC_MIME_TYPE\n", - " );\n", - " this.keyboard_manager.register_events(toinsert);\n", - " // Render to node\n", - " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", - " render(props, toinsert[toinsert.length - 1]);\n", - " element.append(toinsert);\n", - " return toinsert\n", - " }\n", - "\n", - " /* Handle when an output is cleared or removed */\n", - " events.on('clear_output.CodeCell', handleClearOutput);\n", - " events.on('delete.Cell', handleClearOutput);\n", - "\n", - " /* Handle when a new output is added */\n", - " events.on('output_added.OutputArea', handleAddOutput);\n", - "\n", - " /**\n", - " * Register the mime type and append_mime function with output_area\n", - " */\n", - " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", - " /* Is output safe? */\n", - " safe: true,\n", - " /* Index of renderer in `output_area.display_order` */\n", - " index: 0\n", - " });\n", - " }\n", - "\n", - " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", - " if (root.Jupyter !== undefined) {\n", - " const events = require('base/js/events');\n", - " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", - "\n", - " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", - " register_renderer(events, OutputArea);\n", - " }\n", - " }\n", - " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", - " root._bokeh_timeout = Date.now() + 5000;\n", - " root._bokeh_failed_load = false;\n", - " }\n", - "\n", - " const NB_LOAD_WARNING = {'data': {'text/html':\n", - " \"
\\n\"+\n", - " \"

\\n\"+\n", - " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", - " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", - " \"

\\n\"+\n", - " \"
    \\n\"+\n", - " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", - " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", - " \"
\\n\"+\n", - " \"\\n\"+\n", - " \"from bokeh.resources import INLINE\\n\"+\n", - " \"output_notebook(resources=INLINE)\\n\"+\n", - " \"\\n\"+\n", - " \"
\"}};\n", - "\n", - " function display_loaded() {\n", - " const el = document.getElementById(null);\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS is loading...\";\n", - " }\n", - " if (root.Bokeh !== undefined) {\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", - " }\n", - " } else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(display_loaded, 100)\n", - " }\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " try {\n", - " root._bokeh_onload_callbacks.forEach(function(callback) {\n", - " if (callback != null)\n", - " callback();\n", - " });\n", - " } finally {\n", - " delete root._bokeh_onload_callbacks\n", - " }\n", - " console.debug(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(css_urls, js_urls, callback) {\n", - " if (css_urls == null) css_urls = [];\n", - " if (js_urls == null) js_urls = [];\n", - "\n", - " root._bokeh_onload_callbacks.push(callback);\n", - " if (root._bokeh_is_loading > 0) {\n", - " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", - "\n", - " function on_load() {\n", - " root._bokeh_is_loading--;\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", - " run_callbacks()\n", - " }\n", - " }\n", - "\n", - " function on_error(url) {\n", - " console.error(\"failed to load \" + url);\n", - " }\n", - "\n", - " for (let i = 0; i < css_urls.length; i++) {\n", - " const url = css_urls[i];\n", - " const element = document.createElement(\"link\");\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.rel = \"stylesheet\";\n", - " element.type = \"text/css\";\n", - " element.href = url;\n", - " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " for (let i = 0; i < js_urls.length; i++) {\n", - " const url = js_urls[i];\n", - " const element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.async = false;\n", - " element.src = url;\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " };\n", - "\n", - " function inject_raw_css(css) {\n", - " const element = document.createElement(\"style\");\n", - " element.appendChild(document.createTextNode(css));\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n", - " const css_urls = [];\n", - "\n", - " const inline_js = [ function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - "function(Bokeh) {\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " if (root.Bokeh !== undefined || force === true) {\n", - " for (let i = 0; i < inline_js.length; i++) {\n", - " inline_js[i].call(root, root.Bokeh);\n", - " }\n", - "} else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(run_inline_js, 100);\n", - " } else if (!root._bokeh_failed_load) {\n", - " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", - " root._bokeh_failed_load = true;\n", - " } else if (force !== true) {\n", - " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", - " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", - " }\n", - " }\n", - "\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(css_urls, js_urls, function() {\n", - " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(window));" - ], - "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.bokehjs_exec.v0+json": "", - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.bokehjs_exec.v0+json": { - "server_id": "7a1c6a300f4241deaba4ca613e4d18a5" - } - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "ping_pong.plot_comm_matrix()" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "a4aff933", "metadata": {}, - "outputs": [ - { - "data": { - "application/javascript": [ - "(function(root) {\n", - " function now() {\n", - " return new Date();\n", - " }\n", - "\n", - " const force = true;\n", - "\n", - " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", - " root._bokeh_onload_callbacks = [];\n", - " root._bokeh_is_loading = undefined;\n", - " }\n", - "\n", - "const JS_MIME_TYPE = 'application/javascript';\n", - " const HTML_MIME_TYPE = 'text/html';\n", - " const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", - " const CLASS_NAME = 'output_bokeh rendered_html';\n", - "\n", - " /**\n", - " * Render data to the DOM node\n", - " */\n", - " function render(props, node) {\n", - " const script = document.createElement(\"script\");\n", - " node.appendChild(script);\n", - " }\n", - "\n", - " /**\n", - " * Handle when an output is cleared or removed\n", - " */\n", - " function handleClearOutput(event, handle) {\n", - " const cell = handle.cell;\n", - "\n", - " const id = cell.output_area._bokeh_element_id;\n", - " const server_id = cell.output_area._bokeh_server_id;\n", - " // Clean up Bokeh references\n", - " if (id != null && id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - "\n", - " if (server_id !== undefined) {\n", - " // Clean up Bokeh references\n", - " const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", - " cell.notebook.kernel.execute(cmd_clean, {\n", - " iopub: {\n", - " output: function(msg) {\n", - " const id = msg.content.text.trim();\n", - " if (id in Bokeh.index) {\n", - " Bokeh.index[id].model.document.clear();\n", - " delete Bokeh.index[id];\n", - " }\n", - " }\n", - " }\n", - " });\n", - " // Destroy server and session\n", - " const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", - " cell.notebook.kernel.execute(cmd_destroy);\n", - " }\n", - " }\n", - "\n", - " /**\n", - " * Handle when a new output is added\n", - " */\n", - " function handleAddOutput(event, handle) {\n", - " const output_area = handle.output_area;\n", - " const output = handle.output;\n", - "\n", - " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", - " if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n", - " return\n", - " }\n", - "\n", - " const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", - "\n", - " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", - " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", - " // store reference to embed id on output_area\n", - " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", - " }\n", - " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", - " const bk_div = document.createElement(\"div\");\n", - " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", - " const script_attrs = bk_div.children[0].attributes;\n", - " for (let i = 0; i < script_attrs.length; i++) {\n", - " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", - " toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n", - " }\n", - " // store reference to server id on output_area\n", - " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", - " }\n", - " }\n", - "\n", - " function register_renderer(events, OutputArea) {\n", - "\n", - " function append_mime(data, metadata, element) {\n", - " // create a DOM node to render to\n", - " const toinsert = this.create_output_subarea(\n", - " metadata,\n", - " CLASS_NAME,\n", - " EXEC_MIME_TYPE\n", - " );\n", - " this.keyboard_manager.register_events(toinsert);\n", - " // Render to node\n", - " const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", - " render(props, toinsert[toinsert.length - 1]);\n", - " element.append(toinsert);\n", - " return toinsert\n", - " }\n", - "\n", - " /* Handle when an output is cleared or removed */\n", - " events.on('clear_output.CodeCell', handleClearOutput);\n", - " events.on('delete.Cell', handleClearOutput);\n", - "\n", - " /* Handle when a new output is added */\n", - " events.on('output_added.OutputArea', handleAddOutput);\n", - "\n", - " /**\n", - " * Register the mime type and append_mime function with output_area\n", - " */\n", - " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", - " /* Is output safe? */\n", - " safe: true,\n", - " /* Index of renderer in `output_area.display_order` */\n", - " index: 0\n", - " });\n", - " }\n", - "\n", - " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", - " if (root.Jupyter !== undefined) {\n", - " const events = require('base/js/events');\n", - " const OutputArea = require('notebook/js/outputarea').OutputArea;\n", - "\n", - " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", - " register_renderer(events, OutputArea);\n", - " }\n", - " }\n", - " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", - " root._bokeh_timeout = Date.now() + 5000;\n", - " root._bokeh_failed_load = false;\n", - " }\n", - "\n", - " const NB_LOAD_WARNING = {'data': {'text/html':\n", - " \"
\\n\"+\n", - " \"

\\n\"+\n", - " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", - " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", - " \"

\\n\"+\n", - " \"
    \\n\"+\n", - " \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n", - " \"
  • use INLINE resources instead, as so:
    • \\n\"+\n", - " \"
\\n\"+\n", - " \"\\n\"+\n", - " \"from bokeh.resources import INLINE\\n\"+\n", - " \"output_notebook(resources=INLINE)\\n\"+\n", - " \"\\n\"+\n", - " \"
\"}};\n", - "\n", - " function display_loaded() {\n", - " const el = document.getElementById(null);\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS is loading...\";\n", - " }\n", - " if (root.Bokeh !== undefined) {\n", - " if (el != null) {\n", - " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", - " }\n", - " } else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(display_loaded, 100)\n", - " }\n", - " }\n", - "\n", - " function run_callbacks() {\n", - " try {\n", - " root._bokeh_onload_callbacks.forEach(function(callback) {\n", - " if (callback != null)\n", - " callback();\n", - " });\n", - " } finally {\n", - " delete root._bokeh_onload_callbacks\n", - " }\n", - " console.debug(\"Bokeh: all callbacks have finished\");\n", - " }\n", - "\n", - " function load_libs(css_urls, js_urls, callback) {\n", - " if (css_urls == null) css_urls = [];\n", - " if (js_urls == null) js_urls = [];\n", - "\n", - " root._bokeh_onload_callbacks.push(callback);\n", - " if (root._bokeh_is_loading > 0) {\n", - " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", - " return null;\n", - " }\n", - " if (js_urls == null || js_urls.length === 0) {\n", - " run_callbacks();\n", - " return null;\n", - " }\n", - " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", - " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", - "\n", - " function on_load() {\n", - " root._bokeh_is_loading--;\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", - " run_callbacks()\n", - " }\n", - " }\n", - "\n", - " function on_error(url) {\n", - " console.error(\"failed to load \" + url);\n", - " }\n", - "\n", - " for (let i = 0; i < css_urls.length; i++) {\n", - " const url = css_urls[i];\n", - " const element = document.createElement(\"link\");\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.rel = \"stylesheet\";\n", - " element.type = \"text/css\";\n", - " element.href = url;\n", - " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " for (let i = 0; i < js_urls.length; i++) {\n", - " const url = js_urls[i];\n", - " const element = document.createElement('script');\n", - " element.onload = on_load;\n", - " element.onerror = on_error.bind(null, url);\n", - " element.async = false;\n", - " element.src = url;\n", - " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", - " document.head.appendChild(element);\n", - " }\n", - " };\n", - "\n", - " function inject_raw_css(css) {\n", - " const element = document.createElement(\"style\");\n", - " element.appendChild(document.createTextNode(css));\n", - " document.body.appendChild(element);\n", - " }\n", - "\n", - " const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n", - " const css_urls = [];\n", - "\n", - " const inline_js = [ function(Bokeh) {\n", - " Bokeh.set_log_level(\"info\");\n", - " },\n", - "function(Bokeh) {\n", - " }\n", - " ];\n", - "\n", - " function run_inline_js() {\n", - " if (root.Bokeh !== undefined || force === true) {\n", - " for (let i = 0; i < inline_js.length; i++) {\n", - " inline_js[i].call(root, root.Bokeh);\n", - " }\n", - "} else if (Date.now() < root._bokeh_timeout) {\n", - " setTimeout(run_inline_js, 100);\n", - " } else if (!root._bokeh_failed_load) {\n", - " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", - " root._bokeh_failed_load = true;\n", - " } else if (force !== true) {\n", - " const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n", - " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", - " }\n", - " }\n", - "\n", - " if (root._bokeh_is_loading === 0) {\n", - " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", - " run_inline_js();\n", - " } else {\n", - " load_libs(css_urls, js_urls, function() {\n", - " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", - " run_inline_js();\n", - " });\n", - " }\n", - "}(window));" - ], - "application/vnd.bokehjs_load.v0+json": "(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n\n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(null);\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.1.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.1.1.min.js\"];\n const css_urls = [];\n\n const inline_js = [ function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\nfunction(Bokeh) {\n }\n ];\n\n function run_inline_js() {\n if (root.Bokeh !== undefined || force === true) {\n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(null)).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.bokehjs_exec.v0+json": "", - "text/html": [ - "" - ] - }, - "metadata": { - "application/vnd.bokehjs_exec.v0+json": { - "server_id": "c29ef1105a8a4690aa88355a3b32c0e1" - } - }, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "ping_pong.plot_message_histogram()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40bfed1c-5cce-48e0-b220-288b7ac16d26", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/pipit/trace.py b/pipit/trace.py index e438af4e..569c2430 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -29,10 +29,6 @@ def __init__(self, definitions, events, cct=None): self.inc_metrics = [] self.exc_metrics = [] - from .vis.util import generate_palette - - self.palette = generate_palette(self) - def create_cct(self): # adds a column of cct nodes to the events dataframe # and stores the graph object in self.cct diff --git a/pipit/vis/core.py b/pipit/vis/core.py index 19acc8ca..afce4db5 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -70,6 +70,7 @@ def comm_matrix( dw=nranks, dh=nranks, color_mapper=color_mapper, + origin="top_left", ) color_bar = ColorBar( @@ -90,9 +91,9 @@ def comm_matrix( # Configure hover hover = p.select(HoverTool) hover.tooltips = [ - ("sender", "$y{0.}"), - ("receiver", "$x{0.}"), - ("value", "@image") if output == "count" else ("value", "@image{custom}"), + ("Sender", "$y{0.}"), + ("Receiver", "$x{0.}"), + ("Count", "@image") if output == "count" else ("Volume", "@image{custom}"), ] hover.formatters = {"@image": get_size_hover_formatter()} diff --git a/pipit/vis/util.py b/pipit/vis/util.py index ce250692..ea7f204c 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -115,93 +115,3 @@ def get_size_tick_formatter(ignore_range=False): {JS_FORMAT_SIZE} """ ) - - -# Color palette - -# Default color palette is based on bokeh.palettes.Category20_20 -# See https://docs.bokeh.org/en/latest/docs/reference/palettes.html#d3-palettes - -DEFAULT_RESERVED = { - "MPI_Send": "#1f77b4", - "MPI_Isend": "#1f77b4", - "MPI_Recv": "#d62728", - "MPI_Irecv": "#d62728", - "MPI_Wait": "#c7c7c7", - "MPI_Waitany": "#c7c7c7", - "MPI_Waitall": "#c7c7c7", - "Idle": "#c7c7c7", -} - -DEFAULT_LIGHT = [ - "#aec7e8", - "#ffbb78", - "#98df8a", - "#ff9896", - "#c5b0d5", - "#c49c94", - "#f7b6d2", - "#dbdb8d", - "#9edae5", -] - -DEFAULT_DARK = [ - "#ff7f0e", - "#2ca02c", - "#9467bd", - "#8c564b", - "#e377c2", - "#bcbd22", - "#17becf", -] - - -def generate_palette( - trace, - reserved=DEFAULT_RESERVED, - light=DEFAULT_LIGHT, - dark=DEFAULT_DARK, -): - """Generates color palette for a trace. - - Assigns light colors for even depths, and dark colors for odd depths to maximize - contrast in nested functions. - - Returns: - dict: Dictionary mapping function name to CSS color value. - """ - - # Calculate inc time and depth - trace.calc_inc_metrics(["Timestamp (ns)"]) - trace._match_caller_callee() - - # Get all function names from trace - func = trace.events[trace.events["Event Type"] == "Enter"] - names = reversed(trace.flat_profile(["time.inc"]).index.tolist()) - - # Get the depth of each function - depths = ( - func.groupby("Name")["_depth"] - .agg(lambda x: x.value_counts().index[0]) - .to_dict() - ) - - # Start with palette being a copy of reserved colors - palette = reserved.copy() - - # Initialize indices for light and dark colors - dark_index = 0 - light_index = 0 - - # Iterate over function names and assign colors to each - for i, f in enumerate(names): - if f not in palette: - # Assign light color for even-depth, and dark color for odd-depth - if depths[f] % 2 == 0: - palette[f] = light[light_index % len(light)] - light_index += 1 - else: - palette[f] = dark[dark_index % len(dark)] - dark_index += 1 - - return palette From 2603c6ab02bee9eb9f31b29d16d3aa0f3d67af6a Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Wed, 28 Feb 2024 19:56:23 -0500 Subject: [PATCH 19/42] remove files --- docs/examples/dsl2-Copy2.ipynb | 822 --------------------------------- pipit/config.py | 28 -- 2 files changed, 850 deletions(-) delete mode 100644 docs/examples/dsl2-Copy2.ipynb delete mode 100644 pipit/config.py diff --git a/docs/examples/dsl2-Copy2.ipynb b/docs/examples/dsl2-Copy2.ipynb deleted file mode 100644 index 7c47de9e..00000000 --- a/docs/examples/dsl2-Copy2.ipynb +++ /dev/null @@ -1,822 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "8227bf9b-a386-4cb0-9eba-a7e258bf1d00", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload \n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "7c95d344-9406-4c66-978a-e7032df0967b", - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.append(\"../..\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "a8ea89f2-3d70-486c-9901-aeb4900169cb", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import pipit as pp" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "589e87fd-d8d1-4ddf-a426-b911410379c4", - "metadata": {}, - "outputs": [], - "source": [ - "trace = pp.Trace.from_otf2(\"../../pipit/tests/data/ping-pong-otf2/\")\n", - "pandas_df = trace.events[[\"Timestamp (ns)\", \"Name\", \"Event Type\", \"Process\", \"Thread\"]].copy()\n", - "pandas_df[\"Process\"] = pandas_df[\"Process\"].astype(int)\n", - "idx = pandas_df.groupby(['Process']).cumcount()\n", - "pandas_df[\"idx\"] = idx" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3867d2a2-ff3e-4c8c-9c09-46a167a0fde4", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'pandas'" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pp.set_option(\"backend\", \"pandas\")\n", - "pp.get_option(\"backend\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "156df562-4431-40b8-b6b0-b7cfbe6206d0", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "TraceDataset (0 traces, 0 events)\n" - ] - } - ], - "source": [ - "ds = pp.dsl2.TraceDataset()\n", - "ds.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "7fa7fe70-0c61-4503-9b27-3bae52d2f1c3", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Event (rank=1, idx=0, thread=0, timestamp=0.0, event_type='Instant', name='ProgramBegin')" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# simulating a reader\n", - "row = pandas_df.iloc[0]\n", - "evt = pp.dsl2.Event(\n", - " rank=row[\"Process\"],\n", - " idx=row[\"idx\"],\n", - " thread=row[\"Thread\"],\n", - " timestamp=row[\"Timestamp (ns)\"],\n", - " event_type=row[\"Event Type\"],\n", - " name=row[\"Name\"],\n", - ")\n", - "evt" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "5abbffd5-3246-4331-9a40-86b5a6eda0d7", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "TraceDataset (1 trace, 0 events)\n" - ] - } - ], - "source": [ - "ds.push_event(evt)\n", - "ds.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "bca1c384-0239-46b6-94d0-86439635eb80", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+-------------+--------------+--------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+-------------+--------------+--------------|\n", - "| 1 | 0 | 0 | 0 | Instant | ProgramBegin |\n", - "+--------+-------+----------+-------------+--------------+--------------+\n", - "TraceDataset (1 trace, 1 event)\n" - ] - } - ], - "source": [ - "ds.flush()\n", - "ds.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "2910c4d8-e40d-4683-b0d4-7943a6be34a6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+--------------------+--------------+-----------------------|\n", - "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin |\n", - "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) |\n", - "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init |\n", - "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", - "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", - "| ... | ... | ... | ... | ... | ... |\n", - "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", - "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", - "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", - "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", - "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "TraceDataset (2 traces, 120 events)\n" - ] - } - ], - "source": [ - "for i in range(1, len(pandas_df)):\n", - " row = pandas_df.iloc[i]\n", - " evt = pp.dsl2.Event(\n", - " rank=row[\"Process\"],\n", - " idx=row[\"idx\"],\n", - " thread=row[\"Thread\"],\n", - " timestamp=row[\"Timestamp (ns)\"],\n", - " event_type=row[\"Event Type\"],\n", - " name=row[\"Name\"],\n", - " )\n", - " ds.push_event(evt)\n", - "\n", - "ds.flush()\n", - "ds.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "72fac6a1-0486-4ab5-b95d-af594afe0996", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+-------------+--------------+---------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+-------------+--------------+---------------|\n", - "| 1 | 0 | 0 | 0 | Instant | ProgramBegin |\n", - "| 1 | 1 | 0 | 30083.1 | Enter | int main(int, |\n", - "| | | | | | char**) |\n", - "| 1 | 2 | 0 | 40288.3 | Enter | MPI_Init |\n", - "| 0 | 0 | 0 | 307731 | Instant | ProgramBegin |\n", - "| 0 | 1 | 0 | 336980 | Enter | int main(int, |\n", - "| | | | | | char**) |\n", - "+--------+-------+----------+-------------+--------------+---------------+\n", - "TraceDataset (2 traces, 5 events)\n" - ] - } - ], - "source": [ - "ds.head().show()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "3cba6972-052d-41df-a448-43f383fdf04e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+-------------+--------------+---------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+-------------+--------------+---------------|\n", - "| 1 | 57 | 0 | 1.99575e+08 | Leave | MPI_Finalize |\n", - "| 0 | 58 | 0 | 1.99575e+08 | Leave | int main(int, |\n", - "| | | | | | char**) |\n", - "| 1 | 58 | 0 | 1.99577e+08 | Leave | int main(int, |\n", - "| | | | | | char**) |\n", - "| 0 | 59 | 0 | 1.99603e+08 | Instant | ProgramEnd |\n", - "| 1 | 59 | 0 | 1.99604e+08 | Instant | ProgramEnd |\n", - "+--------+-------+----------+-------------+--------------+---------------+\n", - "TraceDataset (2 traces, 5 events)\n" - ] - } - ], - "source": [ - "ds.tail().show()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "c8a7099f-c446-4800-b55a-c263b742a814", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+--------------------+--------------+-----------------------|\n", - "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", - "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", - "| 0 | 2 | 0 | 346054.77444467926 | Enter | MPI_Init |\n", - "| 0 | 3 | 0 | 193643138.1741584 | Leave | MPI_Init |\n", - "| 1 | 3 | 0 | 193643835.4832178 | Leave | MPI_Init |\n", - "| ... | ... | ... | ... | ... | ... |\n", - "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", - "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", - "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", - "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", - "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "TraceDataset (2 traces, 117 events)\n" - ] - } - ], - "source": [ - "ds.filter(\"timestamp > 1e5\").show()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "ebf8f71a-6a30-4c90-a9b6-ecb2215192f7", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------+----------+--------------------+--------------+-----------------------+\n", - "| idx | thread | timestamp | event_type | name |\n", - "|-------+----------+--------------------+--------------+-----------------------|\n", - "| 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", - "| 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", - "| 2 | 0 | 346054.77444467926 | Enter | MPI_Init |\n", - "| 3 | 0 | 193643138.1741584 | Leave | MPI_Init |\n", - "| 4 | 0 | 193651646.20379105 | Enter | MPI_Comm_size |\n", - "| ... | ... | ... | ... | ... |\n", - "| 55 | 0 | 199320512.07918367 | Leave | MPI_Recv |\n", - "| 56 | 0 | 199514778.2785141 | Enter | MPI_Finalize |\n", - "| 57 | 0 | 199573648.15437022 | Leave | MPI_Finalize |\n", - "| 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", - "| 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", - "+-------+----------+--------------------+--------------+-----------------------+\n", - "_Trace (rank=0, 60 events)\n" - ] - } - ], - "source": [ - "ds.traces[0].show()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "bde23e54-6f1b-4620-a90a-903ba8294c27", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Event (rank=0, idx=54, thread=0, timestamp=199319973.70504335, event_type='Instant', name='MpiRecv')" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.traces[0].loc[54]" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "d1562ea5-8048-4c3c-8813-139ab83fcd33", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------+----------+-------------+--------------+----------+\n", - "| idx | thread | timestamp | event_type | name |\n", - "|-------+----------+-------------+--------------+----------|\n", - "| 40 | 0 | 1.94911e+08 | Leave | MPI_Send |\n", - "| 41 | 0 | 1.94912e+08 | Enter | MPI_Recv |\n", - "| 42 | 0 | 1.95132e+08 | Instant | MpiRecv |\n", - "| 43 | 0 | 1.95132e+08 | Leave | MPI_Recv |\n", - "| 44 | 0 | 1.95718e+08 | Enter | MPI_Send |\n", - "| 45 | 0 | 1.95719e+08 | Instant | MpiSend |\n", - "| 46 | 0 | 1.9614e+08 | Leave | MPI_Send |\n", - "| 47 | 0 | 1.9614e+08 | Enter | MPI_Recv |\n", - "| 48 | 0 | 1.96584e+08 | Instant | MpiRecv |\n", - "| 49 | 0 | 1.96584e+08 | Leave | MPI_Recv |\n", - "| 50 | 0 | 1.97613e+08 | Enter | MPI_Send |\n", - "| 51 | 0 | 1.97614e+08 | Instant | MpiSend |\n", - "| 52 | 0 | 1.98506e+08 | Leave | MPI_Send |\n", - "| 53 | 0 | 1.98507e+08 | Enter | MPI_Recv |\n", - "| 54 | 0 | 1.9932e+08 | Instant | MpiRecv |\n", - "+-------+----------+-------------+--------------+----------+\n", - "_Trace (rank=0, 15 events)\n" - ] - } - ], - "source": [ - "ds.traces[0].loc[40:54].show()" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "857acd69-9a15-454c-96c3-bdc4ba8ef955", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+--------------------+--------------+-----------------------|\n", - "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", - "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", - "| 0 | 2 | 0 | 346054.77444467926 | Enter | MPI_Init |\n", - "| 0 | 3 | 0 | 193643138.1741584 | Leave | MPI_Init |\n", - "| 0 | 4 | 0 | 193651646.20379105 | Enter | MPI_Comm_size |\n", - "| ... | ... | ... | ... | ... | ... |\n", - "| 0 | 55 | 0 | 199320512.07918367 | Leave | MPI_Recv |\n", - "| 0 | 56 | 0 | 199514778.2785141 | Enter | MPI_Finalize |\n", - "| 0 | 57 | 0 | 199573648.15437022 | Leave | MPI_Finalize |\n", - "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", - "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "TraceDataset (1 trace, 60 events)\n" - ] - } - ], - "source": [ - "ds.loc[0].show()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "2eeb2fae-2dca-4742-90de-725d45d1e5bb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+--------------------+--------------+-----------------------|\n", - "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin |\n", - "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) |\n", - "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init |\n", - "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", - "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", - "| ... | ... | ... | ... | ... | ... |\n", - "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", - "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", - "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", - "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", - "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "TraceDataset (2 traces, 120 events)\n" - ] - } - ], - "source": [ - "ds.loc[0:2].show()" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "5e843a24-ff1a-44af-975a-15f83df89b07", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Event (rank=0, idx=59, thread=0, timestamp=199603304.5511645, event_type='Instant', name='ProgramEnd')" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.loc[0, 59]" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "2223f5fc-1433-443e-bd5f-1ae1c23faf28", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "120" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.map_traces(lambda trace: trace.count()).reduce(\"sum\")" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "de60e9ad-5aff-4f08-9a83-9d6cd086bc06", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------+--------------------+\n", - "| idx | result |\n", - "|-------+--------------------|\n", - "| 0 | 307730.9358165928 |\n", - "| 1 | 336979.73374932166 |\n", - "| 2 | 346054.77444467926 |\n", - "| 3 | 193643138.1741584 |\n", - "| 4 | 193651646.20379105 |\n", - "| ... | ... |\n", - "| 55 | 199320512.07918367 |\n", - "| 56 | 199514778.2785141 |\n", - "| 57 | 199573648.15437022 |\n", - "| 58 | 199575243.23094556 |\n", - "| 59 | 199603304.5511645 |\n", - "+-------+--------------------+\n", - "DictLike (idx -> result, 60 items)\n" - ] - } - ], - "source": [ - "ds.traces[0].map_events(lambda event: event.timestamp).show()" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "ac27a0b1-26df-4d6d-b9ff-35b77bcfb7c3", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------+--------------------+\n", - "| idx | result |\n", - "|-------+--------------------|\n", - "| 0 | 307730.9358165928 |\n", - "| 1 | 336979.73374932166 |\n", - "| 2 | 346054.77444467926 |\n", - "| 3 | 193643138.1741584 |\n", - "| 4 | 193651646.20379105 |\n", - "| ... | ... |\n", - "| 55 | 199320512.07918367 |\n", - "| 56 | 199514778.2785141 |\n", - "| 57 | 199573648.15437022 |\n", - "| 58 | 199575243.23094556 |\n", - "| 59 | 199603304.5511645 |\n", - "+-------+--------------------+\n", - "DictLike (idx -> result, 60 items)\n" - ] - } - ], - "source": [ - "ds.traces[0].map_events(lambda event: event.timestamp).show()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "7afbf369-7fe4-40c2-97c4-b80d339fcbcb", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "185373520.9350972" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.traces[0].map_events(lambda event: event.timestamp).reduce(\"mean\")" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "44bd35aa-8763-4d11-808b-b42e10a95e81", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "11122246941.50224" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds.map_traces(lambda trace: trace.map_events(lambda event: event.timestamp).reduce(\"sum\")).reduce(\"mean\")" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "8821997b-1154-4e61-93c9-4cca21b948eb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+----------+\n", - "| rank | result |\n", - "|--------+----------|\n", - "| 1 | 60 |\n", - "| 0 | 60 |\n", - "+--------+----------+\n", - "DictLike (rank -> result, 2 items)\n" - ] - } - ], - "source": [ - "ds.map_traces(lambda trace: trace.count()).show()" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "8aa72310-728d-42c9-9e6a-dcc13f9a0299", - "metadata": {}, - "outputs": [], - "source": [ - "# matching_evt = np.full(ds.traces[0].count(), np.nan)\n", - "# matching_ts = np.full(ds.traces[0].count(), np.nan)\n", - "\n", - "# stack = []\n", - "\n", - "# for event in ds.traces[0].iter_events():\n", - "# if event.event_type == \"Enter\":\n", - "# stack.append((event.idx, event.timestamp))\n", - "# elif event.event_type == \"Leave\":\n", - "# enter_idx, enter_ts = stack.pop()\n", - "# matching_evt[enter_idx] = event.idx\n", - "# matching_ts[enter_idx] = event.timestamp\n", - "# matching_evt[event.idx] = enter_idx\n", - "# matching_ts[event.idx] = enter_ts\n", - "\n", - "# ds.traces[0].add_column(\"matching_evt\", matching_evt)\n", - "# ds.traces[0].add_column(\"matching_ts\", matching_ts)\n", - "\n", - "# del matching_evt\n", - "# del matching_ts\n", - "\n", - "# ds.traces[0].show()" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "06005b18-cc7e-4363-9264-ea0c50142575", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "| rank | idx | thread | timestamp | event_type | name |\n", - "|--------+-------+----------+--------------------+--------------+-----------------------|\n", - "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin |\n", - "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) |\n", - "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init |\n", - "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin |\n", - "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) |\n", - "| ... | ... | ... | ... | ... | ... |\n", - "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize |\n", - "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) |\n", - "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) |\n", - "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd |\n", - "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd |\n", - "+--------+-------+----------+--------------------+--------------+-----------------------+\n", - "TraceDataset (2 traces, 120 events)\n" - ] - } - ], - "source": [ - "ds.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "3312389e-71c4-411d-b8e0-9d8c14367359", - "metadata": {}, - "outputs": [], - "source": [ - "analyzer = pp.dsl2.Analyzer(ds)" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "008cdda4-7d70-4501-9f43-55c6b465e1f2", - "metadata": {}, - "outputs": [], - "source": [ - "analyzer._match_events()" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "c5d1a502-d728-4831-a608-9577c2c843b8", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+\n", - "| rank | idx | thread | timestamp | event_type | name | matching_evt | matching_ts |\n", - "|--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------|\n", - "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin | -1 | nan |\n", - "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) | 58 | 199576798.2158296 |\n", - "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init | 3 | 193643835.4832178 |\n", - "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin | -1 | nan |\n", - "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) | 58 | 199575243.23094556 |\n", - "| ... | ... | ... | ... | ... | ... | ... | ... |\n", - "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize | 56 | 199529686.66029385 |\n", - "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) | 1 | 336979.73374932166 |\n", - "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) | 1 | 30083.086937435106 |\n", - "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd | -1 | nan |\n", - "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd | -1 | nan |\n", - "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+\n", - "TraceDataset (2 traces, 120 events)\n" - ] - } - ], - "source": [ - "analyzer.ds.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "ada9f092-fcbc-4f90-bb36-c22e8353d6fe", - "metadata": {}, - "outputs": [], - "source": [ - "analyzer._match_caller_callee()" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "6c856475-0ea0-4acb-8c2b-2b1bfcdb5391", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+---------+-------+--------------------------+\n", - "| rank | idx | thread | timestamp | event_type | name | matching_evt | matching_ts | depth | par | children |\n", - "|--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+---------+-------+--------------------------|\n", - "| 1 | 0 | 0 | 0.0 | Instant | ProgramBegin | -1 | nan | 0 | -1 | [] |\n", - "| 1 | 1 | 0 | 30083.086937435106 | Enter | int main(int, char**) | 58 | 199576798.2158296 | 0 | -1 | [2, 4, 6, 8, 11, 14, 17, |\n", - "| | | | | | | | | | | 20, 23, 26, 29, 32, 35, |\n", - "| | | | | | | | | | | 38, 41, 44, 47, 50, 53, |\n", - "| | | | | | | | | | | 56] |\n", - "| 1 | 2 | 0 | 40288.33150186851 | Enter | MPI_Init | 3 | 193643835.4832178 | 1 | 1 | [] |\n", - "| 0 | 0 | 0 | 307730.9358165928 | Instant | ProgramBegin | -1 | nan | 0 | -1 | [] |\n", - "| 0 | 1 | 0 | 336979.73374932166 | Enter | int main(int, char**) | 58 | 199575243.23094556 | 0 | -1 | [2, 4, 6, 8, 11, 14, 17, |\n", - "| | | | | | | | | | | 20, 23, 26, 29, 32, 35, |\n", - "| | | | | | | | | | | 38, 41, 44, 47, 50, 53, |\n", - "| | | | | | | | | | | 56] |\n", - "| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... |\n", - "| 1 | 57 | 0 | 199574793.63126454 | Leave | MPI_Finalize | 56 | 199529686.66029385 | 0 | -1 | [] |\n", - "| 0 | 58 | 0 | 199575243.23094556 | Leave | int main(int, char**) | 1 | 336979.73374932166 | 0 | -1 | [] |\n", - "| 1 | 58 | 0 | 199576798.2158296 | Leave | int main(int, char**) | 1 | 30083.086937435106 | 0 | -1 | [] |\n", - "| 0 | 59 | 0 | 199603304.5511645 | Instant | ProgramEnd | -1 | nan | 0 | -1 | [] |\n", - "| 1 | 59 | 0 | 199604459.57369962 | Instant | ProgramEnd | -1 | nan | 0 | -1 | [] |\n", - "+--------+-------+----------+--------------------+--------------+-----------------------+----------------+--------------------+---------+-------+--------------------------+\n", - "TraceDataset (2 traces, 120 events)\n" - ] - } - ], - "source": [ - "analyzer.ds.show()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/pipit/config.py b/pipit/config.py deleted file mode 100644 index ff575453..00000000 --- a/pipit/config.py +++ /dev/null @@ -1,28 +0,0 @@ -config = { - "notebook_url": "http://localhost:8888", - "theme": """ - attrs: - Plot: - height: 350 - width: 700 - background_fill_color: "#fafafa" - Axis: - axis_label_text_font_style: "bold" - minor_tick_line_color: null - Toolbar: - autohide: true - logo: null - HoverTool: - point_policy: "follow_mouse" - Legend: - label_text_font_size: "8.5pt" - spacing: 10 - border_line_color: null - glyph_width: 16 - glyph_height: 16 - Scatter: - size: 9 - DataRange1d: - range_padding: 0.05 - """, -} From 1c5037c4e5d44d5091802e1d254165e81eb4ea9e Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 26 Mar 2024 19:04:43 -0400 Subject: [PATCH 20/42] add simple timeline --- pipit/trace.py | 16 +- pipit/vis/__init__.py | 2 + pipit/vis/core.py | 4 +- pipit/vis/timeline.py | 280 +++++++++++++++++++++++++++++ pipit/vis/util.py | 404 +++++++++++++++++++++++++++++++++++++----- 5 files changed, 657 insertions(+), 49 deletions(-) create mode 100644 pipit/vis/__init__.py create mode 100644 pipit/vis/timeline.py diff --git a/pipit/trace.py b/pipit/trace.py index 569c2430..1e2b9482 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -548,7 +548,7 @@ def flat_profile( self.events.loc[self.events["Event Type"] == "Enter"] .groupby([groupby_column, "Process"], observed=True)[metrics] .sum() - .groupby(groupby_column) + .groupby(groupby_column, observed=True) .mean() ) @@ -863,19 +863,25 @@ def detect_pattern( return patterns def plot_comm_matrix(self, output="size", *args, **kwargs): - from .vis import core + from .vis import plot_comm_matrix # Generate the data data = self.comm_matrix(output=output) # Return the Bokeh plot - return core.comm_matrix(data, output=output, *args, **kwargs) + return plot_comm_matrix(data, output=output, *args, **kwargs) def plot_message_histogram(self, bins=20, *args, **kwargs): - from .vis import core + from .vis import plot_message_histogram # Generate the data data = self.message_histogram(bins=bins) # Return the Bokeh plot - return core.message_histogram(data, *args, **kwargs) + return plot_message_histogram(data, *args, **kwargs) + + def plot_timeline(self, *args, **kwargs): + from .vis import plot_timeline + + # Return the Bokeh plot + return plot_timeline(self, *args, **kwargs) diff --git a/pipit/vis/__init__.py b/pipit/vis/__init__.py new file mode 100644 index 00000000..5119ca11 --- /dev/null +++ b/pipit/vis/__init__.py @@ -0,0 +1,2 @@ +from .core import plot_comm_matrix, plot_message_histogram # noqa: F401 +from .timeline import plot_timeline # noqa: F401 diff --git a/pipit/vis/core.py b/pipit/vis/core.py index afce4db5..7b413a12 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -17,7 +17,7 @@ ) -def comm_matrix( +def plot_comm_matrix( data, output="size", cmap="log", palette="Viridis256", return_fig=False ): """Plots the trace's communication matrix. @@ -101,7 +101,7 @@ def comm_matrix( return show(p, return_fig=return_fig) -def message_histogram( +def plot_message_histogram( data, return_fig=False, ): diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py new file mode 100644 index 00000000..98066993 --- /dev/null +++ b/pipit/vis/timeline.py @@ -0,0 +1,280 @@ +import numpy as np +import pandas as pd +from bokeh.models import ( + ColumnDataSource, + HoverTool, + WheelZoomTool, + Grid, + FixedTicker, + CustomJS, +) +from bokeh.plotting import figure +from bokeh.transform import dodge + +from pipit.vis.util import ( + get_time_hover_formatter, + get_html_tooltips, + get_factor_cmap, + trimmed, + show, + factorize_tuples, +) + + +def plot_timeline(trace, show_depth=False, instant_events=False): + """Displays the events of a trace against time + + Instant events are represented by points, functions are represented by horizontal + bars, and MPI messages are represented by lines connecting send/receive events.""" + + # Generate necessary metrics + trace.calc_exc_metrics(["Timestamp (ns)"]) + trace._match_events() + trace._match_caller_callee() + + min_ts = trace.events["Timestamp (ns)"].min() + max_ts = trace.events["Timestamp (ns)"].max() + + # Prepare data for plotting + events = ( + trace.events[ + trace.events["Event Type"].isin( + ["Enter", "Instant"] if instant_events else ["Enter"] + ) + ] + .sort_values(by="time.inc", ascending=False) + .copy(deep=False) + ) + events["_depth"] = events["_depth"].astype(float).fillna("") + + # Determine y-coordinates from process and depth + y_tuples = ( + list(zip(events["Process"], events["_depth"])) + if show_depth + else list(zip(events["Process"])) + ) + + codes, uniques = factorize_tuples(y_tuples) + events["y"] = codes + num_ys = len(uniques) + + depth_ticks = np.arange(0, num_ys) + process_ticks = np.array( + [i for i, v in enumerate(uniques) if len(v) == 1 or v[1] == 0] + ) + + events["name_trimmed"] = trimmed(events["Name"]) + events["_matching_event"] = events["_matching_event"].fillna(-1) + + # Only select a subset of columns for plotting + events = events[ + [ + "Timestamp (ns)", + "_matching_timestamp", + "_matching_event", + "y", + "Name", + "time.inc", + "Process", + "time.exc", + "name_trimmed", + "Event Type", + ] + ] + + # Define CDS for glyphs to be empty + hbar_source = ColumnDataSource(events.head(1)) + scatter_source = ColumnDataSource(events.head(0)) + + # Callback function that updates CDS + def update_cds(event): + x0 = event.x0 if event is not None else min_ts + x1 = event.x1 if event is not None else max_ts + + x0 = x0 - (x1 - x0) * 0.25 + x1 = x1 + (x1 - x0) * 0.25 + + # Remove events that are out of bounds + in_bounds = events[ + ( + (events["Event Type"] == "Instant") + & (events["Timestamp (ns)"] > x0) + & (events["Timestamp (ns)"] < x1) + ) + | ( + (events["Event Type"] == "Enter") + & (events["_matching_timestamp"] > x0) + & (events["Timestamp (ns)"] < x1) + ) + ].copy(deep=False) + + # Update hbar_source to keep 5000 largest functions + func = in_bounds[in_bounds["Event Type"] == "Enter"] + large = func + hbar_source.data = large + + # Update scatter_source to keep sampled events + if instant_events: + inst = in_bounds[in_bounds["Event Type"] == "Instant"].copy(deep=False) + + if len(inst) > 500: + inst["bin"] = pd.cut(x=inst["Timestamp (ns)"], bins=1000, labels=False) + + grouped = inst.groupby(["bin", "y"]) + samples = grouped.first().reset_index() + samples = samples[~samples["Timestamp (ns)"].isna()] + + scatter_source.data = samples + else: + scatter_source.data = inst + + # Create Bokeh plot + # min_height = 50 + 22 * len(events["Name"].unique()) + plot_height = 100 + 22 * num_ys + # height = clamp(plot_height, min_height, 900) + + p = figure( + x_range=(min_ts, max_ts + (max_ts - min_ts) * 0.05), + y_range=(num_ys - 0.5, -0.5), + x_axis_location="above", + tools="hover,xpan,reset,xbox_zoom,xwheel_zoom,save", + output_backend="webgl", + height=min(500, plot_height), + sizing_mode="stretch_width", + toolbar_location=None, + # x_axis_label="Time", + ) + + # p.min_border_bottom = height - plot_height + + # Create color maps + fill_cmap = get_factor_cmap("Name", trace) + line_cmap = get_factor_cmap("Name", trace, scale=0.7) + + # Add lines for each process + # p.segment( + # x0=[0] * len(process_ticks), + # x1=[max_ts] * len(process_ticks), + # y0=process_ticks, + # y1=process_ticks, + # line_dash="dotted", + # line_color="gray", + # ) + + # Add bars for large functions + hbar = p.hbar( + left="Timestamp (ns)", + right="_matching_timestamp", + y="y", + height=0.8 if show_depth else 0.8, + source=hbar_source, + fill_color=fill_cmap, + line_color=line_cmap, + line_width=1, + line_alpha=0.5, + legend_field="name_trimmed", + ) + + # Add raster for small functions + # p.image_rgba(source=image_source) + + if instant_events: + scatter = p.scatter( + x="Timestamp (ns)", + y=dodge("y", -0.2 if show_depth else 0), + # size=9, + line_color="#0868ac", + alpha=1, + color="#ccebc5", + line_width=0.8, + marker="diamond", + source=scatter_source, + # legend_label="Instant event", + ) + + # Add custom grid lines + # p.xgrid.visible = False + p.ygrid.visible = False + + g1 = Grid( + dimension=1, + grid_line_color="white", + grid_line_width=2 if show_depth else 2, + ticker=FixedTicker( + ticks=np.concatenate([depth_ticks - 0.49, depth_ticks + 0.49]) + ), + level="glyph", + ) + g2 = Grid( + dimension=1, + grid_line_width=2, + # band_fill_color="gray", + # band_fill_alpha=0.1, + ticker=FixedTicker(ticks=process_ticks - 0.5), + level="glyph", + ) + p.add_layout(g1) + p.add_layout(g2) + + # Additional plot config + # p.xaxis.formatter = get_time_tick_formatter() + # p.yaxis.formatter = FuncTickFormatter( + # args={ + # "uniques": uniques, + # }, + # code=""" + # return "Process " + uniques[Math.floor(tick)][0]; + # """, + # ) + + p.yaxis.ticker = FixedTicker(ticks=process_ticks + 0.1) + p.yaxis.major_tick_line_color = None + + p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] + # p.on_event(RangesUpdate, update_cds) + # p.on_event(Tap, tap_callback) + + # Move legend to the right + p.add_layout(p.legend[0], "below") + p.legend.orientation = "horizontal" + p.legend.location = "center" + p.legend.nrows = 2 + + # Make initial call to our callback + update_cds(None) + + # Hover config + hover = p.select(HoverTool) + hover.tooltips = get_html_tooltips( + { + "Name": "@Name", + # "Process": "@Process", + "Enter": "@{Timestamp (ns)}{custom} [@{index}]", + "Leave": "@{_matching_timestamp}{custom} [@{_matching_event}]", + "Time (Inc)": "@{time.inc}{custom}", + "Time (Exc)": "@{time.exc}{custom}", + } + ) + hover.formatters = { + "@{Timestamp (ns)}": get_time_hover_formatter(), + "@{_matching_timestamp}": get_time_hover_formatter(), + "@{time.inc}": get_time_hover_formatter(), + "@{time.exc}": get_time_hover_formatter(), + } + hover.renderers = [hbar, scatter] if instant_events else [hbar] + hover.callback = CustomJS( + code=""" + let hbar_tooltip = document.querySelector('.bk-tooltip'); + let scatter_tooltip = hbar_tooltip.nextElementSibling; + + if (hbar_tooltip && scatter_tooltip && + hbar_tooltip.style.display != 'none' && + scatter_tooltip.style.display != 'none') + { + hbar_tooltip.style.display = 'none'; + } + """ + ) + + # Return plot + return show(p) diff --git a/pipit/vis/util.py b/pipit/vis/util.py index ea7f204c..35d601fb 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -1,12 +1,240 @@ -import yaml -from bokeh.models import BasicTicker, CustomJSHover, CustomJSTickFormatter -from bokeh.plotting import output_notebook -from bokeh.plotting import show as bk_show +# Copyright 2022 Parallel Software and Systems Group, University of Maryland. +# See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: MIT + +from bokeh.io import output_notebook, show as bk_show +from bokeh.models import ( + CustomJSHover, + CustomJSTickFormatter, + PrintfTickFormatter, + NumeralTickFormatter, + BasicTicker, +) + +# from bokeh.palettes import Category20_20 +from bokeh.transform import factor_cmap from bokeh.themes import Theme +import yaml + +import math +import numpy as np +import pandas as pd + import pipit as pp +# Formatters +def get_process_ticker(nranks): + return BasicTicker( + base=2, desired_num_ticks=min(nranks, 16), min_interval=1, num_minor_ticks=0 + ) + + +def get_process_tick_formatter(): + return PrintfTickFormatter(format="Process %d") + + +def format_time(n: float) -> str: + """Converts timestamp/timedelta from ns to human-readable time""" + # Adapted from https://github.com/dask/dask/blob/main/dask/utils.py + + if n >= 1e9 * 24 * 60 * 60 * 2: + d = int(n / 1e9 / 3600 / 24) + h = int((n / 1e9 - d * 3600 * 24) / 3600) + return f"{d}d {h}hr" + + if n >= 1e9 * 60 * 60 * 2: + h = int(n / 1e9 / 3600) + m = int((n / 1e9 - h * 3600) / 60) + return f"{h}hr {m}m" + + if n >= 1e9 * 60 * 10: + m = int(n / 1e9 / 60) + s = int(n / 1e9 - m * 60) + return f"{m}m {s}s" + + if n >= 1e9: + return "%.2f s" % (n / 1e9) + + if n >= 1e6: + return "%.2f ms" % (n / 1e6) + + if n >= 1e3: + return "%.2f us" % (n / 1e3) + + return "%.2f ns" % n + + +# JS expression equivalent to `format_time` function above; assumes: +# - `x` is the value (in ns) being compared to determine units +# - `y` is the value (in ns) actually being formatted +JS_FORMAT_TIME = """ + if (x >= 1e9 * 24 * 60 * 60 * 2) { + d = Math.round(y / 1e9 / 3600 / 24) + h = Math.round((y / 1e9 - d * 3600 * 24) / 3600) + return `${d}d ${h}hr` + } + + if (x >= 1e9 * 60 * 60 * 2) { + h = Math.round(y / 1e9 / 3600) + m = Math.round((y / 1e9 - h * 3600) / 60) + return `${h}hr ${m}m` + } + + if (x >= 1e9 * 60 * 10) { + m = Math.round(y / 1e9 / 60) + s = Math.round(y / 1e9 - m * 60) + return `${m}m ${s}s` + } + + if (x >= 1e9) + return (y / 1e9).toFixed(2) + "s" + + if (x >= 1e6) + return (y / 1e6).toFixed(2) + "ms" + + if (x >= 1e3) { + var ms = Math.floor(y / 1e6); + var us = ((y - ms * 1e6) / 1e3); + + var str = ""; + if (ms) str += ms + "ms "; + if (us) str += Math.round(us) + "us"; + + return str; + } + + var ms = Math.floor(y / 1e6); + var us = Math.floor((y - ms * 1e6) / 1e3); + var ns = Math.round(y % 1000); + + var str = ""; + + if (ms) str += ms + "ms "; + if (us) str += us + "us "; + if (ns) str += ns + "ns"; + + return str; +""" + + +# Used to format ticks for time-based axes +def get_time_tick_formatter(): + return CustomJSTickFormatter( + code=f""" + let x = Math.max(...ticks) - Math.min(...ticks); + let y = tick; + {JS_FORMAT_TIME} + """ + ) + + +# Used to format tooltips for time-based values +def get_time_hover_formatter(): + return CustomJSHover( + code=f""" + let x = value; + let y = value; + {JS_FORMAT_TIME} + """, + ) + + +def format_size(b): + """Converts bytes to something more readable""" + + if b < 1e3: # Less than 1 kB -> byte + return f"{b:.2f} B" + if b < 1e6: # Less than 1 MB -> kB + return f"{(b / 1e3):.2f} kB" + if b < 1e9: # Less than 1 GB -> MB + return f"{(b / 1e6):.2f} MB" + if b < 1e12: # Less than 1 TB -> GB + return f"{(b / 1e9):.2f} GB" + if b < 1e15: # Less than 1 PB -> TB + return f"{(b / 1e12):.2f} TB" + else: + return f"{(b / 1e15):.2f} PB" + + +# JS expression equivalent to `format_size` function above; assumes: +# - `x` is the value (in bytes) being compared to determine units +# - `y` is the value (in bytes) actually being formatted +JS_FORMAT_SIZE = """ + if(x < 1e3) + return (y).toFixed(2) + " B"; + if(x < 1e6) + return (y / 1e3).toFixed(2) + " kB"; + if(x < 1e9) + return (y / 1e6).toFixed(2) + " MB"; + if(x < 1e12) + return (y / 1e9).toFixed(2) + " GB"; + if(x < 1e15) + return (y / 1e12).toFixed(2) + " TB"; + else + return (y / 1e15).toFixed(2) + " PB"; +""" + + +# Used to format ticks for size-based axes +def get_size_tick_formatter(ignore_range=False): + x = "tick" if ignore_range else "Math.max(...ticks) - Math.min(...ticks);" + return CustomJSTickFormatter( + code=f""" + let x = {x} + let y = tick; + {JS_FORMAT_SIZE} + """ + ) + + +# Used to format tooltips for size-based values +def get_size_hover_formatter(): + return CustomJSHover( + code=f""" + let x = value; + let y = value; + {JS_FORMAT_SIZE} + """, + ) + + +def get_percent_tick_formatter(): + return NumeralTickFormatter(format="0.0%") + + +def get_percent_hover_formatter(): + return CustomJSHover( + code=""" + return parseFloat(value * 100).toFixed(2)+"%" + """ + ) + + +# TODO: maybe do this client side with transform +def trimmed(names: pd.Series) -> pd.Series: + return np.where( + names.str.len() < 30, names, names.str[0:20] + "..." + names.str[-5:] + ) + + +def get_trimmed_tick_formatter(): + return CustomJSTickFormatter( + code=""" + if (tick.length < 30) { + return tick; + } else { + return tick.substr(0, 20) + "..." + + tick.substr(tick.length - 5, tick.length); + } + """ + ) + + # Helper functions + + def in_notebook(): """Returns True if we are in notebook environment, False otherwise""" try: @@ -69,49 +297,141 @@ def clamp(value, min_val, max_val): return value -# Custom tickers and formatters +def get_html_tooltips(tooltips_dict): + html = "" + for k, v in tooltips_dict.items(): + html += f""" +
+ {k}:  + {v} +
+ """ + html += """ + + """ + return html -# JS expression to convert bytes to human-readable string -# "x" is the value (in bytes) being compared -# "y" is the value (in bytes) being formatted -JS_FORMAT_SIZE = """ - if(x < 1e3) - return (y).toFixed(2) + " B"; - if(x < 1e6) - return (y / 1e3).toFixed(2) + " kB"; - if(x < 1e9) - return (y / 1e6).toFixed(2) + " MB"; - if(x < 1e12) - return (y / 1e9).toFixed(2) + " GB"; - if(x < 1e15) - return (y / 1e12).toFixed(2) + " TB"; - else - return (y / 1e15).toFixed(2) + " PB"; -""" +def factorize_tuples(tuples_list): + unique_values = sorted(set(tuples_list)) + value_to_index = {value: i for i, value in enumerate(unique_values)} + codes = [value_to_index[value] for value in tuples_list] + return codes, list(unique_values) -def get_process_ticker(nranks): - return BasicTicker( - base=2, desired_num_ticks=min(nranks, 16), min_interval=1, num_minor_ticks=0 - ) +def hex_to_rgb(hex): + hex = hex.strip("#") -def get_size_hover_formatter(): - return CustomJSHover( - code=f""" - let x = value; - let y = value; - {JS_FORMAT_SIZE} - """ - ) + r, g, b = int(hex[:2], 16), int(hex[2:4], 16), int(hex[4:], 16) + return (r, g, b) -def get_size_tick_formatter(ignore_range=False): - x = "tick" if ignore_range else "Math.max(...ticks) - Math.min(...ticks);" - return CustomJSTickFormatter( - code=f""" - let x = {x} - let y = tick; - {JS_FORMAT_SIZE} - """ +def rgb_to_hex(rgb): + r, g, b = rgb + return "#%02x%02x%02x" % (int(r), int(g), int(b)) + + +def average_hex(*hex): + """Averages any number of hex colors, returns result in hex""" + colors = [hex_to_rgb(h) for h in hex] + return rgb_to_hex(np.mean(colors, axis=0)) + + +def scale_hex(hex, scale): + """Multiplies a hex color by a scalar, returns result in hex""" + if scale < 0 or len(hex) != 7: + return hex + + r, g, b = hex_to_rgb(hex) + + r = int(clamp(r * scale, 0, 255)) + g = int(clamp(g * scale, 0, 255)) + b = int(clamp(b * scale, 0, 255)) + + return rgb_to_hex((r, g, b)) + + +def get_height(num_yticks, height_per_tick=400): + """Calculates ideal plot height based on number of y ticks""" + return clamp(int(math.log10(num_yticks) * height_per_tick + 50), 200, 900) + + +LIGHT = [ + "#aec7e8", + "#ffbb78", + "#98df8a", + "#ff9896", + "#c5b0d5", + "#c49c94", + "#f7b6d2", + # "#c7c7c7", # reserved for idle + "#dbdb8d", + "#9edae5", +] +DARK = [ + # "#1f77b4", # reserved for send + "#ff7f0e", + "#2ca02c", + # "#d62728", # reserved for recv + "#9467bd", + "#8c564b", + "#e377c2", + # "#7f7f7f", # take out + "#bcbd22", + "#17becf", +] + + +def get_palette(trace, scale=None): + funcs = trace.events[trace.events["Event Type"] == "Enter"] + # names = funcs["Name"].unique().tolist() + names = reversed(trace.flat_profile(["time.inc"]).index.tolist()) + + depths = ( + funcs.groupby("Name", observed=True)["_depth"] + .agg(lambda x: x.value_counts().index[0]) + .to_dict() ) + + palette = {} + + palette["MPI_Send"] = "#1f77b4" + palette["MPI_Isend"] = "#1f77b4" + + palette["MPI_Recv"] = "#d62728" + palette["MPI_Irecv"] = "#d62728" + + palette["MPI_Wait"] = "#c7c7c7" + palette["MPI_Waitany"] = "#c7c7c7" + palette["MPI_Waitall"] = "#c7c7c7" + palette["Idle"] = "#c7c7c7" + + dark_index = 0 + light_index = 0 + + for i, f in enumerate(names): + if f not in palette: + if depths[f] % 2 == 0: + # palette[f] = LIGHT[hash(f) % len(LIGHT)] + palette[f] = LIGHT[light_index % len(LIGHT)] + light_index += 1 + else: + # palette[f] = DARK[hash(f) % len(DARK)] + palette[f] = DARK[dark_index % len(DARK)] + dark_index += 1 + + # apply multiplier + if scale: + for k, v in palette.items(): + palette[k] = scale_hex(v, scale) + + return palette + + +def get_factor_cmap(field_name, trace, **kwargs): + palette = get_palette(trace, **kwargs) + return factor_cmap(field_name, list(palette.values()), list(palette.keys())) From f37e55faf56c4280e0166f9ddc0b8fee7f577e44 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 26 Mar 2024 19:16:01 -0400 Subject: [PATCH 21/42] minor --- docs/examples/vis.ipynb | 12 +++++++++++- pipit/util/config.py | 2 +- pipit/vis/timeline.py | 20 +++++++++++--------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/docs/examples/vis.ipynb b/docs/examples/vis.ipynb index 2109ff99..bb867b2d 100644 --- a/docs/examples/vis.ipynb +++ b/docs/examples/vis.ipynb @@ -54,6 +54,16 @@ "source": [ "ping_pong.plot_message_histogram()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7318b31-b9bf-4a28-ac44-b75614d2ddf4", + "metadata": {}, + "outputs": [], + "source": [ + "ping_pong.plot_timeline()" + ] } ], "metadata": { @@ -72,7 +82,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.11.6" } }, "nbformat": 4, diff --git a/pipit/util/config.py b/pipit/util/config.py index 8dcfa857..c3f77882 100644 --- a/pipit/util/config.py +++ b/pipit/util/config.py @@ -126,7 +126,7 @@ def theme_validator(key, value): point_policy: "follow_mouse" Legend: label_text_font_size: "8.5pt" - spacing: 10 + spacing: 6 border_line_color: null glyph_width: 16 glyph_height: 16 diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 98066993..dc79f628 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -7,6 +7,7 @@ Grid, FixedTicker, CustomJS, + CustomJSTickFormatter ) from bokeh.plotting import figure from bokeh.transform import dodge @@ -18,6 +19,7 @@ trimmed, show, factorize_tuples, + get_time_tick_formatter ) @@ -217,15 +219,15 @@ def update_cds(event): p.add_layout(g2) # Additional plot config - # p.xaxis.formatter = get_time_tick_formatter() - # p.yaxis.formatter = FuncTickFormatter( - # args={ - # "uniques": uniques, - # }, - # code=""" - # return "Process " + uniques[Math.floor(tick)][0]; - # """, - # ) + p.xaxis.formatter = get_time_tick_formatter() + p.yaxis.formatter = CustomJSTickFormatter( + args={ + "uniques": uniques, + }, + code=""" + return "Process " + uniques[Math.floor(tick)][0]; + """, + ) p.yaxis.ticker = FixedTicker(ticks=process_ticks + 0.1) p.yaxis.major_tick_line_color = None From 802c59354a1e78ea71e22c96a1aab0757349dc2d Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 26 Mar 2024 19:21:28 -0400 Subject: [PATCH 22/42] minor --- pipit/trace.py | 10 ++--- pipit/util/config.py | 2 +- pipit/vis/__init__.py | 1 + pipit/vis/core.py | 4 +- pipit/vis/util.py | 95 +++++++++++++++++++++---------------------- 5 files changed, 56 insertions(+), 56 deletions(-) create mode 100644 pipit/vis/__init__.py diff --git a/pipit/trace.py b/pipit/trace.py index 569c2430..63ae4d13 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -548,7 +548,7 @@ def flat_profile( self.events.loc[self.events["Event Type"] == "Enter"] .groupby([groupby_column, "Process"], observed=True)[metrics] .sum() - .groupby(groupby_column) + .groupby(groupby_column, observed=True) .mean() ) @@ -863,19 +863,19 @@ def detect_pattern( return patterns def plot_comm_matrix(self, output="size", *args, **kwargs): - from .vis import core + from .vis import plot_comm_matrix # Generate the data data = self.comm_matrix(output=output) # Return the Bokeh plot - return core.comm_matrix(data, output=output, *args, **kwargs) + return plot_comm_matrix(data, output=output, *args, **kwargs) def plot_message_histogram(self, bins=20, *args, **kwargs): - from .vis import core + from .vis import plot_message_histogram # Generate the data data = self.message_histogram(bins=bins) # Return the Bokeh plot - return core.message_histogram(data, *args, **kwargs) + return plot_message_histogram(data, *args, **kwargs) diff --git a/pipit/util/config.py b/pipit/util/config.py index 8dcfa857..c3f77882 100644 --- a/pipit/util/config.py +++ b/pipit/util/config.py @@ -126,7 +126,7 @@ def theme_validator(key, value): point_policy: "follow_mouse" Legend: label_text_font_size: "8.5pt" - spacing: 10 + spacing: 6 border_line_color: null glyph_width: 16 glyph_height: 16 diff --git a/pipit/vis/__init__.py b/pipit/vis/__init__.py new file mode 100644 index 00000000..12279d52 --- /dev/null +++ b/pipit/vis/__init__.py @@ -0,0 +1 @@ +from .core import plot_comm_matrix, plot_message_histogram # noqa: F401 diff --git a/pipit/vis/core.py b/pipit/vis/core.py index afce4db5..7b413a12 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -17,7 +17,7 @@ ) -def comm_matrix( +def plot_comm_matrix( data, output="size", cmap="log", palette="Viridis256", return_fig=False ): """Plots the trace's communication matrix. @@ -101,7 +101,7 @@ def comm_matrix( return show(p, return_fig=return_fig) -def message_histogram( +def plot_message_histogram( data, return_fig=False, ): diff --git a/pipit/vis/util.py b/pipit/vis/util.py index ea7f204c..647b60d6 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -5,6 +5,53 @@ from bokeh.themes import Theme import pipit as pp +# Custom tickers and formatters + +# JS expression to convert bytes to human-readable string +# "x" is the value (in bytes) being compared +# "y" is the value (in bytes) being formatted +JS_FORMAT_SIZE = """ + if(x < 1e3) + return (y).toFixed(2) + " B"; + if(x < 1e6) + return (y / 1e3).toFixed(2) + " kB"; + if(x < 1e9) + return (y / 1e6).toFixed(2) + " MB"; + if(x < 1e12) + return (y / 1e9).toFixed(2) + " GB"; + if(x < 1e15) + return (y / 1e12).toFixed(2) + " TB"; + else + return (y / 1e15).toFixed(2) + " PB"; +""" + + +def get_process_ticker(nranks): + return BasicTicker( + base=2, desired_num_ticks=min(nranks, 16), min_interval=1, num_minor_ticks=0 + ) + + +def get_size_hover_formatter(): + return CustomJSHover( + code=f""" + let x = value; + let y = value; + {JS_FORMAT_SIZE} + """ + ) + + +def get_size_tick_formatter(ignore_range=False): + x = "tick" if ignore_range else "Math.max(...ticks) - Math.min(...ticks);" + return CustomJSTickFormatter( + code=f""" + let x = {x} + let y = tick; + {JS_FORMAT_SIZE} + """ + ) + # Helper functions def in_notebook(): @@ -67,51 +114,3 @@ def clamp(value, min_val, max_val): if value > max_val: return max_val return value - - -# Custom tickers and formatters - -# JS expression to convert bytes to human-readable string -# "x" is the value (in bytes) being compared -# "y" is the value (in bytes) being formatted -JS_FORMAT_SIZE = """ - if(x < 1e3) - return (y).toFixed(2) + " B"; - if(x < 1e6) - return (y / 1e3).toFixed(2) + " kB"; - if(x < 1e9) - return (y / 1e6).toFixed(2) + " MB"; - if(x < 1e12) - return (y / 1e9).toFixed(2) + " GB"; - if(x < 1e15) - return (y / 1e12).toFixed(2) + " TB"; - else - return (y / 1e15).toFixed(2) + " PB"; -""" - - -def get_process_ticker(nranks): - return BasicTicker( - base=2, desired_num_ticks=min(nranks, 16), min_interval=1, num_minor_ticks=0 - ) - - -def get_size_hover_formatter(): - return CustomJSHover( - code=f""" - let x = value; - let y = value; - {JS_FORMAT_SIZE} - """ - ) - - -def get_size_tick_formatter(ignore_range=False): - x = "tick" if ignore_range else "Math.max(...ticks) - Math.min(...ticks);" - return CustomJSTickFormatter( - code=f""" - let x = {x} - let y = tick; - {JS_FORMAT_SIZE} - """ - ) From cb7c592ccccef54c5b054fc5ed00b492ca3172d0 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 26 Mar 2024 20:17:13 -0400 Subject: [PATCH 23/42] add callbacks --- pipit/vis/timeline.py | 248 ++++++++++++++++++++++++++---------------- 1 file changed, 156 insertions(+), 92 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index dc79f628..1112f7dc 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -7,8 +7,11 @@ Grid, FixedTicker, CustomJS, - CustomJSTickFormatter + CustomJSTickFormatter, + Arrow, + OpenHead, ) +from bokeh.events import RangesUpdate, Tap from bokeh.plotting import figure from bokeh.transform import dodge @@ -19,16 +22,11 @@ trimmed, show, factorize_tuples, - get_time_tick_formatter + get_time_tick_formatter, ) -def plot_timeline(trace, show_depth=False, instant_events=False): - """Displays the events of a trace against time - - Instant events are represented by points, functions are represented by horizontal - bars, and MPI messages are represented by lines connecting send/receive events.""" - +def prepare_data(trace, show_depth, instant_events): # Generate necessary metrics trace.calc_exc_metrics(["Timestamp (ns)"]) trace._match_events() @@ -47,7 +45,6 @@ def plot_timeline(trace, show_depth=False, instant_events=False): .sort_values(by="time.inc", ascending=False) .copy(deep=False) ) - events["_depth"] = events["_depth"].astype(float).fillna("") # Determine y-coordinates from process and depth y_tuples = ( @@ -60,11 +57,7 @@ def plot_timeline(trace, show_depth=False, instant_events=False): events["y"] = codes num_ys = len(uniques) - depth_ticks = np.arange(0, num_ys) - process_ticks = np.array( - [i for i, v in enumerate(uniques) if len(v) == 1 or v[1] == 0] - ) - + events["_depth"] = events["_depth"].astype(float).fillna("") events["name_trimmed"] = trimmed(events["Name"]) events["_matching_event"] = events["_matching_event"].fillna(-1) @@ -84,56 +77,132 @@ def plot_timeline(trace, show_depth=False, instant_events=False): ] ] - # Define CDS for glyphs to be empty - hbar_source = ColumnDataSource(events.head(1)) - scatter_source = ColumnDataSource(events.head(0)) + return events, min_ts, max_ts, uniques, num_ys - # Callback function that updates CDS - def update_cds(event): - x0 = event.x0 if event is not None else min_ts - x1 = event.x1 if event is not None else max_ts - x0 = x0 - (x1 - x0) * 0.25 - x1 = x1 + (x1 - x0) * 0.25 +def update_cds( + event, events, instant_events, min_ts, max_ts, hbar_source, scatter_source +): + """ + Callback function that updates the 3 data sources based on the new range. + Called when user zooms or pans the timeline. + """ - # Remove events that are out of bounds - in_bounds = events[ - ( - (events["Event Type"] == "Instant") - & (events["Timestamp (ns)"] > x0) - & (events["Timestamp (ns)"] < x1) - ) - | ( - (events["Event Type"] == "Enter") - & (events["_matching_timestamp"] > x0) - & (events["Timestamp (ns)"] < x1) - ) - ].copy(deep=False) + x0 = event.x0 if event is not None else min_ts + x1 = event.x1 if event is not None else max_ts + + x0 = x0 - (x1 - x0) * 0.25 + x1 = x1 + (x1 - x0) * 0.25 + + # Remove events that are out of bounds + in_bounds = events[ + ( + (events["Event Type"] == "Instant") + & (events["Timestamp (ns)"] > x0) + & (events["Timestamp (ns)"] < x1) + ) + | ( + (events["Event Type"] == "Enter") + & (events["_matching_timestamp"] > x0) + & (events["Timestamp (ns)"] < x1) + ) + ].copy(deep=False) - # Update hbar_source to keep 5000 largest functions - func = in_bounds[in_bounds["Event Type"] == "Enter"] - large = func - hbar_source.data = large + # Update hbar_source to keep 5000 largest functions + func = in_bounds[in_bounds["Event Type"] == "Enter"] + large = func + hbar_source.data = large + + # Update scatter_source to keep sampled events + if instant_events: + inst = in_bounds[in_bounds["Event Type"] == "Instant"].copy(deep=False) + + if len(inst) > 500: + inst["bin"] = pd.cut(x=inst["Timestamp (ns)"], bins=1000, labels=False) + + grouped = inst.groupby(["bin", "y"]) + samples = grouped.first().reset_index() + samples = samples[~samples["Timestamp (ns)"].isna()] + + scatter_source.data = samples + else: + scatter_source.data = inst + + +def tap_callback(event, events, trace, show_depth, p): + """ + Callback function that adds an MPI message arrow when user clicks + on a send or receive event. + """ + x = event.x + y = event.y + + candidates = events[ + (events["Event Type"] == "Instant") + & (events["Name"].isin(["MpiSend", "MpiRecv", "MpiIsend", "MpiIrecv"])) + & (events["y"] == round(y)) + ] + + dx = candidates["Timestamp (ns)"] - x + distance = pd.Series(dx * dx) + + selected = candidates.iloc[distance.argsort().values] + + if len(selected) >= 1: + selected = selected.iloc[0] + + match = trace._get_matching_p2p_event(selected.name) + send = ( + selected + if selected["Name"] in ["MpiSend", "MpiIsend"] + else events.loc[match] + ) + recv = ( + selected + if selected["Name"] in ["MpiRecv", "MpiIrecv"] + else events.loc[match] + ) + + arrow = Arrow( + end=OpenHead(line_color="#28282B", line_width=1.5, size=8), + line_color="#28282B", + line_width=1.5, + x_start=send["Timestamp (ns)"], + y_start=send["y"] - 0.2 if show_depth else send["y"], + x_end=recv["Timestamp (ns)"], + y_end=recv["y"] - 0.2 if show_depth else recv["y"], + level="overlay", + ) + p.add_layout(arrow) + + +def plot_timeline(trace, show_depth=False, instant_events=False): + """Displays the events of a trace against time - # Update scatter_source to keep sampled events - if instant_events: - inst = in_bounds[in_bounds["Event Type"] == "Instant"].copy(deep=False) + Instant events are represented by points, functions are represented by horizontal + bars, and MPI messages are represented by lines connecting send/receive events.""" - if len(inst) > 500: - inst["bin"] = pd.cut(x=inst["Timestamp (ns)"], bins=1000, labels=False) + # Prepare data to be plotted + events, min_ts, max_ts, uniques, num_ys = prepare_data( + trace, show_depth, instant_events + ) - grouped = inst.groupby(["bin", "y"]) - samples = grouped.first().reset_index() - samples = samples[~samples["Timestamp (ns)"].isna()] + depth_ticks = np.arange(0, num_ys) + process_ticks = np.array( + [i for i, v in enumerate(uniques) if len(v) == 1 or v[1] == 0] + ) - scatter_source.data = samples - else: - scatter_source.data = inst + # Define the data sources (Bokeh ColumnDataSource) + hbar_source = ColumnDataSource(events.head(0)) + scatter_source = ColumnDataSource(events.head(0)) + image_source = ColumnDataSource( + data=dict( + image=[np.zeros((50, 16), dtype=np.uint32)], x=[0], y=[0], dw=[0], dh=[0] + ) + ) # Create Bokeh plot - # min_height = 50 + 22 * len(events["Name"].unique()) - plot_height = 100 + 22 * num_ys - # height = clamp(plot_height, min_height, 900) + plot_height = 120 + 22 * num_ys p = figure( x_range=(min_ts, max_ts + (max_ts - min_ts) * 0.05), @@ -144,42 +213,15 @@ def update_cds(event): height=min(500, plot_height), sizing_mode="stretch_width", toolbar_location=None, - # x_axis_label="Time", + x_axis_label="Time", ) - # p.min_border_bottom = height - plot_height - - # Create color maps + # Define color mappings fill_cmap = get_factor_cmap("Name", trace) line_cmap = get_factor_cmap("Name", trace, scale=0.7) - # Add lines for each process - # p.segment( - # x0=[0] * len(process_ticks), - # x1=[max_ts] * len(process_ticks), - # y0=process_ticks, - # y1=process_ticks, - # line_dash="dotted", - # line_color="gray", - # ) - - # Add bars for large functions - hbar = p.hbar( - left="Timestamp (ns)", - right="_matching_timestamp", - y="y", - height=0.8 if show_depth else 0.8, - source=hbar_source, - fill_color=fill_cmap, - line_color=line_cmap, - line_width=1, - line_alpha=0.5, - legend_field="name_trimmed", - ) - - # Add raster for small functions - # p.image_rgba(source=image_source) - + # Add glyphs + # Scatter for instant events if instant_events: scatter = p.scatter( x="Timestamp (ns)", @@ -194,10 +236,25 @@ def update_cds(event): # legend_label="Instant event", ) + # Bars for "large" functions + hbar = p.hbar( + left="Timestamp (ns)", + right="_matching_timestamp", + y="y", + height=0.8 if show_depth else 0.8, + source=hbar_source, + fill_color=fill_cmap, + line_color=line_cmap, + line_width=1, + line_alpha=0.5, + legend_field="name_trimmed", + ) + + # Image for small functions + p.image_rgba(source=image_source) + # Add custom grid lines - # p.xgrid.visible = False p.ygrid.visible = False - g1 = Grid( dimension=1, grid_line_color="white", @@ -210,8 +267,8 @@ def update_cds(event): g2 = Grid( dimension=1, grid_line_width=2, - # band_fill_color="gray", - # band_fill_alpha=0.1, + band_fill_color="gray", + band_fill_alpha=0.1, ticker=FixedTicker(ticks=process_ticks - 0.5), level="glyph", ) @@ -233,8 +290,13 @@ def update_cds(event): p.yaxis.major_tick_line_color = None p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] - # p.on_event(RangesUpdate, update_cds) - # p.on_event(Tap, tap_callback) + p.on_event( + RangesUpdate, + lambda event: update_cds( + event, events, instant_events, min_ts, max_ts, hbar_source, scatter_source + ), + ) + p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) # Move legend to the right p.add_layout(p.legend[0], "below") @@ -243,7 +305,9 @@ def update_cds(event): p.legend.nrows = 2 # Make initial call to our callback - update_cds(None) + update_cds( + None, events, instant_events, min_ts, max_ts, hbar_source, scatter_source + ) # Hover config hover = p.select(HoverTool) From 1793663bbf513262a21ee102a396a598af509adc Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 26 Mar 2024 20:31:10 -0400 Subject: [PATCH 24/42] clean up --- pipit/vis/timeline.py | 79 +++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 1112f7dc..f7cba95d 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -1,32 +1,37 @@ import numpy as np import pandas as pd from bokeh.models import ( + Arrow, ColumnDataSource, - HoverTool, - WheelZoomTool, - Grid, - FixedTicker, CustomJS, CustomJSTickFormatter, - Arrow, + FixedTicker, + Grid, + HoverTool, OpenHead, + WheelZoomTool, ) from bokeh.events import RangesUpdate, Tap from bokeh.plotting import figure from bokeh.transform import dodge +import pipit as pp from pipit.vis.util import ( - get_time_hover_formatter, - get_html_tooltips, - get_factor_cmap, - trimmed, - show, factorize_tuples, + get_factor_cmap, + get_html_tooltips, + get_time_hover_formatter, get_time_tick_formatter, + show, + trimmed, ) -def prepare_data(trace, show_depth, instant_events): +def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): + """ + Prepare data for plotting the timeline. + """ + # Generate necessary metrics trace.calc_exc_metrics(["Timestamp (ns)"]) trace._match_events() @@ -81,8 +86,14 @@ def prepare_data(trace, show_depth, instant_events): def update_cds( - event, events, instant_events, min_ts, max_ts, hbar_source, scatter_source -): + event: RangesUpdate, + events: pd.DataFrame, + instant_events: bool, + min_ts: float, + max_ts: float, + hbar_source: ColumnDataSource, + scatter_source: ColumnDataSource, +) -> None: """ Callback function that updates the 3 data sources based on the new range. Called when user zooms or pans the timeline. @@ -129,7 +140,13 @@ def update_cds( scatter_source.data = inst -def tap_callback(event, events, trace, show_depth, p): +def tap_callback( + event: Tap, + events: pd.DataFrame, + trace: pp.Trace, + show_depth: bool, + p: figure, +) -> None: """ Callback function that adds an MPI message arrow when user clicks on a send or receive event. @@ -253,7 +270,7 @@ def plot_timeline(trace, show_depth=False, instant_events=False): # Image for small functions p.image_rgba(source=image_source) - # Add custom grid lines + # Additional plot config p.ygrid.visible = False g1 = Grid( dimension=1, @@ -274,8 +291,6 @@ def plot_timeline(trace, show_depth=False, instant_events=False): ) p.add_layout(g1) p.add_layout(g2) - - # Additional plot config p.xaxis.formatter = get_time_tick_formatter() p.yaxis.formatter = CustomJSTickFormatter( args={ @@ -285,30 +300,16 @@ def plot_timeline(trace, show_depth=False, instant_events=False): return "Process " + uniques[Math.floor(tick)][0]; """, ) - p.yaxis.ticker = FixedTicker(ticks=process_ticks + 0.1) p.yaxis.major_tick_line_color = None - p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] - p.on_event( - RangesUpdate, - lambda event: update_cds( - event, events, instant_events, min_ts, max_ts, hbar_source, scatter_source - ), - ) - p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) - # Move legend to the right + # Legend config p.add_layout(p.legend[0], "below") p.legend.orientation = "horizontal" p.legend.location = "center" p.legend.nrows = 2 - # Make initial call to our callback - update_cds( - None, events, instant_events, min_ts, max_ts, hbar_source, scatter_source - ) - # Hover config hover = p.select(HoverTool) hover.tooltips = get_html_tooltips( @@ -342,5 +343,19 @@ def plot_timeline(trace, show_depth=False, instant_events=False): """ ) + # Add interactive Python callbacks + p.on_event( + RangesUpdate, + lambda event: update_cds( + event, events, instant_events, min_ts, max_ts, hbar_source, scatter_source + ), + ) + p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) + + # Make initial call to callback + update_cds( + None, events, instant_events, min_ts, max_ts, hbar_source, scatter_source + ) + # Return plot return show(p) From 8d5e6464018bd4ab4662f516a288bc7431dc8b35 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 26 Mar 2024 20:45:37 -0400 Subject: [PATCH 25/42] cleanup --- pipit/vis/timeline.py | 57 ++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index f7cba95d..f1e8ec9c 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -37,9 +37,6 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): trace._match_events() trace._match_caller_callee() - min_ts = trace.events["Timestamp (ns)"].min() - max_ts = trace.events["Timestamp (ns)"].max() - # Prepare data for plotting events = ( trace.events[ @@ -58,9 +55,9 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): else list(zip(events["Process"])) ) - codes, uniques = factorize_tuples(y_tuples) + codes, y_tuples = factorize_tuples(y_tuples) events["y"] = codes - num_ys = len(uniques) + num_ys = len(y_tuples) events["_depth"] = events["_depth"].astype(float).fillna("") events["name_trimmed"] = trimmed(events["Name"]) @@ -82,15 +79,13 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): ] ] - return events, min_ts, max_ts, uniques, num_ys + return events, y_tuples, num_ys def update_cds( event: RangesUpdate, events: pd.DataFrame, instant_events: bool, - min_ts: float, - max_ts: float, hbar_source: ColumnDataSource, scatter_source: ColumnDataSource, ) -> None: @@ -99,8 +94,8 @@ def update_cds( Called when user zooms or pans the timeline. """ - x0 = event.x0 if event is not None else min_ts - x1 = event.x1 if event is not None else max_ts + x0 = event.x0 if event is not None else events["Timestamp (ns)"].min() + x1 = event.x1 if event is not None else events["Timestamp (ns)"].max() x0 = x0 - (x1 - x0) * 0.25 x1 = x1 + (x1 - x0) * 0.25 @@ -200,13 +195,13 @@ def plot_timeline(trace, show_depth=False, instant_events=False): bars, and MPI messages are represented by lines connecting send/receive events.""" # Prepare data to be plotted - events, min_ts, max_ts, uniques, num_ys = prepare_data( + events, y_tuples, num_ys = prepare_data( trace, show_depth, instant_events ) depth_ticks = np.arange(0, num_ys) process_ticks = np.array( - [i for i, v in enumerate(uniques) if len(v) == 1 or v[1] == 0] + [i for i, v in enumerate(y_tuples) if len(v) == 1 or v[1] == 0] ) # Define the data sources (Bokeh ColumnDataSource) @@ -220,6 +215,7 @@ def plot_timeline(trace, show_depth=False, instant_events=False): # Create Bokeh plot plot_height = 120 + 22 * num_ys + min_ts, max_ts = events["Timestamp (ns)"].min(), events["Timestamp (ns)"].max() p = figure( x_range=(min_ts, max_ts + (max_ts - min_ts) * 0.05), @@ -238,20 +234,6 @@ def plot_timeline(trace, show_depth=False, instant_events=False): line_cmap = get_factor_cmap("Name", trace, scale=0.7) # Add glyphs - # Scatter for instant events - if instant_events: - scatter = p.scatter( - x="Timestamp (ns)", - y=dodge("y", -0.2 if show_depth else 0), - # size=9, - line_color="#0868ac", - alpha=1, - color="#ccebc5", - line_width=0.8, - marker="diamond", - source=scatter_source, - # legend_label="Instant event", - ) # Bars for "large" functions hbar = p.hbar( @@ -270,6 +252,21 @@ def plot_timeline(trace, show_depth=False, instant_events=False): # Image for small functions p.image_rgba(source=image_source) + # Scatter for instant events + if instant_events: + scatter = p.scatter( + x="Timestamp (ns)", + y=dodge("y", -0.2 if show_depth else 0), + # size=9, + line_color="#0868ac", + alpha=1, + color="#ccebc5", + line_width=0.8, + marker="diamond", + source=scatter_source, + legend_label="Instant event", + ) + # Additional plot config p.ygrid.visible = False g1 = Grid( @@ -294,10 +291,10 @@ def plot_timeline(trace, show_depth=False, instant_events=False): p.xaxis.formatter = get_time_tick_formatter() p.yaxis.formatter = CustomJSTickFormatter( args={ - "uniques": uniques, + "y_tuples": y_tuples, }, code=""" - return "Process " + uniques[Math.floor(tick)][0]; + return "Process " + y_tuples[Math.floor(tick)][0]; """, ) p.yaxis.ticker = FixedTicker(ticks=process_ticks + 0.1) @@ -347,14 +344,14 @@ def plot_timeline(trace, show_depth=False, instant_events=False): p.on_event( RangesUpdate, lambda event: update_cds( - event, events, instant_events, min_ts, max_ts, hbar_source, scatter_source + event, events, instant_events, hbar_source, scatter_source ), ) p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) # Make initial call to callback update_cds( - None, events, instant_events, min_ts, max_ts, hbar_source, scatter_source + None, events, instant_events, hbar_source, scatter_source ) # Return plot From 10bf121dfff4c71e55c35610f903e4711c4a1faa Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Wed, 27 Mar 2024 20:38:39 -0400 Subject: [PATCH 26/42] add comments, clean up --- pipit/vis/timeline.py | 63 +++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index f1e8ec9c..132c6c97 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -28,10 +28,7 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): - """ - Prepare data for plotting the timeline. - """ - + """Prepare data for plotting the timeline.""" # Generate necessary metrics trace.calc_exc_metrics(["Timestamp (ns)"]) trace._match_events() @@ -90,8 +87,10 @@ def update_cds( scatter_source: ColumnDataSource, ) -> None: """ - Callback function that updates the 3 data sources based on the new range. - Called when user zooms or pans the timeline. + Callback function that updates the 3 data sources (hbar_source, scatter_source, + image_source) based on the new range. + + Called when user zooms or pans the timeline (and once initially). """ x0 = event.x0 if event is not None else events["Timestamp (ns)"].min() @@ -188,23 +187,30 @@ def tap_callback( p.add_layout(arrow) -def plot_timeline(trace, show_depth=False, instant_events=False): - """Displays the events of a trace against time +def plot_timeline( + trace: pp.Trace, + show_depth: bool = False, + instant_events: bool = False, +): + """ + Displays the events of a trace on a timeline. - Instant events are represented by points, functions are represented by horizontal - bars, and MPI messages are represented by lines connecting send/receive events.""" + Instant events are drawn as points, function calls are drawn as horizontal bars, + and MPI messages are drawn as arrows. - # Prepare data to be plotted - events, y_tuples, num_ys = prepare_data( - trace, show_depth, instant_events - ) + Args: + trace: The trace to be visualized. + show_depth: Whether to show the depth of the function calls. + instant_events: Whether to show instant events. - depth_ticks = np.arange(0, num_ys) - process_ticks = np.array( - [i for i, v in enumerate(y_tuples) if len(v) == 1 or v[1] == 0] - ) + Returns: + The Bokeh plot. + """ - # Define the data sources (Bokeh ColumnDataSource) + # Prepare data to be plotted + events, y_tuples, num_ys = prepare_data(trace, show_depth, instant_events) + + # Define the 3 data sources (Bokeh ColumnDataSource) hbar_source = ColumnDataSource(events.head(0)) scatter_source = ColumnDataSource(events.head(0)) image_source = ColumnDataSource( @@ -234,7 +240,6 @@ def plot_timeline(trace, show_depth=False, instant_events=False): line_cmap = get_factor_cmap("Name", trace, scale=0.7) # Add glyphs - # Bars for "large" functions hbar = p.hbar( left="Timestamp (ns)", @@ -266,8 +271,15 @@ def plot_timeline(trace, show_depth=False, instant_events=False): source=scatter_source, legend_label="Instant event", ) - + # Additional plot config + p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] + + # Grid config + depth_ticks = np.arange(0, num_ys) + process_ticks = np.array( + [i for i, v in enumerate(y_tuples) if len(v) == 1 or v[1] == 0] + ) p.ygrid.visible = False g1 = Grid( dimension=1, @@ -288,6 +300,8 @@ def plot_timeline(trace, show_depth=False, instant_events=False): ) p.add_layout(g1) p.add_layout(g2) + + # Axis config p.xaxis.formatter = get_time_tick_formatter() p.yaxis.formatter = CustomJSTickFormatter( args={ @@ -299,7 +313,6 @@ def plot_timeline(trace, show_depth=False, instant_events=False): ) p.yaxis.ticker = FixedTicker(ticks=process_ticks + 0.1) p.yaxis.major_tick_line_color = None - p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] # Legend config p.add_layout(p.legend[0], "below") @@ -340,7 +353,7 @@ def plot_timeline(trace, show_depth=False, instant_events=False): """ ) - # Add interactive Python callbacks + # Add interactive callbacks (these happen on the Python side) p.on_event( RangesUpdate, lambda event: update_cds( @@ -350,9 +363,7 @@ def plot_timeline(trace, show_depth=False, instant_events=False): p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) # Make initial call to callback - update_cds( - None, events, instant_events, hbar_source, scatter_source - ) + update_cds(None, events, instant_events, hbar_source, scatter_source) # Return plot return show(p) From 7b3a334364fa51865f97fbedb0acd9ff0c74f451 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Thu, 28 Mar 2024 16:14:23 -0400 Subject: [PATCH 27/42] add critical_path_analysis --- pipit/trace.py | 163 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/pipit/trace.py b/pipit/trace.py index 1e2b9482..53f65b85 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -885,3 +885,166 @@ def plot_timeline(self, *args, **kwargs): # Return the Bokeh plot return plot_timeline(self, *args, **kwargs) + + def critical_path_analysis(self): + self._match_events() + + instant_recv_events = ["MpiRecv", "MpiIrecv"] + recv_events = ["MPI_Recv", "MPI_Irecv"] + send_events = ["MPI_Send", "MPI_Isend"] + collective_functions = [ + "MPI_Allgather", + "MPI_Allgatherv", + "MPI_Allreduce", + "MPI_Alltoall", + "MPI_Alltoallv", + "MPI_Alltoallw", + "MPI_Barrier", + "MPI_Bcast", + "MPI_Gather", + "MPI_Gatherv", + "MPI_Iallgather", + "MPI_Iallreduce", + "MPI_Ibarrier", + "MPI_Ibcast", + "MPI_Igather", + "MPI_Igatherv", + "MPI_Ireduce", + "MPI_Iscatter", + "MPI_Iscatterv", + "MPI_Reduce", + "MPI_Scatter", + "MPI_Scatterv", + "MPI_Exscan", + "MPI_Op_create", + "MPI_Op_free", + "MPI_Reduce_local", + "MPI_Reduce_scatter", + "MPI_Scan", + "MPI_File_iread_at_all", + "MPI_File_iwrite_at_all", + "MPI_File_iread_all", + "MPI_File_iwrite_all", + "MPI_File_read_all_begin", + "MPI_File_write_all_begin", + "MPI_File_write_all_end", + "MPI_File_close", + ] + + critical_paths = [] + critical_path = [] + last_event = None + leave_events = self.events[(self.events["Event Type"] == "Leave")] + num_of_processes = leave_events["Process"].astype(int).max() + + if "MPI_Finalize" in self.events["Name"].values: + last_event = self.events[ + (self.events["Event Type"] == "Leave") + & (self.events["Name"] == "MPI_Finalize") + ].iloc[-1] + else: + last_event = self.events[self.events["Event Type"] == "Leave"].iloc[-1] + last_name = last_event["Name"] + last_process = last_event["Process"] + last_timestamp = last_event["Timestamp (ns)"] + critical_path.append(last_event) + after_recieve = False + after_collective = False + # Main loop to trace back + while True: + # Filter for events from the same process before the last timestamp + candidate_events = leave_events[ + (leave_events["Process"] == last_event["Process"]) + & (leave_events["Timestamp (ns)"] < last_event["Timestamp (ns)"]) + ] + + # obtain the latest function after the collective function call. + # we basically do the something similar to starting with MPI_Finalize + # but this time we use a different function. + if after_collective: + candidate_name = candidate_events.iloc[-1]["Name"] + candidate_events = leave_events[ + (leave_events["Name"] == candidate_name) + & (leave_events["Timestamp (ns)"] < last_event["Timestamp (ns)"]) + ] + + # No more events to trace back from. + if candidate_events.empty: + break + + # Select the last event from the candidates if + # we the previous event is not a receive. + if not after_recieve: + last_event = candidate_events.iloc[-1] + critical_path.append(last_event) + + # If we continue after a receive function. + if last_event["Name"] in recv_events: + # Get the corresponding instant event for the recv function. + last_instant_event = self.events[ + (self.events["Event Type"] == "Instant") + & (self.events["Timestamp (ns)"] < last_timestamp) + & (self.events["Process"] == last_process) + & (self.events["Name"].isin(instant_recv_events)) + ] + # Sometimes recv function have some instant events which + # do not include the sender information. We ignore them. + if last_instant_event.empty: + continue + else: + last_instant_event = last_instant_event.iloc[-1] + + # Get the corresponding send event. + last_event = self.events[ + (self.events["Timestamp (ns)"] < last_timestamp) + & ( + self.events["Process"] + == last_instant_event["Attributes"]["sender"] + ) + & ( + self.events["Name"].isin(send_events) + & (self.events["Event Type"] == "Leave") + ) + ].iloc[-1] + last_timestamp = last_event["Timestamp (ns)"] + last_process = last_event["Process"] + + after_receive = True + after_collective = False + critical_path.append(last_event) + pass + + # Restart the detection after a collective function. + if last_event["Name"] in collective_functions: + critical_paths.append(critical_path) + critical_path = [] + after_collective = True + after_receive = False + + start_times = [] + check_if_done = leave_events[leave_events["Name"] == last_event["Name"]] + for start_time in check_if_done.iloc[0 : num_of_processes + 1][ + "Timestamp (ns)" + ]: + start_times.append(start_time) + + # Exit if we have traced back to the beginning + leave_events = leave_events.reset_index(drop=True) + if ( + leave_events[ + (leave_events["Timestamp (ns)"] == last_event["Timestamp (ns)"]) + ].index + <= num_of_processes + 1 + ): + if ( + last_event["Name"] == leave_events.iloc[0]["Name"] + and last_event["Timestamp (ns)"] in start_times + ): + critical_paths.append(critical_path) + break + + critical_dfs = [] + for critical_path in critical_paths: + if len(critical_path) > 1: + critical_dfs.append(pd.DataFrame(critical_path)) + return critical_dfs \ No newline at end of file From b3c828088d1e8bf22a26fe2fad1ee2462c164f02 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Thu, 28 Mar 2024 16:31:25 -0400 Subject: [PATCH 28/42] add critical path plotting --- pipit/vis/timeline.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 132c6c97..9d60ab11 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -191,6 +191,7 @@ def plot_timeline( trace: pp.Trace, show_depth: bool = False, instant_events: bool = False, + critical_path: bool = False, ): """ Displays the events of a trace on a timeline. @@ -202,6 +203,8 @@ def plot_timeline( trace: The trace to be visualized. show_depth: Whether to show the depth of the function calls. instant_events: Whether to show instant events. + critical_path: Whether to show the critical path. NOTE: critical_path currently + only works when show_depth==False. TODO: make it work with show_depth=True. Returns: The Bokeh plot. @@ -272,6 +275,20 @@ def plot_timeline( legend_label="Instant event", ) + # Lines for critical path + if critical_path: + critical_dfs = trace.critical_path_analysis() + for df in critical_dfs: + p.line( + x="Timestamp (ns)", + y="Process", + source=ColumnDataSource(df), + line_width=2, + line_color="black", + legend_label="Critical Path", + level="overlay", + ) + # Additional plot config p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] From e4a89fc3785f25751bc13919a69c154c7f604caa Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Thu, 28 Mar 2024 22:31:37 -0400 Subject: [PATCH 29/42] flake8 --- pipit/trace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipit/trace.py b/pipit/trace.py index 53f65b85..73626bd6 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -1047,4 +1047,4 @@ def critical_path_analysis(self): for critical_path in critical_paths: if len(critical_path) > 1: critical_dfs.append(pd.DataFrame(critical_path)) - return critical_dfs \ No newline at end of file + return critical_dfs From 86fa14cee27d19cac185808fbaeaa9cff6369e68 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Thu, 28 Mar 2024 22:32:05 -0400 Subject: [PATCH 30/42] use arrows for crit. path --- pipit/vis/timeline.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 9d60ab11..0fddc790 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -279,15 +279,21 @@ def plot_timeline( if critical_path: critical_dfs = trace.critical_path_analysis() for df in critical_dfs: - p.line( - x="Timestamp (ns)", - y="Process", - source=ColumnDataSource(df), - line_width=2, - line_color="black", - legend_label="Critical Path", - level="overlay", - ) + # Draw arrows + # TODO: can we vectorize this? + for i in range(len(df) - 1): + p.add_layout( + Arrow( + end=OpenHead(line_color="black", line_width=2, size=8), + line_color="black", + line_width=2, + x_start=df["Timestamp (ns)"].iloc[i], + y_start=df["Process"].iloc[i], + x_end=df["Timestamp (ns)"].iloc[i + 1], + y_end=df["Process"].iloc[i + 1], + level="overlay", + ) + ) # Additional plot config p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] From cf9001d2f269c473c11ba2b4f6b1252c8192bcb2 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Fri, 29 Mar 2024 02:41:01 -0400 Subject: [PATCH 31/42] show all messages --- pipit/vis/timeline.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 0fddc790..3cfb9257 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -10,6 +10,7 @@ HoverTool, OpenHead, WheelZoomTool, + TeeHead ) from bokeh.events import RangesUpdate, Tap from bokeh.plotting import figure @@ -33,12 +34,13 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): trace.calc_exc_metrics(["Timestamp (ns)"]) trace._match_events() trace._match_caller_callee() + trace._match_messages() # Prepare data for plotting events = ( trace.events[ trace.events["Event Type"].isin( - ["Enter", "Instant"] if instant_events else ["Enter"] + ["Enter", "Instant"] ) ] .sort_values(by="time.inc", ascending=False) @@ -75,6 +77,16 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): "Event Type", ] ] + events["first_letter"] = "" + events.loc[events["Name"] == "MpiSend", "first_letter"] = "S" + events.loc[events["Name"] == "MpiRecv", "first_letter"] = "R" + events.loc[events["Name"] == "MpiIsend", "first_letter"] = "IS" + events.loc[events["Name"] == "MpiIrecv", "first_letter"] = "IR" + events.loc[events["Name"] == "MpiIrecvRequest", "first_letter"] = "IRR" + events.loc[events["Name"] == "MpiIsendComplete", "first_letter"] = "ISC" + events.loc[events["Name"] == "MpiCollectiveBegin", "first_letter"] = "CB" + events.loc[events["Name"] == "MpiCollectiveEnd", "first_letter"] = "CE" + return events, y_tuples, num_ys @@ -192,6 +204,7 @@ def plot_timeline( show_depth: bool = False, instant_events: bool = False, critical_path: bool = False, + show_messages: str = "click", ): """ Displays the events of a trace on a timeline. @@ -205,6 +218,7 @@ def plot_timeline( instant_events: Whether to show instant events. critical_path: Whether to show the critical path. NOTE: critical_path currently only works when show_depth==False. TODO: make it work with show_depth=True. + show_messages: Whether to show MPI messages. Can be "click" (default), or "all". Returns: The Bokeh plot. @@ -275,7 +289,28 @@ def plot_timeline( legend_label="Instant event", ) - # Lines for critical path + # Arrows for MPI messages + if show_messages == "all": + sends = events[events["Name"].isin(["MpiSend", "MpiIsend"])] + for i in range(len(sends)): + p.add_layout( + Arrow( + end=OpenHead(line_color="#28282B", line_width=1.5, size=4, line_alpha=0.8), + line_color="#28282B", + line_width=1.5, + x_start=sends["Timestamp (ns)"].iloc[i], + y_start=sends["y"].iloc[i] - 0.2 if show_depth else sends["y"].iloc[i], + x_end=events.loc[sends["_matching_event"].iloc[i]]["Timestamp (ns)"], + y_end=events.loc[sends["_matching_event"].iloc[i]]["y"] + - 0.2 + if show_depth + else events.loc[sends["_matching_event"].iloc[i]]["y"], + level="annotation", + line_alpha=0.8, + ) + ) + + # Arrows for critical path if critical_path: critical_dfs = trace.critical_path_analysis() for df in critical_dfs: @@ -383,7 +418,9 @@ def plot_timeline( event, events, instant_events, hbar_source, scatter_source ), ) - p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) + + if show_messages == "click": + p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) # Make initial call to callback update_cds(None, events, instant_events, hbar_source, scatter_source) From ea2cb018bfddabc14c9a01fc2bb6ce34ce222793 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Fri, 29 Mar 2024 02:41:57 -0400 Subject: [PATCH 32/42] minor --- pipit/vis/timeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 3cfb9257..1f7bf6c6 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -204,7 +204,7 @@ def plot_timeline( show_depth: bool = False, instant_events: bool = False, critical_path: bool = False, - show_messages: str = "click", + messages: str = "click", ): """ Displays the events of a trace on a timeline. @@ -290,7 +290,7 @@ def plot_timeline( ) # Arrows for MPI messages - if show_messages == "all": + if messages == "all": sends = events[events["Name"].isin(["MpiSend", "MpiIsend"])] for i in range(len(sends)): p.add_layout( @@ -419,7 +419,7 @@ def plot_timeline( ), ) - if show_messages == "click": + if messages == "click": p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) # Make initial call to callback From 35380f7fb1eef7a69f09cbb75a6ae83513328f72 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Fri, 29 Mar 2024 02:43:55 -0400 Subject: [PATCH 33/42] match messages --- pipit/trace.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/pipit/trace.py b/pipit/trace.py index 73626bd6..cc867a40 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -291,6 +291,78 @@ def _match_caller_callee(self): {"_depth": "category", "_parent": "category"} ) + def _match_messages(self): + """ + Matches corresponding MpiSend/MpiRecv and MpiIsend/MpiIrecv instant events + """ + if "_matching_event" not in self.events.columns: + self.events["_matching_event"] = None + + if "_matching_timestamp" not in self.events.columns: + self.events["_matching_timestamp"] = np.nan + + matching_events = list(self.events["_matching_event"]) + matching_times = list(self.events["_matching_timestamp"]) + + mpi_events = self.events[ + self.events["Name"].isin(["MpiSend", "MpiRecv", "MpiIsend", "MpiIrecv"]) + ] + + queue = [[] for _ in range(len(self.events["Process"].unique()))] + + df_indices = list(mpi_events.index) + timestamps = list(mpi_events["Timestamp (ns)"]) + names = list(mpi_events["Name"]) + attrs = list(mpi_events["Attributes"]) + processes = list(mpi_events["Process"]) + + # Iterate through all events + for i in range(len(mpi_events)): + curr_df_index = df_indices[i] + curr_timestamp = timestamps[i] + curr_name = names[i] + curr_attrs = attrs[i] + curr_process = processes[i] + + if curr_name == "MpiSend" or curr_name == "MpiIsend": + # Add current dataframe index, timestmap, and process to stack + if "receiver" in curr_attrs: + queue[curr_attrs["receiver"]].append( + (curr_df_index, curr_timestamp, curr_name, curr_process) + ) + elif curr_name == "MpiRecv" or curr_name == "MpiIrecv": + if "sender" in curr_attrs: + send_process = None + i = 0 + + # we want to iterate through the queue in order + # until we find the corresponding "send" event + while send_process != curr_attrs["sender"] and i < len( + queue[curr_process] + ): + send_df_index, send_timestamp, send_name, send_process = queue[ + curr_process + ][i] + i += 1 + + if send_process == curr_attrs["sender"] and i <= len( + queue[curr_process] + ): + # remove matched event from queue + del queue[curr_process][i - 1] + + # Fill in the lists with the matching values if event found + matching_events[send_df_index] = curr_df_index + matching_events[curr_df_index] = send_df_index + + matching_times[send_df_index] = curr_timestamp + matching_times[curr_df_index] = send_timestamp + + self.events["_matching_event"] = matching_events + self.events["_matching_timestamp"] = matching_times + + self.events = self.events.astype({"_matching_event": "Int32"}) + def calc_inc_metrics(self, columns=None): # if no columns are specified by the user, then we calculate # inclusive metrics for all the numeric columns in the trace From 5008a929078ffd4a62e148db314cfcb840e3bc0b Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Fri, 29 Mar 2024 03:02:48 -0400 Subject: [PATCH 34/42] fix bugs with bokeh 3 --- pipit/vis/util.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pipit/vis/util.py b/pipit/vis/util.py index 35d601fb..5db6f87d 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -298,20 +298,33 @@ def clamp(value, min_val, max_val): def get_html_tooltips(tooltips_dict): - html = "" + html = """ +
+
+
+ """ for k, v in tooltips_dict.items(): html += f""" -
- {k}:  - {v} +
+
+ {k}: +
+
+ {v} +
""" html += """ +
+ """ return html From cf72b4637b239576b12b76847845c2ad897f277c Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Fri, 29 Mar 2024 12:31:49 -0400 Subject: [PATCH 35/42] 2 arrows --- pipit/vis/timeline.py | 53 ++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 1f7bf6c6..4732ca97 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -10,7 +10,6 @@ HoverTool, OpenHead, WheelZoomTool, - TeeHead ) from bokeh.events import RangesUpdate, Tap from bokeh.plotting import figure @@ -38,11 +37,7 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): # Prepare data for plotting events = ( - trace.events[ - trace.events["Event Type"].isin( - ["Enter", "Instant"] - ) - ] + trace.events[trace.events["Event Type"].isin(["Enter", "Instant"])] .sort_values(by="time.inc", ascending=False) .copy(deep=False) ) @@ -86,7 +81,6 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): events.loc[events["Name"] == "MpiIsendComplete", "first_letter"] = "ISC" events.loc[events["Name"] == "MpiCollectiveBegin", "first_letter"] = "CB" events.loc[events["Name"] == "MpiCollectiveEnd", "first_letter"] = "CE" - return events, y_tuples, num_ys @@ -295,20 +289,27 @@ def plot_timeline( for i in range(len(sends)): p.add_layout( Arrow( - end=OpenHead(line_color="#28282B", line_width=1.5, size=4, line_alpha=0.8), + end=OpenHead( + line_color="#28282B", line_width=1.5, size=4, line_alpha=0.8 + ), line_color="#28282B", line_width=1.5, x_start=sends["Timestamp (ns)"].iloc[i], - y_start=sends["y"].iloc[i] - 0.2 if show_depth else sends["y"].iloc[i], - x_end=events.loc[sends["_matching_event"].iloc[i]]["Timestamp (ns)"], - y_end=events.loc[sends["_matching_event"].iloc[i]]["y"] - - 0.2 - if show_depth - else events.loc[sends["_matching_event"].iloc[i]]["y"], + y_start=( + sends["y"].iloc[i] - 0.2 if show_depth else sends["y"].iloc[i] + ), + x_end=events.loc[sends["_matching_event"].iloc[i]][ + "Timestamp (ns)" + ], + y_end=( + events.loc[sends["_matching_event"].iloc[i]]["y"] - 0.2 + if show_depth + else events.loc[sends["_matching_event"].iloc[i]]["y"] + ), level="annotation", line_alpha=0.8, ) - ) + ) # Arrows for critical path if critical_path: @@ -316,14 +317,30 @@ def plot_timeline( for df in critical_dfs: # Draw arrows # TODO: can we vectorize this? - for i in range(len(df) - 1): + for i in range(len(df)): + # Step 1) Leave to Enter p.add_layout( Arrow( - end=OpenHead(line_color="black", line_width=2, size=8), + end=OpenHead(line_color="black", line_width=1.5, size=5), line_color="black", - line_width=2, + line_width=1.5, x_start=df["Timestamp (ns)"].iloc[i], y_start=df["Process"].iloc[i], + x_end=df["_matching_timestamp"].iloc[i], + y_end=df["Process"].iloc[i], + level="overlay", + ) + ) + + for i in range(len(df) - 1): + # Step 2) Enter to previous Leave + p.add_layout( + Arrow( + end=OpenHead(line_color="black", line_width=1.5, size=5), + line_color="black", + line_width=1.5, + x_start=df["_matching_timestamp"].iloc[i], + y_start=df["Process"].iloc[i], x_end=df["Timestamp (ns)"].iloc[i + 1], y_end=df["Process"].iloc[i + 1], level="overlay", From 2a4692c5ecf8db68f796dc5eaf590d48b12d80c7 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Fri, 29 Mar 2024 13:50:19 -0400 Subject: [PATCH 36/42] hatch pattern --- pipit/vis/timeline.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 4732ca97..a1d76479 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -86,7 +86,8 @@ def prepare_data(trace: pp.Trace, show_depth: bool, instant_events: bool): def update_cds( - event: RangesUpdate, + x0: float, + x1: float, events: pd.DataFrame, instant_events: bool, hbar_source: ColumnDataSource, @@ -98,10 +99,6 @@ def update_cds( Called when user zooms or pans the timeline (and once initially). """ - - x0 = event.x0 if event is not None else events["Timestamp (ns)"].min() - x1 = event.x1 if event is not None else events["Timestamp (ns)"].max() - x0 = x0 - (x1 - x0) * 0.25 x1 = x1 + (x1 - x0) * 0.25 @@ -199,6 +196,8 @@ def plot_timeline( instant_events: bool = False, critical_path: bool = False, messages: str = "click", + x_start: float = None, + x_end: float = None, ): """ Displays the events of a trace on a timeline. @@ -213,6 +212,8 @@ def plot_timeline( critical_path: Whether to show the critical path. NOTE: critical_path currently only works when show_depth==False. TODO: make it work with show_depth=True. show_messages: Whether to show MPI messages. Can be "click" (default), or "all". + x_start: The start time of the x-axis range. + x_end: The end time of the x-axis range. Returns: The Bokeh plot. @@ -220,7 +221,7 @@ def plot_timeline( # Prepare data to be plotted events, y_tuples, num_ys = prepare_data(trace, show_depth, instant_events) - + # Define the 3 data sources (Bokeh ColumnDataSource) hbar_source = ColumnDataSource(events.head(0)) scatter_source = ColumnDataSource(events.head(0)) @@ -231,11 +232,14 @@ def plot_timeline( ) # Create Bokeh plot - plot_height = 120 + 22 * num_ys - min_ts, max_ts = events["Timestamp (ns)"].min(), events["Timestamp (ns)"].max() + if x_start is None: + x_start = events["Timestamp (ns)"].min() + if x_end is None: + x_end = events["Timestamp (ns)"].max() + (events["Timestamp (ns)"].max() - events["Timestamp (ns)"].min()) * 0.05 + plot_height = 120 + 22 * num_ys p = figure( - x_range=(min_ts, max_ts + (max_ts - min_ts) * 0.05), + x_range=(x_start, x_end), y_range=(num_ys - 0.5, -0.5), x_axis_location="above", tools="hover,xpan,reset,xbox_zoom,xwheel_zoom,save", @@ -315,6 +319,19 @@ def plot_timeline( if critical_path: critical_dfs = trace.critical_path_analysis() for df in critical_dfs: + # Draw hatch pattern + p.hbar( + left="Timestamp (ns)", + right="_matching_timestamp", + y="Process", + height=0.8, + source=df, + fill_color=None, + line_color=None, + hatch_color="white", + hatch_pattern="right_diagonal_line", + ) + # Draw arrows # TODO: can we vectorize this? for i in range(len(df)): @@ -432,7 +449,7 @@ def plot_timeline( p.on_event( RangesUpdate, lambda event: update_cds( - event, events, instant_events, hbar_source, scatter_source + event.x0, event.x1, events, instant_events, hbar_source, scatter_source ), ) @@ -440,7 +457,7 @@ def plot_timeline( p.on_event(Tap, lambda event: tap_callback(event, events, trace, show_depth, p)) # Make initial call to callback - update_cds(None, events, instant_events, hbar_source, scatter_source) + update_cds(x_start, x_end, events, instant_events, hbar_source, scatter_source) # Return plot return show(p) From 7a1a4fd0d9509c44c6456463612261ae28a03958 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Fri, 29 Mar 2024 14:18:43 -0400 Subject: [PATCH 37/42] use old critical path arrow scheme --- pipit/vis/timeline.py | 49 +++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index a1d76479..198fc55e 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -333,36 +333,49 @@ def plot_timeline( ) # Draw arrows - # TODO: can we vectorize this? - for i in range(len(df)): - # Step 1) Leave to Enter - p.add_layout( - Arrow( - end=OpenHead(line_color="black", line_width=1.5, size=5), - line_color="black", - line_width=1.5, - x_start=df["Timestamp (ns)"].iloc[i], - y_start=df["Process"].iloc[i], - x_end=df["_matching_timestamp"].iloc[i], - y_end=df["Process"].iloc[i], - level="overlay", - ) - ) - for i in range(len(df) - 1): - # Step 2) Enter to previous Leave p.add_layout( Arrow( end=OpenHead(line_color="black", line_width=1.5, size=5), line_color="black", line_width=1.5, - x_start=df["_matching_timestamp"].iloc[i], + x_start=df["Timestamp (ns)"].iloc[i], y_start=df["Process"].iloc[i], x_end=df["Timestamp (ns)"].iloc[i + 1], y_end=df["Process"].iloc[i + 1], level="overlay", ) ) + # TODO: can we vectorize this? + # # Step 1) Leave to Enter + # for i in range(len(df)): + # p.add_layout( + # Arrow( + # end=OpenHead(line_color="black", line_width=1.5, size=5), + # line_color="black", + # line_width=1.5, + # x_start=df["Timestamp (ns)"].iloc[i], + # y_start=df["Process"].iloc[i], + # x_end=df["_matching_timestamp"].iloc[i], + # y_end=df["Process"].iloc[i], + # level="overlay", + # ) + # ) + + # # Step 2) Enter to previous Leave + # for i in range(len(df) - 1): + # p.add_layout( + # Arrow( + # end=OpenHead(line_color="black", line_width=1.5, size=5), + # line_color="black", + # line_width=1.5, + # x_start=df["_matching_timestamp"].iloc[i], + # y_start=df["Process"].iloc[i], + # x_end=df["Timestamp (ns)"].iloc[i + 1], + # y_end=df["Process"].iloc[i + 1], + # level="overlay", + # ) + # ) # Additional plot config p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] From ce010e01088b7d0d7890f1728ab01812f8b5f764 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Sat, 30 Mar 2024 01:08:45 -0400 Subject: [PATCH 38/42] format + js bugfix --- pipit/vis/timeline.py | 13 ++++++++----- pipit/vis/util.py | 6 +++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 198fc55e..36363c4a 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -221,7 +221,7 @@ def plot_timeline( # Prepare data to be plotted events, y_tuples, num_ys = prepare_data(trace, show_depth, instant_events) - + # Define the 3 data sources (Bokeh ColumnDataSource) hbar_source = ColumnDataSource(events.head(0)) scatter_source = ColumnDataSource(events.head(0)) @@ -235,9 +235,12 @@ def plot_timeline( if x_start is None: x_start = events["Timestamp (ns)"].min() if x_end is None: - x_end = events["Timestamp (ns)"].max() + (events["Timestamp (ns)"].max() - events["Timestamp (ns)"].min()) * 0.05 + x_end = ( + events["Timestamp (ns)"].max() + + (events["Timestamp (ns)"].max() - events["Timestamp (ns)"].min()) * 0.05 + ) - plot_height = 120 + 22 * num_ys + plot_height = 140 + 22 * num_ys p = figure( x_range=(x_start, x_end), y_range=(num_ys - 0.5, -0.5), @@ -336,9 +339,9 @@ def plot_timeline( for i in range(len(df) - 1): p.add_layout( Arrow( - end=OpenHead(line_color="black", line_width=1.5, size=5), + end=OpenHead(line_color="black", line_width=2, size=9), line_color="black", - line_width=1.5, + line_width=2, x_start=df["Timestamp (ns)"].iloc[i], y_start=df["Process"].iloc[i], x_end=df["Timestamp (ns)"].iloc[i + 1], diff --git a/pipit/vis/util.py b/pipit/vis/util.py index 5db6f87d..fb86b676 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -101,6 +101,7 @@ def format_time(n: float) -> str: var str = ""; if (ms) str += ms + "ms "; if (us) str += Math.round(us) + "us"; + else str += "0us"; return str; } @@ -114,6 +115,7 @@ def format_time(n: float) -> str: if (ms) str += ms + "ms "; if (us) str += us + "us "; if (ns) str += ns + "ns"; + else if (!us) str += "0ns"; return str; """ @@ -375,7 +377,7 @@ def get_height(num_yticks, height_per_tick=400): LIGHT = [ "#aec7e8", - "#ffbb78", + # "#ffbb78", # reserved for sim_life_1d "#98df8a", "#ff9896", "#c5b0d5", @@ -423,6 +425,8 @@ def get_palette(trace, scale=None): palette["MPI_Waitall"] = "#c7c7c7" palette["Idle"] = "#c7c7c7" + palette["sim_life_1d"] = "#ffbb78" + dark_index = 0 light_index = 0 From c78232f3d7eab2271fa253f4ca908d7ed93f098f Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Sat, 30 Mar 2024 01:08:55 -0400 Subject: [PATCH 39/42] add comm over time --- pipit/trace.py | 9 ++++++++ pipit/vis/__init__.py | 6 ++++- pipit/vis/core.py | 51 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/pipit/trace.py b/pipit/trace.py index cc867a40..54460c1b 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -952,6 +952,15 @@ def plot_message_histogram(self, bins=20, *args, **kwargs): # Return the Bokeh plot return plot_message_histogram(data, *args, **kwargs) + def plot_comm_over_time(self, output="size", message_type="send", *args, **kwargs): + from .vis import plot_comm_over_time + + # Generate the data + data = self.comm_over_time(output=output, message_type=message_type, *args, **kwargs) + + # Return the Bokeh plot + return plot_comm_over_time(data, message_type=message_type, output=output) + def plot_timeline(self, *args, **kwargs): from .vis import plot_timeline diff --git a/pipit/vis/__init__.py b/pipit/vis/__init__.py index 5119ca11..733cc0f0 100644 --- a/pipit/vis/__init__.py +++ b/pipit/vis/__init__.py @@ -1,2 +1,6 @@ -from .core import plot_comm_matrix, plot_message_histogram # noqa: F401 +from .core import ( + plot_comm_matrix, + plot_message_histogram, + plot_comm_over_time, +) # noqa: F401 from .timeline import plot_timeline # noqa: F401 diff --git a/pipit/vis/core.py b/pipit/vis/core.py index 7b413a12..e1bd511e 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -14,6 +14,8 @@ get_size_hover_formatter, get_size_tick_formatter, show, + get_time_tick_formatter, + get_time_hover_formatter, ) @@ -146,3 +148,52 @@ def plot_message_histogram( # Return plot return show(p, return_fig=return_fig) + + +def plot_comm_over_time(data, output, message_type, return_fig=False): + """Plots the trace's communication over time. + + Args: + data (hist, edges): Histogram and edges + output (str): Specifies whether the matrix contains "size" or "count" values. + message_type (str): Specifies whether the message is "send" or "receive". + return_fig (bool, optional): Specifies whether to return the Bokeh figure + object. Defaults to False, which displays the result and returns nothing. + + Returns: + Bokeh figure object if return_fig, None otherwise + """ + + hist, edges = data + is_size = output == "size" + + p = figure( + x_axis_label="Time", + y_axis_label="Total volume sent" if is_size else "Number of messages", + tools="hover,save", + ) + p.y_range.start = 0 + p.xaxis.formatter = get_time_tick_formatter() + p.yaxis.formatter = get_size_tick_formatter() + + p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:], line_color="white") + + hover = p.select(HoverTool) + hover.tooltips = ( + [ + ("Bin", "@left{custom} - @right{custom}"), + ("Total volume sent:", "@top{custom}"), + ] + if is_size + else [ + ("Bin", "@left{custom} - @right{custom}"), + ("number of messages:", "@top"), + ] + ) + hover.formatters = { + "@left": get_time_hover_formatter(), + "@right": get_time_hover_formatter(), + "@top": get_size_hover_formatter(), + } + + return show(p, return_fig=return_fig) From 63d934dc632ac8755fb7123960cddab1e077c24e Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 2 Apr 2024 16:04:19 -0400 Subject: [PATCH 40/42] minor --- pipit/vis/timeline.py | 50 ++++++++++--------------------------------- pipit/vis/util.py | 2 +- 2 files changed, 12 insertions(+), 40 deletions(-) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 36363c4a..76fba918 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -198,6 +198,8 @@ def plot_timeline( messages: str = "click", x_start: float = None, x_end: float = None, + width: int = None, + height: int = None, ): """ Displays the events of a trace on a timeline. @@ -214,6 +216,9 @@ def plot_timeline( show_messages: Whether to show MPI messages. Can be "click" (default), or "all". x_start: The start time of the x-axis range. x_end: The end time of the x-axis range. + width: The width of the plot. Default is None, which makes the plot full width. + height: The height of the plot. Default is None, which makes the plot adapt to the + number of ticks on the y-axis. Returns: The Bokeh plot. @@ -240,15 +245,16 @@ def plot_timeline( + (events["Timestamp (ns)"].max() - events["Timestamp (ns)"].min()) * 0.05 ) - plot_height = 140 + 22 * num_ys + height = height if height is not None else 140 + 30 * num_ys p = figure( x_range=(x_start, x_end), y_range=(num_ys - 0.5, -0.5), x_axis_location="above", tools="hover,xpan,reset,xbox_zoom,xwheel_zoom,save", output_backend="webgl", - height=min(500, plot_height), - sizing_mode="stretch_width", + height=min(500, height), + sizing_mode="stretch_width" if width is None else "fixed", + width=width, toolbar_location=None, x_axis_label="Time", ) @@ -296,11 +302,7 @@ def plot_timeline( for i in range(len(sends)): p.add_layout( Arrow( - end=OpenHead( - line_color="#28282B", line_width=1.5, size=4, line_alpha=0.8 - ), - line_color="#28282B", - line_width=1.5, + end=OpenHead(), x_start=sends["Timestamp (ns)"].iloc[i], y_start=( sends["y"].iloc[i] - 0.2 if show_depth else sends["y"].iloc[i] @@ -314,7 +316,6 @@ def plot_timeline( else events.loc[sends["_matching_event"].iloc[i]]["y"] ), level="annotation", - line_alpha=0.8, ) ) @@ -336,6 +337,7 @@ def plot_timeline( ) # Draw arrows + # TODO: can we vectorize this? for i in range(len(df) - 1): p.add_layout( Arrow( @@ -349,36 +351,6 @@ def plot_timeline( level="overlay", ) ) - # TODO: can we vectorize this? - # # Step 1) Leave to Enter - # for i in range(len(df)): - # p.add_layout( - # Arrow( - # end=OpenHead(line_color="black", line_width=1.5, size=5), - # line_color="black", - # line_width=1.5, - # x_start=df["Timestamp (ns)"].iloc[i], - # y_start=df["Process"].iloc[i], - # x_end=df["_matching_timestamp"].iloc[i], - # y_end=df["Process"].iloc[i], - # level="overlay", - # ) - # ) - - # # Step 2) Enter to previous Leave - # for i in range(len(df) - 1): - # p.add_layout( - # Arrow( - # end=OpenHead(line_color="black", line_width=1.5, size=5), - # line_color="black", - # line_width=1.5, - # x_start=df["_matching_timestamp"].iloc[i], - # y_start=df["Process"].iloc[i], - # x_end=df["Timestamp (ns)"].iloc[i + 1], - # y_end=df["Process"].iloc[i + 1], - # level="overlay", - # ) - # ) # Additional plot config p.toolbar.active_scroll = p.select(dict(type=WheelZoomTool))[0] diff --git a/pipit/vis/util.py b/pipit/vis/util.py index fb86b676..e366c177 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -404,7 +404,7 @@ def get_height(num_yticks, height_per_tick=400): def get_palette(trace, scale=None): funcs = trace.events[trace.events["Event Type"] == "Enter"] # names = funcs["Name"].unique().tolist() - names = reversed(trace.flat_profile(["time.inc"]).index.tolist()) + names = reversed(trace.flat_profile(["time.exc"]).index.tolist()) depths = ( funcs.groupby("Name", observed=True)["_depth"] From 326f11b88f06424f9ed5869e1e13bf82a1856f45 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 2 Apr 2024 16:22:25 -0400 Subject: [PATCH 41/42] minor --- pipit/trace.py | 15 ++++++++++++++- pipit/vis/timeline.py | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pipit/trace.py b/pipit/trace.py index 54460c1b..ff1f1e35 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -295,6 +295,17 @@ def _match_messages(self): """ Matches corresponding MpiSend/MpiRecv and MpiIsend/MpiIrecv instant events """ + # if we've already matched -- i.e. if _matching_event column exists AND it's not empty for MpiSend/MpiRecv + if ( + "_matching_event" in self.events.columns + and not self.events[ + self.events["Name"].isin(["MpiSend", "MpiRecv", "MpiIsend", "MpiIrecv"]) + ]["_matching_event"] + .isnull() + .all() + ): + return + if "_matching_event" not in self.events.columns: self.events["_matching_event"] = None @@ -956,7 +967,9 @@ def plot_comm_over_time(self, output="size", message_type="send", *args, **kwarg from .vis import plot_comm_over_time # Generate the data - data = self.comm_over_time(output=output, message_type=message_type, *args, **kwargs) + data = self.comm_over_time( + output=output, message_type=message_type, *args, **kwargs + ) # Return the Bokeh plot return plot_comm_over_time(data, message_type=message_type, output=output) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 76fba918..9bc0b335 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -245,7 +245,7 @@ def plot_timeline( + (events["Timestamp (ns)"].max() - events["Timestamp (ns)"].min()) * 0.05 ) - height = height if height is not None else 140 + 30 * num_ys + height = height if height is not None else 120 + 30 * num_ys p = figure( x_range=(x_start, x_end), y_range=(num_ys - 0.5, -0.5), From ad7495b0ae5156fbc736086133160b8210e53112 Mon Sep 17 00:00:00 2001 From: Rakrish Dhakal Date: Tue, 2 Apr 2024 19:12:11 -0400 Subject: [PATCH 42/42] add comm_by_process plot --- pipit/trace.py | 9 +++++ pipit/vis/__init__.py | 1 + pipit/vis/core.py | 92 ++++++++++++++++++++++++++++++++++++++++++- pipit/vis/timeline.py | 11 ++++-- pipit/vis/util.py | 2 +- 5 files changed, 109 insertions(+), 6 deletions(-) diff --git a/pipit/trace.py b/pipit/trace.py index ff1f1e35..ac4b493f 100644 --- a/pipit/trace.py +++ b/pipit/trace.py @@ -974,6 +974,15 @@ def plot_comm_over_time(self, output="size", message_type="send", *args, **kwarg # Return the Bokeh plot return plot_comm_over_time(data, message_type=message_type, output=output) + def plot_comm_by_process(self, output="size", *args, **kwargs): + from .vis import plot_comm_by_process + + # Generate the data + data = self.comm_by_process(output=output) + + # Return the Bokeh plot + return plot_comm_by_process(data, output=output, *args, **kwargs) + def plot_timeline(self, *args, **kwargs): from .vis import plot_timeline diff --git a/pipit/vis/__init__.py b/pipit/vis/__init__.py index 733cc0f0..03f443fc 100644 --- a/pipit/vis/__init__.py +++ b/pipit/vis/__init__.py @@ -2,5 +2,6 @@ plot_comm_matrix, plot_message_histogram, plot_comm_over_time, + plot_comm_by_process, ) # noqa: F401 from .timeline import plot_timeline # noqa: F401 diff --git a/pipit/vis/core.py b/pipit/vis/core.py index e1bd511e..5c37665a 100644 --- a/pipit/vis/core.py +++ b/pipit/vis/core.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd from bokeh.models import ( ColorBar, HoverTool, @@ -7,6 +8,8 @@ NumeralTickFormatter, ) from bokeh.plotting import figure +from bokeh.models import BasicTicker +from bokeh.transform import dodge from .util import ( clamp, @@ -150,7 +153,9 @@ def plot_message_histogram( return show(p, return_fig=return_fig) -def plot_comm_over_time(data, output, message_type, return_fig=False): +def plot_comm_over_time( + data: tuple, output: str, message_type: str, return_fig: bool = False +): """Plots the trace's communication over time. Args: @@ -197,3 +202,88 @@ def plot_comm_over_time(data, output, message_type, return_fig=False): } return show(p, return_fig=return_fig) + + +def plot_comm_by_process( + data: pd.DataFrame, + output: str, + return_fig: bool = False, + width: int = None, + height: int = 600, +): + """ + Plots the trace's communication by process. + + Args: + data (pd.DataFrame): DataFrame containing the communication data. + output (str): Specifies whether the matrix contains "size" or "count" values. + return_fig (bool, optional): Specifies whether to return the Bokeh figure + object. Defaults to False, which displays the result and returns nothing. + width: The width of the plot. Default is None, which makes the plot full width. + height: The height of the plot. Default is 600. + + Returns: + Bokeh figure object if return_fig, None otherwise + """ + data.reset_index() + is_size = output == "size" + + p = figure( + x_range=(-0.5, len(data) - 0.5), + y_axis_label="Volume", + x_axis_label="Process", + tools="hover,save", + width=width, + sizing_mode="fixed" if width is not None else "stretch_width", + height=height, + ) + p.y_range.start = 0 + p.y_range.range_padding = 0.5 + + p.xgrid.visible = False + p.yaxis.formatter = get_size_tick_formatter() + p.xaxis.ticker = BasicTicker( + base=2, + desired_num_ticks=min(len(data), 16), + min_interval=1, + num_minor_ticks=0, + ) + + p.vbar( + x=dodge("Process", -0.1667, range=p.y_range), + top="Sent", + width=0.2, + source=data, + color="#1f77b4", + legend_label="Total sent", + ) + p.vbar( + x=dodge("Process", 0.1667, range=p.y_range), + top="Received", + width=0.2, + source=data, + color="#d62728", + legend_label="Total received", + ) + p.add_layout(p.legend[0], "below") + p.legend.orientation = "horizontal" + p.legend.location = "center" + + hover = p.select(HoverTool) + hover.tooltips = ( + [ + ("Bin", "@left{custom} - @right{custom}"), + ("Total volume sent:", "@top{custom}"), + ] + if is_size + else [ + ("Bin", "@left{custom} - @right{custom}"), + ("number of messages:", "@top"), + ] + ) + hover.formatters = { + "@Sent": get_size_hover_formatter(), + "@Received": get_size_hover_formatter(), + } + + return show(p, return_fig=return_fig) diff --git a/pipit/vis/timeline.py b/pipit/vis/timeline.py index 9bc0b335..0e4ece5e 100644 --- a/pipit/vis/timeline.py +++ b/pipit/vis/timeline.py @@ -200,6 +200,7 @@ def plot_timeline( x_end: float = None, width: int = None, height: int = None, + legend_nrows: int = None, ): """ Displays the events of a trace on a timeline. @@ -217,8 +218,10 @@ def plot_timeline( x_start: The start time of the x-axis range. x_end: The end time of the x-axis range. width: The width of the plot. Default is None, which makes the plot full width. - height: The height of the plot. Default is None, which makes the plot adapt to the - number of ticks on the y-axis. + height: The height of the plot. Default is None, which makes the plot adapt to + the number of ticks on the y-axis. + legend_nrows: The number of rows in the legend. Default is None, which makes the + legend adapt to the number of items. Returns: The Bokeh plot. @@ -245,7 +248,7 @@ def plot_timeline( + (events["Timestamp (ns)"].max() - events["Timestamp (ns)"].min()) * 0.05 ) - height = height if height is not None else 120 + 30 * num_ys + height = height if height is not None else 150 + 30 * num_ys p = figure( x_range=(x_start, x_end), y_range=(num_ys - 0.5, -0.5), @@ -398,7 +401,7 @@ def plot_timeline( p.add_layout(p.legend[0], "below") p.legend.orientation = "horizontal" p.legend.location = "center" - p.legend.nrows = 2 + p.legend.nrows = legend_nrows if legend_nrows is not None else "auto" # Hover config hover = p.select(HoverTool) diff --git a/pipit/vis/util.py b/pipit/vis/util.py index e366c177..e60df9b0 100644 --- a/pipit/vis/util.py +++ b/pipit/vis/util.py @@ -217,7 +217,7 @@ def get_percent_hover_formatter(): # TODO: maybe do this client side with transform def trimmed(names: pd.Series) -> pd.Series: return np.where( - names.str.len() < 30, names, names.str[0:20] + "..." + names.str[-5:] + names.str.len() < 30, names, names.str[0:10] + "..." + names.str[-5:] )