PerfFlowAspect Reader #101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed

r-yin wants to merge 17 commits into llnl:develop from r-yin:develop

docs/examples/read/perfflowaspect.py

-Original file line number
+Diff line change
@@ -0,0 +1,27 @@
+    #!/usr/bin/env python
+    #
+    # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other
+    # Hatchet Project Developers. See the top-level LICENSE file for details.
+    #
+    # SPDX-License-Identifier: MIT
+    import hatchet as ht
+    if __name__ == "__main__":
+        # pfa_file = "../../../hatchet/tests/data/perfflow.quartz1532.3570764-1iter.pfw"
+        pfa_file = "../../../hatchet/tests/data/laghos_1iter.pfw"
+        gf = ht.GraphFrame.from_perfflowaspect(pfa_file)
+        # Printout the DataFrame component of the GraphFrame.
+        print(gf.dataframe)
+        print(len(gf.graph.roots))
+        for i, node in enumerate(gf.graph.traverse()):
+            print(node._hatchet_nid, node, list(node.parents), list(node.children))
+        # Printout the graph component of the GraphFrame.
+        # Use "ts" as the metric column to be displayed
+        print(gf.tree(metric_column="ts"))

hatchet/graphframe.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -328,6 +328,13 @@ def from_hdf(filename, **kwargs): @@
             return HDF5Reader(filename).read(**kwargs)
+        @staticmethod
+        def from_perfflowaspect(filename):
+            # import this lazily to avoid circular dependencies
+            from .readers.perfflowaspect_reader import PerfFlowAspectReader
+            return PerfFlowAspectReader(filename).read()
         def to_hdf(self, filename, key="hatchet_graphframe", **kwargs):
             # import this lazily to avoid circular dependencies
             from .writers.hdf5_writer import HDF5Writer
@@ Expand Down @@

hatchet/readers/perfflowaspect_reader.py

-Original file line number
+Diff line change
@@ -0,0 +1,92 @@
+    import json
+    import pandas as pd
+    import hatchet.graphframe
+    from hatchet.node import Node
+    from hatchet.graph import Graph
+    from hatchet.frame import Frame
+    class PerfFlowAspectReader:
+        """Create a GraphFrame from JSON array format.
+        Return:
+            (GraphFrame): graphframe containing data from dictionaries
+        """
+        def __init__(self, filename):
+            """Read from a json string specification of a graphframe
+            json (string): Json specification of a graphframe.
+            """
+            with open(filename, "r") as file:
+                content = file.read()
+                self.spec_dict = json.loads(content)
+        def sort(self):
+            # Sort the spec_dict based on the end time (ts + dur) of each function
+            self.spec_dict = sorted(
+                self.spec_dict, key=lambda item: item["ts"] + item["dur"]
+            )
+        def read(self):
+            roots = []
+            node_mapping = {}  # Dictionary to keep track of the nodes
+            node_dicts = []
+            for item in self.spec_dict:
+                name = item["name"]
+                ts = item["ts"]
+                dur = item["dur"]
+                # Create a Frame and Node for the function
+                # Frame stores information about the node
+                # Node represents a node in the hierarchical graph structure
+                frame = Frame({"name": name, "type": "function", "ts": ts, "dur": dur})
+                node = Node(frame, parent=None, hnid=-1)
+                # check the relationships between node and roots
+                for root in reversed(roots):
+                    # if node is a parent of root node
+                    if (ts < root.frame["ts"]) and (
+                        ts + dur > root.frame["ts"] + root.frame["dur"]
+                    ):
+                        node.add_child(root)
+                        root.add_parent(node)
+                        roots.pop()
+                roots.append(node)
+                node_dict = dict(
+                    {
+                        "node": node,
+                        "name": name,
+                        "ts": ts,
+                        "dur": dur,
+                        "pid": item["pid"],
+                        "tid": item["tid"],
+                        "ph": item["ph"],
+                    }
+                )
+                node_dicts.append(node_dict)
+                # Store the Node object with its name for future reference
+                print("Add", name, "to node map")
+                node_mapping[name] = node
+            # Create the Graph object from the root nodes
+            graph = Graph(roots)
+            graph.enumerate_traverse()
+            dataframe = pd.DataFrame(data=node_dicts)
+            dataframe.set_index(["node"], inplace=True)
+            dataframe.sort_index(inplace=True)
+            exc_metrics = []
+            inc_metrics = []
+            for col in dataframe.columns:
+                if "(inc)" in col:
+                    inc_metrics.append(col)
+                else:
+                    exc_metrics.append(col)
+            return hatchet.graphframe.GraphFrame(graph, dataframe)

hatchet/tests/conftest.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -1168,3 +1168,94 @@ def caliper_ordered_json_dup(data_dir, tmpdir): @@
         tmpfile = os.path.join(str(tmpdir), "caliper-region-dup-node-order.json")
         return tmpfile
+    @pytest.fixture
+    def laghos_perfflowaspect_array(data_dir, tmpdir):
+        """Builds a temporary directory containing the laghos PerfFlowAspect file."""
+        pfa_dir = os.path.join(data_dir, "perfflowaspect-laghos")
+        pfa_file = os.path.join(pfa_dir, "laghos_1iter.pfw")
+        shutil.copy(pfa_file, str(tmpdir))
+        tmpfile = os.path.join(str(tmpdir), "laghos_1iter.pfw")
+        return tmpfile
+    @pytest.fixture
+    def foobar_perfflowaspect_array(data_dir, tmpdir):
+        """Builds a temporary directory containing the foobar PerfFlowAspect file."""
+        pfa_dir = os.path.join(data_dir, "perfflowaspect-foobar")
+        pfa_file = os.path.join(pfa_dir, "perfflow.quartz1532.3570764-1iter.pfw")
+        shutil.copy(pfa_file, str(tmpdir))
+        tmpfile = os.path.join(str(tmpdir), "perfflow.quartz1532.3570764-1iter.pfw")
+        return tmpfile
+    def write_file(data_dir):
+        filename = "ams_mpi_allranks.pfw"
+        pfa_dir = os.path.join(data_dir, "perfflowaspect-ams/ams_mpi_test1")
+        path = os.path.join(pfa_dir, filename)
+        print("RRR", path)
+        with open(pfa_dir + "/perfflow.lassen19.108800.pfw", "r") as ifile0:
+            nlines0 = len(ifile0.readlines())
+        with open(pfa_dir + "/perfflow.lassen19.108801.pfw", "r") as ifile1:
+            nlines1 = len(ifile1.readlines())
+        with open(pfa_dir + "/perfflow.lassen19.108802.pfw", "r") as ifile2:
+            nlines2 = len(ifile2.readlines())
+        with open(path, "w") as f:
+            f.write("[")
+            f.write("\n")
+            with open(pfa_dir + "/perfflow.lassen19.108800.pfw", "r") as ifile0:
+                count = 1
+                for line in ifile0:
+                    if count != 1 and count < nlines0:
+                        f.write(line.rstrip())
+                        if count == (nlines0 - 1):
+                            f.write(",")
+                        f.write("\n")
+                    count += 1
+            with open(pfa_dir + "/perfflow.lassen19.108801.pfw", "r") as ifile1:
+                count = 1
+                for line in ifile1:
+                    if count != 1 and count < nlines1:
+                        f.write(line.rstrip())
+                        if count == (nlines1 - 1):
+                            f.write(",")
+                        f.write("\n")
+                    count += 1
+            with open(pfa_dir + "/perfflow.lassen19.108802.pfw", "r") as ifile2:
+                count = 1
+                for line in ifile2:
+                    if count != 1 and count < nlines2:
+                        f.write(line.rstrip())
+                        f.write("\n")
+                    count += 1
+            f.write("]")
+        return path
+    @pytest.fixture
+    def ams_mpi_perfflowaspect_array(data_dir, tmpdir):
+        """Builds a temporary directory containing the AMS PerfFlowAspect file."""
+        pfa_dir = os.path.join(data_dir, "perfflowaspect-ams/ams_mpi_test1")
+        write_file(data_dir)
+        pfa_file = os.path.join(pfa_dir, "ams_mpi_allranks.pfw")
+        with open(pfa_file) as f:
+            s = f.read()
+        shutil.copy(pfa_file, str(tmpdir))
+        tmpfile = os.path.join(str(tmpdir), "ams_mpi_allranks.pfw")
+        return tmpfile

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

PerfFlowAspect Reader #101

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!