diff --git a/params_LinearMHDDriftkineticCC.py b/params_LinearMHDDriftkineticCC.py new file mode 100644 index 000000000..2b95f552d --- /dev/null +++ b/params_LinearMHDDriftkineticCC.py @@ -0,0 +1,116 @@ +from struphy import main +from struphy.fields_background import equils +from struphy.geometry import domains +from struphy.initial import perturbations +from struphy.io.options import BaseUnits, DerhamOptions, EnvironmentOptions, FieldsBackground, Time +from struphy.kinetic_background import maxwellians + +# import model, set verbosity +from struphy.models.hybrid import LinearMHDDriftkineticCC +from struphy.pic.utilities import ( + BinningPlot, + BoundaryParameters, + KernelDensityPlot, + LoadingParameters, + WeightsParameters, +) +from struphy.topology import grids + +# environment options +env = EnvironmentOptions(profiling_activated=True, profiling_trace=True) + +# units +base_units = BaseUnits() + +# time stepping +time_opts = Time() + +# geometry +domain = domains.Cuboid() + +# fluid equilibrium (can be used as part of initial conditions) +equil = equils.HomogenSlab() + +# grid +grid = grids.TensorProductGrid(Nel=(16, 4, 1)) + +# derham options +derham_opts = DerhamOptions() + +# light-weight model instance +model = LinearMHDDriftkineticCC() + +# species parameters +model.mhd.set_phys_params() +model.energetic_ions.set_phys_params() + +loading_params = LoadingParameters(ppc=1000) +weights_params = WeightsParameters() +boundary_params = BoundaryParameters() +model.energetic_ions.set_markers( + loading_params=loading_params, + weights_params=weights_params, + boundary_params=boundary_params, +) +model.energetic_ions.set_sorting_boxes() +model.energetic_ions.set_save_data() + +# propagator options +model.propagators.push_bxe.options = model.propagators.push_bxe.Options( + b_tilde=model.em_fields.b_field, +) +model.propagators.push_parallel.options = model.propagators.push_parallel.Options( + b_tilde=model.em_fields.b_field, +) +model.propagators.shearalfen_cc5d.options = model.propagators.shearalfen_cc5d.Options( + energetic_ions=model.energetic_ions.var, +) +model.propagators.magnetosonic.options = model.propagators.magnetosonic.Options( + b_field=model.em_fields.b_field, +) +model.propagators.cc5d_density.options = model.propagators.cc5d_density.Options( + energetic_ions=model.energetic_ions.var, + b_tilde=model.em_fields.b_field, +) +model.propagators.cc5d_gradb.options = model.propagators.cc5d_gradb.Options( + b_tilde=model.em_fields.b_field, +) +model.propagators.cc5d_curlb.options = model.propagators.cc5d_curlb.Options( + b_tilde=model.em_fields.b_field, +) + +# background, perturbations and initial conditions +model.mhd.velocity.add_background(FieldsBackground()) +model.mhd.velocity.add_perturbation(perturbations.TorusModesCos(given_in_basis="v", comp=0)) +model.mhd.velocity.add_perturbation(perturbations.TorusModesCos(given_in_basis="v", comp=1)) +model.mhd.velocity.add_perturbation(perturbations.TorusModesCos(given_in_basis="v", comp=2)) +maxwellian_1 = maxwellians.GyroMaxwellian2D(n=(1.0, None), equil=equil) +maxwellian_2 = maxwellians.GyroMaxwellian2D(n=(0.1, None), equil=equil) +background = maxwellian_1 + maxwellian_2 +model.energetic_ions.var.add_background(background) + +# if .add_initial_condition is not called, the background is the kinetic initial condition +perturbation = perturbations.TorusModesCos() +maxwellian_1pt = maxwellians.GyroMaxwellian2D(n=(1.0, perturbation), equil=equil) +init = maxwellian_1pt + maxwellian_2 +model.energetic_ions.var.add_initial_condition(init) + +# optional: exclude variables from saving +# model.energetic_ions.var.save_data = False + +if __name__ == "__main__": + # start run + verbose = True + + main.run( + model, + params_path=__file__, + env=env, + base_units=base_units, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + verbose=verbose, + ) diff --git a/pyproject.toml b/pyproject.toml index 201452c06..4f3149bdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ 'pytest', 'pytest-mpi', 'line_profiler', + 'scope-profiler==0.1.6', ] [project.license] diff --git a/src/struphy/io/options.py b/src/struphy/io/options.py index 9be8b3249..e99d31762 100644 --- a/src/struphy/io/options.py +++ b/src/struphy/io/options.py @@ -336,6 +336,12 @@ class EnvironmentOptions: num_clones: int, optional Number of domain clones (default=1) + + profiling_activated: bool, optional + Activate profiling with scope-profiler (default=False) + + profiling_trace: bool, optional + Save time-trace of each profiling region (default=False) """ out_folders: str = os.getcwd() @@ -345,6 +351,8 @@ class EnvironmentOptions: save_step: int = 1 sort_step: int = 0 num_clones: int = 1 + profiling_activated: bool = False + profiling_trace: bool = False def __post_init__(self): self.path_out: str = os.path.join(self.out_folders, self.sim_folder) diff --git a/src/struphy/main.py b/src/struphy/main.py index bae521086..732a83933 100644 --- a/src/struphy/main.py +++ b/src/struphy/main.py @@ -14,6 +14,7 @@ from psydac.ddm.mpi import MockMPI from psydac.ddm.mpi import mpi as MPI from pyevtk.hl import gridToVTK +from scope_profiler import ProfileManager from struphy.fields_background.base import FluidEquilibrium, FluidEquilibriumWithB from struphy.fields_background.equils import HomogenSlab @@ -36,7 +37,6 @@ post_process_markers, post_process_n_sph, ) -from struphy.profiling.profiling import ProfileManager from struphy.topology import grids from struphy.topology.grids import TensorProductGrid from struphy.utils.clone_config import CloneConfig @@ -69,6 +69,12 @@ def run( Absolute path to .py parameter file. """ + ProfileManager.setup( + profiling_activated=env.profiling_activated, + time_trace=env.profiling_trace, + use_likwid=False, + ) + if isinstance(MPI, MockMPI): comm = None rank = 0 @@ -407,6 +413,8 @@ def run( if clone_config is not None: clone_config.free() + ProfileManager.finalize() + def pproc( path: str, @@ -831,154 +839,3 @@ def load_data(path: str) -> SimData: print(f" {kkk}") return simdata - - -if __name__ == "__main__": - import argparse - import os - - import struphy - import struphy.utils.utils as utils - from struphy.profiling.profiling import ( - ProfileManager, - ProfilingConfig, - pylikwid_markerclose, - pylikwid_markerinit, - ) - - # Read struphy state file - state = utils.read_state() - o_path = state["o_path"] - - parser = argparse.ArgumentParser(description="Run an Struphy model.") - - # model - parser.add_argument( - "model", - type=str, - nargs="?", - default=None, - metavar="MODEL", - help="the name of the model to run (default: None)", - ) - - # input (absolute path) - parser.add_argument( - "-i", - "--input", - type=str, - metavar="FILE", - help="absolute path of parameter file", - ) - - # output (absolute path) - parser.add_argument( - "-o", - "--output", - type=str, - metavar="DIR", - help="absolute path of output folder (default=/sim_1)", - default=os.path.join(o_path, "sim_1"), - ) - - # restart - parser.add_argument( - "-r", - "--restart", - help="restart the simulation in the output folder specified under -o", - action="store_true", - ) - - # max_runtime - parser.add_argument( - "--max-runtime", - type=int, - metavar="N", - help="maximum wall-clock time of program in minutes (default=300)", - default=300, - ) - - # save step - parser.add_argument( - "-s", - "--save-step", - type=int, - metavar="N", - help="how often to skip data saving (default=1, which means data is saved every time step)", - default=1, - ) - - # sort step - parser.add_argument( - "--sort-step", - type=int, - metavar="N", - help="sort markers in memory every N time steps (default=0, which means markers are sorted only at the start of simulation)", - default=0, - ) - - parser.add_argument( - "--nclones", - type=int, - metavar="N", - help="number of domain clones (default=1)", - default=1, - ) - - # verbosity (screen output) - parser.add_argument( - "-v", - "--verbose", - help="supress screen output during time integration", - action="store_true", - ) - - parser.add_argument( - "--likwid", - help="run with Likwid", - action="store_true", - ) - - parser.add_argument( - "--time-trace", - help="Measure time traces for each call of the regions measured with ProfileManager", - action="store_true", - ) - - parser.add_argument( - "--sample-duration", - help="Duration of samples when measuring time traces with ProfileManager", - default=1.0, - ) - - parser.add_argument( - "--sample-interval", - help="Time between samples when measuring time traces with ProfileManager", - default=1.0, - ) - - args = parser.parse_args() - config = ProfilingConfig() - config.likwid = args.likwid - config.sample_duration = float(args.sample_duration) - config.sample_interval = float(args.sample_interval) - config.time_trace = args.time_trace - config.simulation_label = "" - pylikwid_markerinit() - with ProfileManager.profile_region("main"): - # solve the model - run( - args.model, - args.input, - args.output, - restart=args.restart, - runtime=args.runtime, - save_step=args.save_step, - verbose=args.verbose, - sort_step=args.sort_step, - num_clones=args.nclones, - ) - pylikwid_markerclose() - if config.time_trace: - ProfileManager.print_summary() - ProfileManager.save_to_pickle(os.path.join(args.output, "profiling_time_trace.pkl")) diff --git a/src/struphy/models/base.py b/src/struphy/models/base.py index e75d281e0..0f7d90401 100644 --- a/src/struphy/models/base.py +++ b/src/struphy/models/base.py @@ -12,6 +12,7 @@ from psydac.ddm.mpi import MockMPI from psydac.ddm.mpi import mpi as MPI from psydac.linalg.stencil import StencilVector +from scope_profiler import ProfileManager import struphy from struphy.feec.basis_projection_ops import BasisProjectionOperators @@ -39,7 +40,6 @@ from struphy.models.variables import FEECVariable, PICVariable, SPHVariable from struphy.pic import particles from struphy.pic.base import Particles -from struphy.profiling.profiling import ProfileManager from struphy.propagators.base import Propagator from struphy.topology.grids import TensorProductGrid from struphy.utils.clone_config import CloneConfig @@ -692,7 +692,7 @@ def integrate(self, dt, split_algo="LieTrotter"): for propagator in self.prop_list: prop_name = propagator.__class__.__name__ - with ProfileManager.profile_region(prop_name): + with ProfileManager.profile_region("prop: " + prop_name): propagator(dt) # second order in time @@ -701,17 +701,17 @@ def integrate(self, dt, split_algo="LieTrotter"): for propagator in self.prop_list[:-1]: prop_name = type(propagator).__name__ - with ProfileManager.profile_region(prop_name): + with ProfileManager.profile_region("prop: " + prop_name): propagator(dt / 2) propagator = self.prop_list[-1] prop_name = type(propagator).__name__ - with ProfileManager.profile_region(prop_name): + with ProfileManager.profile_region("prop: " + prop_name): propagator(dt) for propagator in self.prop_list[:-1][::-1]: prop_name = type(propagator).__name__ - with ProfileManager.profile_region(prop_name): + with ProfileManager.profile_region("prop: " + prop_name): propagator(dt / 2) else: diff --git a/src/struphy/pic/accumulation/particles_to_grid.py b/src/struphy/pic/accumulation/particles_to_grid.py index 06d67a6df..31143a029 100644 --- a/src/struphy/pic/accumulation/particles_to_grid.py +++ b/src/struphy/pic/accumulation/particles_to_grid.py @@ -4,6 +4,7 @@ from psydac.ddm.mpi import mpi as MPI from psydac.linalg.block import BlockVector from psydac.linalg.stencil import StencilMatrix, StencilVector +from scope_profiler import ProfileManager import struphy.pic.accumulation.accum_kernels as accums import struphy.pic.accumulation.accum_kernels_gc as accums_gc @@ -12,7 +13,6 @@ from struphy.kernel_arguments.pusher_args_kernels import DerhamArguments, DomainArguments from struphy.pic.accumulation.filter import AccumFilter, FilterParameters from struphy.pic.base import Particles -from struphy.profiling.profiling import ProfileManager from struphy.utils.pyccel import Pyccelkernel diff --git a/src/struphy/pic/pushing/pusher.py b/src/struphy/pic/pushing/pusher.py index 14c756b31..235fe6cba 100644 --- a/src/struphy/pic/pushing/pusher.py +++ b/src/struphy/pic/pushing/pusher.py @@ -3,10 +3,10 @@ import cunumpy as xp from line_profiler import profile from psydac.ddm.mpi import mpi as MPI +from scope_profiler import ProfileManager from struphy.kernel_arguments.pusher_args_kernels import DerhamArguments, DomainArguments from struphy.pic.base import Particles -from struphy.profiling.profiling import ProfileManager from struphy.utils.pyccel import Pyccelkernel diff --git a/src/struphy/post_processing/likwid/plot_time_traces.py b/src/struphy/post_processing/likwid/plot_time_traces.py index 7451833cb..f4efbe99a 100644 --- a/src/struphy/post_processing/likwid/plot_time_traces.py +++ b/src/struphy/post_processing/likwid/plot_time_traces.py @@ -215,6 +215,13 @@ def plot_gantt_chart_plotly( # Create plotly figure fig = go.Figure() for bar in bars: + if "kernel" in bar["Task"]: + color = "blue" + elif "prop" in bar["Task"]: + color = "red" + else: + color = "black" + # print(bar["Task"]) fig.add_trace( go.Bar( x=[bar["Duration"]], @@ -222,7 +229,8 @@ def plot_gantt_chart_plotly( base=[bar["Start"]], orientation="h", name=bar["Rank"], - marker_color=rank_color_map[bar["Rank"]], + # marker_color=rank_color_map[bar["Rank"]], + marker_color=color, hovertemplate=f"Rank: {bar['Rank']}
Start: {bar['Start']:.3f}s
Duration: {bar['Duration']:.3f}s", ), ) @@ -405,8 +413,10 @@ def plot_gantt_chart( # path = os.path.abspath(args.path) # Convert to absolute path # simulations = parser.simulations - paths = [os.path.join(o_path, simulation, "profiling_time_trace.pkl") for simulation in args.simulations] + # paths = [os.path.join(o_path, simulation, "profiling_time_trace.pkl") for simulation in args.simulations] + paths = [os.path.join(simulation, "profiling_time_trace.pkl") for simulation in args.simulations] # Plot the time trace - plot_time_vs_duration(paths=paths, output_path=o_path, groups_include=args.groups, groups_skip=args.groups_skip) - plot_gantt_chart(paths=paths, output_path=o_path, groups_include=args.groups, groups_skip=args.groups_skip) + plot_gantt_chart_plotly(path=paths[0], output_path=o_path, groups_include=args.groups, groups_skip=args.groups_skip) + # plot_time_vs_duration(paths=paths, output_path=o_path, groups_include=args.groups, groups_skip=args.groups_skip) + # plot_gantt_chart(paths=paths, output_path=o_path, groups_include=args.groups, groups_skip=args.groups_skip) diff --git a/src/struphy/profiling/__init__.py b/src/struphy/profiling/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/struphy/profiling/profiling.py b/src/struphy/profiling/profiling.py deleted file mode 100644 index e96749614..000000000 --- a/src/struphy/profiling/profiling.py +++ /dev/null @@ -1,347 +0,0 @@ -""" -profiling.py - -This module provides a centralized profiling configuration and management system -using LIKWID markers. It includes: -- A singleton class for managing profiling configuration. -- A context manager for profiling specific code regions. -- Initialization and cleanup functions for LIKWID markers. -- Convenience functions for setting and getting the profiling configuration. - -LIKWID is imported only when profiling is enabled to avoid unnecessary overhead. -""" - -import os -import pickle - -# Import the profiling configuration class and context manager -from functools import lru_cache - -import cunumpy as xp -from psydac.ddm.mpi import mpi as MPI - - -@lru_cache(maxsize=None) # Cache the import result to avoid repeated imports -def _import_pylikwid(): - import pylikwid - - return pylikwid - - -class ProfilingConfig: - """Singleton class for managing global profiling configuration.""" - - _instance = None - - def __new__(cls): - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance.likwid = False # Default value (profiling disabled) - cls._instance.simulation_label = "" - cls._instance.sample_duration = None - cls._instance.sample_interval = None - cls._instance.time_trace = False - return cls._instance - - @property - def likwid(self): - return self._likwid - - @likwid.setter - def likwid(self, value): - self._likwid = value - - @property - def simulation_label(self): - return self._simulation_label - - @simulation_label.setter - def simulation_label(self, value): - self._simulation_label = value - - @property - def sample_duration(self): - return self._sample_duration - - @sample_duration.setter - def sample_duration(self, value): - self._sample_duration = value - - @property - def sample_interval(self): - return self._sample_interval - - @sample_interval.setter - def sample_interval(self, value): - self._sample_interval = value - - @property - def time_trace(self): - return self._time_trace - - @time_trace.setter - def time_trace(self, value): - if value: - assert self.sample_interval is not None, "sample_interval must be set first!" - assert self.sample_duration is not None, "sample_duration must be set first!" - self._time_trace = value - - -class ProfileManager: - """ - Singleton class to manage and track all ProfileRegion instances. - """ - - _regions = {} - - @classmethod - def profile_region(cls, region_name): - """ - Get an existing ProfileRegion by name, or create a new one if it doesn't exist. - - Parameters - ---------- - region_name: str - The name of the profiling region. - - Returns - ------- - ProfileRegion: The ProfileRegion instance. - """ - if region_name in cls._regions: - return cls._regions[region_name] - else: - # Check if time profiling is enabled - _config = ProfilingConfig() - # Create and register a new ProfileRegion - new_region = ProfileRegion(region_name, time_trace=_config.time_trace) - cls._regions[region_name] = new_region - return new_region - - @classmethod - def get_region(cls, region_name): - """ - Get a registered ProfileRegion by name. - - Parameters - ---------- - region_name: str - The name of the profiling region. - - Returns - ------- - ProfileRegion or None: The registered ProfileRegion instance or None if not found. - """ - return cls._regions.get(region_name) - - @classmethod - def get_all_regions(cls): - """ - Get all registered ProfileRegion instances. - - Returns - ------- - dict: Dictionary of all registered ProfileRegion instances. - """ - return cls._regions - - @classmethod - def save_to_pickle(cls, file_path): - """ - Save profiling data to a single file using pickle and NumPy arrays in parallel. - - Parameters - ---------- - file_path: str - Path to the file where data will be saved. - """ - - _config = ProfilingConfig() - if not _config.time_trace: - print("time_trace is not set to True --> Time traces are not measured --> Skip saving...") - return - - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - # size = comm.Get_size() - - # Prepare the data to be gathered - local_data = {} - for name, region in cls._regions.items(): - local_data[name] = { - "ncalls": region.ncalls, - "durations": xp.array(region.durations, dtype=xp.float64), - "start_times": xp.array(region.start_times, dtype=xp.float64), - "end_times": xp.array(region.end_times, dtype=xp.float64), - "config": { - "likwid": region.config.likwid, - "simulation_label": region.config.simulation_label, - "sample_duration": region.config.sample_duration, - "sample_interval": region.config.sample_interval, - }, - } - - # Gather all data at the root process (rank 0) - all_data = comm.gather(local_data, root=0) - - # Save the likwid configuration data - likwid_data = {} - if ProfilingConfig().likwid: - pylikwid = _import_pylikwid() - - # Gather LIKWID-specific information - pylikwid.inittopology() - likwid_data["cpu_info"] = pylikwid.getcpuinfo() - likwid_data["cpu_topology"] = pylikwid.getcputopology() - pylikwid.finalizetopology() - - likwid_data["numa_info"] = pylikwid.initnuma() - pylikwid.finalizenuma() - - likwid_data["affinity_info"] = pylikwid.initaffinity() - pylikwid.finalizeaffinity() - - pylikwid.initconfiguration() - likwid_data["configuration"] = pylikwid.getconfiguration() - pylikwid.destroyconfiguration() - - likwid_data["groups"] = pylikwid.getgroups() - - if rank == 0: - # Combine the data from all processes - combined_data = { - "config": None, - "rank_data": {f"rank_{i}": data for i, data in enumerate(all_data)}, - } - - # Add the likwid data - if likwid_data: - combined_data["config"] = likwid_data - - # Convert the file path to an absolute path - absolute_path = os.path.abspath(file_path) - - # Save the combined data using pickle - with open(absolute_path, "wb") as file: - pickle.dump(combined_data, file) - - print(f"Data saved to {absolute_path}") - - @classmethod - def print_summary(cls): - """ - Print a summary of the profiling data for all regions. - """ - - _config = ProfilingConfig() - if not _config.time_trace: - print("time_trace is not set to True --> Time traces are not measured --> Skip printing summary...") - return - - print("Profiling Summary:") - print("=" * 40) - for name, region in cls._regions.items(): - if region.ncalls > 0: - total_duration = sum(region.durations) - average_duration = total_duration / region.ncalls - min_duration = min(region.durations) - max_duration = max(region.durations) - std_duration = xp.std(region.durations) - else: - total_duration = average_duration = min_duration = max_duration = std_duration = 0 - - print(f"Region: {name}") - print(f" Number of Calls: {region.ncalls}") - print(f" Total Duration: {total_duration:.6f} seconds") - print(f" Average Duration: {average_duration:.6f} seconds") - print(f" Min Duration: {min_duration:.6f} seconds") - print(f" Max Duration: {max_duration:.6f} seconds") - print(f" Std Deviation: {std_duration:.6f} seconds") - print("-" * 40) - - -class ProfileRegion: - """Context manager for profiling specific code regions using LIKWID markers.""" - - def __init__(self, region_name, time_trace=False): - if hasattr(self, "_initialized") and self._initialized: - return - self._config = ProfilingConfig() - self._region_name = self.config.simulation_label + region_name - self._time_trace = time_trace - self._ncalls = 0 - self._start_times = xp.empty(1, dtype=float) - self._end_times = xp.empty(1, dtype=float) - self._durations = xp.empty(1, dtype=float) - self._started = False - - def __enter__(self): - if self._ncalls == len(self._start_times): - self._start_times = xp.append(self._start_times, xp.zeros_like(self._start_times)) - self._end_times = xp.append(self._end_times, xp.zeros_like(self._end_times)) - self._durations = xp.append(self._durations, xp.zeros_like(self._durations)) - - if self.config.likwid: - self._pylikwid().markerstartregion(self.region_name) - - if self._time_trace: - self._start_time = MPI.Wtime() - if self._start_time % self.config.sample_interval < self.config.sample_duration or self._ncalls == 0: - self._start_times[self._ncalls] = self._start_time - self._started = True - - self._ncalls += 1 - - return self - - def __exit__(self, exc_type, exc_value, traceback): - if self.config.likwid: - self._pylikwid().markerstopregion(self.region_name) - if self._time_trace and self.started: - end_time = MPI.Wtime() - self._end_times[self._ncalls - 1] = end_time - self._durations[self._ncalls - 1] = end_time - self._start_time - self._started = False - - def _pylikwid(self): - return _import_pylikwid() - - @property - def config(self): - return self._config - - @property - def durations(self): - return self._durations - - @property - def end_times(self): - return self._end_times - - @property - def ncalls(self): - return self._ncalls - - @property - def region_name(self): - return self._region_name - - @property - def start_times(self): - return self._start_times - - @property - def started(self): - return self._started - - -def pylikwid_markerinit(): - """Initialize LIKWID profiling markers.""" - if ProfilingConfig().likwid: - _import_pylikwid().markerinit() - - -def pylikwid_markerclose(): - """Close LIKWID profiling markers.""" - if ProfilingConfig().likwid: - _import_pylikwid().markerclose()