From 74b2678c608887f7b78e5a189ef83d3da300b7be Mon Sep 17 00:00:00 2001 From: Gangwar Date: Mon, 10 Nov 2025 21:34:22 -0500 Subject: [PATCH 01/11] AdiosComm_to_AdiosPtnComm --- redev_adios_channel.h | 4 ++-- redev_comm.h | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/redev_adios_channel.h b/redev_adios_channel.h index 8960fc2..cc78490 100644 --- a/redev_adios_channel.h +++ b/redev_adios_channel.h @@ -95,9 +95,9 @@ class AdiosChannel { // TODO, remove s2c/c2s destinction on variable names then use std::move // name if(comm != MPI_COMM_NULL) { - auto s2c = std::make_unique>(comm, num_client_ranks_, + auto s2c = std::make_unique>(comm, num_client_ranks_, s2c_engine_, s2c_io_, name); - auto c2s = std::make_unique>(comm, num_server_ranks_, + auto c2s = std::make_unique>(comm, num_server_ranks_, c2s_engine_, c2s_io_, name); switch (process_type_) { case ProcessType::Client: diff --git a/redev_comm.h b/redev_comm.h index e519000..8fbad03 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -99,7 +99,7 @@ struct InMessageLayout { size_t start; /** * Number of items (of the user specified type passed to the template - * parameter of AdiosComm) that should be read from the messages array + * parameter of AdiosPtnComm) that should be read from the messages array * (returned by Communicator::Recv). */ size_t count; @@ -131,7 +131,7 @@ class Communicator { */ virtual void Send(T *msgs, Mode mode) = 0; /** - * Receive an array. Use AdiosComm's GetInMessageLayout to retreive + * Receive an array. Use AdiosPtnComm's GetInMessageLayout to retreive * an instance of the InMessageLayout struct containing the layout of * the received array. */ @@ -151,20 +151,20 @@ class NoOpComm : public Communicator { /** - * The AdiosComm class implements the Communicator interface to support sending + * The AdiosPtnComm class implements the Communicator interface to support sending * messages between the clients and server via ADIOS2. The BP4 and SST ADIOS2 * engines are currently supported. - * One AdiosComm object is required for each communication link direction. For + * One AdiosPtnComm object is required for each communication link direction. For * example, for a client and server to both send and receive messages one - * AdiosComm for client->server messaging and another AdiosComm for + * AdiosPtnComm for client->server messaging and another AdiosPtnComm for * server->client messaging are needed. Redev::BidirectionalComm is a helper * class for this use case. */ template -class AdiosComm : public Communicator { +class AdiosPtnComm : public Communicator { public: /** - * Create an AdiosComm object. Collective across sender and receiver ranks. + * Create an AdiosPtnComm object. Collective across sender and receiver ranks. * Calls to the constructor from the sender and receiver ranks must be in * the same order (i.e., first creating the client-to-server object then the * server-to-client link). @@ -172,19 +172,19 @@ class AdiosComm : public Communicator { * @param[in] recvRanks_ number of ranks in the receivers MPI communicator * @param[in] eng_ ADIOS2 engine for writing on the sender side * @param[in] io_ ADIOS2 IO associated with eng_ - * @param[in] name_ unique name among AdiosComm objects + * @param[in] name_ unique name among AdiosPtnComm objects */ - AdiosComm(MPI_Comm comm_, int recvRanks_, adios2::Engine& eng_, adios2::IO& io_, std::string name_) + AdiosPtnComm(MPI_Comm comm_, int recvRanks_, adios2::Engine& eng_, adios2::IO& io_, std::string name_) : comm(comm_), recvRanks(recvRanks_), eng(eng_), io(io_), name(name_), verbose(0) { inMsg.knownSizes = false; } /// We are explicitly not allowing copy/move constructor/assignment as we don't /// know if the ADIOS2 Engine and IO objects can be safely copied/moved. - AdiosComm(const AdiosComm& other) = delete; - AdiosComm(AdiosComm&& other) = delete; - AdiosComm& operator=(const AdiosComm& other) = delete; - AdiosComm& operator=(AdiosComm&& other) = delete; + AdiosPtnComm(const AdiosPtnComm& other) = delete; + AdiosPtnComm(AdiosPtnComm&& other) = delete; + AdiosPtnComm& operator=(const AdiosPtnComm& other) = delete; + AdiosPtnComm& operator=(AdiosPtnComm&& other) = delete; void SetOutMessageLayout(LOs& dest_, LOs& offsets_) { REDEV_FUNCTION_TIMER; @@ -343,7 +343,7 @@ class AdiosComm : public Communicator { return inMsg; } /** - * Control the amount of output from AdiosComm functions. The higher the value the more output is written. + * Control the amount of output from AdiosPtnComm functions. The higher the value the more output is written. * @param[in] lvl valid values are [0:5] where 0 is silent and 5 is produces * the most output */ From cd07262ee2583c18cb78d88b3906c15a2ec8ecea Mon Sep 17 00:00:00 2001 From: Gangwar Date: Tue, 11 Nov 2025 20:26:09 -0500 Subject: [PATCH 02/11] WIP_global_comm --- redev_adios_channel.h | 16 ++++++++++++- redev_comm.h | 54 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/redev_adios_channel.h b/redev_adios_channel.h index cc78490..168d2df 100644 --- a/redev_adios_channel.h +++ b/redev_adios_channel.h @@ -108,7 +108,21 @@ class AdiosChannel { } return {std::make_unique>(), std::make_unique>()}; } - + template + [[nodiscard]] BidirectionalComm CreateGlobalComm( std::string name, MPI_Comm comm) { + REDEV_FUNCTION_TIMER; + if ( comm != MPI_COMM_NULL ) { + auto s2c = std::make_unique>(comm, s2c_engine_, s2c_io_, name); + auto c2s = std::make_unique>(comm, c2s_engine_, c2s_io_, name); + switch (process_type_) { + case ProcessType::Client: + return {std::move(c2s), std::move(s2c)}; + case ProcessType::Server: + return {std::move(s2c), std::move(c2s)}; + } + } + return {std::make_unique>(), std::make_unique>()}; + } // TODO s2c/c2s Engine/IO -> send/receive Engine/IO. This removes need for all // the switch statements... void BeginSendCommunicationPhase() { diff --git a/redev_comm.h b/redev_comm.h index 8fbad03..20bdc51 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -369,5 +369,59 @@ class AdiosPtnComm : public Communicator { //receive side state InMessageLayout inMsg; }; + template + class AdiosGlobalComm : public Communicator { + public: + AdiosGlobalComm(MPI_Comm comm, adios2::Engine& eng_, adios2::IO& io_, std::string name_) : comm(comm), eng(eng_), io(io_), name(name_) { + } + + // copy/move of adios engine and io objects isn't safe. + AdiosGlobalComm(const AdiosGlobalComm& other) = delete; + AdiosGlobalComm(AdiosGlobalComm&& other) = delete; + AdiosGlobalComm& operator=(const AdiosGlobalComm& other) = delete; + AdiosGlobalComm& operator=(AdiosGlobalComm&& other) = delete; + + void Send(T *msg, Mode mode) { + REDEV_FUNCTION_TIMER + const auto varName = name; + auto status = eng.BeginStep(); + REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK ); + auto var = io.InquireVariable(varName); + if (!var) { + var = io.DefineVariable(varName); + } + eng.Put(var, msg); + eng.EndStep(); + } + std::vector Recv(Mode mode) { + REDEV_FUNCTION_TIMER + const auto varName = name; + std::vector msg; + auto status = eng.BeginStep(); + REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK ); + auto var = io.InquireVariable(varName); + assert(var); + if (var) { + eng.Get(var, msg); + eng.PerformGets(); + } + eng.EndStep(); + return msg; + } + void SetOutMessageLayout(LOs& dest, LOs& offsets) {}; + InMessageLayout GetInMessageLayout() { return {}; } + void SetVerbose(int lvl) { + assert(lvl>=0 && lvl<=5); + Verbose = lvl; + } + private: + MPI_Comm comm; + adios2::Engine& eng; + adios2::IO& io; + std::string name; + int Verbose; + + }; + } From f6896de153bfe55f4cae11cbc36d8647e2380c0c Mon Sep 17 00:00:00 2001 From: Gangwar Date: Tue, 11 Nov 2025 22:34:50 -0500 Subject: [PATCH 03/11] WIP_global_comm --- redev_adios_channel.h | 186 ++++++----- redev_channel.h | 249 +++++++++++---- redev_comm.h | 728 +++++++++++++++++++++++------------------- test_sendrecv.cpp | 146 +++++---- 4 files changed, 757 insertions(+), 552 deletions(-) diff --git a/redev_adios_channel.h b/redev_adios_channel.h index 168d2df..718b1c1 100644 --- a/redev_adios_channel.h +++ b/redev_adios_channel.h @@ -4,15 +4,17 @@ #include "redev_profile.h" #include -namespace redev { +namespace redev +{ -class AdiosChannel { +class AdiosChannel +{ public: - AdiosChannel(adios2::ADIOS &adios, MPI_Comm comm, std::string name, + AdiosChannel(adios2::ADIOS& adios, MPI_Comm comm, std::string name, adios2::Params params, TransportType transportType, - ProcessType processType, Partition &partition, std::string path, + ProcessType processType, Partition& partition, std::string path, bool noClients = false) - : comm_(comm), process_type_(processType), partition_(partition) + : comm_(comm), process_type_(processType), partition_(partition) { REDEV_FUNCTION_TIMER; @@ -27,17 +29,17 @@ class AdiosChannel { } std::string engineType; switch (transportType) { - case TransportType::BP4: - engineType = "BP4"; - s2cName = s2cName + ".bp"; - c2sName = c2sName + ".bp"; - break; - case TransportType::SST: - engineType = "SST"; - break; - // no default case. This will cause a compiler error if we do not handle a - // an engine type that has been defined in the TransportType enum. - // (-Werror=switch) + case TransportType::BP4: + engineType = "BP4"; + s2cName = s2cName + ".bp"; + c2sName = c2sName + ".bp"; + break; + case TransportType::SST: + engineType = "SST"; + break; + // no default case. This will cause a compiler error if we do not handle + // a an engine type that has been defined in the TransportType enum. + // (-Werror=switch) } s2c_io_.SetEngine(engineType); c2s_io_.SetEngine(engineType); @@ -45,14 +47,14 @@ class AdiosChannel { c2s_io_.SetParameters(params); REDEV_ALWAYS_ASSERT(s2c_io_.EngineType() == c2s_io_.EngineType()); switch (transportType) { - case TransportType::SST: - openEnginesSST(noClients, s2cName, c2sName, s2c_io_, c2s_io_, s2c_engine_, - c2s_engine_); - break; - case TransportType::BP4: - openEnginesBP4(noClients, s2cName, c2sName, s2c_io_, c2s_io_, s2c_engine_, - c2s_engine_); - break; + case TransportType::SST: + openEnginesSST(noClients, s2cName, c2sName, s2c_io_, c2s_io_, + s2c_engine_, c2s_engine_); + break; + case TransportType::BP4: + openEnginesBP4(noClients, s2cName, c2sName, s2c_io_, c2s_io_, + s2c_engine_, c2s_engine_); + break; } // TODO pull begin/end step out of Setup/SendReceive metadata functions // begin step @@ -63,22 +65,27 @@ class AdiosChannel { // end step } // don't allow copying of class because it creates - AdiosChannel(const AdiosChannel &) = delete; - AdiosChannel operator=(const AdiosChannel &) = delete; + AdiosChannel(const AdiosChannel&) = delete; + AdiosChannel operator=(const AdiosChannel&) = delete; // FIXME - AdiosChannel(AdiosChannel &&o) - : s2c_io_(std::exchange(o.s2c_io_, adios2::IO())), - c2s_io_(std::exchange(o.c2s_io_, adios2::IO())), - c2s_engine_(std::exchange(o.c2s_engine_, adios2::Engine())), - s2c_engine_(std::exchange(o.s2c_engine_, adios2::Engine())), - num_client_ranks_(o.num_client_ranks_), - num_server_ranks_(o.num_server_ranks_), - comm_(std::exchange(o.comm_, MPI_COMM_NULL)), - process_type_(o.process_type_), rank_(o.rank_), - partition_(o.partition_) {REDEV_FUNCTION_TIMER;} - AdiosChannel operator=(AdiosChannel &&) = delete; + AdiosChannel(AdiosChannel&& o) + : s2c_io_(std::exchange(o.s2c_io_, adios2::IO())), + c2s_io_(std::exchange(o.c2s_io_, adios2::IO())), + c2s_engine_(std::exchange(o.c2s_engine_, adios2::Engine())), + s2c_engine_(std::exchange(o.s2c_engine_, adios2::Engine())), + num_client_ranks_(o.num_client_ranks_), + num_server_ranks_(o.num_server_ranks_), + comm_(std::exchange(o.comm_, MPI_COMM_NULL)), + process_type_(o.process_type_), + rank_(o.rank_), + partition_(o.partition_) + { + REDEV_FUNCTION_TIMER; + } + AdiosChannel operator=(AdiosChannel&&) = delete; // FIXME IMPL RULE OF 5 - ~AdiosChannel() { + ~AdiosChannel() + { REDEV_FUNCTION_TIMER; // NEED TO CHECK that the engine exists before trying to close it because it // could be in a moved from state @@ -90,104 +97,93 @@ class AdiosChannel { } } template - [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm) { + [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm) + { REDEV_FUNCTION_TIMER; // TODO, remove s2c/c2s destinction on variable names then use std::move // name - if(comm != MPI_COMM_NULL) { + if (comm != MPI_COMM_NULL) { auto s2c = std::make_unique>(comm, num_client_ranks_, - s2c_engine_, s2c_io_, name); + s2c_engine_, s2c_io_, name); auto c2s = std::make_unique>(comm, num_server_ranks_, - c2s_engine_, c2s_io_, name); + c2s_engine_, c2s_io_, name); switch (process_type_) { - case ProcessType::Client: - return {std::move(c2s), std::move(s2c)}; - case ProcessType::Server: - return {std::move(s2c), std::move(c2s)}; + case ProcessType::Client: return {std::move(c2s), std::move(s2c)}; + case ProcessType::Server: return {std::move(s2c), std::move(c2s)}; } } return {std::make_unique>(), std::make_unique>()}; } template - [[nodiscard]] BidirectionalComm CreateGlobalComm( std::string name, MPI_Comm comm) { + [[nodiscard]] BidirectionalComm CreateGlobalComm(std::string name, + MPI_Comm comm) + { REDEV_FUNCTION_TIMER; - if ( comm != MPI_COMM_NULL ) { - auto s2c = std::make_unique>(comm, s2c_engine_, s2c_io_, name); - auto c2s = std::make_unique>(comm, c2s_engine_, c2s_io_, name); + if (comm != MPI_COMM_NULL) { + auto s2c = + std::make_unique>(comm, s2c_engine_, s2c_io_, name); + auto c2s = + std::make_unique>(comm, c2s_engine_, c2s_io_, name); switch (process_type_) { - case ProcessType::Client: - return {std::move(c2s), std::move(s2c)}; - case ProcessType::Server: - return {std::move(s2c), std::move(c2s)}; + case ProcessType::Client: return {std::move(c2s), std::move(s2c)}; + case ProcessType::Server: return {std::move(s2c), std::move(c2s)}; } } return {std::make_unique>(), std::make_unique>()}; } // TODO s2c/c2s Engine/IO -> send/receive Engine/IO. This removes need for all // the switch statements... - void BeginSendCommunicationPhase() { + void BeginSendCommunicationPhase() + { REDEV_FUNCTION_TIMER; adios2::StepStatus status; switch (process_type_) { - case ProcessType::Client: - status = c2s_engine_.BeginStep(); - break; - case ProcessType::Server: - status = s2c_engine_.BeginStep(); - break; + case ProcessType::Client: status = c2s_engine_.BeginStep(); break; + case ProcessType::Server: status = s2c_engine_.BeginStep(); break; } REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); } - void EndSendCommunicationPhase() { + void EndSendCommunicationPhase() + { switch (process_type_) { - case ProcessType::Client: - c2s_engine_.EndStep(); - break; - case ProcessType::Server: - s2c_engine_.EndStep(); - break; + case ProcessType::Client: c2s_engine_.EndStep(); break; + case ProcessType::Server: s2c_engine_.EndStep(); break; } } - void BeginReceiveCommunicationPhase() { + void BeginReceiveCommunicationPhase() + { REDEV_FUNCTION_TIMER; adios2::StepStatus status; switch (process_type_) { - case ProcessType::Client: - status = s2c_engine_.BeginStep(); - break; - case ProcessType::Server: - status = c2s_engine_.BeginStep(); - break; + case ProcessType::Client: status = s2c_engine_.BeginStep(); break; + case ProcessType::Server: status = c2s_engine_.BeginStep(); break; } REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); } - void EndReceiveCommunicationPhase() { + void EndReceiveCommunicationPhase() + { REDEV_FUNCTION_TIMER; switch (process_type_) { - case ProcessType::Client: - s2c_engine_.EndStep(); - break; - case ProcessType::Server: - c2s_engine_.EndStep(); - break; + case ProcessType::Client: s2c_engine_.EndStep(); break; + case ProcessType::Server: c2s_engine_.EndStep(); break; } } private: void openEnginesBP4(bool noClients, std::string s2cName, std::string c2sName, - adios2::IO &s2cIO, adios2::IO &c2sIO, - adios2::Engine &s2cEngine, adios2::Engine &c2sEngine); + adios2::IO& s2cIO, adios2::IO& c2sIO, + adios2::Engine& s2cEngine, adios2::Engine& c2sEngine); void openEnginesSST(bool noClients, std::string s2cName, std::string c2sName, - adios2::IO &s2cIO, adios2::IO &c2sIO, - adios2::Engine &s2cEngine, adios2::Engine &c2sEngine); - [[nodiscard]] redev::LO SendServerCommSizeToClient(adios2::IO &s2cIO, - adios2::Engine &s2cEngine); - [[nodiscard]] redev::LO SendClientCommSizeToServer(adios2::IO &c2sIO, - adios2::Engine &c2sEngine); - [[nodiscard]] std::size_t - SendPartitionTypeToClient(adios2::IO &s2cIO, adios2::Engine &s2cEngine); - void Setup(adios2::IO &s2cIO, adios2::Engine &s2cEngine); - void CheckVersion(adios2::Engine &eng, adios2::IO &io); + adios2::IO& s2cIO, adios2::IO& c2sIO, + adios2::Engine& s2cEngine, adios2::Engine& c2sEngine); + [[nodiscard]] redev::LO SendServerCommSizeToClient(adios2::IO& s2cIO, + adios2::Engine& s2cEngine); + [[nodiscard]] redev::LO SendClientCommSizeToServer(adios2::IO& c2sIO, + adios2::Engine& c2sEngine); + [[nodiscard]] std::size_t SendPartitionTypeToClient( + adios2::IO& s2cIO, adios2::Engine& s2cEngine); + void Setup(adios2::IO& s2cIO, adios2::Engine& s2cEngine); + void CheckVersion(adios2::Engine& eng, adios2::IO& io); void ConstructPartitionFromIndex(size_t partition_index); adios2::IO s2c_io_; @@ -199,7 +195,7 @@ class AdiosChannel { MPI_Comm comm_; ProcessType process_type_; int rank_; - Partition &partition_; + Partition& partition_; }; } // namespace redev diff --git a/redev_channel.h b/redev_channel.h index 6a1f58a..6c62338 100644 --- a/redev_channel.h +++ b/redev_channel.h @@ -3,171 +3,272 @@ #include "redev_bidirectional_comm.h" #include -namespace redev { +namespace redev +{ -class Channel { +class Channel +{ public: template - Channel(T &&channel) - : pimpl_{new ChannelModel>>( - std::forward(channel))}, - send_communication_phase_active_(false), - receive_communication_phase_active_(false) { - REDEV_FUNCTION_TIMER; - } + Channel(T&& channel) + : pimpl_{new ChannelModel>>( + std::forward(channel))}, + send_communication_phase_active_(false), + receive_communication_phase_active_(false) + { + REDEV_FUNCTION_TIMER; + } // For cases where we may be interested in storing the comm variant rather // than the exact type this function can be used to reduce the runtime // overhead of converting from the variant to the explicit type back to the // variant - template [[nodiscard]] CommV CreateCommV(std::string name, MPI_Comm comm) { + template + [[nodiscard]] CommV CreateCommV(std::string name, MPI_Comm comm) + { REDEV_FUNCTION_TIMER; return pimpl_->CreateComm(std::move(name), comm, InvCommunicatorTypeMap::value); } + template + [[nodiscard]] CommV CreateGlobalCommV(std::string name, MPI_Comm comm) + { + REDEV_FUNCTION_TIMER; + return pimpl_->CreateGlobalComm(std::move(name), comm, + InvCommunicatorTypeMap::value); + } // convenience typesafe wrapper to get back the specific communicator type // rather than the variant. This is here to simplify updating legacy code // that expects a typed communicator to be created. template - [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm) { + [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm) + { REDEV_FUNCTION_TIMER; - return std::get>(CreateCommV(std::move(name), comm)); + return std::get>( + CreateCommV(std::move(name), comm)); } - void BeginSendCommunicationPhase() { + // Typesafe wrapper to get the specific global communicator + template + [[nodiscard]] BidirectionalComm CreateGlobalComm(std::string name, + MPI_Comm comm) + { + REDEV_FUNCTION_TIMER; + return std::get>( + CreateCommV(std::move(name), MPI_COMM_WORLD)); + } + void BeginSendCommunicationPhase() + { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InSendCommunicationPhase() == false); pimpl_->BeginSendCommunicationPhase(); send_communication_phase_active_ = true; } - void EndSendCommunicationPhase() { + void EndSendCommunicationPhase() + { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InSendCommunicationPhase() == true); pimpl_->EndSendCommunicationPhase(); send_communication_phase_active_ = false; } - void BeginReceiveCommunicationPhase() { + void BeginReceiveCommunicationPhase() + { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InReceiveCommunicationPhase() == false); pimpl_->BeginReceiveCommunicationPhase(); receive_communication_phase_active_ = true; } - void EndReceiveCommunicationPhase() { + void EndReceiveCommunicationPhase() + { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InReceiveCommunicationPhase() == true); pimpl_->EndReceiveCommunicationPhase(); receive_communication_phase_active_ = false; } - [[nodiscard]] bool InSendCommunicationPhase() const noexcept { + [[nodiscard]] bool InSendCommunicationPhase() const noexcept + { REDEV_FUNCTION_TIMER; return send_communication_phase_active_; } - [[nodiscard]] bool InReceiveCommunicationPhase() const noexcept { + [[nodiscard]] bool InReceiveCommunicationPhase() const noexcept + { REDEV_FUNCTION_TIMER; return receive_communication_phase_active_; } template - auto SendPhase(const Func &f, Args &&...args) { + auto SendPhase(const Func& f, Args&&... args) + { auto sg = SendPhaseScope(*this); return f(std::forward(args)...); } template - auto ReceivePhase(const Func &f, Args &&...args) { + auto ReceivePhase(const Func& f, Args&&... args) + { auto sg = ReceivePhaseScope(*this); return f(std::forward(args)...); } private: - class ChannelConcept { + class ChannelConcept + { public: - virtual CommV CreateComm(std::string &&, MPI_Comm, CommunicatorDataType) = 0; + virtual CommV CreateComm(std::string&&, MPI_Comm, CommunicatorDataType) = 0; + virtual CommV CreateGlobalComm(std::string&&, MPI_Comm, + CommunicatorDataType) = 0; virtual void BeginSendCommunicationPhase() = 0; virtual void EndSendCommunicationPhase() = 0; virtual void BeginReceiveCommunicationPhase() = 0; virtual void EndReceiveCommunicationPhase() = 0; virtual ~ChannelConcept() noexcept {} }; - template class ChannelModel final : public ChannelConcept { + template + class ChannelModel final : public ChannelConcept + { public: - ChannelModel(T &&impl) : ChannelConcept(), impl_(std::forward(impl)) {} + ChannelModel(T&& impl) : ChannelConcept(), impl_(std::forward(impl)) {} // since we don't have templated virtual functions, we convert the type to a // runtime parameter then extract out the appropriate type based on the // typemap. Assuming the typemap/inverse type map are correct, this is // entirely safe unlike doing it in user code. Although, it's not the most // beautiful construction in the world. ~ChannelModel() noexcept final {} - [[nodiscard]] CommV CreateComm(std::string &&name, MPI_Comm comm, - CommunicatorDataType type) final { + [[nodiscard]] CommV CreateComm(std::string&& name, MPI_Comm comm, + CommunicatorDataType type) final + { REDEV_FUNCTION_TIMER; switch (type) { - case CommunicatorDataType::INT8: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT8: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::INT16: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT16: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::INT32: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT32: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::INT64: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT64: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::UINT8: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT8: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::UINT16: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT16: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::UINT32: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT32: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::UINT64: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT64: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::LONG_INT: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::LONG_INT: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::FLOAT: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::FLOAT: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::DOUBLE: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::DOUBLE: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::LONG_DOUBLE: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::LONG_DOUBLE: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; - case CommunicatorDataType::COMPLEX_DOUBLE: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::COMPLEX_DOUBLE: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm)}; } return {}; } - void BeginSendCommunicationPhase() final { + [[nodiscard]] CommV CreateGlobalComm(std::string&& name, MPI_Comm comm, + CommunicatorDataType type) final + { + REDEV_FUNCTION_TIMER; + switch (type) { + case CommunicatorDataType::INT8: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::INT16: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::INT32: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::INT64: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::UINT8: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::UINT16: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::UINT32: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::UINT64: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::LONG_INT: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::FLOAT: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::DOUBLE: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::LONG_DOUBLE: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + case CommunicatorDataType::COMPLEX_DOUBLE: + return CommV{impl_.template CreateGlobalComm< + CommunicatorTypeMap::type>( + std::move(name), comm)}; + } + return {}; + } + void BeginSendCommunicationPhase() final + { REDEV_FUNCTION_TIMER; impl_.BeginSendCommunicationPhase(); } - void EndSendCommunicationPhase() final { + void EndSendCommunicationPhase() final + { REDEV_FUNCTION_TIMER; impl_.EndSendCommunicationPhase(); } - void BeginReceiveCommunicationPhase() final { + void BeginReceiveCommunicationPhase() final + { REDEV_FUNCTION_TIMER; impl_.BeginReceiveCommunicationPhase(); } - void EndReceiveCommunicationPhase() final { + void EndReceiveCommunicationPhase() final + { REDEV_FUNCTION_TIMER; impl_.EndReceiveCommunicationPhase(); } @@ -175,25 +276,29 @@ class Channel { private: T impl_; }; - class SendPhaseScope { + class SendPhaseScope + { public: - explicit SendPhaseScope(Channel &channel) : channel_(channel) { + explicit SendPhaseScope(Channel& channel) : channel_(channel) + { channel_.BeginSendCommunicationPhase(); } ~SendPhaseScope() { channel_.EndSendCommunicationPhase(); } private: - Channel &channel_; + Channel& channel_; }; - class ReceivePhaseScope { + class ReceivePhaseScope + { public: - explicit ReceivePhaseScope(Channel &channel) : channel_(channel) { + explicit ReceivePhaseScope(Channel& channel) : channel_(channel) + { channel_.BeginReceiveCommunicationPhase(); } ~ReceivePhaseScope() { channel_.EndReceiveCommunicationPhase(); } private: - Channel &channel_; + Channel& channel_; }; std::unique_ptr pimpl_; @@ -201,17 +306,25 @@ class Channel { bool receive_communication_phase_active_; }; -class NoOpChannel { +class NoOpChannel +{ public: template [[nodiscard]] - BidirectionalComm CreateComm(std::string, MPI_Comm) { + BidirectionalComm CreateComm(std::string, MPI_Comm) + { + return {std::make_unique>(), std::make_unique>()}; + } + template + [[nodiscard]] + BidirectionalComm CreateGlobalComm(std::string, MPI_Comm) + { return {std::make_unique>(), std::make_unique>()}; } - void BeginSendCommunicationPhase(){} - void EndSendCommunicationPhase(){} - void BeginReceiveCommunicationPhase(){} - void EndReceiveCommunicationPhase(){} + void BeginSendCommunicationPhase() {} + void EndSendCommunicationPhase() {} + void BeginReceiveCommunicationPhase() {} + void EndReceiveCommunicationPhase() {} }; } // namespace redev diff --git a/redev_comm.h b/redev_comm.h index 20bdc51..cbff14b 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -10,60 +10,100 @@ #include #include "redev_time.h" -namespace { -void checkStep(adios2::StepStatus status) { +namespace +{ +void checkStep(adios2::StepStatus status) +{ REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); } -} +} // namespace -namespace redev { +namespace redev +{ - namespace detail { - template struct dependent_always_false : std::false_type {}; - } +namespace detail +{ +template +struct dependent_always_false : std::false_type +{}; +} // namespace detail -enum class Mode { +enum class Mode +{ Deferred, Synchronous }; -template -[[ nodiscard ]] -constexpr MPI_Datatype getMpiType(T) noexcept { - if constexpr (std::is_same_v) { return MPI_CHAR; } - else if constexpr (std::is_same_v) { return MPI_SHORT; } - else if constexpr (std::is_same_v) { return MPI_INT; } - else if constexpr (std::is_same_v) { return MPI_LONG; } - else if constexpr (std::is_same_v) { return MPI_LONG_LONG; } - else if constexpr (std::is_same_v) { return MPI_SIGNED_CHAR; } - else if constexpr (std::is_same_v) { return MPI_UNSIGNED_CHAR; } - else if constexpr (std::is_same_v) { return MPI_UNSIGNED_SHORT; } - else if constexpr (std::is_same_v) { return MPI_UNSIGNED; } - else if constexpr (std::is_same_v) { return MPI_UNSIGNED_LONG; } - else if constexpr (std::is_same_v) { return MPI_UNSIGNED_LONG_LONG; } - else if constexpr (std::is_same_v) { return MPI_FLOAT; } - else if constexpr (std::is_same_v) { return MPI_DOUBLE; } - else if constexpr (std::is_same_v) { return MPI_LONG_DOUBLE; } - else if constexpr (std::is_same_v) { return MPI_WCHAR; } - else if constexpr (std::is_same_v) { return MPI_INT8_T; } - else if constexpr (std::is_same_v) { return MPI_INT16_T; } - else if constexpr (std::is_same_v) { return MPI_INT32_T; } - else if constexpr (std::is_same_v) { return MPI_INT64_T; } - else if constexpr (std::is_same_v) { return MPI_UINT8_T; } - else if constexpr (std::is_same_v) { return MPI_UINT16_T; } - else if constexpr (std::is_same_v) { return MPI_UINT32_T; } - else if constexpr (std::is_same_v) { return MPI_UINT64_T; } - else if constexpr (std::is_same_v) { return MPI_CXX_BOOL; } - else if constexpr (std::is_same_v>) { return MPI_CXX_FLOAT_COMPLEX; } - else if constexpr (std::is_same_v>) { return MPI_CXX_DOUBLE_COMPLEX; } - else if constexpr (std::is_same_v>) { return MPI_CXX_LONG_DOUBLE_COMPLEX; } - else{ static_assert(detail::dependent_always_false::value, "type has unkown map to MPI_Type"); return {}; } +template +[[nodiscard]] +constexpr MPI_Datatype getMpiType(T) noexcept +{ + if constexpr (std::is_same_v) { + return MPI_CHAR; + } else if constexpr (std::is_same_v) { + return MPI_SHORT; + } else if constexpr (std::is_same_v) { + return MPI_INT; + } else if constexpr (std::is_same_v) { + return MPI_LONG; + } else if constexpr (std::is_same_v) { + return MPI_LONG_LONG; + } else if constexpr (std::is_same_v) { + return MPI_SIGNED_CHAR; + } else if constexpr (std::is_same_v) { + return MPI_UNSIGNED_CHAR; + } else if constexpr (std::is_same_v) { + return MPI_UNSIGNED_SHORT; + } else if constexpr (std::is_same_v) { + return MPI_UNSIGNED; + } else if constexpr (std::is_same_v) { + return MPI_UNSIGNED_LONG; + } else if constexpr (std::is_same_v) { + return MPI_UNSIGNED_LONG_LONG; + } else if constexpr (std::is_same_v) { + return MPI_FLOAT; + } else if constexpr (std::is_same_v) { + return MPI_DOUBLE; + } else if constexpr (std::is_same_v) { + return MPI_LONG_DOUBLE; + } else if constexpr (std::is_same_v) { + return MPI_WCHAR; + } else if constexpr (std::is_same_v) { + return MPI_INT8_T; + } else if constexpr (std::is_same_v) { + return MPI_INT16_T; + } else if constexpr (std::is_same_v) { + return MPI_INT32_T; + } else if constexpr (std::is_same_v) { + return MPI_INT64_T; + } else if constexpr (std::is_same_v) { + return MPI_UINT8_T; + } else if constexpr (std::is_same_v) { + return MPI_UINT16_T; + } else if constexpr (std::is_same_v) { + return MPI_UINT32_T; + } else if constexpr (std::is_same_v) { + return MPI_UINT64_T; + } else if constexpr (std::is_same_v) { + return MPI_CXX_BOOL; + } else if constexpr (std::is_same_v>) { + return MPI_CXX_FLOAT_COMPLEX; + } else if constexpr (std::is_same_v>) { + return MPI_CXX_DOUBLE_COMPLEX; + } else if constexpr (std::is_same_v>) { + return MPI_CXX_LONG_DOUBLE_COMPLEX; + } else { + static_assert(detail::dependent_always_false::value, + "type has unkown map to MPI_Type"); + return {}; + } // empty return statement needed to avoid compiler warning return {}; } -template -void Broadcast(T* data, int count, int root, MPI_Comm comm) { +template +void Broadcast(T* data, int count, int root, MPI_Comm comm) +{ REDEV_FUNCTION_TIMER; auto type = getMpiType(T()); MPI_Bcast(data, count, type, root, comm); @@ -73,7 +113,8 @@ void Broadcast(T* data, int count, int root, MPI_Comm comm) { * The InMessageLayout struct contains the arrays defining the arrangement of * data in the array returned by Communicator::Recv. */ -struct InMessageLayout { +struct InMessageLayout +{ /** * Array of source ranks sized NumberOfClientRanks*NumberOfServerRanks. Each * rank reads the entire array once at the start of a communication round. @@ -88,13 +129,13 @@ struct InMessageLayout { */ redev::GOs offset; /** - * Set to true if Communicator::Recv has been called and the message layout data set; - * false otherwise. + * Set to true if Communicator::Recv has been called and the message layout + * data set; false otherwise. */ bool knownSizes; /** - * Index into the messages array (returned by Communicator::Recv) where the current process should start - * reading. + * Index into the messages array (returned by Communicator::Recv) where the + * current process should start reading. */ size_t start; /** @@ -108,320 +149,351 @@ struct InMessageLayout { /** * The Communicator class provides an abstract interface for sending and * receiving messages to/from the client and server. - * TODO: Split Communicator into Send/Recieve Communicators, bidirectional constructed by composition and can perform both send and receive + * TODO: Split Communicator into Send/Recieve Communicators, bidirectional + * constructed by composition and can perform both send and receive */ -template -class Communicator { - public: - /** - * Set the arrangement of data in the messages array so that its segments, - * defined by the offsets array, are sent to the correct destination ranks, - * defined by the dest array. - * @param[in] dest array of integers specifying the destination rank for a - * portion of the msgs array - * @param[in] offsets array of length |dest|+1 defining the segment of the - * msgs array (passed to the Send function) being sent to each destination rank. - * the segment [ msgs[offsets[i]] : msgs[offsets[i+1]] } is sent to rank dest[i] - */ - virtual void SetOutMessageLayout(LOs& dest, LOs& offsets) = 0; - /** - * Send the array. - * @param[in] msgs array of data to be sent according to the layout specified - * with SetOutMessageLayout - */ - virtual void Send(T *msgs, Mode mode) = 0; - /** - * Receive an array. Use AdiosPtnComm's GetInMessageLayout to retreive - * an instance of the InMessageLayout struct containing the layout of - * the received array. - */ - virtual std::vector Recv(Mode mode) = 0; +template +class Communicator +{ +public: + /** + * Set the arrangement of data in the messages array so that its segments, + * defined by the offsets array, are sent to the correct destination ranks, + * defined by the dest array. + * @param[in] dest array of integers specifying the destination rank for a + * portion of the msgs array + * @param[in] offsets array of length |dest|+1 defining the segment of the + * msgs array (passed to the Send function) being sent to each destination + * rank. the segment [ msgs[offsets[i]] : msgs[offsets[i+1]] } is sent to rank + * dest[i] + */ + virtual void SetOutMessageLayout(LOs& dest, LOs& offsets) = 0; + /** + * Send the array. + * @param[in] msgs array of data to be sent according to the layout specified + * with SetOutMessageLayout + */ + virtual void Send(T* msgs, Mode mode) = 0; + /** + * Receive an array. Use AdiosPtnComm's GetInMessageLayout to retreive + * an instance of the InMessageLayout struct containing the layout of + * the received array. + */ + virtual std::vector Recv(Mode mode) = 0; - virtual InMessageLayout GetInMessageLayout() = 0; - virtual ~Communicator() = default; + virtual InMessageLayout GetInMessageLayout() = 0; + virtual ~Communicator() = default; }; template -class NoOpComm : public Communicator { - void SetOutMessageLayout(LOs& dest, LOs& offsets) final {}; - void Send(T *msgs, Mode /*unused*/) final {}; - std::vector Recv(Mode /*unused*/) final { return {}; } - InMessageLayout GetInMessageLayout() final { return {}; } +class NoOpComm : public Communicator +{ + void SetOutMessageLayout(LOs& dest, LOs& offsets) final {}; + void Send(T* msgs, Mode /*unused*/) final {}; + std::vector Recv(Mode /*unused*/) final { return {}; } + InMessageLayout GetInMessageLayout() final { return {}; } }; - /** - * The AdiosPtnComm class implements the Communicator interface to support sending - * messages between the clients and server via ADIOS2. The BP4 and SST ADIOS2 - * engines are currently supported. - * One AdiosPtnComm object is required for each communication link direction. For - * example, for a client and server to both send and receive messages one - * AdiosPtnComm for client->server messaging and another AdiosPtnComm for - * server->client messaging are needed. Redev::BidirectionalComm is a helper - * class for this use case. + * The AdiosPtnComm class implements the Communicator interface to support + * sending messages between the clients and server via ADIOS2. The BP4 and SST + * ADIOS2 engines are currently supported. One AdiosPtnComm object is required + * for each communication link direction. For example, for a client and server + * to both send and receive messages one AdiosPtnComm for client->server + * messaging and another AdiosPtnComm for server->client messaging are needed. + * Redev::BidirectionalComm is a helper class for this use case. */ -template -class AdiosPtnComm : public Communicator { - public: - /** - * Create an AdiosPtnComm object. Collective across sender and receiver ranks. - * Calls to the constructor from the sender and receiver ranks must be in - * the same order (i.e., first creating the client-to-server object then the - * server-to-client link). - * @param[in] comm_ MPI communicator for sender ranks - * @param[in] recvRanks_ number of ranks in the receivers MPI communicator - * @param[in] eng_ ADIOS2 engine for writing on the sender side - * @param[in] io_ ADIOS2 IO associated with eng_ - * @param[in] name_ unique name among AdiosPtnComm objects - */ - AdiosPtnComm(MPI_Comm comm_, int recvRanks_, adios2::Engine& eng_, adios2::IO& io_, std::string name_) - : comm(comm_), recvRanks(recvRanks_), eng(eng_), io(io_), name(name_), verbose(0) { - inMsg.knownSizes = false; - } - - /// We are explicitly not allowing copy/move constructor/assignment as we don't - /// know if the ADIOS2 Engine and IO objects can be safely copied/moved. - AdiosPtnComm(const AdiosPtnComm& other) = delete; - AdiosPtnComm(AdiosPtnComm&& other) = delete; - AdiosPtnComm& operator=(const AdiosPtnComm& other) = delete; - AdiosPtnComm& operator=(AdiosPtnComm&& other) = delete; +template +class AdiosPtnComm : public Communicator +{ +public: + /** + * Create an AdiosPtnComm object. Collective across sender and receiver + * ranks. Calls to the constructor from the sender and receiver ranks must be + * in the same order (i.e., first creating the client-to-server object then + * the server-to-client link). + * @param[in] comm_ MPI communicator for sender ranks + * @param[in] recvRanks_ number of ranks in the receivers MPI communicator + * @param[in] eng_ ADIOS2 engine for writing on the sender side + * @param[in] io_ ADIOS2 IO associated with eng_ + * @param[in] name_ unique name among AdiosPtnComm objects + */ + AdiosPtnComm(MPI_Comm comm_, int recvRanks_, adios2::Engine& eng_, + adios2::IO& io_, std::string name_) + : comm(comm_), + recvRanks(recvRanks_), + eng(eng_), + io(io_), + name(name_), + verbose(0) + { + inMsg.knownSizes = false; + } - void SetOutMessageLayout(LOs& dest_, LOs& offsets_) { - REDEV_FUNCTION_TIMER; - outMsg = OutMessageLayout{dest_, offsets_}; + /// We are explicitly not allowing copy/move constructor/assignment as we + /// don't know if the ADIOS2 Engine and IO objects can be safely copied/moved. + AdiosPtnComm(const AdiosPtnComm& other) = delete; + AdiosPtnComm(AdiosPtnComm&& other) = delete; + AdiosPtnComm& operator=(const AdiosPtnComm& other) = delete; + AdiosPtnComm& operator=(AdiosPtnComm&& other) = delete; + + void SetOutMessageLayout(LOs& dest_, LOs& offsets_) + { + REDEV_FUNCTION_TIMER; + outMsg = OutMessageLayout{dest_, offsets_}; + } + void Send(T* msgs, Mode mode) + { + REDEV_FUNCTION_TIMER; + int rank, commSz; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &commSz); + GOs degree(recvRanks, 0); // TODO ideally, this would not be needed + for (size_t i = 0; i < outMsg.dest.size(); i++) { + auto destRank = outMsg.dest[i]; + assert(destRank < recvRanks); + degree[destRank] += outMsg.offsets[i + 1] - outMsg.offsets[i]; + } + GOs rdvRankStart(recvRanks, 0); + auto ret = MPI_Exscan(degree.data(), rdvRankStart.data(), recvRanks, + getMpiType(redev::GO()), MPI_SUM, comm); + assert(ret == MPI_SUCCESS); + if (!rank) { + // on rank 0 the result of MPI_Exscan is undefined, set it to zero + rdvRankStart = GOs(recvRanks, 0); } - void Send(T *msgs, Mode mode) { - REDEV_FUNCTION_TIMER; - int rank, commSz; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &commSz); - GOs degree(recvRanks,0); //TODO ideally, this would not be needed - for( size_t i=0; i(std::accumulate(gDegree.begin(), gDegree.end(), redev::GO(0))); + GOs gDegree(recvRanks, 0); + ret = MPI_Allreduce(degree.data(), gDegree.data(), recvRanks, + getMpiType(redev::GO()), MPI_SUM, comm); + assert(ret == MPI_SUCCESS); + const size_t gDegreeTot = static_cast( + std::accumulate(gDegree.begin(), gDegree.end(), redev::GO(0))); - GOs gStart(recvRanks,0); - redev::exclusive_scan(gDegree.begin(), gDegree.end(), gStart.begin(), redev::GO(0)); + GOs gStart(recvRanks, 0); + redev::exclusive_scan(gDegree.begin(), gDegree.end(), gStart.begin(), + redev::GO(0)); - //The messages array has a different length on each rank ('irregular') so we don't - //define local size and count here. - adios2::Dims shape{static_cast(gDegreeTot)}; - adios2::Dims start{}; - adios2::Dims count{}; - if(!rdvVar) { - rdvVar = io.DefineVariable(name, shape, start, count); - } - assert(rdvVar); - const auto srcRanksName = name+"_srcRanks"; - //The source rank offsets array is the same on each process ('regular'). - adios2::Dims srShape{static_cast(commSz*recvRanks)}; - adios2::Dims srStart{static_cast(recvRanks*rank)}; - adios2::Dims srCount{static_cast(recvRanks)}; + // The messages array has a different length on each rank ('irregular') so + // we don't define local size and count here. + adios2::Dims shape{static_cast(gDegreeTot)}; + adios2::Dims start{}; + adios2::Dims count{}; + if (!rdvVar) { + rdvVar = io.DefineVariable(name, shape, start, count); + } + assert(rdvVar); + const auto srcRanksName = name + "_srcRanks"; + // The source rank offsets array is the same on each process ('regular'). + adios2::Dims srShape{static_cast(commSz * recvRanks)}; + adios2::Dims srStart{static_cast(recvRanks * rank)}; + adios2::Dims srCount{static_cast(recvRanks)}; - //send dest rank offsets array from rank 0 - auto offsets = gStart; - offsets.push_back(gDegreeTot); - if(!rank) { - const auto offsetsName = name+"_offsets"; - const auto oShape = offsets.size(); - const auto oStart = 0; - const auto oCount = offsets.size(); - if(!offsetsVar) { - offsetsVar = io.DefineVariable(offsetsName,{oShape},{oStart},{oCount}); - // if we are in sync mode we will peform all puts at the end of the function, otherwise we need to put this now before - // offsets data goes out of scope - eng.Put(offsetsVar, offsets.data(), (mode==Mode::Deferred)?adios2::Mode::Sync:adios2::Mode::Deferred); - } + // send dest rank offsets array from rank 0 + auto offsets = gStart; + offsets.push_back(gDegreeTot); + if (!rank) { + const auto offsetsName = name + "_offsets"; + const auto oShape = offsets.size(); + const auto oStart = 0; + const auto oCount = offsets.size(); + if (!offsetsVar) { + offsetsVar = io.DefineVariable(offsetsName, {oShape}, + {oStart}, {oCount}); + // if we are in sync mode we will peform all puts at the end of the + // function, otherwise we need to put this now before offsets data goes + // out of scope + eng.Put(offsetsVar, offsets.data(), + (mode == Mode::Deferred) ? adios2::Mode::Sync + : adios2::Mode::Deferred); } + } - //send source rank offsets array 'rdvRankStart' - if(!srcRanksVar) { - srcRanksVar = io.DefineVariable(srcRanksName, srShape, srStart, srCount); - assert(srcRanksVar); - // if we are in sync mode we will peform all puts at the end of the function, otherwise we need to put this now before - // ranks data goes out of scope - eng.Put(srcRanksVar, rdvRankStart.data(),(mode==Mode::Deferred)?adios2::Mode::Sync:adios2::Mode::Deferred); - - } + // send source rank offsets array 'rdvRankStart' + if (!srcRanksVar) { + srcRanksVar = + io.DefineVariable(srcRanksName, srShape, srStart, srCount); + assert(srcRanksVar); + // if we are in sync mode we will peform all puts at the end of the + // function, otherwise we need to put this now before ranks data goes out + // of scope + eng.Put(srcRanksVar, rdvRankStart.data(), + (mode == Mode::Deferred) ? adios2::Mode::Sync + : adios2::Mode::Deferred); + } - //assume one call to pack from each rank for now - for( size_t i=0; i 0 ) { - start = adios2::Dims{static_cast(lStart)}; - count = adios2::Dims{static_cast(lCount)}; - rdvVar.SetSelection({start,count}); - eng.Put(rdvVar, &(msgs[outMsg.offsets[i]])); - } - } - if(mode == Mode::Synchronous) { - eng.PerformPuts(); + // assume one call to pack from each rank for now + for (size_t i = 0; i < outMsg.dest.size(); i++) { + const auto destRank = outMsg.dest[i]; + const auto lStart = gStart[destRank] + rdvRankStart[destRank]; + const auto lCount = outMsg.offsets[i + 1] - outMsg.offsets[i]; + if (lCount > 0) { + start = adios2::Dims{static_cast(lStart)}; + count = adios2::Dims{static_cast(lCount)}; + rdvVar.SetSelection({start, count}); + eng.Put(rdvVar, &(msgs[outMsg.offsets[i]])); } } - std::vector Recv(Mode mode) { - REDEV_FUNCTION_TIMER; - int rank, commSz; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &commSz); - auto t1 = redev::getTime(); - - if(!inMsg.knownSizes) { - auto rdvRanksVar = io.InquireVariable(name+"_srcRanks"); - assert(rdvRanksVar); - auto offsetsVar = io.InquireVariable(name+"_offsets"); - assert(offsetsVar); + if (mode == Mode::Synchronous) { + eng.PerformPuts(); + } + } + std::vector Recv(Mode mode) + { + REDEV_FUNCTION_TIMER; + int rank, commSz; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &commSz); + auto t1 = redev::getTime(); - auto offsetsShape = offsetsVar.Shape(); - assert(offsetsShape.size() == 1); - const auto offSz = offsetsShape[0]; - inMsg.offset.resize(offSz); - offsetsVar.SetSelection({{0}, {offSz}}); - eng.Get(offsetsVar, inMsg.offset.data()); + if (!inMsg.knownSizes) { + auto rdvRanksVar = io.InquireVariable(name + "_srcRanks"); + assert(rdvRanksVar); + auto offsetsVar = io.InquireVariable(name + "_offsets"); + assert(offsetsVar); - auto rdvRanksShape = rdvRanksVar.Shape(); - assert(rdvRanksShape.size() == 1); - const auto rsrSz = rdvRanksShape[0]; - inMsg.srcRanks.resize(rsrSz); - rdvRanksVar.SetSelection({{0},{rsrSz}}); - eng.Get(rdvRanksVar, inMsg.srcRanks.data()); + auto offsetsShape = offsetsVar.Shape(); + assert(offsetsShape.size() == 1); + const auto offSz = offsetsShape[0]; + inMsg.offset.resize(offSz); + offsetsVar.SetSelection({{0}, {offSz}}); + eng.Get(offsetsVar, inMsg.offset.data()); - // TODO: Can remove in synchronous mode? - eng.PerformGets(); - inMsg.start = static_cast(inMsg.offset[rank]); - inMsg.count = static_cast(inMsg.offset[rank+1]-inMsg.start); - inMsg.knownSizes = true; - } - auto t2 = redev::getTime(); + auto rdvRanksShape = rdvRanksVar.Shape(); + assert(rdvRanksShape.size() == 1); + const auto rsrSz = rdvRanksShape[0]; + inMsg.srcRanks.resize(rsrSz); + rdvRanksVar.SetSelection({{0}, {rsrSz}}); + eng.Get(rdvRanksVar, inMsg.srcRanks.data()); - auto msgsVar = io.InquireVariable(name); - assert(msgsVar); - std::vector msgs(inMsg.count); - if(inMsg.count) { - //only call Get with non-zero sized reads - msgsVar.SetSelection({{inMsg.start}, {inMsg.count}}); - eng.Get(msgsVar, msgs.data()); - } - if(mode == Mode::Synchronous) { - eng.PerformGets(); - } + // TODO: Can remove in synchronous mode? + eng.PerformGets(); + inMsg.start = static_cast(inMsg.offset[rank]); + inMsg.count = static_cast(inMsg.offset[rank + 1] - inMsg.start); + inMsg.knownSizes = true; + } + auto t2 = redev::getTime(); - //if(mode == Mode::Synchronous) { - // eng.EndStep(); - //} - auto t3 = redev::getTime(); - std::chrono::duration r1 = t2-t1; - std::chrono::duration r2 = t3-t2; - if(!rank && verbose) { - fprintf(stderr, "recv knownSizes %d r1(sec.) r2(sec.) %f %f\n", - inMsg.knownSizes, r1.count(), r2.count()); - } - return msgs; + auto msgsVar = io.InquireVariable(name); + assert(msgsVar); + std::vector msgs(inMsg.count); + if (inMsg.count) { + // only call Get with non-zero sized reads + msgsVar.SetSelection({{inMsg.start}, {inMsg.count}}); + eng.Get(msgsVar, msgs.data()); } - /** - * Return the InMessageLayout object. - * @todo should return const object - */ - InMessageLayout GetInMessageLayout() { - return inMsg; + if (mode == Mode::Synchronous) { + eng.PerformGets(); } - /** - * Control the amount of output from AdiosPtnComm functions. The higher the value the more output is written. - * @param[in] lvl valid values are [0:5] where 0 is silent and 5 is produces - * the most output - */ - void SetVerbose(int lvl) { - assert(lvl>=0 && lvl<=5); - verbose = lvl; + + // if(mode == Mode::Synchronous) { + // eng.EndStep(); + // } + auto t3 = redev::getTime(); + std::chrono::duration r1 = t2 - t1; + std::chrono::duration r2 = t3 - t2; + if (!rank && verbose) { + fprintf(stderr, "recv knownSizes %d r1(sec.) r2(sec.) %f %f\n", + inMsg.knownSizes, r1.count(), r2.count()); } - private: - MPI_Comm comm; - int recvRanks; - adios2::Engine& eng; - adios2::IO& io; - adios2::Variable rdvVar; - adios2::Variable srcRanksVar; - adios2::Variable offsetsVar; - std::string name; - //support only one call to pack for now... - struct OutMessageLayout { - LOs dest; - LOs offsets; - } outMsg; - int verbose; - //receive side state - InMessageLayout inMsg; + return msgs; + } + /** + * Return the InMessageLayout object. + * @todo should return const object + */ + InMessageLayout GetInMessageLayout() { return inMsg; } + /** + * Control the amount of output from AdiosPtnComm functions. The higher the + * value the more output is written. + * @param[in] lvl valid values are [0:5] where 0 is silent and 5 is produces + * the most output + */ + void SetVerbose(int lvl) + { + assert(lvl >= 0 && lvl <= 5); + verbose = lvl; + } + +private: + MPI_Comm comm; + int recvRanks; + adios2::Engine& eng; + adios2::IO& io; + adios2::Variable rdvVar; + adios2::Variable srcRanksVar; + adios2::Variable offsetsVar; + std::string name; + // support only one call to pack for now... + struct OutMessageLayout + { + LOs dest; + LOs offsets; + } outMsg; + int verbose; + // receive side state + InMessageLayout inMsg; }; - template - class AdiosGlobalComm : public Communicator { - public: - AdiosGlobalComm(MPI_Comm comm, adios2::Engine& eng_, adios2::IO& io_, std::string name_) : comm(comm), eng(eng_), io(io_), name(name_) { - } - // copy/move of adios engine and io objects isn't safe. - AdiosGlobalComm(const AdiosGlobalComm& other) = delete; - AdiosGlobalComm(AdiosGlobalComm&& other) = delete; - AdiosGlobalComm& operator=(const AdiosGlobalComm& other) = delete; - AdiosGlobalComm& operator=(AdiosGlobalComm&& other) = delete; +template +class AdiosGlobalComm : public Communicator +{ +public: + AdiosGlobalComm(MPI_Comm comm, adios2::Engine& eng_, adios2::IO& io_, + std::string name_) + : comm(comm), eng(eng_), io(io_), name(name_) + { + } - void Send(T *msg, Mode mode) { - REDEV_FUNCTION_TIMER - const auto varName = name; - auto status = eng.BeginStep(); - REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK ); - auto var = io.InquireVariable(varName); - if (!var) { - var = io.DefineVariable(varName); - } - eng.Put(var, msg); - eng.EndStep(); - } - std::vector Recv(Mode mode) { - REDEV_FUNCTION_TIMER - const auto varName = name; - std::vector msg; - auto status = eng.BeginStep(); - REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK ); - auto var = io.InquireVariable(varName); - assert(var); - if (var) { - eng.Get(var, msg); - eng.PerformGets(); - } - eng.EndStep(); - return msg; + // copy/move of adios engine and io objects isn't safe. + AdiosGlobalComm(const AdiosGlobalComm& other) = delete; + AdiosGlobalComm(AdiosGlobalComm&& other) = delete; + AdiosGlobalComm& operator=(const AdiosGlobalComm& other) = delete; + AdiosGlobalComm& operator=(AdiosGlobalComm&& other) = delete; + + void Send(T* msg, Mode mode) + { + REDEV_FUNCTION_TIMER + const auto varName = name; + auto status = eng.BeginStep(); + REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); + auto var = io.InquireVariable(varName); + if (!var) { + var = io.DefineVariable(varName); } - void SetOutMessageLayout(LOs& dest, LOs& offsets) {}; - InMessageLayout GetInMessageLayout() { return {}; } - void SetVerbose(int lvl) { - assert(lvl>=0 && lvl<=5); - Verbose = lvl; + eng.Put(var, msg); + eng.EndStep(); + } + std::vector Recv(Mode mode) + { + REDEV_FUNCTION_TIMER + const auto varName = name; + std::vector msg; + auto status = eng.BeginStep(); + REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); + auto var = io.InquireVariable(varName); + assert(var); + if (var) { + eng.Get(var, msg); + eng.PerformGets(); } - private: - MPI_Comm comm; - adios2::Engine& eng; - adios2::IO& io; - std::string name; - int Verbose; - - }; + eng.EndStep(); + return msg; + } + void SetOutMessageLayout(LOs& dest, LOs& offsets) {}; + InMessageLayout GetInMessageLayout() { return {}; } + void SetVerbose(int lvl) + { + assert(lvl >= 0 && lvl <= 5); + Verbose = lvl; + } +private: + MPI_Comm comm; + adios2::Engine& eng; + adios2::IO& io; + std::string name; + int Verbose; +}; -} +} // namespace redev diff --git a/test_sendrecv.cpp b/test_sendrecv.cpp index 49e6b62..c5f14f5 100644 --- a/test_sendrecv.cpp +++ b/test_sendrecv.cpp @@ -11,81 +11,105 @@ * Coming soon... */ -//If the name of the variable being sent ("foo"), the communication -//pattern, or data being sent is changed then also update the cmake -//test using the adios2 utility bpls to check the array. +// If the name of the variable being sent ("foo"), the communication +// pattern, or data being sent is changed then also update the cmake +// test using the adios2 utility bpls to check the array. -int main(int argc, char** argv) { +int main(int argc, char** argv) +{ int rank, nproc; MPI_Init(&argc, &argv); - if(argc != 2) { - std::cerr << "Usage: " << argv[0] << " <1=isRendezvousApp,0=isParticipant>\n"; + if (argc != 2) { + std::cerr << "Usage: " << argv[0] + << " <1=isRendezvousApp,0=isParticipant>\n"; exit(EXIT_FAILURE); } MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); auto isRdv = atoi(argv[1]); fprintf(stderr, "rank %d isRdv %d\n", rank, isRdv); - if(isRdv && nproc != 4) { - std::cerr << "There must be exactly 4 rendezvous processes for this test.\n"; - exit(EXIT_FAILURE); + if (isRdv && nproc != 4) { + std::cerr + << "There must be exactly 4 rendezvous processes for this test.\n"; + exit(EXIT_FAILURE); } - if(!isRdv && nproc != 3) { - std::cerr << "There must be exactly 3 non-rendezvous processes for this test.\n"; - exit(EXIT_FAILURE); + if (!isRdv && nproc != 3) { + std::cerr + << "There must be exactly 3 non-rendezvous processes for this test.\n"; + exit(EXIT_FAILURE); } { - //dummy partition vector data - const auto dim = 2; - auto ranks = isRdv ? redev::LOs({0,1,2,3}) : redev::LOs(4); - auto cuts = isRdv ? redev::Reals({0,0.5,0.75,0.25}) : redev::Reals(4); - auto ptn = redev::RCBPtn(dim,ranks,cuts); - redev::Redev rdv(MPI_COMM_WORLD,redev::Partition{std::move(ptn)},static_cast(isRdv)); - std::string name = "foo"; - adios2::Params params{ {"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; - auto channel = rdv.CreateAdiosChannel(name, params, - redev::TransportType::BP4); - auto commPair = channel.CreateComm(name, MPI_COMM_WORLD); - // the non-rendezvous app sends to the rendezvous app - if(!isRdv) { - redev::LOs dest; - redev::LOs offsets; - redev::LOs msgs; - if(rank==0) { - dest = redev::LOs{0,2}; - offsets = redev::LOs{0,2,6}; - msgs = redev::LOs(6,0); //write the src rank as the msg for now - } else if (rank==1) { - dest = redev::LOs{0,1,2,3}; - offsets = redev::LOs{0,1,4,8,10}; - msgs = redev::LOs(10,1); - } else if (rank==2) { - dest = redev::LOs{0,1,2,3}; - offsets = redev::LOs{0,4,5,7,11}; - msgs = redev::LOs(11,2); - } - commPair.SetOutMessageLayout(dest, offsets); - channel.BeginSendCommunicationPhase(); - commPair.Send(msgs.data(),redev::Mode::Deferred); - channel.EndSendCommunicationPhase(); - } else { - channel.BeginReceiveCommunicationPhase(); - auto msgVec = commPair.Recv(redev::Mode::Deferred); - channel.EndReceiveCommunicationPhase(); - auto inMsg = commPair.GetInMessageLayout(); - REDEV_ALWAYS_ASSERT(inMsg.offset == redev::GOs({0,7,11,21,27})); - REDEV_ALWAYS_ASSERT(inMsg.srcRanks == redev::GOs({0,0,0,0,2,0,4,0,3,3,8,2})); - if(rank == 0) { - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0,0,1,2,2,2,2})); - } else if(rank == 1) { - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1,1,1,2})); - } else if(rank == 2) { - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0,0,0,0,1,1,1,1,2,2})); - } else if(rank == 3) { - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1,1,2,2,2,2})); + // dummy partition vector data + const auto dim = 2; + auto ranks = isRdv ? redev::LOs({0, 1, 2, 3}) : redev::LOs(4); + auto cuts = isRdv ? redev::Reals({0, 0.5, 0.75, 0.25}) : redev::Reals(4); + auto ptn = redev::RCBPtn(dim, ranks, cuts); + redev::Redev rdv(MPI_COMM_WORLD, redev::Partition{std::move(ptn)}, + static_cast(isRdv)); + std::string name = "foo"; + adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; + auto channel = + rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); + auto commPair = channel.CreateComm(name, MPI_COMM_WORLD); + + auto commGlob = + channel.CreateGlobalComm("global_" + name, MPI_COMM_WORLD); + // test the ptn comm + // the non-rendezvous app sends to the rendezvous app + if (!isRdv) { + redev::LOs dest; + redev::LOs offsets; + redev::LOs msgs; + if (rank == 0) { + dest = redev::LOs{0, 2}; + offsets = redev::LOs{0, 2, 6}; + msgs = redev::LOs(6, 0); // write the src rank as the msg for now + } else if (rank == 1) { + dest = redev::LOs{0, 1, 2, 3}; + offsets = redev::LOs{0, 1, 4, 8, 10}; + msgs = redev::LOs(10, 1); + } else if (rank == 2) { + dest = redev::LOs{0, 1, 2, 3}; + offsets = redev::LOs{0, 4, 5, 7, 11}; + msgs = redev::LOs(11, 2); + } + commPair.SetOutMessageLayout(dest, offsets); + channel.BeginSendCommunicationPhase(); + commPair.Send(msgs.data(), redev::Mode::Deferred); + channel.EndSendCommunicationPhase(); + // send data to test global comm + msgs = redev::LOs{0, 2}; + if (rank == 0) { + channel.BeginSendCommunicationPhase(); + commGlob.Send(msgs.data(), redev::Mode::Deferred); + channel.EndSendCommunicationPhase(); + } + } else { + channel.BeginReceiveCommunicationPhase(); + auto msgVec = commPair.Recv(redev::Mode::Deferred); + channel.EndReceiveCommunicationPhase(); + auto inMsg = commPair.GetInMessageLayout(); + REDEV_ALWAYS_ASSERT(inMsg.offset == redev::GOs({0, 7, 11, 21, 27})); + REDEV_ALWAYS_ASSERT(inMsg.srcRanks == + redev::GOs({0, 0, 0, 0, 2, 0, 4, 0, 3, 3, 8, 2})); + if (rank == 0) { + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0, 0, 1, 2, 2, 2, 2})); + } else if (rank == 1) { + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1, 1, 1, 2})); + } else if (rank == 2) { + REDEV_ALWAYS_ASSERT(msgVec == + redev::LOs({0, 0, 0, 0, 1, 1, 1, 1, 2, 2})); + } else if (rank == 3) { + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1, 1, 2, 2, 2, 2})); + } + // receive global date + channel.BeginReceiveCommunicationPhase(); + msgVec = commGlob.Recv(redev::Mode::Deferred); + channel.EndReceiveCommunicationPhase(); + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0, 2})); } } - } + MPI_Finalize(); return 0; } From 3463800fc9f617607f06d8822461b3ae8ca8d519 Mon Sep 17 00:00:00 2001 From: Gangwar Date: Wed, 12 Nov 2025 18:02:13 -0500 Subject: [PATCH 04/11] WIP_comm_testing --- CMakeLists.txt | 1 + redev_adios_channel.h | 41 +++++++------- redev_channel.h | 127 ++++++++---------------------------------- redev_types.h | 38 +++++++++---- test_global_comm.cpp | 58 +++++++++++++++++++ 5 files changed, 128 insertions(+), 137 deletions(-) create mode 100644 test_global_comm.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f89dd23..c6fe72d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,7 @@ if(BUILD_TESTING) NAME2 client1 EXE2 ./test_twoClients PROCS2 1 ARGS2 ${isSST} 1 NAME3 rdv EXE3 ./test_twoClients PROCS3 1 ARGS3 ${isSST} -1) endif() + add_exe(test_global_comm ./test_global_comm.cpp) endif(BUILD_TESTING) ## export the library diff --git a/redev_adios_channel.h b/redev_adios_channel.h index 718b1c1..9e03085 100644 --- a/redev_adios_channel.h +++ b/redev_adios_channel.h @@ -4,6 +4,8 @@ #include "redev_profile.h" #include +#include "redev_types.h" + namespace redev { @@ -97,33 +99,28 @@ class AdiosChannel } } template - [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm) + [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm, + CommType ctype) { REDEV_FUNCTION_TIMER; - // TODO, remove s2c/c2s destinction on variable names then use std::move + // TODO, remove s2c/c2s distinction on variable names then use std::move // name if (comm != MPI_COMM_NULL) { - auto s2c = std::make_unique>(comm, num_client_ranks_, - s2c_engine_, s2c_io_, name); - auto c2s = std::make_unique>(comm, num_server_ranks_, - c2s_engine_, c2s_io_, name); - switch (process_type_) { - case ProcessType::Client: return {std::move(c2s), std::move(s2c)}; - case ProcessType::Server: return {std::move(s2c), std::move(c2s)}; + std::unique_ptr> s2c, c2s; + switch (ctype) { + case CommType::Ptn: + s2c = std::make_unique>(comm, num_client_ranks_, + s2c_engine_, s2c_io_, name); + c2s = std::make_unique>(comm, num_server_ranks_, + c2s_engine_, c2s_io_, name); + break; + case CommType::Global: + s2c = std::make_unique>(comm, s2c_engine_, s2c_io_, + name); + c2s = std::make_unique>(comm, c2s_engine_, c2s_io_, + name); + break; } - } - return {std::make_unique>(), std::make_unique>()}; - } - template - [[nodiscard]] BidirectionalComm CreateGlobalComm(std::string name, - MPI_Comm comm) - { - REDEV_FUNCTION_TIMER; - if (comm != MPI_COMM_NULL) { - auto s2c = - std::make_unique>(comm, s2c_engine_, s2c_io_, name); - auto c2s = - std::make_unique>(comm, c2s_engine_, c2s_io_, name); switch (process_type_) { case ProcessType::Client: return {std::move(c2s), std::move(s2c)}; case ProcessType::Server: return {std::move(s2c), std::move(c2s)}; diff --git a/redev_channel.h b/redev_channel.h index 6c62338..4a10454 100644 --- a/redev_channel.h +++ b/redev_channel.h @@ -24,38 +24,25 @@ class Channel // overhead of converting from the variant to the explicit type back to the // variant template - [[nodiscard]] CommV CreateCommV(std::string name, MPI_Comm comm) + [[nodiscard]] CommV CreateCommV(std::string name, MPI_Comm comm, + CommType ctype) { REDEV_FUNCTION_TIMER; - return pimpl_->CreateComm(std::move(name), comm, + return pimpl_->CreateComm(std::move(name), comm, std::move(ctype), InvCommunicatorTypeMap::value); } - template - [[nodiscard]] CommV CreateGlobalCommV(std::string name, MPI_Comm comm) - { - REDEV_FUNCTION_TIMER; - return pimpl_->CreateGlobalComm(std::move(name), comm, - InvCommunicatorTypeMap::value); - } // convenience typesafe wrapper to get back the specific communicator type // rather than the variant. This is here to simplify updating legacy code // that expects a typed communicator to be created. template - [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm) - { - REDEV_FUNCTION_TIMER; - return std::get>( - CreateCommV(std::move(name), comm)); - } - // Typesafe wrapper to get the specific global communicator - template - [[nodiscard]] BidirectionalComm CreateGlobalComm(std::string name, - MPI_Comm comm) + [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm, + CommType ctype = CommType::Ptn) { REDEV_FUNCTION_TIMER; return std::get>( - CreateCommV(std::move(name), MPI_COMM_WORLD)); + CreateCommV(std::move(name), comm, std::move(ctype))); } + void BeginSendCommunicationPhase() { REDEV_FUNCTION_TIMER; @@ -112,9 +99,8 @@ class Channel class ChannelConcept { public: - virtual CommV CreateComm(std::string&&, MPI_Comm, CommunicatorDataType) = 0; - virtual CommV CreateGlobalComm(std::string&&, MPI_Comm, - CommunicatorDataType) = 0; + virtual CommV CreateComm(std::string&&, MPI_Comm, CommType, + CommunicatorDataType) = 0; virtual void BeginSendCommunicationPhase() = 0; virtual void EndSendCommunicationPhase() = 0; virtual void BeginReceiveCommunicationPhase() = 0; @@ -133,6 +119,7 @@ class Channel // beautiful construction in the world. ~ChannelModel() noexcept final {} [[nodiscard]] CommV CreateComm(std::string&& name, MPI_Comm comm, + CommType ctype, CommunicatorDataType type) final { REDEV_FUNCTION_TIMER; @@ -140,115 +127,55 @@ class Channel case CommunicatorDataType::INT8: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::INT16: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::INT32: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::INT64: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::UINT8: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::UINT16: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::UINT32: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::UINT64: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::LONG_INT: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::FLOAT: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::DOUBLE: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::LONG_DOUBLE: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; case CommunicatorDataType::COMPLEX_DOUBLE: return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( - std::move(name), comm)}; - } - return {}; - } - [[nodiscard]] CommV CreateGlobalComm(std::string&& name, MPI_Comm comm, - CommunicatorDataType type) final - { - REDEV_FUNCTION_TIMER; - switch (type) { - case CommunicatorDataType::INT8: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::INT16: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::INT32: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::INT64: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::UINT8: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::UINT16: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::UINT32: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::UINT64: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::LONG_INT: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::FLOAT: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::DOUBLE: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::LONG_DOUBLE: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; - case CommunicatorDataType::COMPLEX_DOUBLE: - return CommV{impl_.template CreateGlobalComm< - CommunicatorTypeMap::type>( - std::move(name), comm)}; + std::move(name), comm, std::move(ctype))}; } return {}; } @@ -311,13 +238,7 @@ class NoOpChannel public: template [[nodiscard]] - BidirectionalComm CreateComm(std::string, MPI_Comm) - { - return {std::make_unique>(), std::make_unique>()}; - } - template - [[nodiscard]] - BidirectionalComm CreateGlobalComm(std::string, MPI_Comm) + BidirectionalComm CreateComm(std::string, MPI_Comm, CommType) { return {std::make_unique>(), std::make_unique>()}; } diff --git a/redev_types.h b/redev_types.h index c85f72c..834f5ef 100644 --- a/redev_types.h +++ b/redev_types.h @@ -4,26 +4,40 @@ #include #include -namespace redev { -///Local ordinate, used to count items local to a process +namespace redev +{ +/// Local ordinate, used to count items local to a process typedef std::int32_t LO; -///Vector of local ordinates +/// Vector of local ordinates using LOs = std::vector; -///Global ordinate, used to count items across multiple processes +/// Global ordinate, used to count items across multiple processes typedef std::int64_t GO; -///Vector of global ordinates +/// Vector of global ordinates using GOs = std::vector; -///Floating point values +/// Floating point values typedef double Real; -///Vector of floating point values +/// Vector of floating point values using Reals = std::vector; -///Complex values +/// Complex values typedef std::complex CV; -///Vector of complex values +/// Vector of complex values using CVs = std::vector; -enum class ProcessType { Client = 0, Server = 1 }; -enum class TransportType { BP4 = 0, SST = 1 }; +enum class ProcessType +{ + Client = 0, + Server = 1 +}; +enum class TransportType +{ + BP4 = 0, + SST = 1 +}; +enum class CommType +{ + Ptn = 0, + Global = 1 +}; -} +} // namespace redev #endif diff --git a/test_global_comm.cpp b/test_global_comm.cpp new file mode 100644 index 0000000..a4942b3 --- /dev/null +++ b/test_global_comm.cpp @@ -0,0 +1,58 @@ +#include +#include +#include "redev.h" + +int main(int argc, char** argv) +{ + int rank, nproc; + MPI_Init(&argc, &argv); + if (argc != 2) { + std::cerr << "Usage: " << argv[0] + << " <1=isRendezvousApp,0=isParticipant>\n"; + exit(EXIT_FAILURE); + } + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + auto isRdv = atoi(argv[1]); + fprintf(stderr, "rank %d isRdv %d\n", rank, isRdv); + { + // dummy partition vector data + const auto dim = 2; + auto ranks = isRdv ? redev::LOs({0, 1, 2, 3}) : redev::LOs(4); + auto cuts = isRdv ? redev::Reals({0, 0.5, 0.75, 0.25}) : redev::Reals(4); + auto ptn = redev::RCBPtn(dim, ranks, cuts); + redev::Redev rdv(MPI_COMM_WORLD, redev::Partition{std::move(ptn)}, + static_cast(isRdv)); + adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; + std::string name = "bar"; + auto channel = + rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); + auto commPair = channel.CreateComm(name, MPI_COMM_WORLD, + redev::CommType::Global); + + // test the ptn comm + // the non-rendezvous app sends to the rendezvous app + if (!isRdv) { + // send data to test global comm + auto msgs = redev::LOs{0, 2}; + if (rank == 0) { + channel.BeginSendCommunicationPhase(); + commPair.Send(msgs.data(), redev::Mode::Deferred); + channel.EndSendCommunicationPhase(); + } + } else { + channel.BeginReceiveCommunicationPhase(); + auto msgVec = commPair.Recv(redev::Mode::Deferred); + channel.EndReceiveCommunicationPhase(); + + // receive global date + channel.BeginReceiveCommunicationPhase(); + msgVec = commPair.Recv(redev::Mode::Deferred); + channel.EndReceiveCommunicationPhase(); + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0, 2})); + } + } + + MPI_Finalize(); + return 0; +} \ No newline at end of file From 5420fa3a9606aeb7be350572d9764567a62f515b Mon Sep 17 00:00:00 2001 From: Gangwar Date: Wed, 12 Nov 2025 20:44:21 -0500 Subject: [PATCH 05/11] WIP_ctest_failing --- CMakeLists.txt | 4 +++ redev_comm.h | 7 +--- test_global_comm.cpp | 82 ++++++++++++++++++++++++-------------------- test_sendrecv.cpp | 17 --------- 4 files changed, 49 insertions(+), 61 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6fe72d..e24410b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -216,6 +216,10 @@ if(BUILD_TESTING) NAME3 rdv EXE3 ./test_twoClients PROCS3 1 ARGS3 ${isSST} -1) endif() add_exe(test_global_comm ./test_global_comm.cpp) + dual_mpi_test(TESTNAME test_global_comm TIMEOUT ${test_timeout} + NAME1 rdv PROCS1 1 EXE1 ./test_global_comm ARGS1 0 + NAME2 app PROCS2 1 EXE2 ./test_global_comm ARGS2 1) + endif(BUILD_TESTING) ## export the library diff --git a/redev_comm.h b/redev_comm.h index cbff14b..7a1069b 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -455,29 +455,24 @@ class AdiosGlobalComm : public Communicator { REDEV_FUNCTION_TIMER const auto varName = name; - auto status = eng.BeginStep(); - REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); + auto var = io.InquireVariable(varName); if (!var) { var = io.DefineVariable(varName); } eng.Put(var, msg); - eng.EndStep(); } std::vector Recv(Mode mode) { REDEV_FUNCTION_TIMER const auto varName = name; std::vector msg; - auto status = eng.BeginStep(); - REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); auto var = io.InquireVariable(varName); assert(var); if (var) { eng.Get(var, msg); eng.PerformGets(); } - eng.EndStep(); return msg; } void SetOutMessageLayout(LOs& dest, LOs& offsets) {}; diff --git a/test_global_comm.cpp b/test_global_comm.cpp index a4942b3..6b30f5a 100644 --- a/test_global_comm.cpp +++ b/test_global_comm.cpp @@ -11,48 +11,54 @@ int main(int argc, char** argv) << " <1=isRendezvousApp,0=isParticipant>\n"; exit(EXIT_FAILURE); } - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nproc); + + MPI_Comm world = MPI_COMM_WORLD; + MPI_Comm_rank(world, &rank); + MPI_Comm_size(world, &nproc); auto isRdv = atoi(argv[1]); fprintf(stderr, "rank %d isRdv %d\n", rank, isRdv); - { - // dummy partition vector data - const auto dim = 2; - auto ranks = isRdv ? redev::LOs({0, 1, 2, 3}) : redev::LOs(4); - auto cuts = isRdv ? redev::Reals({0, 0.5, 0.75, 0.25}) : redev::Reals(4); - auto ptn = redev::RCBPtn(dim, ranks, cuts); - redev::Redev rdv(MPI_COMM_WORLD, redev::Partition{std::move(ptn)}, - static_cast(isRdv)); - adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; - std::string name = "bar"; - auto channel = - rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); - auto commPair = channel.CreateComm(name, MPI_COMM_WORLD, - redev::CommType::Global); - - // test the ptn comm - // the non-rendezvous app sends to the rendezvous app - if (!isRdv) { - // send data to test global comm - auto msgs = redev::LOs{0, 2}; - if (rank == 0) { - channel.BeginSendCommunicationPhase(); - commPair.Send(msgs.data(), redev::Mode::Deferred); - channel.EndSendCommunicationPhase(); - } - } else { - channel.BeginReceiveCommunicationPhase(); - auto msgVec = commPair.Recv(redev::Mode::Deferred); - channel.EndReceiveCommunicationPhase(); - - // receive global date - channel.BeginReceiveCommunicationPhase(); - msgVec = commPair.Recv(redev::Mode::Deferred); - channel.EndReceiveCommunicationPhase(); - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0, 2})); + + const auto dim = 2; + auto ranks = isRdv ? redev::LOs({0}) : redev::LOs(1); + auto cuts = isRdv ? redev::Reals({0}) : redev::Reals(1); + auto ptn = redev::RCBPtn(dim, ranks, cuts); + + int color = isRdv ? 0 : 1; + + // Split communicator into two groups + MPI_Comm localComm; + MPI_Comm_split(world, color, rank, &localComm); + + // dummy partition vector data + redev::Redev rdv(localComm, redev::Partition{std::move(ptn)}, + static_cast(isRdv)); + adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; + std::string name = "bar"; + + auto channel = + rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); + auto commPair = + channel.CreateComm(name, localComm, redev::CommType::Global); + + // test the ptn comm + // the non-rendezvous app sends to the rendezvous app + if (!isRdv) { + // send data to test global comm + redev::Real val = 3.14; + auto* msgs = &val; + if (rank == 0) { + channel.BeginSendCommunicationPhase(); + commPair.Send(msgs, redev::Mode::Deferred); + channel.EndSendCommunicationPhase(); } + } else { + // receive global date + channel.BeginReceiveCommunicationPhase(); + auto msgVec = commPair.Recv(redev::Mode::Deferred); + channel.EndReceiveCommunicationPhase(); + REDEV_ALWAYS_ASSERT(msgVec[0] == redev::Real{3.14}); + printf("\nTest passed."); } - MPI_Finalize(); return 0; } \ No newline at end of file diff --git a/test_sendrecv.cpp b/test_sendrecv.cpp index c5f14f5..07e5a73 100644 --- a/test_sendrecv.cpp +++ b/test_sendrecv.cpp @@ -51,10 +51,6 @@ int main(int argc, char** argv) auto channel = rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); auto commPair = channel.CreateComm(name, MPI_COMM_WORLD); - - auto commGlob = - channel.CreateGlobalComm("global_" + name, MPI_COMM_WORLD); - // test the ptn comm // the non-rendezvous app sends to the rendezvous app if (!isRdv) { redev::LOs dest; @@ -77,13 +73,6 @@ int main(int argc, char** argv) channel.BeginSendCommunicationPhase(); commPair.Send(msgs.data(), redev::Mode::Deferred); channel.EndSendCommunicationPhase(); - // send data to test global comm - msgs = redev::LOs{0, 2}; - if (rank == 0) { - channel.BeginSendCommunicationPhase(); - commGlob.Send(msgs.data(), redev::Mode::Deferred); - channel.EndSendCommunicationPhase(); - } } else { channel.BeginReceiveCommunicationPhase(); auto msgVec = commPair.Recv(redev::Mode::Deferred); @@ -102,14 +91,8 @@ int main(int argc, char** argv) } else if (rank == 3) { REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1, 1, 2, 2, 2, 2})); } - // receive global date - channel.BeginReceiveCommunicationPhase(); - msgVec = commGlob.Recv(redev::Mode::Deferred); - channel.EndReceiveCommunicationPhase(); - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0, 2})); } } - MPI_Finalize(); return 0; } From f8cf322d549779eab07b065232484578337bb915 Mon Sep 17 00:00:00 2001 From: Gangwar Date: Mon, 17 Nov 2025 23:59:48 -0500 Subject: [PATCH 06/11] WIP_remove_fromatting --- redev_adios_channel.h | 171 +++++----- redev_channel.h | 170 ++++------ redev_comm.h | 744 +++++++++++++++++++----------------------- redev_types.h | 40 +-- test_sendrecv.cpp | 129 ++++---- 5 files changed, 568 insertions(+), 686 deletions(-) diff --git a/redev_adios_channel.h b/redev_adios_channel.h index 9e03085..8a434ca 100644 --- a/redev_adios_channel.h +++ b/redev_adios_channel.h @@ -4,19 +4,15 @@ #include "redev_profile.h" #include -#include "redev_types.h" +namespace redev { -namespace redev -{ - -class AdiosChannel -{ +class AdiosChannel { public: - AdiosChannel(adios2::ADIOS& adios, MPI_Comm comm, std::string name, + AdiosChannel(adios2::ADIOS &adios, MPI_Comm comm, std::string name, adios2::Params params, TransportType transportType, - ProcessType processType, Partition& partition, std::string path, + ProcessType processType, Partition &partition, std::string path, bool noClients = false) - : comm_(comm), process_type_(processType), partition_(partition) + : comm_(comm), process_type_(processType), partition_(partition) { REDEV_FUNCTION_TIMER; @@ -31,17 +27,17 @@ class AdiosChannel } std::string engineType; switch (transportType) { - case TransportType::BP4: - engineType = "BP4"; - s2cName = s2cName + ".bp"; - c2sName = c2sName + ".bp"; - break; - case TransportType::SST: - engineType = "SST"; - break; - // no default case. This will cause a compiler error if we do not handle - // a an engine type that has been defined in the TransportType enum. - // (-Werror=switch) + case TransportType::BP4: + engineType = "BP4"; + s2cName = s2cName + ".bp"; + c2sName = c2sName + ".bp"; + break; + case TransportType::SST: + engineType = "SST"; + break; + // no default case. This will cause a compiler error if we do not handle a + // an engine type that has been defined in the TransportType enum. + // (-Werror=switch) } s2c_io_.SetEngine(engineType); c2s_io_.SetEngine(engineType); @@ -49,14 +45,14 @@ class AdiosChannel c2s_io_.SetParameters(params); REDEV_ALWAYS_ASSERT(s2c_io_.EngineType() == c2s_io_.EngineType()); switch (transportType) { - case TransportType::SST: - openEnginesSST(noClients, s2cName, c2sName, s2c_io_, c2s_io_, - s2c_engine_, c2s_engine_); - break; - case TransportType::BP4: - openEnginesBP4(noClients, s2cName, c2sName, s2c_io_, c2s_io_, - s2c_engine_, c2s_engine_); - break; + case TransportType::SST: + openEnginesSST(noClients, s2cName, c2sName, s2c_io_, c2s_io_, s2c_engine_, + c2s_engine_); + break; + case TransportType::BP4: + openEnginesBP4(noClients, s2cName, c2sName, s2c_io_, c2s_io_, s2c_engine_, + c2s_engine_); + break; } // TODO pull begin/end step out of Setup/SendReceive metadata functions // begin step @@ -67,27 +63,22 @@ class AdiosChannel // end step } // don't allow copying of class because it creates - AdiosChannel(const AdiosChannel&) = delete; - AdiosChannel operator=(const AdiosChannel&) = delete; + AdiosChannel(const AdiosChannel &) = delete; + AdiosChannel operator=(const AdiosChannel &) = delete; // FIXME - AdiosChannel(AdiosChannel&& o) - : s2c_io_(std::exchange(o.s2c_io_, adios2::IO())), - c2s_io_(std::exchange(o.c2s_io_, adios2::IO())), - c2s_engine_(std::exchange(o.c2s_engine_, adios2::Engine())), - s2c_engine_(std::exchange(o.s2c_engine_, adios2::Engine())), - num_client_ranks_(o.num_client_ranks_), - num_server_ranks_(o.num_server_ranks_), - comm_(std::exchange(o.comm_, MPI_COMM_NULL)), - process_type_(o.process_type_), - rank_(o.rank_), - partition_(o.partition_) - { - REDEV_FUNCTION_TIMER; - } - AdiosChannel operator=(AdiosChannel&&) = delete; + AdiosChannel(AdiosChannel &&o) + : s2c_io_(std::exchange(o.s2c_io_, adios2::IO())), + c2s_io_(std::exchange(o.c2s_io_, adios2::IO())), + c2s_engine_(std::exchange(o.c2s_engine_, adios2::Engine())), + s2c_engine_(std::exchange(o.s2c_engine_, adios2::Engine())), + num_client_ranks_(o.num_client_ranks_), + num_server_ranks_(o.num_server_ranks_), + comm_(std::exchange(o.comm_, MPI_COMM_NULL)), + process_type_(o.process_type_), rank_(o.rank_), + partition_(o.partition_) {REDEV_FUNCTION_TIMER;} + AdiosChannel operator=(AdiosChannel &&) = delete; // FIXME IMPL RULE OF 5 - ~AdiosChannel() - { + ~AdiosChannel() { REDEV_FUNCTION_TIMER; // NEED TO CHECK that the engine exists before trying to close it because it // could be in a moved from state @@ -100,8 +91,7 @@ class AdiosChannel } template [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm, - CommType ctype) - { + CommType ctype){ REDEV_FUNCTION_TIMER; // TODO, remove s2c/c2s distinction on variable names then use std::move // name @@ -111,15 +101,15 @@ class AdiosChannel case CommType::Ptn: s2c = std::make_unique>(comm, num_client_ranks_, s2c_engine_, s2c_io_, name); - c2s = std::make_unique>(comm, num_server_ranks_, - c2s_engine_, c2s_io_, name); - break; + c2s = std::make_unique>(comm, num_server_ranks_, + c2s_engine_, c2s_io_, name); + break; case CommType::Global: s2c = std::make_unique>(comm, s2c_engine_, s2c_io_, name); - c2s = std::make_unique>(comm, c2s_engine_, c2s_io_, - name); - break; + c2s = std::make_unique>(comm, c2s_engine_, c2s_io_, + name); + break; } switch (process_type_) { case ProcessType::Client: return {std::move(c2s), std::move(s2c)}; @@ -128,59 +118,72 @@ class AdiosChannel } return {std::make_unique>(), std::make_unique>()}; } + // TODO s2c/c2s Engine/IO -> send/receive Engine/IO. This removes need for all // the switch statements... - void BeginSendCommunicationPhase() - { + void BeginSendCommunicationPhase() { REDEV_FUNCTION_TIMER; adios2::StepStatus status; switch (process_type_) { - case ProcessType::Client: status = c2s_engine_.BeginStep(); break; - case ProcessType::Server: status = s2c_engine_.BeginStep(); break; + case ProcessType::Client: + status = c2s_engine_.BeginStep(); + break; + case ProcessType::Server: + status = s2c_engine_.BeginStep(); + break; } REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); } - void EndSendCommunicationPhase() - { + void EndSendCommunicationPhase() { switch (process_type_) { - case ProcessType::Client: c2s_engine_.EndStep(); break; - case ProcessType::Server: s2c_engine_.EndStep(); break; + case ProcessType::Client: + c2s_engine_.EndStep(); + break; + case ProcessType::Server: + s2c_engine_.EndStep(); + break; } } - void BeginReceiveCommunicationPhase() - { + void BeginReceiveCommunicationPhase() { REDEV_FUNCTION_TIMER; adios2::StepStatus status; switch (process_type_) { - case ProcessType::Client: status = s2c_engine_.BeginStep(); break; - case ProcessType::Server: status = c2s_engine_.BeginStep(); break; + case ProcessType::Client: + status = s2c_engine_.BeginStep(); + break; + case ProcessType::Server: + status = c2s_engine_.BeginStep(); + break; } REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); } - void EndReceiveCommunicationPhase() - { + void EndReceiveCommunicationPhase() { REDEV_FUNCTION_TIMER; switch (process_type_) { - case ProcessType::Client: s2c_engine_.EndStep(); break; - case ProcessType::Server: c2s_engine_.EndStep(); break; + case ProcessType::Client: + s2c_engine_.EndStep(); + break; + case ProcessType::Server: + c2s_engine_.EndStep(); + break; } } private: void openEnginesBP4(bool noClients, std::string s2cName, std::string c2sName, - adios2::IO& s2cIO, adios2::IO& c2sIO, - adios2::Engine& s2cEngine, adios2::Engine& c2sEngine); + adios2::IO &s2cIO, adios2::IO &c2sIO, + adios2::Engine &s2cEngine, adios2::Engine &c2sEngine); void openEnginesSST(bool noClients, std::string s2cName, std::string c2sName, - adios2::IO& s2cIO, adios2::IO& c2sIO, - adios2::Engine& s2cEngine, adios2::Engine& c2sEngine); - [[nodiscard]] redev::LO SendServerCommSizeToClient(adios2::IO& s2cIO, - adios2::Engine& s2cEngine); - [[nodiscard]] redev::LO SendClientCommSizeToServer(adios2::IO& c2sIO, - adios2::Engine& c2sEngine); - [[nodiscard]] std::size_t SendPartitionTypeToClient( - adios2::IO& s2cIO, adios2::Engine& s2cEngine); - void Setup(adios2::IO& s2cIO, adios2::Engine& s2cEngine); - void CheckVersion(adios2::Engine& eng, adios2::IO& io); + adios2::IO &s2cIO, adios2::IO &c2sIO, + adios2::Engine &s2cEngine, adios2::Engine &c2sEngine); + [[nodiscard]] redev::LO SendServerCommSizeToClient(adios2::IO &s2cIO, + adios2::Engine &s2cEngine); + [[nodiscard]] redev::LO SendClientCommSizeToServer(adios2::IO &c2sIO, + adios2::Engine &c2sEngine); + [[nodiscard]] std::size_t + SendPartitionTypeToClient(adios2::IO &s2cIO, adios2::Engine &s2cEngine); + void Setup(adios2::IO &s2cIO, adios2::Engine &s2cEngine); + void CheckVersion(adios2::Engine &eng, adios2::IO &io); void ConstructPartitionFromIndex(size_t partition_index); adios2::IO s2c_io_; @@ -192,7 +195,7 @@ class AdiosChannel MPI_Comm comm_; ProcessType process_type_; int rank_; - Partition& partition_; + Partition &partition_; }; } // namespace redev diff --git a/redev_channel.h b/redev_channel.h index 4a10454..8f45157 100644 --- a/redev_channel.h +++ b/redev_channel.h @@ -3,30 +3,24 @@ #include "redev_bidirectional_comm.h" #include -namespace redev -{ +namespace redev { -class Channel -{ +class Channel { public: template - Channel(T&& channel) - : pimpl_{new ChannelModel>>( - std::forward(channel))}, - send_communication_phase_active_(false), - receive_communication_phase_active_(false) - { - REDEV_FUNCTION_TIMER; - } + Channel(T &&channel) + : pimpl_{new ChannelModel>>( + std::forward(channel))}, + send_communication_phase_active_(false), + receive_communication_phase_active_(false) { + REDEV_FUNCTION_TIMER; + } // For cases where we may be interested in storing the comm variant rather // than the exact type this function can be used to reduce the runtime // overhead of converting from the variant to the explicit type back to the // variant - template - [[nodiscard]] CommV CreateCommV(std::string name, MPI_Comm comm, - CommType ctype) - { + template [[nodiscard]] CommV CreateCommV(std::string name, MPI_Comm comm, CommType ctype) { REDEV_FUNCTION_TIMER; return pimpl_->CreateComm(std::move(name), comm, std::move(ctype), InvCommunicatorTypeMap::value); @@ -35,167 +29,145 @@ class Channel // rather than the variant. This is here to simplify updating legacy code // that expects a typed communicator to be created. template - [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm, - CommType ctype = CommType::Ptn) - { + [[nodiscard]] BidirectionalComm CreateComm(std::string name, MPI_Comm comm, CommType ctype = CommType::Ptn) { REDEV_FUNCTION_TIMER; - return std::get>( - CreateCommV(std::move(name), comm, std::move(ctype))); + return std::get>(CreateCommV(std::move(name), comm, std::move(ctype))); } - - void BeginSendCommunicationPhase() - { + void BeginSendCommunicationPhase() { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InSendCommunicationPhase() == false); pimpl_->BeginSendCommunicationPhase(); send_communication_phase_active_ = true; } - void EndSendCommunicationPhase() - { + void EndSendCommunicationPhase() { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InSendCommunicationPhase() == true); pimpl_->EndSendCommunicationPhase(); send_communication_phase_active_ = false; } - void BeginReceiveCommunicationPhase() - { + void BeginReceiveCommunicationPhase() { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InReceiveCommunicationPhase() == false); pimpl_->BeginReceiveCommunicationPhase(); receive_communication_phase_active_ = true; } - void EndReceiveCommunicationPhase() - { + void EndReceiveCommunicationPhase() { REDEV_FUNCTION_TIMER; REDEV_ALWAYS_ASSERT(InReceiveCommunicationPhase() == true); pimpl_->EndReceiveCommunicationPhase(); receive_communication_phase_active_ = false; } - [[nodiscard]] bool InSendCommunicationPhase() const noexcept - { + [[nodiscard]] bool InSendCommunicationPhase() const noexcept { REDEV_FUNCTION_TIMER; return send_communication_phase_active_; } - [[nodiscard]] bool InReceiveCommunicationPhase() const noexcept - { + [[nodiscard]] bool InReceiveCommunicationPhase() const noexcept { REDEV_FUNCTION_TIMER; return receive_communication_phase_active_; } template - auto SendPhase(const Func& f, Args&&... args) - { + auto SendPhase(const Func &f, Args &&...args) { auto sg = SendPhaseScope(*this); return f(std::forward(args)...); } template - auto ReceivePhase(const Func& f, Args&&... args) - { + auto ReceivePhase(const Func &f, Args &&...args) { auto sg = ReceivePhaseScope(*this); return f(std::forward(args)...); } private: - class ChannelConcept - { + class ChannelConcept { public: - virtual CommV CreateComm(std::string&&, MPI_Comm, CommType, - CommunicatorDataType) = 0; + virtual CommV CreateComm(std::string &&, MPI_Comm, CommType, CommunicatorDataType) = 0; virtual void BeginSendCommunicationPhase() = 0; virtual void EndSendCommunicationPhase() = 0; virtual void BeginReceiveCommunicationPhase() = 0; virtual void EndReceiveCommunicationPhase() = 0; virtual ~ChannelConcept() noexcept {} }; - template - class ChannelModel final : public ChannelConcept - { + template class ChannelModel final : public ChannelConcept { public: - ChannelModel(T&& impl) : ChannelConcept(), impl_(std::forward(impl)) {} + ChannelModel(T &&impl) : ChannelConcept(), impl_(std::forward(impl)) {} // since we don't have templated virtual functions, we convert the type to a // runtime parameter then extract out the appropriate type based on the // typemap. Assuming the typemap/inverse type map are correct, this is // entirely safe unlike doing it in user code. Although, it's not the most // beautiful construction in the world. ~ChannelModel() noexcept final {} - [[nodiscard]] CommV CreateComm(std::string&& name, MPI_Comm comm, - CommType ctype, - CommunicatorDataType type) final - { + [[nodiscard]] CommV CreateComm(std::string &&name, MPI_Comm comm, CommType ctype, + CommunicatorDataType type) final { REDEV_FUNCTION_TIMER; switch (type) { - case CommunicatorDataType::INT8: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT8: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::INT16: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT16: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::INT32: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT32: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::INT64: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::INT64: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::UINT8: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT8: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::UINT16: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT16: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::UINT32: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT32: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::UINT64: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::UINT64: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::LONG_INT: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::LONG_INT: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::FLOAT: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::FLOAT: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::DOUBLE: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::DOUBLE: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::LONG_DOUBLE: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::LONG_DOUBLE: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; - case CommunicatorDataType::COMPLEX_DOUBLE: - return CommV{impl_.template CreateComm< + case CommunicatorDataType::COMPLEX_DOUBLE: + return CommV{impl_.template CreateComm< CommunicatorTypeMap::type>( std::move(name), comm, std::move(ctype))}; } return {}; } - void BeginSendCommunicationPhase() final - { + void BeginSendCommunicationPhase() final { REDEV_FUNCTION_TIMER; impl_.BeginSendCommunicationPhase(); } - void EndSendCommunicationPhase() final - { + void EndSendCommunicationPhase() final { REDEV_FUNCTION_TIMER; impl_.EndSendCommunicationPhase(); } - void BeginReceiveCommunicationPhase() final - { + void BeginReceiveCommunicationPhase() final { REDEV_FUNCTION_TIMER; impl_.BeginReceiveCommunicationPhase(); } - void EndReceiveCommunicationPhase() final - { + void EndReceiveCommunicationPhase() final { REDEV_FUNCTION_TIMER; impl_.EndReceiveCommunicationPhase(); } @@ -203,29 +175,25 @@ class Channel private: T impl_; }; - class SendPhaseScope - { + class SendPhaseScope { public: - explicit SendPhaseScope(Channel& channel) : channel_(channel) - { + explicit SendPhaseScope(Channel &channel) : channel_(channel) { channel_.BeginSendCommunicationPhase(); } ~SendPhaseScope() { channel_.EndSendCommunicationPhase(); } private: - Channel& channel_; + Channel &channel_; }; - class ReceivePhaseScope - { + class ReceivePhaseScope { public: - explicit ReceivePhaseScope(Channel& channel) : channel_(channel) - { + explicit ReceivePhaseScope(Channel &channel) : channel_(channel) { channel_.BeginReceiveCommunicationPhase(); } ~ReceivePhaseScope() { channel_.EndReceiveCommunicationPhase(); } private: - Channel& channel_; + Channel &channel_; }; std::unique_ptr pimpl_; @@ -233,19 +201,17 @@ class Channel bool receive_communication_phase_active_; }; -class NoOpChannel -{ +class NoOpChannel { public: template [[nodiscard]] - BidirectionalComm CreateComm(std::string, MPI_Comm, CommType) - { + BidirectionalComm CreateComm(std::string, MPI_Comm) { return {std::make_unique>(), std::make_unique>()}; } - void BeginSendCommunicationPhase() {} - void EndSendCommunicationPhase() {} - void BeginReceiveCommunicationPhase() {} - void EndReceiveCommunicationPhase() {} + void BeginSendCommunicationPhase(){} + void EndSendCommunicationPhase(){} + void BeginReceiveCommunicationPhase(){} + void EndReceiveCommunicationPhase(){} }; } // namespace redev diff --git a/redev_comm.h b/redev_comm.h index 7a1069b..32061d5 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -10,100 +10,60 @@ #include #include "redev_time.h" -namespace -{ -void checkStep(adios2::StepStatus status) -{ +namespace { +void checkStep(adios2::StepStatus status) { REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK); } -} // namespace +} -namespace redev -{ +namespace redev { -namespace detail -{ -template -struct dependent_always_false : std::false_type -{}; -} // namespace detail + namespace detail { + template struct dependent_always_false : std::false_type {}; + } -enum class Mode -{ +enum class Mode { Deferred, Synchronous }; -template -[[nodiscard]] -constexpr MPI_Datatype getMpiType(T) noexcept -{ - if constexpr (std::is_same_v) { - return MPI_CHAR; - } else if constexpr (std::is_same_v) { - return MPI_SHORT; - } else if constexpr (std::is_same_v) { - return MPI_INT; - } else if constexpr (std::is_same_v) { - return MPI_LONG; - } else if constexpr (std::is_same_v) { - return MPI_LONG_LONG; - } else if constexpr (std::is_same_v) { - return MPI_SIGNED_CHAR; - } else if constexpr (std::is_same_v) { - return MPI_UNSIGNED_CHAR; - } else if constexpr (std::is_same_v) { - return MPI_UNSIGNED_SHORT; - } else if constexpr (std::is_same_v) { - return MPI_UNSIGNED; - } else if constexpr (std::is_same_v) { - return MPI_UNSIGNED_LONG; - } else if constexpr (std::is_same_v) { - return MPI_UNSIGNED_LONG_LONG; - } else if constexpr (std::is_same_v) { - return MPI_FLOAT; - } else if constexpr (std::is_same_v) { - return MPI_DOUBLE; - } else if constexpr (std::is_same_v) { - return MPI_LONG_DOUBLE; - } else if constexpr (std::is_same_v) { - return MPI_WCHAR; - } else if constexpr (std::is_same_v) { - return MPI_INT8_T; - } else if constexpr (std::is_same_v) { - return MPI_INT16_T; - } else if constexpr (std::is_same_v) { - return MPI_INT32_T; - } else if constexpr (std::is_same_v) { - return MPI_INT64_T; - } else if constexpr (std::is_same_v) { - return MPI_UINT8_T; - } else if constexpr (std::is_same_v) { - return MPI_UINT16_T; - } else if constexpr (std::is_same_v) { - return MPI_UINT32_T; - } else if constexpr (std::is_same_v) { - return MPI_UINT64_T; - } else if constexpr (std::is_same_v) { - return MPI_CXX_BOOL; - } else if constexpr (std::is_same_v>) { - return MPI_CXX_FLOAT_COMPLEX; - } else if constexpr (std::is_same_v>) { - return MPI_CXX_DOUBLE_COMPLEX; - } else if constexpr (std::is_same_v>) { - return MPI_CXX_LONG_DOUBLE_COMPLEX; - } else { - static_assert(detail::dependent_always_false::value, - "type has unkown map to MPI_Type"); - return {}; - } +template +[[ nodiscard ]] +constexpr MPI_Datatype getMpiType(T) noexcept { + if constexpr (std::is_same_v) { return MPI_CHAR; } + else if constexpr (std::is_same_v) { return MPI_SHORT; } + else if constexpr (std::is_same_v) { return MPI_INT; } + else if constexpr (std::is_same_v) { return MPI_LONG; } + else if constexpr (std::is_same_v) { return MPI_LONG_LONG; } + else if constexpr (std::is_same_v) { return MPI_SIGNED_CHAR; } + else if constexpr (std::is_same_v) { return MPI_UNSIGNED_CHAR; } + else if constexpr (std::is_same_v) { return MPI_UNSIGNED_SHORT; } + else if constexpr (std::is_same_v) { return MPI_UNSIGNED; } + else if constexpr (std::is_same_v) { return MPI_UNSIGNED_LONG; } + else if constexpr (std::is_same_v) { return MPI_UNSIGNED_LONG_LONG; } + else if constexpr (std::is_same_v) { return MPI_FLOAT; } + else if constexpr (std::is_same_v) { return MPI_DOUBLE; } + else if constexpr (std::is_same_v) { return MPI_LONG_DOUBLE; } + else if constexpr (std::is_same_v) { return MPI_WCHAR; } + else if constexpr (std::is_same_v) { return MPI_INT8_T; } + else if constexpr (std::is_same_v) { return MPI_INT16_T; } + else if constexpr (std::is_same_v) { return MPI_INT32_T; } + else if constexpr (std::is_same_v) { return MPI_INT64_T; } + else if constexpr (std::is_same_v) { return MPI_UINT8_T; } + else if constexpr (std::is_same_v) { return MPI_UINT16_T; } + else if constexpr (std::is_same_v) { return MPI_UINT32_T; } + else if constexpr (std::is_same_v) { return MPI_UINT64_T; } + else if constexpr (std::is_same_v) { return MPI_CXX_BOOL; } + else if constexpr (std::is_same_v>) { return MPI_CXX_FLOAT_COMPLEX; } + else if constexpr (std::is_same_v>) { return MPI_CXX_DOUBLE_COMPLEX; } + else if constexpr (std::is_same_v>) { return MPI_CXX_LONG_DOUBLE_COMPLEX; } + else{ static_assert(detail::dependent_always_false::value, "type has unkown map to MPI_Type"); return {}; } // empty return statement needed to avoid compiler warning return {}; } -template -void Broadcast(T* data, int count, int root, MPI_Comm comm) -{ +template +void Broadcast(T* data, int count, int root, MPI_Comm comm) { REDEV_FUNCTION_TIMER; auto type = getMpiType(T()); MPI_Bcast(data, count, type, root, comm); @@ -113,8 +73,7 @@ void Broadcast(T* data, int count, int root, MPI_Comm comm) * The InMessageLayout struct contains the arrays defining the arrangement of * data in the array returned by Communicator::Recv. */ -struct InMessageLayout -{ +struct InMessageLayout { /** * Array of source ranks sized NumberOfClientRanks*NumberOfServerRanks. Each * rank reads the entire array once at the start of a communication round. @@ -129,13 +88,13 @@ struct InMessageLayout */ redev::GOs offset; /** - * Set to true if Communicator::Recv has been called and the message layout - * data set; false otherwise. + * Set to true if Communicator::Recv has been called and the message layout data set; + * false otherwise. */ bool knownSizes; /** - * Index into the messages array (returned by Communicator::Recv) where the - * current process should start reading. + * Index into the messages array (returned by Communicator::Recv) where the current process should start + * reading. */ size_t start; /** @@ -149,346 +108,321 @@ struct InMessageLayout /** * The Communicator class provides an abstract interface for sending and * receiving messages to/from the client and server. - * TODO: Split Communicator into Send/Recieve Communicators, bidirectional - * constructed by composition and can perform both send and receive + * TODO: Split Communicator into Send/Recieve Communicators, bidirectional constructed by composition and can perform both send and receive */ -template -class Communicator -{ -public: - /** - * Set the arrangement of data in the messages array so that its segments, - * defined by the offsets array, are sent to the correct destination ranks, - * defined by the dest array. - * @param[in] dest array of integers specifying the destination rank for a - * portion of the msgs array - * @param[in] offsets array of length |dest|+1 defining the segment of the - * msgs array (passed to the Send function) being sent to each destination - * rank. the segment [ msgs[offsets[i]] : msgs[offsets[i+1]] } is sent to rank - * dest[i] - */ - virtual void SetOutMessageLayout(LOs& dest, LOs& offsets) = 0; - /** - * Send the array. - * @param[in] msgs array of data to be sent according to the layout specified - * with SetOutMessageLayout - */ - virtual void Send(T* msgs, Mode mode) = 0; - /** - * Receive an array. Use AdiosPtnComm's GetInMessageLayout to retreive - * an instance of the InMessageLayout struct containing the layout of - * the received array. - */ - virtual std::vector Recv(Mode mode) = 0; - - virtual InMessageLayout GetInMessageLayout() = 0; - virtual ~Communicator() = default; +template +class Communicator { + public: + /** + * Set the arrangement of data in the messages array so that its segments, + * defined by the offsets array, are sent to the correct destination ranks, + * defined by the dest array. + * @param[in] dest array of integers specifying the destination rank for a + * portion of the msgs array + * @param[in] offsets array of length |dest|+1 defining the segment of the + * msgs array (passed to the Send function) being sent to each destination rank. + * the segment [ msgs[offsets[i]] : msgs[offsets[i+1]] } is sent to rank dest[i] + */ + virtual void SetOutMessageLayout(LOs& dest, LOs& offsets) = 0; + /** + * Send the array. + * @param[in] msgs array of data to be sent according to the layout specified + * with SetOutMessageLayout + */ + virtual void Send(T *msgs, Mode mode) = 0; + /** + * Receive an array. Use AdiosPtnComm's GetInMessageLayout to retreive + * an instance of the InMessageLayout struct containing the layout of + * the received array. + */ + virtual std::vector Recv(Mode mode) = 0; + + virtual InMessageLayout GetInMessageLayout() = 0; + virtual ~Communicator() = default; }; template -class NoOpComm : public Communicator -{ - void SetOutMessageLayout(LOs& dest, LOs& offsets) final {}; - void Send(T* msgs, Mode /*unused*/) final {}; - std::vector Recv(Mode /*unused*/) final { return {}; } - InMessageLayout GetInMessageLayout() final { return {}; } +class NoOpComm : public Communicator { + void SetOutMessageLayout(LOs& dest, LOs& offsets) final {}; + void Send(T *msgs, Mode /*unused*/) final {}; + std::vector Recv(Mode /*unused*/) final { return {}; } + InMessageLayout GetInMessageLayout() final { return {}; } }; + /** - * The AdiosPtnComm class implements the Communicator interface to support - * sending messages between the clients and server via ADIOS2. The BP4 and SST - * ADIOS2 engines are currently supported. One AdiosPtnComm object is required - * for each communication link direction. For example, for a client and server - * to both send and receive messages one AdiosPtnComm for client->server - * messaging and another AdiosPtnComm for server->client messaging are needed. - * Redev::BidirectionalComm is a helper class for this use case. + * The AdiosPtnComm class implements the Communicator interface to support sending + * messages between the clients and server via ADIOS2. The BP4 and SST ADIOS2 + * engines are currently supported. + * One AdiosPtnComm object is required for each communication link direction. For + * example, for a client and server to both send and receive messages one + * AdiosPtnComm for client->server messaging and another AdiosPtnComm for + * server->client messaging are needed. Redev::BidirectionalComm is a helper + * class for this use case. */ -template -class AdiosPtnComm : public Communicator -{ -public: - /** - * Create an AdiosPtnComm object. Collective across sender and receiver - * ranks. Calls to the constructor from the sender and receiver ranks must be - * in the same order (i.e., first creating the client-to-server object then - * the server-to-client link). - * @param[in] comm_ MPI communicator for sender ranks - * @param[in] recvRanks_ number of ranks in the receivers MPI communicator - * @param[in] eng_ ADIOS2 engine for writing on the sender side - * @param[in] io_ ADIOS2 IO associated with eng_ - * @param[in] name_ unique name among AdiosPtnComm objects - */ - AdiosPtnComm(MPI_Comm comm_, int recvRanks_, adios2::Engine& eng_, - adios2::IO& io_, std::string name_) - : comm(comm_), - recvRanks(recvRanks_), - eng(eng_), - io(io_), - name(name_), - verbose(0) - { - inMsg.knownSizes = false; - } - - /// We are explicitly not allowing copy/move constructor/assignment as we - /// don't know if the ADIOS2 Engine and IO objects can be safely copied/moved. - AdiosPtnComm(const AdiosPtnComm& other) = delete; - AdiosPtnComm(AdiosPtnComm&& other) = delete; - AdiosPtnComm& operator=(const AdiosPtnComm& other) = delete; - AdiosPtnComm& operator=(AdiosPtnComm&& other) = delete; - - void SetOutMessageLayout(LOs& dest_, LOs& offsets_) - { - REDEV_FUNCTION_TIMER; - outMsg = OutMessageLayout{dest_, offsets_}; - } - void Send(T* msgs, Mode mode) - { - REDEV_FUNCTION_TIMER; - int rank, commSz; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &commSz); - GOs degree(recvRanks, 0); // TODO ideally, this would not be needed - for (size_t i = 0; i < outMsg.dest.size(); i++) { - auto destRank = outMsg.dest[i]; - assert(destRank < recvRanks); - degree[destRank] += outMsg.offsets[i + 1] - outMsg.offsets[i]; +template +class AdiosPtnComm : public Communicator { + public: + /** + * Create an AdiosPtnComm object. Collective across sender and receiver ranks. + * Calls to the constructor from the sender and receiver ranks must be in + * the same order (i.e., first creating the client-to-server object then the + * server-to-client link). + * @param[in] comm_ MPI communicator for sender ranks + * @param[in] recvRanks_ number of ranks in the receivers MPI communicator + * @param[in] eng_ ADIOS2 engine for writing on the sender side + * @param[in] io_ ADIOS2 IO associated with eng_ + * @param[in] name_ unique name among AdiosPtnComm objects + */ + AdiosPtnComm(MPI_Comm comm_, int recvRanks_, adios2::Engine& eng_, adios2::IO& io_, std::string name_) + : comm(comm_), recvRanks(recvRanks_), eng(eng_), io(io_), name(name_), verbose(0) { + inMsg.knownSizes = false; } - GOs rdvRankStart(recvRanks, 0); - auto ret = MPI_Exscan(degree.data(), rdvRankStart.data(), recvRanks, - getMpiType(redev::GO()), MPI_SUM, comm); - assert(ret == MPI_SUCCESS); - if (!rank) { - // on rank 0 the result of MPI_Exscan is undefined, set it to zero - rdvRankStart = GOs(recvRanks, 0); - } - - GOs gDegree(recvRanks, 0); - ret = MPI_Allreduce(degree.data(), gDegree.data(), recvRanks, - getMpiType(redev::GO()), MPI_SUM, comm); - assert(ret == MPI_SUCCESS); - const size_t gDegreeTot = static_cast( - std::accumulate(gDegree.begin(), gDegree.end(), redev::GO(0))); - - GOs gStart(recvRanks, 0); - redev::exclusive_scan(gDegree.begin(), gDegree.end(), gStart.begin(), - redev::GO(0)); - - // The messages array has a different length on each rank ('irregular') so - // we don't define local size and count here. - adios2::Dims shape{static_cast(gDegreeTot)}; - adios2::Dims start{}; - adios2::Dims count{}; - if (!rdvVar) { - rdvVar = io.DefineVariable(name, shape, start, count); + + /// We are explicitly not allowing copy/move constructor/assignment as we don't + /// know if the ADIOS2 Engine and IO objects can be safely copied/moved. + AdiosPtnComm(const AdiosPtnComm& other) = delete; + AdiosPtnComm(AdiosPtnComm&& other) = delete; + AdiosPtnComm& operator=(const AdiosPtnComm& other) = delete; + AdiosPtnComm& operator=(AdiosPtnComm&& other) = delete; + + void SetOutMessageLayout(LOs& dest_, LOs& offsets_) { + REDEV_FUNCTION_TIMER; + outMsg = OutMessageLayout{dest_, offsets_}; } - assert(rdvVar); - const auto srcRanksName = name + "_srcRanks"; - // The source rank offsets array is the same on each process ('regular'). - adios2::Dims srShape{static_cast(commSz * recvRanks)}; - adios2::Dims srStart{static_cast(recvRanks * rank)}; - adios2::Dims srCount{static_cast(recvRanks)}; - - // send dest rank offsets array from rank 0 - auto offsets = gStart; - offsets.push_back(gDegreeTot); - if (!rank) { - const auto offsetsName = name + "_offsets"; - const auto oShape = offsets.size(); - const auto oStart = 0; - const auto oCount = offsets.size(); - if (!offsetsVar) { - offsetsVar = io.DefineVariable(offsetsName, {oShape}, - {oStart}, {oCount}); - // if we are in sync mode we will peform all puts at the end of the - // function, otherwise we need to put this now before offsets data goes - // out of scope - eng.Put(offsetsVar, offsets.data(), - (mode == Mode::Deferred) ? adios2::Mode::Sync - : adios2::Mode::Deferred); + void Send(T *msgs, Mode mode) { + REDEV_FUNCTION_TIMER; + int rank, commSz; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &commSz); + GOs degree(recvRanks,0); //TODO ideally, this would not be needed + for( size_t i=0; i(srcRanksName, srShape, srStart, srCount); - assert(srcRanksVar); - // if we are in sync mode we will peform all puts at the end of the - // function, otherwise we need to put this now before ranks data goes out - // of scope - eng.Put(srcRanksVar, rdvRankStart.data(), - (mode == Mode::Deferred) ? adios2::Mode::Sync - : adios2::Mode::Deferred); - } - - // assume one call to pack from each rank for now - for (size_t i = 0; i < outMsg.dest.size(); i++) { - const auto destRank = outMsg.dest[i]; - const auto lStart = gStart[destRank] + rdvRankStart[destRank]; - const auto lCount = outMsg.offsets[i + 1] - outMsg.offsets[i]; - if (lCount > 0) { - start = adios2::Dims{static_cast(lStart)}; - count = adios2::Dims{static_cast(lCount)}; - rdvVar.SetSelection({start, count}); - eng.Put(rdvVar, &(msgs[outMsg.offsets[i]])); + GOs rdvRankStart(recvRanks,0); + auto ret = MPI_Exscan(degree.data(), rdvRankStart.data(), recvRanks, + getMpiType(redev::GO()), MPI_SUM, comm); + assert(ret == MPI_SUCCESS); + if(!rank) { + //on rank 0 the result of MPI_Exscan is undefined, set it to zero + rdvRankStart = GOs(recvRanks,0); } - } - if (mode == Mode::Synchronous) { - eng.PerformPuts(); - } - } - std::vector Recv(Mode mode) - { - REDEV_FUNCTION_TIMER; - int rank, commSz; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &commSz); - auto t1 = redev::getTime(); - if (!inMsg.knownSizes) { - auto rdvRanksVar = io.InquireVariable(name + "_srcRanks"); - assert(rdvRanksVar); - auto offsetsVar = io.InquireVariable(name + "_offsets"); - assert(offsetsVar); + GOs gDegree(recvRanks,0); + ret = MPI_Allreduce(degree.data(), gDegree.data(), recvRanks, + getMpiType(redev::GO()), MPI_SUM, comm); + assert(ret == MPI_SUCCESS); + const size_t gDegreeTot = static_cast(std::accumulate(gDegree.begin(), gDegree.end(), redev::GO(0))); + + GOs gStart(recvRanks,0); + redev::exclusive_scan(gDegree.begin(), gDegree.end(), gStart.begin(), redev::GO(0)); + + //The messages array has a different length on each rank ('irregular') so we don't + //define local size and count here. + adios2::Dims shape{static_cast(gDegreeTot)}; + adios2::Dims start{}; + adios2::Dims count{}; + if(!rdvVar) { + rdvVar = io.DefineVariable(name, shape, start, count); + } + assert(rdvVar); + const auto srcRanksName = name+"_srcRanks"; + //The source rank offsets array is the same on each process ('regular'). + adios2::Dims srShape{static_cast(commSz*recvRanks)}; + adios2::Dims srStart{static_cast(recvRanks*rank)}; + adios2::Dims srCount{static_cast(recvRanks)}; + + //send dest rank offsets array from rank 0 + auto offsets = gStart; + offsets.push_back(gDegreeTot); + if(!rank) { + const auto offsetsName = name+"_offsets"; + const auto oShape = offsets.size(); + const auto oStart = 0; + const auto oCount = offsets.size(); + if(!offsetsVar) { + offsetsVar = io.DefineVariable(offsetsName,{oShape},{oStart},{oCount}); + // if we are in sync mode we will peform all puts at the end of the function, otherwise we need to put this now before + // offsets data goes out of scope + eng.Put(offsetsVar, offsets.data(), (mode==Mode::Deferred)?adios2::Mode::Sync:adios2::Mode::Deferred); + } + } - auto offsetsShape = offsetsVar.Shape(); - assert(offsetsShape.size() == 1); - const auto offSz = offsetsShape[0]; - inMsg.offset.resize(offSz); - offsetsVar.SetSelection({{0}, {offSz}}); - eng.Get(offsetsVar, inMsg.offset.data()); + //send source rank offsets array 'rdvRankStart' + if(!srcRanksVar) { + srcRanksVar = io.DefineVariable(srcRanksName, srShape, srStart, srCount); + assert(srcRanksVar); + // if we are in sync mode we will peform all puts at the end of the function, otherwise we need to put this now before + // ranks data goes out of scope + eng.Put(srcRanksVar, rdvRankStart.data(),(mode==Mode::Deferred)?adios2::Mode::Sync:adios2::Mode::Deferred); - auto rdvRanksShape = rdvRanksVar.Shape(); - assert(rdvRanksShape.size() == 1); - const auto rsrSz = rdvRanksShape[0]; - inMsg.srcRanks.resize(rsrSz); - rdvRanksVar.SetSelection({{0}, {rsrSz}}); - eng.Get(rdvRanksVar, inMsg.srcRanks.data()); + } - // TODO: Can remove in synchronous mode? - eng.PerformGets(); - inMsg.start = static_cast(inMsg.offset[rank]); - inMsg.count = static_cast(inMsg.offset[rank + 1] - inMsg.start); - inMsg.knownSizes = true; + //assume one call to pack from each rank for now + for( size_t i=0; i 0 ) { + start = adios2::Dims{static_cast(lStart)}; + count = adios2::Dims{static_cast(lCount)}; + rdvVar.SetSelection({start,count}); + eng.Put(rdvVar, &(msgs[outMsg.offsets[i]])); + } + } + if(mode == Mode::Synchronous) { + eng.PerformPuts(); + } } - auto t2 = redev::getTime(); + std::vector Recv(Mode mode) { + REDEV_FUNCTION_TIMER; + int rank, commSz; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &commSz); + auto t1 = redev::getTime(); + + if(!inMsg.knownSizes) { + auto rdvRanksVar = io.InquireVariable(name+"_srcRanks"); + assert(rdvRanksVar); + auto offsetsVar = io.InquireVariable(name+"_offsets"); + assert(offsetsVar); + + auto offsetsShape = offsetsVar.Shape(); + assert(offsetsShape.size() == 1); + const auto offSz = offsetsShape[0]; + inMsg.offset.resize(offSz); + offsetsVar.SetSelection({{0}, {offSz}}); + eng.Get(offsetsVar, inMsg.offset.data()); + + auto rdvRanksShape = rdvRanksVar.Shape(); + assert(rdvRanksShape.size() == 1); + const auto rsrSz = rdvRanksShape[0]; + inMsg.srcRanks.resize(rsrSz); + rdvRanksVar.SetSelection({{0},{rsrSz}}); + eng.Get(rdvRanksVar, inMsg.srcRanks.data()); + + // TODO: Can remove in synchronous mode? + eng.PerformGets(); + inMsg.start = static_cast(inMsg.offset[rank]); + inMsg.count = static_cast(inMsg.offset[rank+1]-inMsg.start); + inMsg.knownSizes = true; + } + auto t2 = redev::getTime(); + + auto msgsVar = io.InquireVariable(name); + assert(msgsVar); + std::vector msgs(inMsg.count); + if(inMsg.count) { + //only call Get with non-zero sized reads + msgsVar.SetSelection({{inMsg.start}, {inMsg.count}}); + eng.Get(msgsVar, msgs.data()); + } + if(mode == Mode::Synchronous) { + eng.PerformGets(); + } - auto msgsVar = io.InquireVariable(name); - assert(msgsVar); - std::vector msgs(inMsg.count); - if (inMsg.count) { - // only call Get with non-zero sized reads - msgsVar.SetSelection({{inMsg.start}, {inMsg.count}}); - eng.Get(msgsVar, msgs.data()); + //if(mode == Mode::Synchronous) { + // eng.EndStep(); + //} + auto t3 = redev::getTime(); + std::chrono::duration r1 = t2-t1; + std::chrono::duration r2 = t3-t2; + if(!rank && verbose) { + fprintf(stderr, "recv knownSizes %d r1(sec.) r2(sec.) %f %f\n", + inMsg.knownSizes, r1.count(), r2.count()); + } + return msgs; } - if (mode == Mode::Synchronous) { - eng.PerformGets(); + /** + * Return the InMessageLayout object. + * @todo should return const object + */ + InMessageLayout GetInMessageLayout() { + return inMsg; } - - // if(mode == Mode::Synchronous) { - // eng.EndStep(); - // } - auto t3 = redev::getTime(); - std::chrono::duration r1 = t2 - t1; - std::chrono::duration r2 = t3 - t2; - if (!rank && verbose) { - fprintf(stderr, "recv knownSizes %d r1(sec.) r2(sec.) %f %f\n", - inMsg.knownSizes, r1.count(), r2.count()); + /** + * Control the amount of output from AdiosPtnComm functions. The higher the value the more output is written. + * @param[in] lvl valid values are [0:5] where 0 is silent and 5 is produces + * the most output + */ + void SetVerbose(int lvl) { + assert(lvl>=0 && lvl<=5); + verbose = lvl; } - return msgs; - } - /** - * Return the InMessageLayout object. - * @todo should return const object - */ - InMessageLayout GetInMessageLayout() { return inMsg; } - /** - * Control the amount of output from AdiosPtnComm functions. The higher the - * value the more output is written. - * @param[in] lvl valid values are [0:5] where 0 is silent and 5 is produces - * the most output - */ - void SetVerbose(int lvl) - { - assert(lvl >= 0 && lvl <= 5); - verbose = lvl; - } - -private: - MPI_Comm comm; - int recvRanks; - adios2::Engine& eng; - adios2::IO& io; - adios2::Variable rdvVar; - adios2::Variable srcRanksVar; - adios2::Variable offsetsVar; - std::string name; - // support only one call to pack for now... - struct OutMessageLayout - { - LOs dest; - LOs offsets; - } outMsg; - int verbose; - // receive side state - InMessageLayout inMsg; + private: + MPI_Comm comm; + int recvRanks; + adios2::Engine& eng; + adios2::IO& io; + adios2::Variable rdvVar; + adios2::Variable srcRanksVar; + adios2::Variable offsetsVar; + std::string name; + //support only one call to pack for now... + struct OutMessageLayout { + LOs dest; + LOs offsets; + } outMsg; + int verbose; + //receive side state + InMessageLayout inMsg; }; template class AdiosGlobalComm : public Communicator -{ -public: - AdiosGlobalComm(MPI_Comm comm, adios2::Engine& eng_, adios2::IO& io_, - std::string name_) - : comm(comm), eng(eng_), io(io_), name(name_) - { - } - - // copy/move of adios engine and io objects isn't safe. - AdiosGlobalComm(const AdiosGlobalComm& other) = delete; - AdiosGlobalComm(AdiosGlobalComm&& other) = delete; - AdiosGlobalComm& operator=(const AdiosGlobalComm& other) = delete; - AdiosGlobalComm& operator=(AdiosGlobalComm&& other) = delete; - - void Send(T* msg, Mode mode) - { - REDEV_FUNCTION_TIMER - const auto varName = name; - - auto var = io.InquireVariable(varName); - if (!var) { - var = io.DefineVariable(varName); - } - eng.Put(var, msg); - } - std::vector Recv(Mode mode) - { - REDEV_FUNCTION_TIMER - const auto varName = name; - std::vector msg; - auto var = io.InquireVariable(varName); - assert(var); - if (var) { - eng.Get(var, msg); - eng.PerformGets(); - } - return msg; - } - void SetOutMessageLayout(LOs& dest, LOs& offsets) {}; - InMessageLayout GetInMessageLayout() { return {}; } - void SetVerbose(int lvl) - { - assert(lvl >= 0 && lvl <= 5); - Verbose = lvl; - } - -private: - MPI_Comm comm; - adios2::Engine& eng; - adios2::IO& io; - std::string name; - int Verbose; -}; - -} // namespace redev + { + public: + AdiosGlobalComm(MPI_Comm comm, adios2::Engine& eng_, adios2::IO& io_, + std::string name_) + : comm(comm), eng(eng_), io(io_), name(name_) + { + } + + // copy/move of adios engine and io objects isn't safe. + AdiosGlobalComm(const AdiosGlobalComm& other) = delete; + AdiosGlobalComm(AdiosGlobalComm&& other) = delete; + AdiosGlobalComm& operator=(const AdiosGlobalComm& other) = delete; + AdiosGlobalComm& operator=(AdiosGlobalComm&& other) = delete; + + void Send(T* msg, Mode mode) + { + REDEV_FUNCTION_TIMER + const auto varName = name; + + auto var = io.InquireVariable(varName); + if (!var) { + var = io.DefineVariable(varName); + } + eng.Put(var, msg); + } + std::vector Recv(Mode mode) + { + REDEV_FUNCTION_TIMER + const auto varName = name; + std::vector msg; + auto var = io.InquireVariable(varName); + assert(var); + if (var) { + eng.Get(var, msg); + eng.PerformGets(); + } + return msg; + } + void SetOutMessageLayout(LOs& dest, LOs& offsets) {}; + InMessageLayout GetInMessageLayout() { return {}; } + void SetVerbose(int lvl) + { + assert(lvl >= 0 && lvl <= 5); + Verbose = lvl; + } + + private: + MPI_Comm comm; + adios2::Engine& eng; + adios2::IO& io; + std::string name; + int Verbose; + }; +} diff --git a/redev_types.h b/redev_types.h index 834f5ef..40c6e68 100644 --- a/redev_types.h +++ b/redev_types.h @@ -4,40 +4,26 @@ #include #include -namespace redev -{ -/// Local ordinate, used to count items local to a process +namespace redev { +///Local ordinate, used to count items local to a process typedef std::int32_t LO; -/// Vector of local ordinates +///Vector of local ordinates using LOs = std::vector; -/// Global ordinate, used to count items across multiple processes +///Global ordinate, used to count items across multiple processes typedef std::int64_t GO; -/// Vector of global ordinates +///Vector of global ordinates using GOs = std::vector; -/// Floating point values +///Floating point values typedef double Real; -/// Vector of floating point values +///Vector of floating point values using Reals = std::vector; -/// Complex values +///Complex values typedef std::complex CV; -/// Vector of complex values +///Vector of complex values using CVs = std::vector; -enum class ProcessType -{ - Client = 0, - Server = 1 -}; -enum class TransportType -{ - BP4 = 0, - SST = 1 -}; -enum class CommType -{ - Ptn = 0, - Global = 1 -}; - -} // namespace redev +enum class ProcessType { Client = 0, Server = 1 }; +enum class TransportType { BP4 = 0, SST = 1 }; +enum class CommType{ Ptn = 0, Global = 1 }; +} #endif diff --git a/test_sendrecv.cpp b/test_sendrecv.cpp index 07e5a73..49e6b62 100644 --- a/test_sendrecv.cpp +++ b/test_sendrecv.cpp @@ -11,87 +11,80 @@ * Coming soon... */ -// If the name of the variable being sent ("foo"), the communication -// pattern, or data being sent is changed then also update the cmake -// test using the adios2 utility bpls to check the array. +//If the name of the variable being sent ("foo"), the communication +//pattern, or data being sent is changed then also update the cmake +//test using the adios2 utility bpls to check the array. -int main(int argc, char** argv) -{ +int main(int argc, char** argv) { int rank, nproc; MPI_Init(&argc, &argv); - if (argc != 2) { - std::cerr << "Usage: " << argv[0] - << " <1=isRendezvousApp,0=isParticipant>\n"; + if(argc != 2) { + std::cerr << "Usage: " << argv[0] << " <1=isRendezvousApp,0=isParticipant>\n"; exit(EXIT_FAILURE); } MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); auto isRdv = atoi(argv[1]); fprintf(stderr, "rank %d isRdv %d\n", rank, isRdv); - if (isRdv && nproc != 4) { - std::cerr - << "There must be exactly 4 rendezvous processes for this test.\n"; - exit(EXIT_FAILURE); + if(isRdv && nproc != 4) { + std::cerr << "There must be exactly 4 rendezvous processes for this test.\n"; + exit(EXIT_FAILURE); } - if (!isRdv && nproc != 3) { - std::cerr - << "There must be exactly 3 non-rendezvous processes for this test.\n"; - exit(EXIT_FAILURE); + if(!isRdv && nproc != 3) { + std::cerr << "There must be exactly 3 non-rendezvous processes for this test.\n"; + exit(EXIT_FAILURE); } { - // dummy partition vector data - const auto dim = 2; - auto ranks = isRdv ? redev::LOs({0, 1, 2, 3}) : redev::LOs(4); - auto cuts = isRdv ? redev::Reals({0, 0.5, 0.75, 0.25}) : redev::Reals(4); - auto ptn = redev::RCBPtn(dim, ranks, cuts); - redev::Redev rdv(MPI_COMM_WORLD, redev::Partition{std::move(ptn)}, - static_cast(isRdv)); - std::string name = "foo"; - adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; - auto channel = - rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); - auto commPair = channel.CreateComm(name, MPI_COMM_WORLD); - // the non-rendezvous app sends to the rendezvous app - if (!isRdv) { - redev::LOs dest; - redev::LOs offsets; - redev::LOs msgs; - if (rank == 0) { - dest = redev::LOs{0, 2}; - offsets = redev::LOs{0, 2, 6}; - msgs = redev::LOs(6, 0); // write the src rank as the msg for now - } else if (rank == 1) { - dest = redev::LOs{0, 1, 2, 3}; - offsets = redev::LOs{0, 1, 4, 8, 10}; - msgs = redev::LOs(10, 1); - } else if (rank == 2) { - dest = redev::LOs{0, 1, 2, 3}; - offsets = redev::LOs{0, 4, 5, 7, 11}; - msgs = redev::LOs(11, 2); - } - commPair.SetOutMessageLayout(dest, offsets); - channel.BeginSendCommunicationPhase(); - commPair.Send(msgs.data(), redev::Mode::Deferred); - channel.EndSendCommunicationPhase(); - } else { - channel.BeginReceiveCommunicationPhase(); - auto msgVec = commPair.Recv(redev::Mode::Deferred); - channel.EndReceiveCommunicationPhase(); - auto inMsg = commPair.GetInMessageLayout(); - REDEV_ALWAYS_ASSERT(inMsg.offset == redev::GOs({0, 7, 11, 21, 27})); - REDEV_ALWAYS_ASSERT(inMsg.srcRanks == - redev::GOs({0, 0, 0, 0, 2, 0, 4, 0, 3, 3, 8, 2})); - if (rank == 0) { - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0, 0, 1, 2, 2, 2, 2})); - } else if (rank == 1) { - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1, 1, 1, 2})); - } else if (rank == 2) { - REDEV_ALWAYS_ASSERT(msgVec == - redev::LOs({0, 0, 0, 0, 1, 1, 1, 1, 2, 2})); - } else if (rank == 3) { - REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1, 1, 2, 2, 2, 2})); - } + //dummy partition vector data + const auto dim = 2; + auto ranks = isRdv ? redev::LOs({0,1,2,3}) : redev::LOs(4); + auto cuts = isRdv ? redev::Reals({0,0.5,0.75,0.25}) : redev::Reals(4); + auto ptn = redev::RCBPtn(dim,ranks,cuts); + redev::Redev rdv(MPI_COMM_WORLD,redev::Partition{std::move(ptn)},static_cast(isRdv)); + std::string name = "foo"; + adios2::Params params{ {"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; + auto channel = rdv.CreateAdiosChannel(name, params, + redev::TransportType::BP4); + auto commPair = channel.CreateComm(name, MPI_COMM_WORLD); + // the non-rendezvous app sends to the rendezvous app + if(!isRdv) { + redev::LOs dest; + redev::LOs offsets; + redev::LOs msgs; + if(rank==0) { + dest = redev::LOs{0,2}; + offsets = redev::LOs{0,2,6}; + msgs = redev::LOs(6,0); //write the src rank as the msg for now + } else if (rank==1) { + dest = redev::LOs{0,1,2,3}; + offsets = redev::LOs{0,1,4,8,10}; + msgs = redev::LOs(10,1); + } else if (rank==2) { + dest = redev::LOs{0,1,2,3}; + offsets = redev::LOs{0,4,5,7,11}; + msgs = redev::LOs(11,2); } + commPair.SetOutMessageLayout(dest, offsets); + channel.BeginSendCommunicationPhase(); + commPair.Send(msgs.data(),redev::Mode::Deferred); + channel.EndSendCommunicationPhase(); + } else { + channel.BeginReceiveCommunicationPhase(); + auto msgVec = commPair.Recv(redev::Mode::Deferred); + channel.EndReceiveCommunicationPhase(); + auto inMsg = commPair.GetInMessageLayout(); + REDEV_ALWAYS_ASSERT(inMsg.offset == redev::GOs({0,7,11,21,27})); + REDEV_ALWAYS_ASSERT(inMsg.srcRanks == redev::GOs({0,0,0,0,2,0,4,0,3,3,8,2})); + if(rank == 0) { + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0,0,1,2,2,2,2})); + } else if(rank == 1) { + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1,1,1,2})); + } else if(rank == 2) { + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({0,0,0,0,1,1,1,1,2,2})); + } else if(rank == 3) { + REDEV_ALWAYS_ASSERT(msgVec == redev::LOs({1,1,2,2,2,2})); + } + } } MPI_Finalize(); return 0; From 1edd1c347eabdde3087c1060444ec23270b0090a Mon Sep 17 00:00:00 2001 From: Gangwar Date: Tue, 18 Nov 2025 00:14:34 -0500 Subject: [PATCH 07/11] WIP_GlobalComm_desc --- redev_channel.h | 2 +- redev_comm.h | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/redev_channel.h b/redev_channel.h index 8f45157..75cf317 100644 --- a/redev_channel.h +++ b/redev_channel.h @@ -205,7 +205,7 @@ class NoOpChannel { public: template [[nodiscard]] - BidirectionalComm CreateComm(std::string, MPI_Comm) { + BidirectionalComm CreateComm(std::string, MPI_Comm, CommType) { return {std::make_unique>(), std::make_unique>()}; } void BeginSendCommunicationPhase(){} diff --git a/redev_comm.h b/redev_comm.h index 32061d5..e4054d6 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -370,6 +370,18 @@ class AdiosPtnComm : public Communicator { InMessageLayout inMsg; }; +/** + * The AdiosGlobalComm class implements the Communicator interface to enable + * message exchange between clients and the server through ADIOS2. + * Similar to AdiosPtnComm, it provides bidirectional communication, + * but the key distinction is that the global communicator is shared + * across all ranks and partitions. + * + * It is primarily used for transferring global data and metadata + * relevant to coupled applications. + * + * Currently, the BP4 and SST ADIOS2 engines are supported. + */ template class AdiosGlobalComm : public Communicator { From bff8f2c7b4fbb3d3eb2acc76649da0b314060f30 Mon Sep 17 00:00:00 2001 From: Gangwar Date: Fri, 21 Nov 2025 03:36:38 -0500 Subject: [PATCH 08/11] WIP_added_SetCommParams --- redev_adios_channel.h | 8 ++++---- redev_bidirectional_comm.h | 5 +++++ redev_comm.h | 39 +++++++++++++++++++++----------------- test_global_comm.cpp | 7 +++++-- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/redev_adios_channel.h b/redev_adios_channel.h index 8a434ca..d295aa2 100644 --- a/redev_adios_channel.h +++ b/redev_adios_channel.h @@ -101,15 +101,15 @@ class AdiosChannel { case CommType::Ptn: s2c = std::make_unique>(comm, num_client_ranks_, s2c_engine_, s2c_io_, name); - c2s = std::make_unique>(comm, num_server_ranks_, + c2s = std::make_unique>(comm, num_server_ranks_, c2s_engine_, c2s_io_, name); - break; + break; case CommType::Global: s2c = std::make_unique>(comm, s2c_engine_, s2c_io_, name); - c2s = std::make_unique>(comm, c2s_engine_, c2s_io_, + c2s = std::make_unique>(comm, c2s_engine_, c2s_io_, name); - break; + break; } switch (process_type_) { case ProcessType::Client: return {std::move(c2s), std::move(s2c)}; diff --git a/redev_bidirectional_comm.h b/redev_bidirectional_comm.h index 60f054c..2cbde8f 100644 --- a/redev_bidirectional_comm.h +++ b/redev_bidirectional_comm.h @@ -42,6 +42,11 @@ template class BidirectionalComm { REDEV_ALWAYS_ASSERT(receiver != nullptr); return receiver->Recv(mode); } + void SetCommParams(std::string &varName, size_t &msgSize) { + REDEV_FUNCTION_TIMER; + sender->SetCommParams(varName, msgSize); + receiver->SetCommParams(varName, msgSize); + } private: std::unique_ptr> sender; diff --git a/redev_comm.h b/redev_comm.h index e4054d6..94ac8f7 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -138,6 +138,9 @@ class Communicator { virtual std::vector Recv(Mode mode) = 0; virtual InMessageLayout GetInMessageLayout() = 0; + + virtual void SetCommParams(std::string VarName, size_t msgSize ) {} + virtual ~Communicator() = default; }; @@ -398,43 +401,45 @@ class AdiosGlobalComm : public Communicator AdiosGlobalComm& operator=(const AdiosGlobalComm& other) = delete; AdiosGlobalComm& operator=(AdiosGlobalComm&& other) = delete; - void Send(T* msg, Mode mode) + void SetCommParams(std::string varName, size_t msgSize){ + varName_ = varName; + msgSize_ = msgSize; + } + void Send(T* ptr, Mode mode) { REDEV_FUNCTION_TIMER - const auto varName = name; - - auto var = io.InquireVariable(varName); + auto var = io.InquireVariable(varName_); + auto msg = std::vector(ptr, ptr + msgSize_); if (!var) { - var = io.DefineVariable(varName); + var = io.DefineVariable(varName_,{} ,{},{msgSize_}); + } + assert(var); + eng.Put(var, msg.data()); + if(mode == Mode::Synchronous) { + eng.PerformPuts(); } - eng.Put(var, msg); } std::vector Recv(Mode mode) { REDEV_FUNCTION_TIMER - const auto varName = name; std::vector msg; - auto var = io.InquireVariable(varName); + auto var = io.InquireVariable(varName_); assert(var); - if (var) { - eng.Get(var, msg); - eng.PerformGets(); + eng.Get(var, msg.data()); + if(mode == Mode::Synchronous) { + eng.PerformGets(); } return msg; } void SetOutMessageLayout(LOs& dest, LOs& offsets) {}; InMessageLayout GetInMessageLayout() { return {}; } - void SetVerbose(int lvl) - { - assert(lvl >= 0 && lvl <= 5); - Verbose = lvl; - } private: MPI_Comm comm; adios2::Engine& eng; adios2::IO& io; std::string name; - int Verbose; + std::string varName_ = ""; + std::size_t msgSize_ = 0; }; } diff --git a/test_global_comm.cpp b/test_global_comm.cpp index 6b30f5a..fd2672c 100644 --- a/test_global_comm.cpp +++ b/test_global_comm.cpp @@ -44,8 +44,11 @@ int main(int argc, char** argv) // the non-rendezvous app sends to the rendezvous app if (!isRdv) { // send data to test global comm - redev::Real val = 3.14; - auto* msgs = &val; + redev::Reals vals = {3.14}; + auto* msgs = &vals[0]; + std::string varName = "barVar"; + size_t n= vals.size(); + commPair.SetCommParams(varName, n); if (rank == 0) { channel.BeginSendCommunicationPhase(); commPair.Send(msgs, redev::Mode::Deferred); From a8230a5805abc722cb98ae39f77b54038b556b28 Mon Sep 17 00:00:00 2001 From: Gangwar Date: Sat, 22 Nov 2025 17:55:47 -0500 Subject: [PATCH 09/11] WIP_msgSize_fix --- redev_comm.h | 27 ++++++++++++++------------- test_global_comm.cpp | 19 +++++++++++-------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/redev_comm.h b/redev_comm.h index 94ac8f7..d30c229 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -389,9 +389,9 @@ template class AdiosGlobalComm : public Communicator { public: - AdiosGlobalComm(MPI_Comm comm, adios2::Engine& eng_, adios2::IO& io_, + AdiosGlobalComm(MPI_Comm comm_, adios2::Engine& eng_, adios2::IO& io_, std::string name_) - : comm(comm), eng(eng_), io(io_), name(name_) + : comm(comm_), eng(eng_), io(io_), name(name_) { } @@ -401,17 +401,17 @@ class AdiosGlobalComm : public Communicator AdiosGlobalComm& operator=(const AdiosGlobalComm& other) = delete; AdiosGlobalComm& operator=(AdiosGlobalComm&& other) = delete; - void SetCommParams(std::string varName, size_t msgSize){ - varName_ = varName; - msgSize_ = msgSize; + void SetCommParams(std::string varName_, size_t msgSize_){ + varName = varName_; + msgSize = msgSize_; } void Send(T* ptr, Mode mode) { - REDEV_FUNCTION_TIMER - auto var = io.InquireVariable(varName_); - auto msg = std::vector(ptr, ptr + msgSize_); + REDEV_FUNCTION_TIMER; + auto var = io.InquireVariable(varName); + auto msg = std::vector(ptr, ptr + msgSize); if (!var) { - var = io.DefineVariable(varName_,{} ,{},{msgSize_}); + var = io.DefineVariable(varName,{} ,{},{msgSize}); } assert(var); eng.Put(var, msg.data()); @@ -421,10 +421,11 @@ class AdiosGlobalComm : public Communicator } std::vector Recv(Mode mode) { - REDEV_FUNCTION_TIMER + REDEV_FUNCTION_TIMER; std::vector msg; - auto var = io.InquireVariable(varName_); + auto var = io.InquireVariable(varName); assert(var); + msg.resize(msgSize); eng.Get(var, msg.data()); if(mode == Mode::Synchronous) { eng.PerformGets(); @@ -439,7 +440,7 @@ class AdiosGlobalComm : public Communicator adios2::Engine& eng; adios2::IO& io; std::string name; - std::string varName_ = ""; - std::size_t msgSize_ = 0; + std::string varName = ""; + std::size_t msgSize = 0; }; } diff --git a/test_global_comm.cpp b/test_global_comm.cpp index fd2672c..87fdd0a 100644 --- a/test_global_comm.cpp +++ b/test_global_comm.cpp @@ -32,32 +32,35 @@ int main(int argc, char** argv) // dummy partition vector data redev::Redev rdv(localComm, redev::Partition{std::move(ptn)}, static_cast(isRdv)); - adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "2"}}; + adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "4"}}; + std::string name = "bar"; + auto channel = rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); auto commPair = channel.CreateComm(name, localComm, redev::CommType::Global); + // send data to test global comm + redev::Reals vals = {3.14}; + auto* msgs = &vals[0]; + std::string varName = "barVar"; + size_t n = vals.size(); // test the ptn comm // the non-rendezvous app sends to the rendezvous app if (!isRdv) { - // send data to test global comm - redev::Reals vals = {3.14}; - auto* msgs = &vals[0]; - std::string varName = "barVar"; - size_t n= vals.size(); commPair.SetCommParams(varName, n); if (rank == 0) { channel.BeginSendCommunicationPhase(); - commPair.Send(msgs, redev::Mode::Deferred); + commPair.Send(msgs, redev::Mode::Synchronous); channel.EndSendCommunicationPhase(); } } else { // receive global date channel.BeginReceiveCommunicationPhase(); - auto msgVec = commPair.Recv(redev::Mode::Deferred); + commPair.SetCommParams(varName, n); + auto msgVec = commPair.Recv(redev::Mode::Synchronous); channel.EndReceiveCommunicationPhase(); REDEV_ALWAYS_ASSERT(msgVec[0] == redev::Real{3.14}); printf("\nTest passed."); From 8d3689fd184690809d3e5c713af47afb140855a5 Mon Sep 17 00:00:00 2001 From: Gangwar Date: Tue, 25 Nov 2025 19:23:55 -0500 Subject: [PATCH 10/11] MPI_warning_fix --- CMakeLists.txt | 9 ++-- test_global_comm.cpp | 108 ++++++++++++++++++++----------------------- 2 files changed, 53 insertions(+), 64 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e24410b..b2cdf53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,7 +198,10 @@ if(BUILD_TESTING) dual_mpi_test(TESTNAME test_pingpong TIMEOUT ${test_timeout} NAME1 rdv PROCS1 1 EXE1 ./test_pingpong ARGS1 1 NAME2 app PROCS2 1 EXE2 ./test_pingpong ARGS2 0) - + add_exe(test_global_comm ./test_global_comm.cpp) + dual_mpi_test(TESTNAME test_global_comm TIMEOUT ${test_timeout} + NAME1 rdv PROCS1 1 EXE1 ./test_global_comm ARGS1 0 + NAME2 app PROCS2 1 EXE2 ./test_global_comm ARGS2 1) set(isSST 0) add_exe(test_twoClients test_twoClients.cpp) tri_mpi_test(TESTNAME test_twoClients @@ -215,10 +218,6 @@ if(BUILD_TESTING) NAME2 client1 EXE2 ./test_twoClients PROCS2 1 ARGS2 ${isSST} 1 NAME3 rdv EXE3 ./test_twoClients PROCS3 1 ARGS3 ${isSST} -1) endif() - add_exe(test_global_comm ./test_global_comm.cpp) - dual_mpi_test(TESTNAME test_global_comm TIMEOUT ${test_timeout} - NAME1 rdv PROCS1 1 EXE1 ./test_global_comm ARGS1 0 - NAME2 app PROCS2 1 EXE2 ./test_global_comm ARGS2 1) endif(BUILD_TESTING) diff --git a/test_global_comm.cpp b/test_global_comm.cpp index 87fdd0a..4cad7d3 100644 --- a/test_global_comm.cpp +++ b/test_global_comm.cpp @@ -4,67 +4,57 @@ int main(int argc, char** argv) { - int rank, nproc; MPI_Init(&argc, &argv); - if (argc != 2) { - std::cerr << "Usage: " << argv[0] - << " <1=isRendezvousApp,0=isParticipant>\n"; - exit(EXIT_FAILURE); - } - - MPI_Comm world = MPI_COMM_WORLD; - MPI_Comm_rank(world, &rank); - MPI_Comm_size(world, &nproc); - auto isRdv = atoi(argv[1]); - fprintf(stderr, "rank %d isRdv %d\n", rank, isRdv); - - const auto dim = 2; - auto ranks = isRdv ? redev::LOs({0}) : redev::LOs(1); - auto cuts = isRdv ? redev::Reals({0}) : redev::Reals(1); - auto ptn = redev::RCBPtn(dim, ranks, cuts); - - int color = isRdv ? 0 : 1; - - // Split communicator into two groups - MPI_Comm localComm; - MPI_Comm_split(world, color, rank, &localComm); - - // dummy partition vector data - redev::Redev rdv(localComm, redev::Partition{std::move(ptn)}, - static_cast(isRdv)); - adios2::Params params{{"Streaming", "On"}, {"OpenTimeoutSecs", "4"}}; - - std::string name = "bar"; - - - auto channel = - rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); - auto commPair = - channel.CreateComm(name, localComm, redev::CommType::Global); - - // send data to test global comm - redev::Reals vals = {3.14}; - auto* msgs = &vals[0]; - std::string varName = "barVar"; - size_t n = vals.size(); - // test the ptn comm - // the non-rendezvous app sends to the rendezvous app - if (!isRdv) { - commPair.SetCommParams(varName, n); - if (rank == 0) { - channel.BeginSendCommunicationPhase(); - commPair.Send(msgs, redev::Mode::Synchronous); - channel.EndSendCommunicationPhase(); + { + if (argc != 2) { + std::cerr << "Usage: " << argv[0] + << " <1=isRendezvousApp,0=isParticipant>\n"; + exit(EXIT_FAILURE); + } + auto isRdv = atoi(argv[1]); + MPI_Comm localComm = MPI_COMM_WORLD; + + // dummy partition vector data + const auto dim = 2; + auto cuts = isRdv ? redev::Reals({0}) : redev::Reals(1); + auto ranks = isRdv ? redev::LOs({0}) : redev::LOs(1); + auto ptn = redev::RCBPtn(dim, ranks, cuts); + redev::Redev rdv(localComm, redev::Partition{std::move(ptn)}, + static_cast(isRdv)); + + // Initialize the Adios Channel + adios2::Params params{{"Streaming", "On"}, + {"OpenTimeoutSecs", "4"}}; + std::string name = "bar"; + auto channel = + rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); + auto commPair = + channel.CreateComm(name, localComm, redev::CommType::Global); + + // send data to test global comm + redev::Reals vals = {3.14}; + auto *msgs = &vals[0]; + std::string varName = "barVar"; + size_t n = vals.size(); + // test the ptn comm + // the non-rendezvous app sends to the rendezvous app + if (!isRdv) { + commPair.SetCommParams(varName, n); + + channel.BeginSendCommunicationPhase(); + commPair.Send(msgs, redev::Mode::Synchronous); + channel.EndSendCommunicationPhase(); + + } else { + // receive global date + channel.BeginReceiveCommunicationPhase(); + commPair.SetCommParams(varName, n); + auto msgVec = commPair.Recv(redev::Mode::Synchronous); + channel.EndReceiveCommunicationPhase(); + REDEV_ALWAYS_ASSERT(msgVec[0] == redev::Real{3.14}); + printf("\nTest passed."); + } } - } else { - // receive global date - channel.BeginReceiveCommunicationPhase(); - commPair.SetCommParams(varName, n); - auto msgVec = commPair.Recv(redev::Mode::Synchronous); - channel.EndReceiveCommunicationPhase(); - REDEV_ALWAYS_ASSERT(msgVec[0] == redev::Real{3.14}); - printf("\nTest passed."); - } MPI_Finalize(); return 0; } \ No newline at end of file From c95c9aab605175183494f67f4c894ead90657211 Mon Sep 17 00:00:00 2001 From: Gangwar Date: Tue, 25 Nov 2025 19:35:03 -0500 Subject: [PATCH 11/11] handle_no_CommParms --- redev_comm.h | 4 +- test_global_comm.cpp | 89 ++++++++++++++++++++------------------------ 2 files changed, 44 insertions(+), 49 deletions(-) diff --git a/redev_comm.h b/redev_comm.h index d30c229..ddf01ea 100644 --- a/redev_comm.h +++ b/redev_comm.h @@ -139,7 +139,9 @@ class Communicator { virtual InMessageLayout GetInMessageLayout() = 0; - virtual void SetCommParams(std::string VarName, size_t msgSize ) {} + virtual void SetCommParams(std::string VarName, size_t msgSize ) { + throw std::logic_error("Communicator::SetCommParams() called — must be overridden in the derived Comm class"); + } virtual ~Communicator() = default; }; diff --git a/test_global_comm.cpp b/test_global_comm.cpp index 4cad7d3..ba00311 100644 --- a/test_global_comm.cpp +++ b/test_global_comm.cpp @@ -5,56 +5,49 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); - { - if (argc != 2) { - std::cerr << "Usage: " << argv[0] - << " <1=isRendezvousApp,0=isParticipant>\n"; - exit(EXIT_FAILURE); - } - auto isRdv = atoi(argv[1]); - MPI_Comm localComm = MPI_COMM_WORLD; + { + if (argc != 2) { + std::cerr << "Usage: " << argv[0] + << " <1=isRendezvousApp,0=isParticipant>\n"; + exit(EXIT_FAILURE); + } + auto isRdv = atoi(argv[1]); + MPI_Comm localComm = MPI_COMM_WORLD; - // dummy partition vector data - const auto dim = 2; - auto cuts = isRdv ? redev::Reals({0}) : redev::Reals(1); - auto ranks = isRdv ? redev::LOs({0}) : redev::LOs(1); - auto ptn = redev::RCBPtn(dim, ranks, cuts); - redev::Redev rdv(localComm, redev::Partition{std::move(ptn)}, - static_cast(isRdv)); + // dummy partition vector data + const auto dim = 2; + auto cuts = isRdv ? redev::Reals({0}) : redev::Reals(1); + auto ranks = isRdv ? redev::LOs({0}) : redev::LOs(1); + auto ptn = redev::RCBPtn(dim, ranks, cuts); + redev::Redev rdv(localComm, redev::Partition{std::move(ptn)}, static_cast(isRdv)); + // Initialize the Adios Channel + adios2::Params params{{"Streaming", "On"},{"OpenTimeoutSecs", "4"}}; + std::string name = "bar"; + auto channel = rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); + auto commPair = channel.CreateComm(name, localComm, redev::CommType::Global); - // Initialize the Adios Channel - adios2::Params params{{"Streaming", "On"}, - {"OpenTimeoutSecs", "4"}}; - std::string name = "bar"; - auto channel = - rdv.CreateAdiosChannel(name, params, redev::TransportType::BP4); - auto commPair = - channel.CreateComm(name, localComm, redev::CommType::Global); - - // send data to test global comm - redev::Reals vals = {3.14}; - auto *msgs = &vals[0]; - std::string varName = "barVar"; - size_t n = vals.size(); - // test the ptn comm - // the non-rendezvous app sends to the rendezvous app - if (!isRdv) { - commPair.SetCommParams(varName, n); - - channel.BeginSendCommunicationPhase(); - commPair.Send(msgs, redev::Mode::Synchronous); - channel.EndSendCommunicationPhase(); - - } else { - // receive global date - channel.BeginReceiveCommunicationPhase(); - commPair.SetCommParams(varName, n); - auto msgVec = commPair.Recv(redev::Mode::Synchronous); - channel.EndReceiveCommunicationPhase(); - REDEV_ALWAYS_ASSERT(msgVec[0] == redev::Real{3.14}); - printf("\nTest passed."); - } - } + // send data to test global comm + redev::Reals vals = {3.14}; + auto *msgs = &vals[0]; + std::string varName = "barVar"; + size_t n = vals.size(); + // test the ptn comm + // the non-rendezvous app sends to the rendezvous app + if (!isRdv) { + commPair.SetCommParams(varName, n); + channel.BeginSendCommunicationPhase(); + commPair.Send(msgs, redev::Mode::Synchronous); + channel.EndSendCommunicationPhase(); + } else { + // receive global date + channel.BeginReceiveCommunicationPhase(); + commPair.SetCommParams(varName, n); + auto msgVec = commPair.Recv(redev::Mode::Synchronous); + channel.EndReceiveCommunicationPhase(); + REDEV_ALWAYS_ASSERT(msgVec[0] == redev::Real{3.14}); + printf("\nTest passed."); + } + } MPI_Finalize(); return 0; } \ No newline at end of file