From a392561a4bb882867caa1baff1f4746e663691e3 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Thu, 2 Jul 2015 23:15:29 +0200 Subject: [PATCH 01/25] Initial commit with basic functionality working --- CMakeLists.txt | 8 ++ examples/CMakeLists.txt | 87 ++++++++-------- examples/zeromq.cpp | 20 ++++ src/CMakeLists.txt | 8 ++ src/pagmo.h | 4 + src/zmq_island.cpp | 213 ++++++++++++++++++++++++++++++++++++++++ src/zmq_island.h | 160 ++++++++++++++++++++++++++++++ 7 files changed, 458 insertions(+), 42 deletions(-) create mode 100644 examples/zeromq.cpp create mode 100644 src/zmq_island.cpp create mode 100644 src/zmq_island.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 37dc41d9..ea2bdb8a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,9 @@ OPTION(ENABLE_WORHP "Enable support for WORHP minimiser." OFF) # Build Option: support for MPI clustering. OPTION(ENABLE_MPI "Enable support for the Message Passage Interface (MPI)." OFF) +# Build Option: ZeroMQ islands. +OPTION(ENABLE_ZMQ "Enable support for ZeroMQ islands." OFF) + # Build option: enable test set. OPTION(ENABLE_TESTS "Build test set." OFF) @@ -199,6 +202,11 @@ IF(ENABLE_MPI) ADD_DEFINITIONS(-DPAGMO_ENABLE_MPI) ENDIF(ENABLE_MPI) +IF(ENABLE_ZMQ) + ADD_DEFINITIONS(-DPAGMO_ENABLE_ZMQ) + SET(MANDATORY_LIBRARIES ${MANDATORY_LIBRARIES} zmq redox ev hiredis) +ENDIF(ENABLE_ZMQ) + # If GSL support is requested, look for the library. IF(ENABLE_GSL) FIND_LIBRARY(GSL_GSL_LIBRARY NAMES gsl) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 582921a9..85032dc9 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,42 +1,45 @@ -IF(ENABLE_GTOP_DATABASE) - -ADD_EXECUTABLE(hm_2_asteroids hm_2_asteroids.cpp) -TARGET_LINK_LIBRARIES(hm_2_asteroids ${MANDATORY_LIBRARIES} pagmo_static) - -ADD_EXECUTABLE(migrate_or_not migrate_or_not.cpp) -TARGET_LINK_LIBRARIES(migrate_or_not ${MANDATORY_LIBRARIES} pagmo_static) - -IF(ENABLE_SNOPT) - ADD_EXECUTABLE(gtoc_2_turin gtoc_2_turin.cpp) - TARGET_LINK_LIBRARIES(gtoc_2_turin ${MANDATORY_LIBRARIES} pagmo_static) -ENDIF(ENABLE_SNOPT) - -ENDIF(ENABLE_GTOP_DATABASE) - -IF(ENABLE_GSL) - ADD_EXECUTABLE(evolve_spheres evolve_spheres.cpp) - TARGET_LINK_LIBRARIES(evolve_spheres ${MANDATORY_LIBRARIES} pagmo_static) - ADD_EXECUTABLE(evolve_spheres_racing evolve_spheres_racing.cpp) - TARGET_LINK_LIBRARIES(evolve_spheres_racing ${MANDATORY_LIBRARIES} pagmo_static) -ENDIF(ENABLE_GSL) - -ADD_EXECUTABLE(constraints_handling constraints_handling.cpp) -TARGET_LINK_LIBRARIES(constraints_handling ${MANDATORY_LIBRARIES} pagmo_static) - -ADD_EXECUTABLE(cstrs_death_penalty cstrs_death_penalty.cpp) -TARGET_LINK_LIBRARIES(cstrs_death_penalty ${MANDATORY_LIBRARIES} pagmo_static) - -ADD_EXECUTABLE(cstrs_co_evolution cstrs_co_evolution.cpp) -TARGET_LINK_LIBRARIES(cstrs_co_evolution ${MANDATORY_LIBRARIES} pagmo_static) - -ADD_EXECUTABLE(cstrs_self_adaptive cstrs_self_adaptive.cpp) -TARGET_LINK_LIBRARIES(cstrs_self_adaptive ${MANDATORY_LIBRARIES} pagmo_static) - -ADD_EXECUTABLE(cstrs_self_adaptive_island cstrs_self_adaptive_island.cpp) -TARGET_LINK_LIBRARIES(cstrs_self_adaptive_island ${MANDATORY_LIBRARIES} pagmo_static) - -ADD_EXECUTABLE(cstrs_immune_system cstrs_immune_system.cpp) -TARGET_LINK_LIBRARIES(cstrs_immune_system ${MANDATORY_LIBRARIES} pagmo_static) - -ADD_EXECUTABLE(cstrs_core cstrs_core.cpp) -TARGET_LINK_LIBRARIES(cstrs_core ${MANDATORY_LIBRARIES} pagmo_static) +ADD_EXECUTABLE(zeromq zeromq.cpp) +TARGET_LINK_LIBRARIES(zeromq ${MANDATORY_LIBRARIES} pagmo_static) + +#IF(ENABLE_GTOP_DATABASE) +# +#ADD_EXECUTABLE(hm_2_asteroids hm_2_asteroids.cpp) +#TARGET_LINK_LIBRARIES(hm_2_asteroids ${MANDATORY_LIBRARIES} pagmo_static) +# +#ADD_EXECUTABLE(migrate_or_not migrate_or_not.cpp) +#TARGET_LINK_LIBRARIES(migrate_or_not ${MANDATORY_LIBRARIES} pagmo_static) +# +#IF(ENABLE_SNOPT) +# ADD_EXECUTABLE(gtoc_2_turin gtoc_2_turin.cpp) +# TARGET_LINK_LIBRARIES(gtoc_2_turin ${MANDATORY_LIBRARIES} pagmo_static) +#ENDIF(ENABLE_SNOPT) +# +#ENDIF(ENABLE_GTOP_DATABASE) +# +#IF(ENABLE_GSL) +# ADD_EXECUTABLE(evolve_spheres evolve_spheres.cpp) +# TARGET_LINK_LIBRARIES(evolve_spheres ${MANDATORY_LIBRARIES} pagmo_static) +# ADD_EXECUTABLE(evolve_spheres_racing evolve_spheres_racing.cpp) +# TARGET_LINK_LIBRARIES(evolve_spheres_racing ${MANDATORY_LIBRARIES} pagmo_static) +#ENDIF(ENABLE_GSL) +# +#ADD_EXECUTABLE(constraints_handling constraints_handling.cpp) +#TARGET_LINK_LIBRARIES(constraints_handling ${MANDATORY_LIBRARIES} pagmo_static) +# +#ADD_EXECUTABLE(cstrs_death_penalty cstrs_death_penalty.cpp) +#TARGET_LINK_LIBRARIES(cstrs_death_penalty ${MANDATORY_LIBRARIES} pagmo_static) +# +#ADD_EXECUTABLE(cstrs_co_evolution cstrs_co_evolution.cpp) +#TARGET_LINK_LIBRARIES(cstrs_co_evolution ${MANDATORY_LIBRARIES} pagmo_static) +# +#ADD_EXECUTABLE(cstrs_self_adaptive cstrs_self_adaptive.cpp) +#TARGET_LINK_LIBRARIES(cstrs_self_adaptive ${MANDATORY_LIBRARIES} pagmo_static) +# +#ADD_EXECUTABLE(cstrs_self_adaptive_island cstrs_self_adaptive_island.cpp) +#TARGET_LINK_LIBRARIES(cstrs_self_adaptive_island ${MANDATORY_LIBRARIES} pagmo_static) +# +#ADD_EXECUTABLE(cstrs_immune_system cstrs_immune_system.cpp) +#TARGET_LINK_LIBRARIES(cstrs_immune_system ${MANDATORY_LIBRARIES} pagmo_static) +# +#ADD_EXECUTABLE(cstrs_core cstrs_core.cpp) +#TARGET_LINK_LIBRARIES(cstrs_core ${MANDATORY_LIBRARIES} pagmo_static) diff --git a/examples/zeromq.cpp b/examples/zeromq.cpp new file mode 100644 index 00000000..571a62ca --- /dev/null +++ b/examples/zeromq.cpp @@ -0,0 +1,20 @@ +#include "../src/pagmo.h" + +using namespace pagmo; + +int main() { + problem::dejong p(10); + algorithm::monte_carlo a(100); + zmq_island i(a, p, 1); + + i.set_broker_details("192.168.1.36", 6379); + i.set_token("zeromq_test"); + i.initialise(); + + //i.evolve(10); + // + + i.close(); + + return 0; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0a7727dd..5a775c13 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -242,6 +242,14 @@ IF(ENABLE_MPI) ) ENDIF(ENABLE_MPI) +# ZeroMQ support + +IF(ENABLE_ZMQ) + SET(PAGMO_LIB_SRC_LIST ${PAGMO_LIB_SRC_LIST} + ${CMAKE_CURRENT_SOURCE_DIR}/zmq_island.cpp + ) +ENDIF(ENABLE_ZMQ) + # Create a pagmo_static library if main or tests or examples are requested. IF(BUILD_MAIN OR ENABLE_TESTS OR BUILD_EXAMPLES) ADD_LIBRARY(pagmo_static STATIC ${PAGMO_LIB_SRC_LIST}) diff --git a/src/pagmo.h b/src/pagmo.h index c03a734a..9d00d942 100644 --- a/src/pagmo.h +++ b/src/pagmo.h @@ -44,4 +44,8 @@ #include "mpi_island.h" #endif +#ifdef PAGMO_ENABLE_ZMQ + #include "zmq_island.h" +#endif + #endif diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp new file mode 100644 index 00000000..97b26a45 --- /dev/null +++ b/src/zmq_island.cpp @@ -0,0 +1,213 @@ +/***************************************************************************** + * Copyright (C) 2004-2015 The PaGMO development team, * + * Advanced Concepts Team (ACT), European Space Agency (ESA) * + * * + * https://github.com/esa/pagmo * + * * + * act@esa.int * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the * + * Free Software Foundation, Inc., * + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "algorithm/base.h" +#include "base_island.h" +#include "exceptions.h" +#include "zmq_island.h" +#include "migration/base_r_policy.h" +#include "migration/base_s_policy.h" +#include "population.h" +#include "problem/base.h" + +namespace pagmo +{ + +/// Constructor from problem::base, algorithm::base, number of individuals, migration probability and selection/replacement policies. +/** + * @see pagmo::base_island constructors. + */ +zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, + const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): + base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_receiveSocket(m_zmqContext, ZMQ_REQ) +{} + +/// Constructor from population. +/** + * @see pagmo::base_island constructors. + */ +zmq_island::zmq_island(const algorithm::base &a, const population &pop, + const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): + base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_receiveSocket(m_zmqContext, ZMQ_REQ) +{} + +/// Copy constructor. +/** + * @see pagmo::base_island constructors. + */ +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_receiveSocket(m_zmqContext, ZMQ_REQ) // TODO: does this make sense? +{} + +/// Assignment operator. +zmq_island &zmq_island::operator=(const zmq_island &isl) +{ + base_island::operator=(isl); + return *this; +} + +base_island_ptr zmq_island::clone() const +{ + return base_island_ptr(new zmq_island(*this)); +} + +// This method performs the local evolution for this island's population. +void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) const +{ + // TODO: Magic +} + +/// Return a string identifying the island's type. +// TODO: Add topic string +/** + * @return the string "ZMQ island". + */ +std::string zmq_island::get_name() const +{ + return "ZMQ island"; +} + +// The following methods should go in the constructor, but +// I'm separating them out for easier debugging. +// They will eventually be merged into the constructor. + +std::string zmq_island::get_ip(std::string broker) { + // TODO(important): Command injection vulnerability. + // Look into mitigation strategies: sanitising input, etc. + std::string command = "ip route get " + broker; + FILE* result_stream = popen(command.c_str(), "r"); + char result_buffer[100]; + + fgets(result_buffer, 100, result_stream); + fclose(result_stream); + + std::string output(result_buffer); + boost::algorithm::trim_right(output); + + auto pos = output.find("src"); + if(pos == -1) { + return "127.0.0.1"; // TODO: What to do if there's no route to the broker? + } + + pos += 4; // "src " + return output.substr(pos, -1); +} + +void zmq_island::set_broker_details(std::string host, int port) { + m_brokerHost = host; + m_brokerPort = port; +} + +void zmq_island::set_token(std::string token) { + m_token = token; +} + +void zmq_island::connect(std::string host) { + zmq::socket_t* sck = new zmq::socket_t(m_zmqContext, ZMQ_REQ); + std::cout << "DEBUG: Opening connection to " << host << std::endl; + + sck->connect(("tcp://" + host).c_str()); + m_remoteConnections.push_back(sck); +} + +bool zmq_island::initialise() { + if(m_brokerHost == "" || m_brokerPort == -1 || m_token == "") { + return false; // Can't initialise if we're missing those parameters + } + + // Connect to the broker + if(!m_brokerConn.connect(m_brokerHost, m_brokerPort)) { + std::cout << "ERROR: Can't connect to broker" << std::endl; // TODO: better error reporting + return false; // can't connect to broker + } + + // Obtain IP address to advertise to the peers + m_IP = get_ip(m_brokerHost); + + // Choose a port between 1000 and 2000 + m_localPort = rand() % 2000 + 1000; + m_IP += ":" + std::to_string(m_localPort); + + std::cout << "DEBUG: IP: '" << m_IP << "'" << std::endl; + + std::string brokerKey = "pagmo.islands." + m_token; + // Get list of peers + redox::Command >& result = + m_brokerConn.commandSync >({"SMEMBERS", brokerKey}); + + if(!result.ok()) { + std::cout << "ERROR: Unable to get list of peers" << std::endl; + return false; + } + + // Connect to peers + auto peers = result.reply(); + for(auto it = peers.begin(); it != peers.end(); ++it) { + connect(*it); + } + + // Add ourselves to the list of islands on the chosen topic. + m_brokerConn.commandSync({"SADD", brokerKey, m_IP}); + + // Broadcast that we've added ourselves to the list + m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "connected:" + m_IP}); + + // Open incoming socket + m_receiveSocket.bind(("tcp://" + m_IP).c_str()); + // TODO: What to do if port is already in use? + + return true; +} + +// TODO: This should go in the destructor (obviously) +void zmq_island::close() { + std::string brokerKey = "pagmo.islands." + m_token; + + m_brokerConn.commandSync({"SREM", brokerKey, m_IP}); + m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "disconnected:" + m_IP}); + + m_brokerConn.disconnect(); + + std::cout << "DEBUG: Closed" << std::endl; +} + +} + +BOOST_CLASS_EXPORT_IMPLEMENT(pagmo::zmq_island) diff --git a/src/zmq_island.h b/src/zmq_island.h new file mode 100644 index 00000000..584ff709 --- /dev/null +++ b/src/zmq_island.h @@ -0,0 +1,160 @@ +/***************************************************************************** + * Copyright (C) 2004-2015 The PaGMO development team, * + * Advanced Concepts Team (ACT), European Space Agency (ESA) * + * * + * https://github.com/esa/pagmo * + * * + * act@esa.int * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the * + * Free Software Foundation, Inc., * + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * + *****************************************************************************/ + +#ifndef PAGMO_ZMQ_ISLAND_H +#define PAGMO_ZMQ_ISLAND_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base_island.h" +#include "config.h" +#include "algorithm/base.h" +#include "migration/base_r_policy.h" +#include "migration/base_s_policy.h" +#include "migration/best_s_policy.h" +#include "migration/fair_r_policy.h" +#include "population.h" +#include "problem/base.h" +#include "serialization.h" + +// Forward declarations. +namespace pagmo { + +class zmq_island; + +} + +namespace boost { namespace serialization { + +template +void save_construct_data(Archive &, const pagmo::zmq_island *, const unsigned int); + +template +inline void load_construct_data(Archive &, pagmo::zmq_island *, const unsigned int); + +}} + +namespace pagmo +{ + +/// ZMQ island class. +/** + * This island can communicate with other ZeroMQ islands active on as well as local islands. + * The intended use of this island class is to exchange solutions with remote archipelagos + * through a ZeroMQ socket. The functionality is similar to the MPI island (TODO: add ref), + * but using a different transport protocol. + * + * NOTE: this class is available only if PaGMO was compiled with ZeroMQ support. + * + * @author Jose Diez (me@jdiez.me) + */ +class __PAGMO_VISIBLE zmq_island: public base_island +{ + template + friend void boost::serialization::save_construct_data(Archive &, const pagmo::zmq_island *, const unsigned int); + template + friend void boost::serialization::load_construct_data(Archive &, pagmo::zmq_island *, const unsigned int); + public: + zmq_island(const zmq_island &); + explicit zmq_island(const algorithm::base &, const problem::base &, int = 0, + const migration::base_s_policy & = migration::best_s_policy(), + const migration::base_r_policy & = migration::fair_r_policy()); + explicit zmq_island(const algorithm::base &, const population &, + const migration::base_s_policy & = migration::best_s_policy(), + const migration::base_r_policy & = migration::fair_r_policy()); + zmq_island &operator=(const zmq_island &); + base_island_ptr clone() const; + protected: + void perform_evolution(const algorithm::base &, population &) const; + public: + std::string get_name() const; + + void set_broker_details(std::string, int); + void set_token(std::string); + bool initialise(); + void close(); + + private: + std::string m_brokerHost; + int m_brokerPort; + std::string m_token; + std::string m_IP; + int m_localPort; + + redox::Redox m_brokerConn; + + zmq::context_t m_zmqContext; + zmq::socket_t m_receiveSocket; + std::vector m_remoteConnections; + + std::string get_ip(std::string); + void connect(std::string); + + friend class boost::serialization::access; + template + void serialize(Archive &ar, const unsigned int) + { + // Join is already done in base_island. + ar & boost::serialization::base_object(*this); + } +}; + +} + +namespace boost { namespace serialization { + +template +inline void save_construct_data(Archive &ar, const pagmo::zmq_island *isl, const unsigned int) +{ + // Save data required to construct instance. + pagmo::algorithm::base_ptr algo = isl->m_algo->clone(); + pagmo::problem::base_ptr prob = isl->m_pop.problem().clone(); + ar << algo; + ar << prob; +} + +template +inline void load_construct_data(Archive &ar, pagmo::zmq_island *isl, const unsigned int) +{ + // Retrieve data from archive required to construct new instance. + pagmo::algorithm::base_ptr algo; + pagmo::problem::base_ptr prob; + ar >> algo; + ar >> prob; + // Invoke inplace constructor to initialize instance of the algorithm. + ::new(isl)pagmo::zmq_island(*algo,*prob); +} + +}} //namespaces + +BOOST_CLASS_EXPORT_KEY(pagmo::zmq_island) + +#endif From fa95007920dc292d88d5e8afaedaa71915939da5 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Mon, 17 Aug 2015 22:08:53 +0200 Subject: [PATCH 02/25] zmq_islands: able to exchange solutions through pubsub --- CMakeLists.txt | 1 + examples/zeromq.cpp | 31 ++++++++++--- src/zmq_island.cpp | 110 ++++++++++++++++++++++++++------------------ src/zmq_island.h | 12 +++-- 4 files changed, 98 insertions(+), 56 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ea2bdb8a..8fc863b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,7 @@ ENDIF(ENABLE_MPI) IF(ENABLE_ZMQ) ADD_DEFINITIONS(-DPAGMO_ENABLE_ZMQ) SET(MANDATORY_LIBRARIES ${MANDATORY_LIBRARIES} zmq redox ev hiredis) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") # workaround for void perform_evolution() const ENDIF(ENABLE_ZMQ) # If GSL support is requested, look for the library. diff --git a/examples/zeromq.cpp b/examples/zeromq.cpp index 571a62ca..b3ade1c7 100644 --- a/examples/zeromq.cpp +++ b/examples/zeromq.cpp @@ -1,20 +1,39 @@ #include "../src/pagmo.h" +#include +#include +#include +#include using namespace pagmo; +std::atomic quit; + +void sig(int) { + quit.store(true); +} + int main() { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = &sig; + sigfillset(&sa.sa_mask); + sigaction(SIGINT, &sa, NULL); // register signal handler + + quit.store(false); + problem::dejong p(10); algorithm::monte_carlo a(100); zmq_island i(a, p, 1); - i.set_broker_details("192.168.1.36", 6379); + i.set_broker_details("192.168.1.39", 6379); i.set_token("zeromq_test"); - i.initialise(); + i.initialise("192.168.1.39"); - //i.evolve(10); - // - - i.close(); + while(!quit) { + i.evolve(1); + std::cout << "Best: " << i.get_population().champion().x << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } return 0; } diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 97b26a45..788327c4 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include "algorithm/base.h" #include "base_island.h" @@ -57,7 +58,7 @@ namespace pagmo */ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_receiveSocket(m_zmqContext, ZMQ_REQ) + base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) {} /// Constructor from population. @@ -66,16 +67,21 @@ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, */ zmq_island::zmq_island(const algorithm::base &a, const population &pop, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_receiveSocket(m_zmqContext, ZMQ_REQ) + base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) {} /// Copy constructor. /** * @see pagmo::base_island constructors. */ -zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_receiveSocket(m_zmqContext, ZMQ_REQ) // TODO: does this make sense? +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) // TODO: does this make sense? {} +/// Destructor. +zmq_island::~zmq_island() { + disconnect(); +} + /// Assignment operator. zmq_island &zmq_island::operator=(const zmq_island &isl) { @@ -91,7 +97,40 @@ base_island_ptr zmq_island::clone() const // This method performs the local evolution for this island's population. void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) const { - // TODO: Magic + const boost::shared_ptr pop_copy(new population(pop)); + const algorithm::base_ptr algo_copy = algo.clone(); + //const std::pair,algorithm::base_ptr> out(pop_copy,algo_copy); + const boost::shared_ptr out = pop_copy; + + // First, we send a copy of our population and algorithm + std::stringstream ss; + boost::archive::text_oarchive oa(ss); + oa << out; + std::string buffer(ss.str()); + zmq::message_t msg(buffer.size()); + memcpy((void *) msg.data(), buffer.c_str(), buffer.size() - 1); + m_publisherSocket.send(msg); + + // See if there is any data available + zmq::message_t incoming; + if(m_subscriptionSocket.recv(&incoming, ZMQ_DONTWAIT) > 0) { + if(incoming.size()) { + try { + std::string bytes_in((char *) incoming.data(), incoming.size()); + + std::stringstream incoming_ss(bytes_in); + boost::archive::text_iarchive ia(incoming_ss); + boost::shared_ptr in; + + ia >> in; + pop = *in; + } catch (const boost::archive::archive_exception &e) { + std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ": " << e.what() << std::endl; + } catch (...) { + std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ", unknown exception caught. :(" << std::endl; + } + } + } } /// Return a string identifying the island's type. @@ -104,32 +143,6 @@ std::string zmq_island::get_name() const return "ZMQ island"; } -// The following methods should go in the constructor, but -// I'm separating them out for easier debugging. -// They will eventually be merged into the constructor. - -std::string zmq_island::get_ip(std::string broker) { - // TODO(important): Command injection vulnerability. - // Look into mitigation strategies: sanitising input, etc. - std::string command = "ip route get " + broker; - FILE* result_stream = popen(command.c_str(), "r"); - char result_buffer[100]; - - fgets(result_buffer, 100, result_stream); - fclose(result_stream); - - std::string output(result_buffer); - boost::algorithm::trim_right(output); - - auto pos = output.find("src"); - if(pos == -1) { - return "127.0.0.1"; // TODO: What to do if there's no route to the broker? - } - - pos += 4; // "src " - return output.substr(pos, -1); -} - void zmq_island::set_broker_details(std::string host, int port) { m_brokerHost = host; m_brokerPort = port; @@ -140,30 +153,29 @@ void zmq_island::set_token(std::string token) { } void zmq_island::connect(std::string host) { - zmq::socket_t* sck = new zmq::socket_t(m_zmqContext, ZMQ_REQ); std::cout << "DEBUG: Opening connection to " << host << std::endl; - sck->connect(("tcp://" + host).c_str()); - m_remoteConnections.push_back(sck); + m_subscriptionSocket.connect(("tcp://" + host).c_str()); } -bool zmq_island::initialise() { +bool zmq_island::initialise(std::string ip) { if(m_brokerHost == "" || m_brokerPort == -1 || m_token == "") { return false; // Can't initialise if we're missing those parameters } // Connect to the broker - if(!m_brokerConn.connect(m_brokerHost, m_brokerPort)) { + if(!m_brokerConn.connect(m_brokerHost, m_brokerPort) || !m_brokerSubscriber.connect(m_brokerHost, m_brokerPort)) { std::cout << "ERROR: Can't connect to broker" << std::endl; // TODO: better error reporting return false; // can't connect to broker } - // Obtain IP address to advertise to the peers - m_IP = get_ip(m_brokerHost); + // Initialise subscription socket + m_subscriptionSocket.setsockopt(ZMQ_SUBSCRIBE, "", 0); // Choose a port between 1000 and 2000 + srand(time(0)); m_localPort = rand() % 2000 + 1000; - m_IP += ":" + std::to_string(m_localPort); + m_IP += ip + ":" + std::to_string(m_localPort); std::cout << "DEBUG: IP: '" << m_IP << "'" << std::endl; @@ -187,23 +199,31 @@ bool zmq_island::initialise() { m_brokerConn.commandSync({"SADD", brokerKey, m_IP}); // Broadcast that we've added ourselves to the list - m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "connected:" + m_IP}); + m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "connected/" + m_IP}); // Open incoming socket - m_receiveSocket.bind(("tcp://" + m_IP).c_str()); - // TODO: What to do if port is already in use? + m_publisherSocket.bind(("tcp://" + m_IP).c_str()); + + // Connect to new peers when they advertise on the control channel + m_brokerSubscriber.subscribe(brokerKey + ".control", [&](const std::string&, const std::string& msg) { + std::vector data; + boost::split(data, msg, boost::is_any_of("/")); + if(data[0] == "connected") { + connect(data[1]); + } else { /* disconnect */ } + }); return true; } -// TODO: This should go in the destructor (obviously) -void zmq_island::close() { +void zmq_island::disconnect() { std::string brokerKey = "pagmo.islands." + m_token; - + m_brokerConn.commandSync({"SREM", brokerKey, m_IP}); - m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "disconnected:" + m_IP}); + m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "disconnected/" + m_IP}); m_brokerConn.disconnect(); + m_brokerSubscriber.disconnect(); std::cout << "DEBUG: Closed" << std::endl; } diff --git a/src/zmq_island.h b/src/zmq_island.h index 584ff709..344c09ee 100644 --- a/src/zmq_island.h +++ b/src/zmq_island.h @@ -84,6 +84,7 @@ class __PAGMO_VISIBLE zmq_island: public base_island friend void boost::serialization::load_construct_data(Archive &, pagmo::zmq_island *, const unsigned int); public: zmq_island(const zmq_island &); + ~zmq_island(); explicit zmq_island(const algorithm::base &, const problem::base &, int = 0, const migration::base_s_policy & = migration::best_s_policy(), const migration::base_r_policy & = migration::fair_r_policy()); @@ -99,7 +100,7 @@ class __PAGMO_VISIBLE zmq_island: public base_island void set_broker_details(std::string, int); void set_token(std::string); - bool initialise(); + bool initialise(std::string); void close(); private: @@ -110,13 +111,14 @@ class __PAGMO_VISIBLE zmq_island: public base_island int m_localPort; redox::Redox m_brokerConn; + redox::Subscriber m_brokerSubscriber; - zmq::context_t m_zmqContext; - zmq::socket_t m_receiveSocket; - std::vector m_remoteConnections; + zmq::context_t m_zmqContext; + zmq::socket_t m_publisherSocket; + zmq::socket_t m_subscriptionSocket; - std::string get_ip(std::string); void connect(std::string); + void disconnect(); friend class boost::serialization::access; template From 72ffb5127a6d8e8c8428e2e788343d1bc96291a4 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Mon, 17 Aug 2015 22:48:18 +0200 Subject: [PATCH 03/25] ZMQ bindings --- PyGMO/core/core.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index 3a189bef..a773f112 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -49,6 +49,7 @@ #include "../../src/algorithm/base.h" #include "../../src/archipelago.h" #include "../../src/base_island.h" +#include "../../src/zmq_island.h" #include "../../src/config.h" #include "../../src/exceptions.h" #include "../../src/migration/base_r_policy.h" @@ -380,6 +381,16 @@ BOOST_PYTHON_MODULE(_core) // Register to_python conversion from smart pointer. register_ptr_to_python(); + // ZMQ island class. + class_ >("zmq_island", "ZMQ island class.",init >()) + .def(init >()) + .def(init()) + .def("__copy__", &Py_copy_from_ctor) + .def("__deepcopy__", &Py_deepcopy_from_ctor) + .def("set_broker_details", &zmq_island::set_broker_details) + .def("set_token", &zmq_island::set_token) + .def("initialise", &zmq_island::initialise); + // Expose archipelago class. class_("archipelago", "Archipelago class.", init >()) From ace85b4d3a7499f038fa1216c8525a78ae424de9 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 12:13:43 +0200 Subject: [PATCH 04/25] examples: Re-enabled other examples --- examples/CMakeLists.txt | 92 +++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 85032dc9..68957b88 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,45 +1,47 @@ -ADD_EXECUTABLE(zeromq zeromq.cpp) -TARGET_LINK_LIBRARIES(zeromq ${MANDATORY_LIBRARIES} pagmo_static) - -#IF(ENABLE_GTOP_DATABASE) -# -#ADD_EXECUTABLE(hm_2_asteroids hm_2_asteroids.cpp) -#TARGET_LINK_LIBRARIES(hm_2_asteroids ${MANDATORY_LIBRARIES} pagmo_static) -# -#ADD_EXECUTABLE(migrate_or_not migrate_or_not.cpp) -#TARGET_LINK_LIBRARIES(migrate_or_not ${MANDATORY_LIBRARIES} pagmo_static) -# -#IF(ENABLE_SNOPT) -# ADD_EXECUTABLE(gtoc_2_turin gtoc_2_turin.cpp) -# TARGET_LINK_LIBRARIES(gtoc_2_turin ${MANDATORY_LIBRARIES} pagmo_static) -#ENDIF(ENABLE_SNOPT) -# -#ENDIF(ENABLE_GTOP_DATABASE) -# -#IF(ENABLE_GSL) -# ADD_EXECUTABLE(evolve_spheres evolve_spheres.cpp) -# TARGET_LINK_LIBRARIES(evolve_spheres ${MANDATORY_LIBRARIES} pagmo_static) -# ADD_EXECUTABLE(evolve_spheres_racing evolve_spheres_racing.cpp) -# TARGET_LINK_LIBRARIES(evolve_spheres_racing ${MANDATORY_LIBRARIES} pagmo_static) -#ENDIF(ENABLE_GSL) -# -#ADD_EXECUTABLE(constraints_handling constraints_handling.cpp) -#TARGET_LINK_LIBRARIES(constraints_handling ${MANDATORY_LIBRARIES} pagmo_static) -# -#ADD_EXECUTABLE(cstrs_death_penalty cstrs_death_penalty.cpp) -#TARGET_LINK_LIBRARIES(cstrs_death_penalty ${MANDATORY_LIBRARIES} pagmo_static) -# -#ADD_EXECUTABLE(cstrs_co_evolution cstrs_co_evolution.cpp) -#TARGET_LINK_LIBRARIES(cstrs_co_evolution ${MANDATORY_LIBRARIES} pagmo_static) -# -#ADD_EXECUTABLE(cstrs_self_adaptive cstrs_self_adaptive.cpp) -#TARGET_LINK_LIBRARIES(cstrs_self_adaptive ${MANDATORY_LIBRARIES} pagmo_static) -# -#ADD_EXECUTABLE(cstrs_self_adaptive_island cstrs_self_adaptive_island.cpp) -#TARGET_LINK_LIBRARIES(cstrs_self_adaptive_island ${MANDATORY_LIBRARIES} pagmo_static) -# -#ADD_EXECUTABLE(cstrs_immune_system cstrs_immune_system.cpp) -#TARGET_LINK_LIBRARIES(cstrs_immune_system ${MANDATORY_LIBRARIES} pagmo_static) -# -#ADD_EXECUTABLE(cstrs_core cstrs_core.cpp) -#TARGET_LINK_LIBRARIES(cstrs_core ${MANDATORY_LIBRARIES} pagmo_static) +IF(ENABLE_ZMQ) + ADD_EXECUTABLE(zeromq zeromq.cpp) + TARGET_LINK_LIBRARIES(zeromq ${MANDATORY_LIBRARIES} pagmo_static) +ENDIF(ENABLE_ZMQ) + +IF(ENABLE_GTOP_DATABASE) + +ADD_EXECUTABLE(hm_2_asteroids hm_2_asteroids.cpp) +TARGET_LINK_LIBRARIES(hm_2_asteroids ${MANDATORY_LIBRARIES} pagmo_static) + +ADD_EXECUTABLE(migrate_or_not migrate_or_not.cpp) +TARGET_LINK_LIBRARIES(migrate_or_not ${MANDATORY_LIBRARIES} pagmo_static) + +IF(ENABLE_SNOPT) + ADD_EXECUTABLE(gtoc_2_turin gtoc_2_turin.cpp) + TARGET_LINK_LIBRARIES(gtoc_2_turin ${MANDATORY_LIBRARIES} pagmo_static) +ENDIF(ENABLE_SNOPT) + +ENDIF(ENABLE_GTOP_DATABASE) + +IF(ENABLE_GSL) + ADD_EXECUTABLE(evolve_spheres evolve_spheres.cpp) + TARGET_LINK_LIBRARIES(evolve_spheres ${MANDATORY_LIBRARIES} pagmo_static) + ADD_EXECUTABLE(evolve_spheres_racing evolve_spheres_racing.cpp) + TARGET_LINK_LIBRARIES(evolve_spheres_racing ${MANDATORY_LIBRARIES} pagmo_static) +ENDIF(ENABLE_GSL) + +ADD_EXECUTABLE(constraints_handling constraints_handling.cpp) +TARGET_LINK_LIBRARIES(constraints_handling ${MANDATORY_LIBRARIES} pagmo_static) + +ADD_EXECUTABLE(cstrs_death_penalty cstrs_death_penalty.cpp) +TARGET_LINK_LIBRARIES(cstrs_death_penalty ${MANDATORY_LIBRARIES} pagmo_static) + +ADD_EXECUTABLE(cstrs_co_evolution cstrs_co_evolution.cpp) +TARGET_LINK_LIBRARIES(cstrs_co_evolution ${MANDATORY_LIBRARIES} pagmo_static) + +ADD_EXECUTABLE(cstrs_self_adaptive cstrs_self_adaptive.cpp) +TARGET_LINK_LIBRARIES(cstrs_self_adaptive ${MANDATORY_LIBRARIES} pagmo_static) + +ADD_EXECUTABLE(cstrs_self_adaptive_island cstrs_self_adaptive_island.cpp) +TARGET_LINK_LIBRARIES(cstrs_self_adaptive_island ${MANDATORY_LIBRARIES} pagmo_static) + +ADD_EXECUTABLE(cstrs_immune_system cstrs_immune_system.cpp) +TARGET_LINK_LIBRARIES(cstrs_immune_system ${MANDATORY_LIBRARIES} pagmo_static) + +ADD_EXECUTABLE(cstrs_core cstrs_core.cpp) +TARGET_LINK_LIBRARIES(cstrs_core ${MANDATORY_LIBRARIES} pagmo_static) From 7fe8bf1ddcf30e0b50dddaa9af424c264efe2039 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 12:27:15 +0200 Subject: [PATCH 05/25] pygmo: fixed clashing namespace, retabbed zmq example --- PyGMO/utils.h | 2 +- examples/zeromq.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PyGMO/utils.h b/PyGMO/utils.h index 80bd84ec..7cbeda58 100644 --- a/PyGMO/utils.h +++ b/PyGMO/utils.h @@ -90,7 +90,7 @@ struct python_class_pickle_suite: boost::python::pickle_suite throw_error_already_set(); } // Restore the object's __dict__. - dict d = extract(obj.attr("__dict__"))(); + boost::python::dict d = extract(obj.attr("__dict__"))(); d.update(state[0]); // Restore the internal state of the C++ object. const std::string str = extract(state[1]); diff --git a/examples/zeromq.cpp b/examples/zeromq.cpp index b3ade1c7..fb42ea2e 100644 --- a/examples/zeromq.cpp +++ b/examples/zeromq.cpp @@ -21,9 +21,9 @@ int main() { quit.store(false); - problem::dejong p(10); - algorithm::monte_carlo a(100); - zmq_island i(a, p, 1); + problem::dejong p(10); + algorithm::monte_carlo a(100); + zmq_island i(a, p, 1); i.set_broker_details("192.168.1.39", 6379); i.set_token("zeromq_test"); @@ -35,5 +35,5 @@ int main() { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } - return 0; + return 0; } From b2cdfc32b63715bbb3e58ee9cc6da1780fa9167e Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 13:29:46 +0200 Subject: [PATCH 06/25] Added Python documentation for zmq_island Also cleared some trailing whitespace and added a check to prevent zmq_island from sending solutions without having been initialised. trailing whitespace and --- PyGMO/core/__init__.py | 96 +++++++++++++++++++++++ PyGMO/core/core.cpp | 2 +- doc/sphinx/documentation/island.rst | 114 ++++++++++++++++++++++------ examples/zeromq.cpp | 2 +- src/zmq_island.cpp | 74 +++++++++--------- src/zmq_island.h | 1 + 6 files changed, 230 insertions(+), 59 deletions(-) diff --git a/PyGMO/core/__init__.py b/PyGMO/core/__init__.py index 12f2eefc..91565d41 100644 --- a/PyGMO/core/__init__.py +++ b/PyGMO/core/__init__.py @@ -14,6 +14,7 @@ 'ipy_island', 'island', 'local_island', + 'zmq_island', 'migration_direction', 'population', 'py_island'] @@ -136,6 +137,101 @@ def _generic_island_ctor(self, *args, **kwargs): local_island.__original_init__ = local_island.__init__ local_island.__init__ = _generic_island_ctor +# Raw C++ ZMQ island class. +_zmq_island = _core._zmq_island + +class zmq_island(_core._zmq_island): + def __init__(self, *args, **kwargs): + """ZMQ Island, yo. Unnamed arguments: + + #. algorithm + #. problem or population + #. number of individuals (optional and valid only if the second argument is a problem, defaults to 0 if not specified) + + Keyword arguments: + + * *s_policy* -- migration selection policy (defaults to 'best selection' policy) + * *r_policy* -- migration replacement policy (defaults to 'fair replacement' policy) + + """ + if len(args) == 0: + raise ValueError( + "Cannot initialise ZeroMQ island without parameters for the constructor.") + + from PyGMO.algorithm._algorithm import _base as _base_algorithm + from PyGMO.algorithm import base as base_algorithm + from PyGMO.problem._problem import _base as _base_problem + from PyGMO.problem._problem import _base_stochastic as _base_problem_stochastic + from PyGMO.problem import base as base_problem + from PyGMO.problem import base_stochastic as base_problem_stochastic + from PyGMO.migration._migration import best_s_policy, fair_r_policy, _base_s_policy, _base_r_policy + + if len(args) < 2 or len(args) > 3: + raise ValueError( + "Unnamed arguments list must have either 2 or three elements, but %d elements were found instead." % + (len(args),)) + if not isinstance(args[0], _base_algorithm): + raise TypeError("The first unnamed argument must be an algorithm.") + ctor_args = [args[0]] + if isinstance(args[1], _base_problem) or isinstance(args[1], _base_problem_stochastic): + ctor_args.append(args[1]) + if len(args) == 3: + if not isinstance(args[2], int): + raise TypeError( + "Please provide an integer for the number of individuals in the island.") + ctor_args.append(args[2]) + else: + ctor_args.append(0) + elif isinstance(args[1], population): + if len(args) == 3: + raise ValueError( + "When the second unnamed argument is a population, there cannot be a third unnamed argument.") + ctor_args.append(args[1]) + else: + raise TypeError( + "The second unnamed argument must be either a problem or a population.") + + if 's_policy' in kwargs: + ctor_args.append(kwargs['s_policy']) + else: + ctor_args.append(best_s_policy()) + if not isinstance(ctor_args[-1], _base_s_policy): + raise TypeError("s_policy must be a migration selection policy.") + + if 'r_policy' in kwargs: + ctor_args.append(kwargs['r_policy']) + else: + ctor_args.append(fair_r_policy()) + if not isinstance(ctor_args[-1], _base_r_policy): + raise TypeError("r_policy must be a migration replacement policy.") + + super(type(self), self).__init__(*ctor_args) + #if isinstance(self, zmq_island): + # super(type(self), self).__init__(*ctor_args) + #elif isinstance(self, _zmq_island): + # self.__original_init__(*ctor_args) + #else: + # assert(self is None) + # n_pythonic_items = 0 + # if isinstance(args[0], base_algorithm): + # n_pythonic_items += 1 + # if isinstance(args[1], base_problem) or isinstance(args[1], base_problem_stochastic): + # n_pythonic_items += 1 + # elif isinstance(args[1], population) and (isinstance(args[1].problem, base_problem) or isinstance(args[1], base_problem_stochastic)): + # n_pythonic_items += 1 + # if n_pythonic_items > 0: + # return py_island(*args, **kwargs) + # else: + # return local_island(*args, **kwargs) + # _core._base_island.__init__(self, *args) + + def get_name(self): + return str(type(self)) + + def __get_deepcopy__(self): + from copy import deepcopy + return deepcopy(self) + # This is the function that will be called by the separate process # spawned from py_island. diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index a773f112..5ba6ddf0 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -382,7 +382,7 @@ BOOST_PYTHON_MODULE(_core) register_ptr_to_python(); // ZMQ island class. - class_ >("zmq_island", "ZMQ island class.",init >()) + class_ >("_zmq_island", "ZMQ island class.",init >()) .def(init >()) .def(init()) .def("__copy__", &Py_copy_from_ctor) diff --git a/doc/sphinx/documentation/island.rst b/doc/sphinx/documentation/island.rst index 6b39412e..f8586392 100644 --- a/doc/sphinx/documentation/island.rst +++ b/doc/sphinx/documentation/island.rst @@ -4,7 +4,7 @@ Island and Archipelago NOTE: There are two different types of islands in PyGMO. The PyGMO.local_island and the PyGMO.py_island. The need for two different types of island is purely technical (it is necessary to give the user the possibility of implementing the problem or the algorithm in python directly) and the user needs not to know any details. We will thus here document only one class that we call -island and that, in reality, is a helper function returning automatically the correct object type. +island and that, in reality, is a helper function returning automatically the correct object type. .. class:: PyGMO.island @@ -27,7 +27,7 @@ island and that, in reality, is a helper function returning automatically the co .. method:: PyGMO.island.__init__((PyGMO.algorithm)algo, (PyGMO.problem)prob, (int)N=0 [, s_policy = best_s_policy(1), r_policy=fair_r_policy(1)]) - Constructs an island directly from a problem. The resulting population (of size N) will be evolved by algo. + Constructs an island directly from a problem. The resulting population (of size N) will be evolved by algo. Migration occurs at the end of each evolution if the *island* belongs to an :class:`PyGMO.archipelago` .. code-block:: python @@ -38,17 +38,17 @@ island and that, in reality, is a helper function returning automatically the co isl = island(algo,prob,20) .. method:: PyGMO.island.evolve((int)n) - + Evolves the :class:`PyGMO.population` in the *island* performing n calls to :class:`PyGMO.algorithm`. At the end of each call migration occurs if the *island* belongs to a :class:`PyGMO.archipelago`. Evolution happens in the background on a dedicated thread while the program flows continues. - + .. code-block:: python from PyGMO import * prob = problem.schwefel(1500) algo = algorithm.de(100) #instantiates differential evolution with default params and 100 generations - pop = population(prob,20) + pop = population(prob,20) isl = island(algo,pop) isl.evolve(1) #calls algo to evolve pop. The program flow continues @@ -62,7 +62,7 @@ island and that, in reality, is a helper function returning automatically the co from PyGMO import * prob = problem.schwefel(1500) algo = algorithm.de(100) #instantiates differential evolution with default params and 100 generations - pop = population(prob,20) + pop = population(prob,20) isl = island(algo,pop) isl.evolve(1) #calls algo to evolve pop. The program flow continues isl.join() #Waits for the evolve to finish (i.e. waits for completion of the 100 generations of differential evolution) @@ -73,7 +73,7 @@ island and that, in reality, is a helper function returning automatically the co .. method:: PyGMO.island.set_x((int)idx,(list) x) - Sets a new chromosome for the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` + Sets a new chromosome for the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` of the *island* to x. .. code-block:: python @@ -82,14 +82,14 @@ island and that, in reality, is a helper function returning automatically the co prob = problem.ackley(5) algo = algorithm.de(10) #instantiates differential evolution with default params and 10 generations isl = island(algo,prob,10) - isl.population.set_x(0,[1,2,3,4,5]) # This is completely uneffective + isl.population.set_x(0,[1,2,3,4,5]) # This is completely uneffective # as the 'attribute' population returns a copy isl.set_x(0,[1,2,3,4,5]) # This works!! - + .. method:: PyGMO.island.set_v((int)idx,(list) v) - Sets the velocity of the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` + Sets the velocity of the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` of the *island* to v. .. code-block:: python @@ -98,7 +98,7 @@ island and that, in reality, is a helper function returning automatically the co prob = problem.ackley(5) algo = algorithm.de(10) #instantiates differential evolution with default params and 10 generations isl = island(algo,prob,10) - isl.population.set_v(0,[0.02,0.03,-0.3,0.12,0.1]) # This is completely uneffective + isl.population.set_v(0,[0.02,0.03,-0.3,0.12,0.1]) # This is completely uneffective # as the 'attribute' population returns a copy isl.set_v(0,[0.02,0.03,-0.3,0.12,0.1]) # This works!! @@ -129,7 +129,7 @@ island and that, in reality, is a helper function returning automatically the co .. class:: PyGMO.archipelago - Probably the most important object in all PyGMO. An *Archipelago* is a container of :class:`PyGMO.island` + Probably the most important object in all PyGMO. An *Archipelago* is a container of :class:`PyGMO.island` and is responsible to start the asynchronous island model. The solutions exchange among :class:`PyGMO.island` is done following the routes allowed by the underlying topology @@ -149,7 +149,7 @@ island and that, in reality, is a helper function returning automatically the co Constructs an empty *archipelago* from a topology (defaults to :class:`PyGMO.topology.unconnected()`) a distribution type (defaults to :class:`PyGMO.distribution_type.point_to_point`) and a migration - direction (defaults to :class:`PyGMO.migration_direction.destination`). + direction (defaults to :class:`PyGMO.migration_direction.destination`). It then pushes back into the archipelago n_isl :class:`PyGMO.island` constructed using defaults values for the kwargs and (algo,prob,n_ind) as args. @@ -163,10 +163,10 @@ island and that, in reality, is a helper function returning automatically the co #and a copy of the griewank(30) problem .. method:: PyGMO.archipelago.evolve((int)n) - - Calls the method :class:`PyGMO.island.evolve` (n) on all the :class:`PyGMO.island` of the *archipelago*. + + Calls the method :class:`PyGMO.island.evolve` (n) on all the :class:`PyGMO.island` of the *archipelago*. In other words, it starts the asynchronous generalized island model that is at the core of PyGMO. - + .. code-block:: python from PyGMO import * @@ -174,9 +174,9 @@ island and that, in reality, is a helper function returning automatically the co algo = algorithm.de(500) #instantiates differential evolution with default #params and 100 generations archi = archipelago(algo,prob,8,20) - archi.evolve(10) #starts the asynchronous generalized island model. + archi.evolve(10) #starts the asynchronous generalized island model. #each of the 8 islands will call algo 10 times and try to migrate in between calls - + .. method:: PyGMO.archipelago.push_back((PyGMO.island) isl) Pushes back isl in the archipelago taking also to also update the topological links between islands. @@ -195,17 +195,17 @@ island and that, in reality, is a helper function returning automatically the co algo = algorithm.de(500) #instantiates differential evolution with default #params and 100 generations archi = archipelago(algo,prob,8,20) - archi.evolve(10) #starts the asynchronous generalized island model. + archi.evolve(10) #starts the asynchronous generalized island model. archi.join() #waits for it to finish [isl.population.champion.f for isl in archi] #builds a list with the best fittnesses found .. method:: PyGMO.archipelago.busy() - Returns True if evolution is ongoing in the *archipelago*. + Returns True if evolution is ongoing in the *archipelago*. .. method:: PyGMO.archipelago.interrupt() - Halts evolution at the first occasion in all islands. + Halts evolution at the first occasion in all islands. .. automethod:: PyGMO.archipelago.draw() @@ -219,8 +219,76 @@ island and that, in reality, is a helper function returning automatically the co .. method:: PyGMO.archipelago.dump_migration_history() - Returns a temporal history of all the archipelago migrations in one string. Each entry is in the form + Returns a temporal history of all the archipelago migrations in one string. Each entry is in the form (n,src_isl,dest_isl) and logs that n individuals, from the *island* having the index src_isl, successfully replaced n individuals in the *population* of the *island* having the index dest_isl. - + +.. class:: PyGMO.zmq_island + + The ZeroMQ island behaves like any :class:`PyGMO.island` because it manages the algorithm, population and migration strategies, + but it also communicates with other ZeroMQ islands that are in the same configuration. This is used to distribute the computation + workload between networked machines. If it is not initialised, it will behave exactly like a local island. + + .. method:: PyGMO.zmq_island.__init__((PyGMO.algorithm)algo, (PyGMO.population)pop [, s_policy = best_s_policy(1), r_policy=fair_r_policy(1)]) + + Instantiates a :class:`PyGMO.zmq_island`, with the given population and algorithm. After every evolution, the island will broadcast a copy of the population to the ZeroMQ network. + + .. code-block:: python + + from PyGMO import * + prob = problem.schwefel(2) + algo = algorithm.de(10) + pop = population(prob,20) + + isl = zmq_island(algo,pop) + + Note that because the parameters specific to the ZMQ island are not set yet, this island cannot communicate and will work exactly like a local island. + + .. method:: PyGMO.zmq_island.set_broker_details((string) host, (int) port) + + Stores the broker host and port. At the moment, the broker software is a Redis server. + + .. code-block:: python + + from PyGMO import * + isl = zmq_island(a, p) + isl.set_broker_details("127.0.0.1", 6379) + + .. method:: PyGMO.zmq_island.set_token((string) token) + + Sets the token (also known as channel topic), which is used to determine which nodes are working on a problem with similar configurations. + + .. code-block:: python + + isl.set_token("schwefel2_de10_pop20") + + The token can be any string, but describing the algorithm and problem in the token is useful to monitor the progress of the computation. A way of monitoring the progress is described in the tutorial (TODO: link), where a user connects to the ZeroMQ network but does not perform evolution tasks. + + .. method:: PyGMO.zmq_island.initialise((string) ip) + + This method will initialise the communication capabilities of the ZeroMQ island by communicating with the network, connecting to its peers and broadcasting the new connection. + + The IP address given as an argument will be used to bind the receiving socket, so it must be accessible by all of the nodes in the network. A receiving port will be chosen at random, from 1000 to 2000. + + .. code-block:: python + + from PyGMO import * + import time + + prob = problem.schwefel(10) + algo = algorithm.de(10) + pop = population(prob,20) + + isl = zmq_island(algo,pop) + isl.set_broker_details("127.0.0.1", 6379) + isl.set_token("schwefel2_de10_pop10") + isl.initialise("127.0.0.1") + + while True: + isl.evolve(10) + print("Best: ", isl.population.champion.x) + + time.sleep(1) + + The given example will solve the 10-dimensional Schwefel problem locally, but if another instance of this program is started up in the same computer, they will exchange solutions until they converge on a solution. diff --git a/examples/zeromq.cpp b/examples/zeromq.cpp index fb42ea2e..e2157f85 100644 --- a/examples/zeromq.cpp +++ b/examples/zeromq.cpp @@ -23,7 +23,7 @@ int main() { problem::dejong p(10); algorithm::monte_carlo a(100); - zmq_island i(a, p, 1); + zmq_island i(a, p, 100); i.set_broker_details("192.168.1.39", 6379); i.set_token("zeromq_test"); diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 788327c4..d9e1d702 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -58,7 +58,7 @@ namespace pagmo */ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) + base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false) {} /// Constructor from population. @@ -67,14 +67,14 @@ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, */ zmq_island::zmq_island(const algorithm::base &a, const population &pop, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) + base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false) {} /// Copy constructor. /** * @see pagmo::base_island constructors. */ -zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) // TODO: does this make sense? +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false) // TODO: does this make sense? {} /// Destructor. @@ -97,37 +97,41 @@ base_island_ptr zmq_island::clone() const // This method performs the local evolution for this island's population. void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) const { - const boost::shared_ptr pop_copy(new population(pop)); - const algorithm::base_ptr algo_copy = algo.clone(); - //const std::pair,algorithm::base_ptr> out(pop_copy,algo_copy); - const boost::shared_ptr out = pop_copy; - - // First, we send a copy of our population and algorithm - std::stringstream ss; - boost::archive::text_oarchive oa(ss); - oa << out; - std::string buffer(ss.str()); - zmq::message_t msg(buffer.size()); - memcpy((void *) msg.data(), buffer.c_str(), buffer.size() - 1); - m_publisherSocket.send(msg); - - // See if there is any data available - zmq::message_t incoming; - if(m_subscriptionSocket.recv(&incoming, ZMQ_DONTWAIT) > 0) { - if(incoming.size()) { - try { - std::string bytes_in((char *) incoming.data(), incoming.size()); - - std::stringstream incoming_ss(bytes_in); - boost::archive::text_iarchive ia(incoming_ss); - boost::shared_ptr in; - - ia >> in; - pop = *in; - } catch (const boost::archive::archive_exception &e) { - std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ": " << e.what() << std::endl; - } catch (...) { - std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ", unknown exception caught. :(" << std::endl; + algo.evolve(pop); + + if(m_initialised) { + const boost::shared_ptr pop_copy(new population(pop)); + const algorithm::base_ptr algo_copy = algo.clone(); + //const std::pair,algorithm::base_ptr> out(pop_copy,algo_copy); + const boost::shared_ptr out = pop_copy; + + // First, we send a copy of our population and algorithm + std::stringstream ss; + boost::archive::text_oarchive oa(ss); + oa << out; + std::string buffer(ss.str()); + zmq::message_t msg(buffer.size()); + memcpy((void *) msg.data(), buffer.c_str(), buffer.size() - 1); + m_publisherSocket.send(msg); + + // See if there is any data available + zmq::message_t incoming; + if(m_subscriptionSocket.recv(&incoming, ZMQ_DONTWAIT) > 0) { + if(incoming.size()) { + try { + std::string bytes_in((char *) incoming.data(), incoming.size()); + + std::stringstream incoming_ss(bytes_in); + boost::archive::text_iarchive ia(incoming_ss); + boost::shared_ptr in; + + ia >> in; + pop = *in; + } catch (const boost::archive::archive_exception &e) { + std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ": " << e.what() << std::endl; + } catch (...) { + std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ", unknown exception caught. :(" << std::endl; + } } } } @@ -213,6 +217,8 @@ bool zmq_island::initialise(std::string ip) { } else { /* disconnect */ } }); + m_initialised = true; + return true; } diff --git a/src/zmq_island.h b/src/zmq_island.h index 344c09ee..f30a828d 100644 --- a/src/zmq_island.h +++ b/src/zmq_island.h @@ -109,6 +109,7 @@ class __PAGMO_VISIBLE zmq_island: public base_island std::string m_token; std::string m_IP; int m_localPort; + bool m_initialised; redox::Redox m_brokerConn; redox::Subscriber m_brokerSubscriber; From ff974614802b5d37b6c09e683f9f390eceefb855 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 13:45:40 +0200 Subject: [PATCH 07/25] zmq_island: add callback for zmq messages --- examples/zeromq.cpp | 6 ++++++ src/zmq_island.cpp | 31 +++++++++++++++++++++++++++---- src/zmq_island.h | 13 ++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/examples/zeromq.cpp b/examples/zeromq.cpp index e2157f85..98ac5754 100644 --- a/examples/zeromq.cpp +++ b/examples/zeromq.cpp @@ -12,6 +12,10 @@ void sig(int) { quit.store(true); } +void callback(zmq::message_t& msg) { + std::cout << "Got a " << msg.size() << "byte message from the network." << std::endl; +} + int main() { struct sigaction sa; memset(&sa, 0, sizeof(sa)); @@ -29,6 +33,8 @@ int main() { i.set_token("zeromq_test"); i.initialise("192.168.1.39"); + i.set_callback(&callback); + while(!quit) { i.evolve(1); std::cout << "Best: " << i.get_population().champion().x << std::endl; diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index d9e1d702..dc5d6dc8 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -58,7 +58,8 @@ namespace pagmo */ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false) + base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), + m_evolve(true), m_callback(NULL) {} /// Constructor from population. @@ -67,14 +68,16 @@ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, */ zmq_island::zmq_island(const algorithm::base &a, const population &pop, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false) + base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), + m_evolve(true), m_callback(NULL) {} /// Copy constructor. /** * @see pagmo::base_island constructors. */ -zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false) // TODO: does this make sense? +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), + m_evolve(true), m_callback(NULL) // TODO: does this make sense? {} /// Destructor. @@ -97,7 +100,9 @@ base_island_ptr zmq_island::clone() const // This method performs the local evolution for this island's population. void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) const { - algo.evolve(pop); + if(m_evolve) { + algo.evolve(pop); + } if(m_initialised) { const boost::shared_ptr pop_copy(new population(pop)); @@ -118,6 +123,8 @@ void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) zmq::message_t incoming; if(m_subscriptionSocket.recv(&incoming, ZMQ_DONTWAIT) > 0) { if(incoming.size()) { + m_callback(incoming); + try { std::string bytes_in((char *) incoming.data(), incoming.size()); @@ -234,6 +241,22 @@ void zmq_island::disconnect() { std::cout << "DEBUG: Closed" << std::endl; } +void zmq_island::set_evolve(bool e) { + m_evolve = e; +} + +bool zmq_island::get_evolve() { + return m_evolve; +} + +void zmq_island::set_callback(zmq_island::callback c) { + m_callback = c; +} + +void zmq_island::disable_callback() { + m_callback = NULL; +} + } BOOST_CLASS_EXPORT_IMPLEMENT(pagmo::zmq_island) diff --git a/src/zmq_island.h b/src/zmq_island.h index f30a828d..e2ff9bbe 100644 --- a/src/zmq_island.h +++ b/src/zmq_island.h @@ -96,11 +96,18 @@ class __PAGMO_VISIBLE zmq_island: public base_island protected: void perform_evolution(const algorithm::base &, population &) const; public: + typedef void (*callback)(zmq::message_t&); + std::string get_name() const; void set_broker_details(std::string, int); void set_token(std::string); bool initialise(std::string); + void set_evolve(bool); + bool get_evolve(); + + void set_callback(zmq_island::callback); + void disable_callback(); void close(); private: @@ -109,7 +116,6 @@ class __PAGMO_VISIBLE zmq_island: public base_island std::string m_token; std::string m_IP; int m_localPort; - bool m_initialised; redox::Redox m_brokerConn; redox::Subscriber m_brokerSubscriber; @@ -118,6 +124,11 @@ class __PAGMO_VISIBLE zmq_island: public base_island zmq::socket_t m_publisherSocket; zmq::socket_t m_subscriptionSocket; + bool m_initialised; + bool m_evolve; + + zmq_island::callback m_callback; + void connect(std::string); void disconnect(); From 37eaa86c27948771d523b87c85e219d505b0040c Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 14:43:30 +0200 Subject: [PATCH 08/25] zmq_island: fix compilation issue in debian --- doc/sphinx/documentation/island.rst | 2 +- doc/sphinx/tutorials/index.rst | 1 + src/zmq_island.cpp | 11 ++++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/sphinx/documentation/island.rst b/doc/sphinx/documentation/island.rst index f8586392..8ac7b784 100644 --- a/doc/sphinx/documentation/island.rst +++ b/doc/sphinx/documentation/island.rst @@ -265,7 +265,7 @@ island and that, in reality, is a helper function returning automatically the co The token can be any string, but describing the algorithm and problem in the token is useful to monitor the progress of the computation. A way of monitoring the progress is described in the tutorial (TODO: link), where a user connects to the ZeroMQ network but does not perform evolution tasks. - .. method:: PyGMO.zmq_island.initialise((string) ip) + .. method:: PyGMO.zmq_island.initialise((string) ip) This method will initialise the communication capabilities of the ZeroMQ island by communicating with the network, connecting to its peers and broadcasting the new connection. diff --git a/doc/sphinx/tutorials/index.rst b/doc/sphinx/tutorials/index.rst index 5d6af46f..554db478 100644 --- a/doc/sphinx/tutorials/index.rst +++ b/doc/sphinx/tutorials/index.rst @@ -15,3 +15,4 @@ This is a collection of tutorials on PyGMO's basics: racing tsp landscape_analysis_with_DrPyGMO + distributed_computation_with_zeromq diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index dc5d6dc8..9859e448 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -58,7 +58,9 @@ namespace pagmo */ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), + base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), + m_zmqContext(1), + m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), m_evolve(true), m_callback(NULL) {} @@ -68,7 +70,9 @@ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, */ zmq_island::zmq_island(const algorithm::base &a, const population &pop, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), + base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), + m_zmqContext(1), + m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), m_evolve(true), m_callback(NULL) {} @@ -76,7 +80,8 @@ zmq_island::zmq_island(const algorithm::base &a, const population &pop, /** * @see pagmo::base_island constructors. */ -zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_zmqContext(1), + m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), m_evolve(true), m_callback(NULL) // TODO: does this make sense? {} From d5e1a00e33534208b5af336a3449b59224fd10d2 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 14:49:26 +0200 Subject: [PATCH 09/25] zmq_island: use NOBLOCK for versions earlier than 3.0 --- src/zmq_island.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 9859e448..3173d497 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -58,8 +58,8 @@ namespace pagmo */ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): - base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), - m_zmqContext(1), + base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), m_evolve(true), m_callback(NULL) {} @@ -71,7 +71,7 @@ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, zmq_island::zmq_island(const algorithm::base &a, const population &pop, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), - m_zmqContext(1), + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), m_evolve(true), m_callback(NULL) {} @@ -80,7 +80,7 @@ zmq_island::zmq_island(const algorithm::base &a, const population &pop, /** * @see pagmo::base_island constructors. */ -zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_zmqContext(1), +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), m_evolve(true), m_callback(NULL) // TODO: does this make sense? {} @@ -102,7 +102,7 @@ base_island_ptr zmq_island::clone() const return base_island_ptr(new zmq_island(*this)); } -// This method performs the local evolution for this island's population. +// This method performs the local evolution for this island's population. void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) const { if(m_evolve) { @@ -126,8 +126,13 @@ void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) // See if there is any data available zmq::message_t incoming; - if(m_subscriptionSocket.recv(&incoming, ZMQ_DONTWAIT) > 0) { - if(incoming.size()) { +#if ZMQ_VERSION >= ZMQ_MAKE_VERSION(3, 0, 0) + // https://github.com/zeromq/libzmq/blob/master/NEWS#L548 + if(m_subscriptionSocket.recv(&incoming, ZMQ_DONTWAIT) > 0) { +#else + if(m_subscriptionSocket.recv(&incoming, ZMQ_NOBLOCK) > 0) { +#endif + if(incoming.size()) { m_callback(incoming); try { @@ -197,7 +202,7 @@ bool zmq_island::initialise(std::string ip) { std::string brokerKey = "pagmo.islands." + m_token; // Get list of peers - redox::Command >& result = + redox::Command >& result = m_brokerConn.commandSync >({"SMEMBERS", brokerKey}); if(!result.ok()) { @@ -224,7 +229,7 @@ bool zmq_island::initialise(std::string ip) { m_brokerSubscriber.subscribe(brokerKey + ".control", [&](const std::string&, const std::string& msg) { std::vector data; boost::split(data, msg, boost::is_any_of("/")); - if(data[0] == "connected") { + if(data[0] == "connected") { connect(data[1]); } else { /* disconnect */ } }); From 4ecd21a665431e63098e74ca4d6ed35f0c65d241 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 15:28:13 +0200 Subject: [PATCH 10/25] tutorials: added several docs about zeromq islands --- .../distributed_computation_with_zeromq.rst | 12 +++ doc/sphinx/tutorials/zmq_first_contact.rst | 79 +++++++++++++++++++ doc/sphinx/tutorials/zmq_setup.rst | 67 ++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 doc/sphinx/tutorials/distributed_computation_with_zeromq.rst create mode 100644 doc/sphinx/tutorials/zmq_first_contact.rst create mode 100644 doc/sphinx/tutorials/zmq_setup.rst diff --git a/doc/sphinx/tutorials/distributed_computation_with_zeromq.rst b/doc/sphinx/tutorials/distributed_computation_with_zeromq.rst new file mode 100644 index 00000000..1e2cc285 --- /dev/null +++ b/doc/sphinx/tutorials/distributed_computation_with_zeromq.rst @@ -0,0 +1,12 @@ +.. _zeromq: + +=================================== +Distributed computation with ZeroMQ +=================================== + +.. toctree:: + :maxdepth: 2 + + zmq_setup + zmq_first_contact + zmq_archipelago diff --git a/doc/sphinx/tutorials/zmq_first_contact.rst b/doc/sphinx/tutorials/zmq_first_contact.rst new file mode 100644 index 00000000..80813c78 --- /dev/null +++ b/doc/sphinx/tutorials/zmq_first_contact.rst @@ -0,0 +1,79 @@ +.. _zeromq_first_contact: + +============= +First contact +============= + +Noe that we have a Redis server running and we have compiled PaGMO and PyGMO with ZeroMQ support (see :ref:`zeromq_setup`), we can run an example program that leverages the functionality of this type of island. + +.. code-block:: python + :linenos: + + from PyGMO import * + import time + + prob = problem.schwefel(10) + algo = algorithm.de(10) + pop = population(prob, 20) + + isl = zmq_island(algo, pop) + isl.set_broker_details("127.0.0.1", 6379) + isl.set_token("schwefel10_de10_pop20") + isl.initialise("127.0.0.1") + + while True: + isl.evolve(10) + print("Best: ", isl.population.champion.x) + + time.sleep(1) + +This code will set up a single ZeroMQ island working on the 10-dimensional Schwefel problem and using the Differential Evolution algorithm. The population of the island will be 20. Lines 4-6 set up the problem, and lines 8-11 set up the island itself. + +In this tutorial we assume that the broker is at 127.0.0.1:6379, and we will be binding the receiving port only on the loopback inteface, but in a networked setup you would use IPs that are accessible beyond localhost. + +The channel token is set to `schwefel10_de10_pop20`, which can be any string, but choosing a token relevant to the actual properties of the computation will help you determine which group of islands is working on what problem. If we start this program, it will connect to the broker and advertise itself as an island working on that token, but because there are no other peers it will simply perform the evolution locally. This is a typical output: + +.. code-block:: bash + + root@f3428b4e74bd:/# python /tmp/zmq.py + DEBUG: IP: '127.0.0.1:2401' + ('Best: ', (401.9594173543314, 424.7239031087063, -489.08965281305683, 192.79310987286283, 422.3702354040747, -280.2352091575768, -304.1105751377624, 419.61971508325234, -304.4306512922853, 412.3797020751874)) + ('Best: ', (401.9594173543314, 422.9252917377888, 397.6750493053389, -308.787292270248, 417.6016092014828, 415.24467267670747, -304.1105751377624, 419.61971508325234, 443.02530229342995, 412.3797020751874)) + ('Best: ', (417.7624473989758, 419.38998223162383, 423.9121066735651, 420.9581335823198, 423.1917112361371, 416.75217338872466, 416.5656801251844, 423.65078535816195, 427.8092589385355, 417.23729521493794)) + ('Best: ', (421.19930412300704, 420.80770338293433, 420.95461980350495, 421.1387381497945, 420.9882620455983, 420.7632718115494, 420.9945625003906, 420.68798717006837, 421.2449946368809, 420.9255606400046)) + ('Best: ', (420.97784637271286, 420.95030001558206, 420.97695438787514, 420.9925460783401, 420.9588231827722, 420.9747030586656, 420.96113438989653, 420.9744491052427, 420.9570307864317, 420.9708116286286)) + ('Best: ', (420.96796502949985, 420.9692843203323, 420.9674981715592, 420.96888513790157, 420.96902668827914, 420.96711830078385, 420.96698080790026, 420.9675721284126, 420.9690259384316, 420.9693847247998)) + +However, if we run two instances of this program simultaneously, they will connect to each other and start exchanging solutions. + + +.. code-block:: bash + :linenos: + + root@f3428b4e74bd:/# python /tmp/zmq.py + DEBUG: IP: '127.0.0.1:2839' + Best: (446.330415191082, -318.57151650616345, -319.141745784881, 409.37490624414585, 434.9522671118881, 436.11092724710943, 412.9042329474612, -327.3902497012161, 433.12563296150233, 405.2335813101326) + Best: (406.4818160886365, 429.4589205156737, 411.6918195936836, 412.4947146394673, 439.92422758511657, 413.4096775345739, 402.7337709186839, 410.8803588738704, 458.24230339934326, 436.5535606636335) + Best: (416.6611528308124, 424.2092721283991, 418.8522866953207, 422.8026613418712, 423.3550577222186, 420.2964268628981, 422.0640382301092, 422.3657526856015, 422.8089197437483, 421.1248410730861) + DEBUG: Opening connection to 127.0.0.1:2758 + Best: (420.8118964319755, 420.97105998402435, 420.8675085594526, 421.0028607982087, 420.95935903890796, 420.991509807542, 420.84368415020265, 421.08197535480474, 420.81508001828564, 421.10958460178466) + Best: (421.07749509648613, 420.8546192606146, 420.7665760480669, 421.17780038893375, 421.0733383225067, 421.0745532974556, 421.1227771446935, 421.23088854902136, 420.9196459791927, 421.0675596920381) + Best: (421.02737839902096, 421.37184378009175, 421.25678912369284, 421.0148941900929, 421.1551021658338, 421.0769746858749, 421.14611943756057, 420.829339492172, 421.08317952008963, 420.76028074583166) + Best: (420.98011516631004, 420.7954519611739, 420.99948716750913, 420.83980802618726, 420.9522553458674, 421.28867861574423, 420.7846133663843, 421.114115625814, 421.0752711472328, 421.1199157748332) + Best: (420.8864797873097, 420.93967731273574, 420.80247869292685, 420.8796939441704, 421.04690058491394, 420.9704882856439, 420.9380577878827, 420.98689425859016, 420.9460027704123, 421.2146020009918) + Best: (420.6410077561692, 420.8774213590491, 420.9770237342764, 420.8169234318866, 421.0771442746173, 420.7741545848519, 421.2015181934143, 420.75764749753006, 421.08444174574305, 421.2578651544008) + +And on the other side: + +.. code-block:: bash + :linenos: + + root@f3428b4e74bd:/# python /tmp/zmq.py + DEBUG: IP: '127.0.0.1:2758' + DEBUG: Opening connection to 127.0.0.1:2839 + Best: (170.02156365996962, 432.0808563954199, -299.9044328266113, 449.57827719040404, 414.32650241479433, 442.74256832933975, -497.56415018442954, 409.9770554543145, -128.49045206283606, 437.2655093302242) + Best: (420.8118964319755, 420.97105998402435, 420.8675085594526, 421.0028607982087, 420.95935903890796, 420.991509807542, 420.84368415020265, 421.08197535480474, 420.81508001828564, 421.10958460178466) + Best: (421.1237678412085, 420.9910361828901, 420.80333273611603, 421.0852572932901, 421.0638265999589, 420.97221532173234, 420.84697129401894, 421.1221559791297, 420.94034652647935, 421.01529447856126) + Best: (421.01592546566957, 421.2374973637021, 420.932682470239, 420.9163032282556, 421.02010123092407, 421.16968911726184, 421.09670372915707, 420.9404867930009, 420.94383616409635, 420.9957624510262) + +Here we can see that the last best value from the first instance is 416.6611528308124 (line 5), and the second program produces a better candidate, 420.8118964319755 (line 5 on the second program), which is sent over the ZeroMQ sockets and appears on the output of the first program after it opens a connection to the new peer (lines 6 and 7). diff --git a/doc/sphinx/tutorials/zmq_setup.rst b/doc/sphinx/tutorials/zmq_setup.rst new file mode 100644 index 00000000..feb24218 --- /dev/null +++ b/doc/sphinx/tutorials/zmq_setup.rst @@ -0,0 +1,67 @@ +.. _zeromq_setup: + +========= +The Setup +========= + +In order for ZeroMQ islands to find their peers and announce when they connect to and disconnect from the network, we must first set up the broker server. Our broker software of choice is Redis. This server acts as a bootstrap and communication channel. + +There are packages available in most distributions of Linux, but in this tutorial we will use the `debian:latest` Docker image. It's not necessary to install it in a container, but it makes it easier to reproduce the setup steps. + +First we will update the package list and install the Redis server: + +.. code-block:: bash + + root@dbc834f0d0aa:/# apt-get -qq update + root@dbc834f0d0aa:/# apt-get -qq install redis-server -y + (output omitted...) + root@dbc834f0d0aa:/# + +Now we'll install the compilation tools and necessary libraries (this will take a while) + +.. code-block:: bash + + root@dbc834f0d0aa:/# apt-get -qq install build-essential cmake git libboost-dev libboost-system-dev libboost-serialization-dev libboost-thread-dev libboost-python-dev libhiredis-dev libev-dev libzmq5-dev -y + (output omitted...) + +We also need to compile and install `redox` because Debian does not provide a binary package. + +.. code-block:: bash + root@dbc834f0d0aa:~# git clone https://github.com/hmartiro/redox /tmp/redox + (output omitted...) + root@dbc834f0d0aa:~# cd /tmp/redox/ + root@dbc834f0d0aa:/tmp/redox# mkdir build && cd build + root@dbc834f0d0aa:/tmp/redox/build# cmake .. -DLIB_SUFFIX="/" + (output omitted...) + root@dbc834f0d0aa:/tmp/redox/build# make && make install + +Next, we'll clone the PaGMO source tree and compile it with PyGMO and ZeroMQ enabled. The `make` command takes around 15-20 minutes to complete (could take longer), so this is the perfect time to grab a caffeinated beverage. + +.. code-block:: bash + + root@dbc834f0d0aa:/# git clone https://github.com/jdiez17/pagmo /tmp/pagmo + root@dbc834f0d0aa:/# cd /tmp/pagmo + root@dbc834f0d0aa:/tmp/pagmo# mkdir build && cd build + root@dbc834f0d0aa:/tmp/pagmo/build# cmake .. -DENABLE_ZMQ=ON -DBUILD_PYGMO=ON + root@dbc834f0d0aa:/tmp/pagmo/build# make && make install && ldconfig + +If everything went well, you should be able to import PyGMO and zmq_island from Python! + +.. code-block:: bash + + root@dbc834f0d0aa:/tmp/pagmo/build# python + Python 2.7.9 (default, Mar 1 2015, 12:57:24) + [GCC 4.9.2] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import PyGMO + >>> 'zmq_island' in dir(PyGMO) + True + >>> + +Finally, we'll start a Redis server and daemonize it so it runs in the background. + +.. code-block:: bash + + root@dbc834f0d0aa:/# redis-server --daemonize yes + +At this point, you are ready to run ZMQ islands! From 8db93a0e191a059d68592c81ba758f050094b06e Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 16:08:33 +0200 Subject: [PATCH 11/25] Changed visibility of some fields and fixed compilation in PyGMO without ZeroMQ --- PyGMO/core/core.cpp | 4 +++- src/zmq_island.cpp | 23 +++++++++++++---------- src/zmq_island.h | 13 +++++++------ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index 5ba6ddf0..ee1e3289 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -49,7 +49,9 @@ #include "../../src/algorithm/base.h" #include "../../src/archipelago.h" #include "../../src/base_island.h" -#include "../../src/zmq_island.h" +#ifdef PAGMO_ENABLE_ZMQ + #include "zmq_island.h" +#endif #include "../../src/config.h" #include "../../src/exceptions.h" #include "../../src/migration/base_r_policy.h" diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 3173d497..096aba74 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -59,9 +59,8 @@ namespace pagmo zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), - m_zmqContext(1), - m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), - m_evolve(true), m_callback(NULL) + m_initialised(false), m_evolve(true), m_callback(NULL), + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) {} /// Constructor from population. @@ -71,18 +70,17 @@ zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, zmq_island::zmq_island(const algorithm::base &a, const population &pop, const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), - m_zmqContext(1), - m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), - m_evolve(true), m_callback(NULL) + m_initialised(false), m_evolve(true), m_callback(NULL), + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) {} /// Copy constructor. /** * @see pagmo::base_island constructors. */ -zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_zmqContext(1), - m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB), m_initialised(false), - m_evolve(true), m_callback(NULL) // TODO: does this make sense? +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), + m_initialised(false), m_evolve(true), m_callback(NULL), // TODO: does this make sense? + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) {} /// Destructor. @@ -99,7 +97,10 @@ zmq_island &zmq_island::operator=(const zmq_island &isl) base_island_ptr zmq_island::clone() const { - return base_island_ptr(new zmq_island(*this)); + disconnect(); + auto ret = new zmq_island(*this); + ret->m_IP = m_IP; + return base_island_ptr(ret); } // This method performs the local evolution for this island's population. @@ -248,6 +249,8 @@ void zmq_island::disconnect() { m_brokerConn.disconnect(); m_brokerSubscriber.disconnect(); + m_publisherSocket.disconnect(("tcp://" + m_IP).c_str()); + std::cout << "DEBUG: Closed" << std::endl; } diff --git a/src/zmq_island.h b/src/zmq_island.h index e2ff9bbe..1bb16d04 100644 --- a/src/zmq_island.h +++ b/src/zmq_island.h @@ -110,13 +110,19 @@ class __PAGMO_VISIBLE zmq_island: public base_island void disable_callback(); void close(); - private: + protected: std::string m_brokerHost; int m_brokerPort; std::string m_token; std::string m_IP; int m_localPort; + bool m_initialised; + bool m_evolve; + + zmq_island::callback m_callback; + + private: redox::Redox m_brokerConn; redox::Subscriber m_brokerSubscriber; @@ -124,11 +130,6 @@ class __PAGMO_VISIBLE zmq_island: public base_island zmq::socket_t m_publisherSocket; zmq::socket_t m_subscriptionSocket; - bool m_initialised; - bool m_evolve; - - zmq_island::callback m_callback; - void connect(std::string); void disconnect(); From 8d26a844f23c34198757fb51f902ef0bea8973b6 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 16:09:44 +0200 Subject: [PATCH 12/25] pygmo: fix zmq path --- PyGMO/core/core.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index ee1e3289..7b83890c 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -50,7 +50,7 @@ #include "../../src/archipelago.h" #include "../../src/base_island.h" #ifdef PAGMO_ENABLE_ZMQ - #include "zmq_island.h" + #include "../../zmq_island.h" #endif #include "../../src/config.h" #include "../../src/exceptions.h" From a7378afe10c8631bd8a62ff8dbb43d0cffab40f9 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 16:10:41 +0200 Subject: [PATCH 13/25] Path is definitely fixed now. --- PyGMO/core/core.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index 7b83890c..a4af5951 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -50,7 +50,7 @@ #include "../../src/archipelago.h" #include "../../src/base_island.h" #ifdef PAGMO_ENABLE_ZMQ - #include "../../zmq_island.h" + #include "../../src/zmq_island.h" #endif #include "../../src/config.h" #include "../../src/exceptions.h" From bc9f9b872a5d81e442a61121eef6e198241fd264 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 16:20:50 +0200 Subject: [PATCH 14/25] Only define the ZMQ island if it's enabled --- PyGMO/core/__init__.py | 171 +++++++++++++++++++++-------------------- PyGMO/core/core.cpp | 2 + src/zmq_island.cpp | 6 ++ 3 files changed, 95 insertions(+), 84 deletions(-) diff --git a/PyGMO/core/__init__.py b/PyGMO/core/__init__.py index 91565d41..9778a3f6 100644 --- a/PyGMO/core/__init__.py +++ b/PyGMO/core/__init__.py @@ -14,11 +14,13 @@ 'ipy_island', 'island', 'local_island', - 'zmq_island', 'migration_direction', 'population', 'py_island'] +if '_zmq_island' in dir(_core): + __all__.append('zmq_island') + _orig_signal = _signal.getsignal(_signal.SIGINT) _main_pid = _os.getpid() @@ -137,100 +139,101 @@ def _generic_island_ctor(self, *args, **kwargs): local_island.__original_init__ = local_island.__init__ local_island.__init__ = _generic_island_ctor +if '_zmq_island' in dir(_core): # Raw C++ ZMQ island class. -_zmq_island = _core._zmq_island - -class zmq_island(_core._zmq_island): - def __init__(self, *args, **kwargs): - """ZMQ Island, yo. Unnamed arguments: - - #. algorithm - #. problem or population - #. number of individuals (optional and valid only if the second argument is a problem, defaults to 0 if not specified) + _zmq_island = _core._zmq_island - Keyword arguments: + class zmq_island(_core._zmq_island): + def __init__(self, *args, **kwargs): + """ZMQ Island, yo. Unnamed arguments: - * *s_policy* -- migration selection policy (defaults to 'best selection' policy) - * *r_policy* -- migration replacement policy (defaults to 'fair replacement' policy) + #. algorithm + #. problem or population + #. number of individuals (optional and valid only if the second argument is a problem, defaults to 0 if not specified) - """ - if len(args) == 0: - raise ValueError( - "Cannot initialise ZeroMQ island without parameters for the constructor.") + Keyword arguments: - from PyGMO.algorithm._algorithm import _base as _base_algorithm - from PyGMO.algorithm import base as base_algorithm - from PyGMO.problem._problem import _base as _base_problem - from PyGMO.problem._problem import _base_stochastic as _base_problem_stochastic - from PyGMO.problem import base as base_problem - from PyGMO.problem import base_stochastic as base_problem_stochastic - from PyGMO.migration._migration import best_s_policy, fair_r_policy, _base_s_policy, _base_r_policy + * *s_policy* -- migration selection policy (defaults to 'best selection' policy) + * *r_policy* -- migration replacement policy (defaults to 'fair replacement' policy) - if len(args) < 2 or len(args) > 3: - raise ValueError( - "Unnamed arguments list must have either 2 or three elements, but %d elements were found instead." % - (len(args),)) - if not isinstance(args[0], _base_algorithm): - raise TypeError("The first unnamed argument must be an algorithm.") - ctor_args = [args[0]] - if isinstance(args[1], _base_problem) or isinstance(args[1], _base_problem_stochastic): - ctor_args.append(args[1]) - if len(args) == 3: - if not isinstance(args[2], int): - raise TypeError( - "Please provide an integer for the number of individuals in the island.") - ctor_args.append(args[2]) - else: - ctor_args.append(0) - elif isinstance(args[1], population): - if len(args) == 3: + """ + if len(args) == 0: raise ValueError( - "When the second unnamed argument is a population, there cannot be a third unnamed argument.") - ctor_args.append(args[1]) - else: - raise TypeError( - "The second unnamed argument must be either a problem or a population.") + "Cannot initialise ZeroMQ island without parameters for the constructor.") - if 's_policy' in kwargs: - ctor_args.append(kwargs['s_policy']) - else: - ctor_args.append(best_s_policy()) - if not isinstance(ctor_args[-1], _base_s_policy): - raise TypeError("s_policy must be a migration selection policy.") - - if 'r_policy' in kwargs: - ctor_args.append(kwargs['r_policy']) - else: - ctor_args.append(fair_r_policy()) - if not isinstance(ctor_args[-1], _base_r_policy): - raise TypeError("r_policy must be a migration replacement policy.") + from PyGMO.algorithm._algorithm import _base as _base_algorithm + from PyGMO.algorithm import base as base_algorithm + from PyGMO.problem._problem import _base as _base_problem + from PyGMO.problem._problem import _base_stochastic as _base_problem_stochastic + from PyGMO.problem import base as base_problem + from PyGMO.problem import base_stochastic as base_problem_stochastic + from PyGMO.migration._migration import best_s_policy, fair_r_policy, _base_s_policy, _base_r_policy - super(type(self), self).__init__(*ctor_args) - #if isinstance(self, zmq_island): - # super(type(self), self).__init__(*ctor_args) - #elif isinstance(self, _zmq_island): - # self.__original_init__(*ctor_args) - #else: - # assert(self is None) - # n_pythonic_items = 0 - # if isinstance(args[0], base_algorithm): - # n_pythonic_items += 1 - # if isinstance(args[1], base_problem) or isinstance(args[1], base_problem_stochastic): - # n_pythonic_items += 1 - # elif isinstance(args[1], population) and (isinstance(args[1].problem, base_problem) or isinstance(args[1], base_problem_stochastic)): - # n_pythonic_items += 1 - # if n_pythonic_items > 0: - # return py_island(*args, **kwargs) - # else: - # return local_island(*args, **kwargs) - # _core._base_island.__init__(self, *args) + if len(args) < 2 or len(args) > 3: + raise ValueError( + "Unnamed arguments list must have either 2 or three elements, but %d elements were found instead." % + (len(args),)) + if not isinstance(args[0], _base_algorithm): + raise TypeError("The first unnamed argument must be an algorithm.") + ctor_args = [args[0]] + if isinstance(args[1], _base_problem) or isinstance(args[1], _base_problem_stochastic): + ctor_args.append(args[1]) + if len(args) == 3: + if not isinstance(args[2], int): + raise TypeError( + "Please provide an integer for the number of individuals in the island.") + ctor_args.append(args[2]) + else: + ctor_args.append(0) + elif isinstance(args[1], population): + if len(args) == 3: + raise ValueError( + "When the second unnamed argument is a population, there cannot be a third unnamed argument.") + ctor_args.append(args[1]) + else: + raise TypeError( + "The second unnamed argument must be either a problem or a population.") - def get_name(self): - return str(type(self)) + if 's_policy' in kwargs: + ctor_args.append(kwargs['s_policy']) + else: + ctor_args.append(best_s_policy()) + if not isinstance(ctor_args[-1], _base_s_policy): + raise TypeError("s_policy must be a migration selection policy.") - def __get_deepcopy__(self): - from copy import deepcopy - return deepcopy(self) + if 'r_policy' in kwargs: + ctor_args.append(kwargs['r_policy']) + else: + ctor_args.append(fair_r_policy()) + if not isinstance(ctor_args[-1], _base_r_policy): + raise TypeError("r_policy must be a migration replacement policy.") + + super(type(self), self).__init__(*ctor_args) + #if isinstance(self, zmq_island): + # super(type(self), self).__init__(*ctor_args) + #elif isinstance(self, _zmq_island): + # self.__original_init__(*ctor_args) + #else: + # assert(self is None) + # n_pythonic_items = 0 + # if isinstance(args[0], base_algorithm): + # n_pythonic_items += 1 + # if isinstance(args[1], base_problem) or isinstance(args[1], base_problem_stochastic): + # n_pythonic_items += 1 + # elif isinstance(args[1], population) and (isinstance(args[1].problem, base_problem) or isinstance(args[1], base_problem_stochastic)): + # n_pythonic_items += 1 + # if n_pythonic_items > 0: + # return py_island(*args, **kwargs) + # else: + # return local_island(*args, **kwargs) + # _core._base_island.__init__(self, *args) + + def get_name(self): + return str(type(self)) + + def __get_deepcopy__(self): + from copy import deepcopy + return deepcopy(self) # This is the function that will be called by the separate process # spawned from py_island. diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index a4af5951..12da5dd4 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -383,6 +383,7 @@ BOOST_PYTHON_MODULE(_core) // Register to_python conversion from smart pointer. register_ptr_to_python(); +#ifdef PAGMO_ENABLE_ZMQ // ZMQ island class. class_ >("_zmq_island", "ZMQ island class.",init >()) .def(init >()) @@ -392,6 +393,7 @@ BOOST_PYTHON_MODULE(_core) .def("set_broker_details", &zmq_island::set_broker_details) .def("set_token", &zmq_island::set_token) .def("initialise", &zmq_island::initialise); +#endif // Expose archipelago class. class_("archipelago", "Archipelago class.", initm_brokerHost = m_brokerHost; + ret->m_brokerPort = m_brokerPort; + ret->m_token = m_token; ret->m_IP = m_IP; + ret->m_localPort = m_localPort; + ret->m_evolve = m_evolve; + return base_island_ptr(ret); } From b3162b81fd31b6a3b33460825e9914fa9f570e62 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 17:11:34 +0200 Subject: [PATCH 15/25] Fixed formatting in tutorial, more initialisation protection, don't clone zmq islands from archipelago on access --- PyGMO/core/core.cpp | 3 +- doc/sphinx/tutorials/zmq_setup.rst | 1 + examples/zeromq.cpp | 3 +- src/archipelago.cpp | 25 +++++++++++-- src/archipelago.h | 2 + src/zmq_island.cpp | 59 ++++++++++++++++-------------- src/zmq_island.h | 5 ++- 7 files changed, 64 insertions(+), 34 deletions(-) diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index 12da5dd4..3d8a3f32 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -392,7 +392,8 @@ BOOST_PYTHON_MODULE(_core) .def("__deepcopy__", &Py_deepcopy_from_ctor) .def("set_broker_details", &zmq_island::set_broker_details) .def("set_token", &zmq_island::set_token) - .def("initialise", &zmq_island::initialise); + .def("set_ip", &zmq_island::set_ip) + .def("connect", &zmq_island::connect); #endif // Expose archipelago class. diff --git a/doc/sphinx/tutorials/zmq_setup.rst b/doc/sphinx/tutorials/zmq_setup.rst index feb24218..5c12d003 100644 --- a/doc/sphinx/tutorials/zmq_setup.rst +++ b/doc/sphinx/tutorials/zmq_setup.rst @@ -27,6 +27,7 @@ Now we'll install the compilation tools and necessary libraries (this will take We also need to compile and install `redox` because Debian does not provide a binary package. .. code-block:: bash + root@dbc834f0d0aa:~# git clone https://github.com/hmartiro/redox /tmp/redox (output omitted...) root@dbc834f0d0aa:~# cd /tmp/redox/ diff --git a/examples/zeromq.cpp b/examples/zeromq.cpp index 98ac5754..d0e3b9b0 100644 --- a/examples/zeromq.cpp +++ b/examples/zeromq.cpp @@ -31,7 +31,8 @@ int main() { i.set_broker_details("192.168.1.39", 6379); i.set_token("zeromq_test"); - i.initialise("192.168.1.39"); + i.set_ip("192.168.1.39"); + i.connect(); i.set_callback(&callback); diff --git a/src/archipelago.cpp b/src/archipelago.cpp index 69679fab..c9eab88a 100644 --- a/src/archipelago.cpp +++ b/src/archipelago.cpp @@ -207,6 +207,18 @@ archipelago::size_type archipelago::locate_island(const base_island &isl) const return std::distance(m_container.begin(),it); } +/// Determine whether we should return a copy of the island, or a pointer to it. +/** + * @param[in] isl island to be checked + */ +bool archipelago::should_clone(base_island_ptr isl) const { + if(isl->get_name().find("ZMQ") != std::string::npos) { + // we don't want to clone ZMQ islands + return false; + } + return true; +} + /// Add an island to the archipelago. /** * Both the island and the archipelago will be synchronised before any operation takes place. The island will then @@ -711,9 +723,16 @@ base_island_ptr archipelago::get_island(const size_type &idx) const if (idx >= m_container.size()) { pagmo_throw(index_error,"invalid island index"); } - base_island_ptr retval = m_container[idx]->clone(); - // The island is no more in an archipelago. - retval->m_archi = 0; + base_island_ptr retval; + + if(should_clone(m_container[idx])) { + retval = m_container[idx]->clone(); + // The island is no more in an archipelago. + retval->m_archi = 0; + } else { + retval = m_container[idx]; + } + return retval; } diff --git a/src/archipelago.h b/src/archipelago.h index 50a5b8d5..7b817cfd 100644 --- a/src/archipelago.h +++ b/src/archipelago.h @@ -162,6 +162,8 @@ class __PAGMO_VISIBLE archipelago bool destruction_checks() const; void reevaluate_immigrants(std::vector > &, const base_island &) const; + + bool should_clone(base_island_ptr) const; private: friend class boost::serialization::access; template diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 6f0f0c0a..5a245859 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -81,7 +81,17 @@ zmq_island::zmq_island(const algorithm::base &a, const population &pop, zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_initialised(false), m_evolve(true), m_callback(NULL), // TODO: does this make sense? m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) -{} +{ + m_brokerHost = isl.m_brokerHost; + m_brokerPort = isl.m_brokerPort; + m_token = isl.m_token; + m_IP = isl.m_IP; + m_localPort = isl.m_localPort; + m_evolve = isl.m_evolve; + + connect(); + std::cout << "yep this is copy" << std::endl; +} /// Destructor. zmq_island::~zmq_island() { @@ -98,15 +108,7 @@ zmq_island &zmq_island::operator=(const zmq_island &isl) base_island_ptr zmq_island::clone() const { disconnect(); - auto ret = new zmq_island(*this); - ret->m_brokerHost = m_brokerHost; - ret->m_brokerPort = m_brokerPort; - ret->m_token = m_token; - ret->m_IP = m_IP; - ret->m_localPort = m_localPort; - ret->m_evolve = m_evolve; - - return base_island_ptr(ret); + return base_island_ptr(new zmq_island(*this)); } // This method performs the local evolution for this island's population. @@ -180,14 +182,14 @@ void zmq_island::set_token(std::string token) { m_token = token; } -void zmq_island::connect(std::string host) { +void zmq_island::connect_host(std::string host) { std::cout << "DEBUG: Opening connection to " << host << std::endl; m_subscriptionSocket.connect(("tcp://" + host).c_str()); } -bool zmq_island::initialise(std::string ip) { - if(m_brokerHost == "" || m_brokerPort == -1 || m_token == "") { +bool zmq_island::connect() { + if(m_brokerHost == "" || m_brokerPort == -1 || m_token == "" || m_IP == "") { return false; // Can't initialise if we're missing those parameters } @@ -200,11 +202,6 @@ bool zmq_island::initialise(std::string ip) { // Initialise subscription socket m_subscriptionSocket.setsockopt(ZMQ_SUBSCRIBE, "", 0); - // Choose a port between 1000 and 2000 - srand(time(0)); - m_localPort = rand() % 2000 + 1000; - m_IP += ip + ":" + std::to_string(m_localPort); - std::cout << "DEBUG: IP: '" << m_IP << "'" << std::endl; std::string brokerKey = "pagmo.islands." + m_token; @@ -220,7 +217,7 @@ bool zmq_island::initialise(std::string ip) { // Connect to peers auto peers = result.reply(); for(auto it = peers.begin(); it != peers.end(); ++it) { - connect(*it); + connect_host(*it); } // Add ourselves to the list of islands on the chosen topic. @@ -237,7 +234,7 @@ bool zmq_island::initialise(std::string ip) { std::vector data; boost::split(data, msg, boost::is_any_of("/")); if(data[0] == "connected") { - connect(data[1]); + connect_host(data[1]); } else { /* disconnect */ } }); @@ -247,17 +244,19 @@ bool zmq_island::initialise(std::string ip) { } void zmq_island::disconnect() { - std::string brokerKey = "pagmo.islands." + m_token; + if(m_initialised) { + std::string brokerKey = "pagmo.islands." + m_token; - m_brokerConn.commandSync({"SREM", brokerKey, m_IP}); - m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "disconnected/" + m_IP}); + m_brokerConn.commandSync({"SREM", brokerKey, m_IP}); + m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "disconnected/" + m_IP}); - m_brokerConn.disconnect(); - m_brokerSubscriber.disconnect(); + m_brokerConn.disconnect(); + m_brokerSubscriber.disconnect(); - m_publisherSocket.disconnect(("tcp://" + m_IP).c_str()); + m_publisherSocket.disconnect(("tcp://" + m_IP).c_str()); - std::cout << "DEBUG: Closed" << std::endl; + std::cout << "DEBUG: Closed" << std::endl; + } } void zmq_island::set_evolve(bool e) { @@ -276,6 +275,12 @@ void zmq_island::disable_callback() { m_callback = NULL; } +void zmq_island::set_ip(std::string ip) { + srand(time(0)); + m_localPort = rand() % 2000 + 1000; + m_IP += ip + ":" + std::to_string(m_localPort); +} + } BOOST_CLASS_EXPORT_IMPLEMENT(pagmo::zmq_island) diff --git a/src/zmq_island.h b/src/zmq_island.h index 1bb16d04..f24e4cba 100644 --- a/src/zmq_island.h +++ b/src/zmq_island.h @@ -102,12 +102,13 @@ class __PAGMO_VISIBLE zmq_island: public base_island void set_broker_details(std::string, int); void set_token(std::string); - bool initialise(std::string); + bool connect(); void set_evolve(bool); bool get_evolve(); void set_callback(zmq_island::callback); void disable_callback(); + void set_ip(std::string ip); void close(); protected: @@ -130,7 +131,7 @@ class __PAGMO_VISIBLE zmq_island: public base_island zmq::socket_t m_publisherSocket; zmq::socket_t m_subscriptionSocket; - void connect(std::string); + void connect_host(std::string); void disconnect(); friend class boost::serialization::access; From 6cd66161cba59edd400c1ef0097a14e01c55279f Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 17:30:04 +0200 Subject: [PATCH 16/25] zmq_island: null check --- src/zmq_island.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 5a245859..168ddeaf 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -133,6 +133,7 @@ void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) memcpy((void *) msg.data(), buffer.c_str(), buffer.size() - 1); m_publisherSocket.send(msg); + // See if there is any data available zmq::message_t incoming; #if ZMQ_VERSION >= ZMQ_MAKE_VERSION(3, 0, 0) @@ -142,7 +143,9 @@ void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) if(m_subscriptionSocket.recv(&incoming, ZMQ_NOBLOCK) > 0) { #endif if(incoming.size()) { - m_callback(incoming); + if(m_callback != NULL) { + m_callback(incoming); + } try { std::string bytes_in((char *) incoming.data(), incoming.size()); From 9d39f5ac9ae8b5a048b61b6d842899652185504f Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 18:11:45 +0200 Subject: [PATCH 17/25] notebook: add first zmq tutorial --- .../tutorials/first_contact_zmq.ipynb | 191 ++++++++++++++++++ doc/notebooks/tutorials/index.ipynb | 106 ++++++---- doc/sphinx/tutorials/zmq_first_contact.rst | 5 +- src/zmq_island.cpp | 3 +- 4 files changed, 258 insertions(+), 47 deletions(-) create mode 100644 doc/notebooks/tutorials/first_contact_zmq.ipynb diff --git a/doc/notebooks/tutorials/first_contact_zmq.ipynb b/doc/notebooks/tutorials/first_contact_zmq.ipynb new file mode 100644 index 00000000..af01b744 --- /dev/null +++ b/doc/notebooks/tutorials/first_contact_zmq.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First contact\n", + "===\n", + "\n", + "Now that we have a Redis server running and we have compiled PaGMO and PyGMO with ZeroMQ support (see The Setup (TODO: link)), we can run an example program that leverages the functionality of this type of island.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from PyGMO import *\n", + "import time\n", + "\n", + "prob = problem.schwefel(10)\n", + "algo = algorithm.de(10)\n", + "pop = population(prob, 20)\n", + "\n", + "isl = zmq_island(algo, pop)\n", + "isl.set_broker_details(\"127.0.0.1\", 6379)\n", + "isl.set_token(\"schwefel10_de10_pop20\")\n", + "isl.set_ip(\"127.0.0.1\")\n", + "\n", + "def evolve():\n", + " isl.evolve(10)\n", + " print(\"Best: \", isl.population.champion.x)\n", + "\n", + " time.sleep(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This code will set up a single ZeroMQ island working on the 10-dimensional Schwefel problem and using the Differential Evolution algorithm. The population of the island will be 20. Lines 4-6 set up the problem, and lines 8-12 set up the island itself. \n", + "\n", + "In this tutorial we assume that the broker is at 127.0.0.1:6379, and we will be binding the receiving port only on the loopback inteface, but in a networked setup you would use IPs that are accessible beyond localhost.\n", + "\n", + "The channel token is set to `schwefel10_de10_pop20`, which can be any string, but choosing a token relevant to the actual properties of the computation will help you determine which group of islands is working on what problem. If we start this program, it will connect to the broker and advertise itself as an island working on that token, but because there are no other peers it will simply perform the evolution locally. This is a typical output:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best: (424.4920569537634, 440.7206390429582, 399.43102290788795, -299.52078183929916, 424.4274999225018, 395.96019766970727, 396.61431073790675, 414.7987356103403, -293.15767979233385, -152.01178644903854)\n" + ] + } + ], + "source": [ + "evolve()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that because the island is not actually connected to the ZeroMQ network yet, it's not exchanging solutions with peers." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, if we call the `connect()` function on this island, we'll be ready to accept migrants and send our own as well." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "isl.connect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Because IPython doesn't have an easy way of executing two threads and displaying the result easily, I'll start up another program just like this outside IPython and call evolve() here -- it should receive the population from the other program, and display it." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best: (420.9687728816528, 420.9687569400687, 420.9687642651889, 420.968743361018, 420.96875116870063, 420.968771074309, 420.96871292112354, 420.9687127857178, 420.96875246113143, 420.9687562620334)\n" + ] + } + ], + "source": [ + "evolve()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, our current solution is much better than the original and because the program in the background had been running for a little while before that, and it had already converged on the optimum for the Schwefel function." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's a snippet of the output on the other side: \n", + "\n", + " Best: (420.96880347940544, 420.9687194811639, 420.96880839006843, 420.9687540217262, 420.9687810359755, 420.9687609702579, 420.968730572192, 420.96875630500364, 420.9687648447646, 420.96880903640675)\n", + " Best: (420.9687351908159, 420.96872163933637, 420.9687399639167, 420.9687069931382, 420.9686997709129, 420.9687976850297, 420.9687312413354, 420.9687104977846, 420.9687392971252, 420.9687677879039)\n", + " DEBUG: Opening connection to 127.0.0.1:2609\n", + " Best: (420.9687701356386, 420.9687781629428, 420.9687691217583, 420.9687483448487, 420.96878827364174, 420.96875668743473, 420.96870707941497, 420.9687079359776, 420.96874266075383, 420.96872310305565)\n", + " Best: (420.9687728816528, 420.9687569400687, 420.9687642651889, 420.968743361018, 420.96875116870063, 420.968771074309, 420.96871292112354, 420.9687127857178, 420.96875246113143, 420.9687562620334)\n", + " Best: (420.9687533680281, 420.96876243183203, 420.9687590126924, 420.9687372408334, 420.96875017480545, 420.96877891439186, 420.9687558066539, 420.9687460272027, 420.9687420838824, 420.9687647092545)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note how it opens a connection to our island as soon as we call `isl.connect()`. Then it sent its population, which had already converged on the answer and was better than our original value." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/doc/notebooks/tutorials/index.ipynb b/doc/notebooks/tutorials/index.ipynb index c083b52b..d38f8be1 100644 --- a/doc/notebooks/tutorials/index.ipynb +++ b/doc/notebooks/tutorials/index.ipynb @@ -1,47 +1,65 @@ { - "metadata": { - "name": "", - "signature": "sha256:4e3c0951d9d81090c3c1c2520fd6e4170f76fedd5b98af109335e7e59a85e1d4" - }, - "nbformat": 3, - "nbformat_minor": 0, - "worksheets": [ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PyGMO Tutorials" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this section you can get yourself acquainted with the very basics of using PyGMO." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Content" + ] + }, { - "cells": [ - { - "cell_type": "heading", - "level": 1, - "metadata": {}, - "source": [ - "PyGMO Tutorials" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this section you can get yourself acquainted with the very basics of using PyGMO." - ] - }, - { - "cell_type": "heading", - "level": 1, - "metadata": {}, - "source": [ - "Content" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* [Adding new problems and algorithms to PyGMO](adding_new_problems_and_algorithms_to_pygmo.ipynb)\n", - "* [Meta problems](meta_problems.ipynb)\n", - "* [Multi objective optimization](multi_objective_optimization.ipynb)" - ] - } - ], - "metadata": {} + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* [Adding new problems and algorithms to PyGMO](adding_new_problems_and_algorithms_to_pygmo.ipynb)\n", + "* [Meta problems](meta_problems.ipynb)\n", + "* [Multi objective optimization](multi_objective_optimization.ipynb)\n", + "* [First contact with ZMQ Islands](first_contact_zmq.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/doc/sphinx/tutorials/zmq_first_contact.rst b/doc/sphinx/tutorials/zmq_first_contact.rst index 80813c78..b196e303 100644 --- a/doc/sphinx/tutorials/zmq_first_contact.rst +++ b/doc/sphinx/tutorials/zmq_first_contact.rst @@ -19,7 +19,8 @@ Noe that we have a Redis server running and we have compiled PaGMO and PyGMO wit isl = zmq_island(algo, pop) isl.set_broker_details("127.0.0.1", 6379) isl.set_token("schwefel10_de10_pop20") - isl.initialise("127.0.0.1") + isl.set_ip("127.0.0.1") + isl.connect() while True: isl.evolve(10) @@ -27,7 +28,7 @@ Noe that we have a Redis server running and we have compiled PaGMO and PyGMO wit time.sleep(1) -This code will set up a single ZeroMQ island working on the 10-dimensional Schwefel problem and using the Differential Evolution algorithm. The population of the island will be 20. Lines 4-6 set up the problem, and lines 8-11 set up the island itself. +This code will set up a single ZeroMQ island working on the 10-dimensional Schwefel problem and using the Differential Evolution algorithm. The population of the island will be 20. Lines 4-6 set up the problem, and lines 8-12 set up the island itself. In this tutorial we assume that the broker is at 127.0.0.1:6379, and we will be binding the receiving port only on the loopback inteface, but in a networked setup you would use IPs that are accessible beyond localhost. diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 168ddeaf..8ff57755 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -72,6 +72,7 @@ zmq_island::zmq_island(const algorithm::base &a, const population &pop, base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), m_initialised(false), m_evolve(true), m_callback(NULL), m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) + {} /// Copy constructor. @@ -88,9 +89,9 @@ zmq_island::zmq_island(const zmq_island &isl):base_island(isl), m_IP = isl.m_IP; m_localPort = isl.m_localPort; m_evolve = isl.m_evolve; + m_callback = isl.m_callback; connect(); - std::cout << "yep this is copy" << std::endl; } /// Destructor. From 453dfb52ba1b5452a045b25e154fe6f2966b4681 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 18:36:47 +0200 Subject: [PATCH 18/25] Add tutorial using ZeroMQ islands in an archipelago --- doc/notebooks/tutorials/index.ipynb | 3 +- .../tutorials/zmq_islands_archipelago.ipynb | 206 ++++++++++++++++++ 2 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 doc/notebooks/tutorials/zmq_islands_archipelago.ipynb diff --git a/doc/notebooks/tutorials/index.ipynb b/doc/notebooks/tutorials/index.ipynb index d38f8be1..5107b30a 100644 --- a/doc/notebooks/tutorials/index.ipynb +++ b/doc/notebooks/tutorials/index.ipynb @@ -28,7 +28,8 @@ "* [Adding new problems and algorithms to PyGMO](adding_new_problems_and_algorithms_to_pygmo.ipynb)\n", "* [Meta problems](meta_problems.ipynb)\n", "* [Multi objective optimization](multi_objective_optimization.ipynb)\n", - "* [First contact with ZMQ Islands](first_contact_zmq.ipynb)" + "* [First contact with ZMQ Islands](first_contact_zmq.ipynb)\n", + "* [Using ZMQ Islands in an archipelago](zmq_islands_archipelago.ipynb)" ] }, { diff --git a/doc/notebooks/tutorials/zmq_islands_archipelago.ipynb b/doc/notebooks/tutorials/zmq_islands_archipelago.ipynb new file mode 100644 index 00000000..1430546e --- /dev/null +++ b/doc/notebooks/tutorials/zmq_islands_archipelago.ipynb @@ -0,0 +1,206 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using ZeroMQ islands in an archipelago\n", + "====================\n", + "\n", + "While ZeroMQ islands can be used standalone (without being part of an archipelago), they are used most effectively inside one. The idea is that through archipelago migration strategies, local islands exchange individuals with a single ZeroMQ inside it, which acts as a nexus to send solutions to other archipelagos with ZeroMQ islands inside them. This also works in reverse; a ZeroMQ island is constantly receiving population broadcasts from its peers, so the archipelago can migrate them from there to other islands.\n", + "\n", + "\n", + "\n", + "In this image there are three archipelagos, each with a different topology. The dotted lines represent the ZeroMQ network, which is always fully connected.\n", + "\n", + "To simplify the example, the following program instantiates an archipelago with three local islands and one ZeroMQ island." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from PyGMO import *\n", + "import time\n", + "\n", + "prob = problem.schwefel(10)\n", + "algo = algorithm.de(10)\n", + "\n", + "zmqisl = zmq_island(algo, prob, 20)\n", + "zmqisl.set_broker_details(\"127.0.0.1\", 6379)\n", + "zmqisl.set_token(\"schwefel10_de10_pop20\")\n", + "zmqisl.set_ip(\"127.0.0.1\")\n", + "\n", + "archi = archipelago(algo, prob, 3, 20)\n", + "archi.push_back(zmqisl)\n", + "\n", + "def evolve():\n", + " archi.evolve(1)\n", + " for isl in archi:\n", + " if isl.get_name().startswith(\"ZMQ\"):\n", + " return(isl.population.champion.x[0])\n", + "\n", + " time.sleep(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following demonstrations will use graphics and plots, so we need to load matplotlib:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we will evolve the archipelago 20 times and record the value of the best individual in the ZeroMQ island over time, and plot the results." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEACAYAAAC57G0KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGitJREFUeJzt3XuUVNWZ9/Hv0yBEMqKCSmwYQYlIBsTwGhTBaEE0gzHB\ne5jlmGXiODoRb7xLx8S8GkbfzEwMy3klIWBIYswko8RLVBwyOlFqbBKYdEPwzkXsAWmjcg0aAZF+\n3j92NZQtdNflVJ9L/z5r1aLr1DmnHmoVv97ss/c+5u6IiEj21MVdgIiI1IYCXkQkoxTwIiIZpYAX\nEckoBbyISEYp4EVEMqqkgDezOjNbYmYN7bZfbWatZnZM0bZpZrbJzFrM7PyoCxYRkdKU2oK/ClgF\n7Bk0b2b1wOTC9rZtQ4EbgbHAhcAsMzswsmpFRKRknQZ8IcjPA2YDVvTSncBNFIU+IfDvd/fV7r4Y\naAQmRleuiIiUqpQWfFuQt7ZtMLOzgbfc/fft9q0H1hU9bwYGVlukiIiUr8OAN7NJwFZ3b6LQei90\nudwK3GJmbS16288pREQkJj07eX08cIWZXVG0bR3QD9hStG2VmY0CWoDBRduPAZ5of1Iz0wI4IiIV\ncPeSG9QdtuDd/RZ3r3P3OuAU4Dfufri79yjavhI41t1fBOYDU8xsmJmNA8YAT+/n3HpE9PjmN78Z\new1Zeujz1GeZ1Ee5OmvBFzM+eEF1X6G9xsxmAEuAHcBUd99RdlUiIlK1kgPe3ZcAp+1j+yfaPb+T\ncGFWRERipJmsGZDL5eIuIVP0eUZHn2W8rJJ+narf1MzjeF8RkTQzMzyqi6wiIpJeCngRkYxSwIuI\nZJQCXkQkoxTwIiIZpYAXEckoBbyISEYp4EVEMkoBLyKSUQp4EZGMUsCLiGSUAl5EJKMU8CIiGaWA\nFxHJKAW8iEhGKeBFRDJKAS8iklEKeBGRjFLAi4hkVM+4CxARSbLWVpg5EzZvru48Q4fCpZdGU1Op\n1IIXEenAvHnwwx9Cz57VPepiSFtz965/UzOP431FRMrx3nvwF38Bc+fChAlxVwNmhrtbqfurBS8i\nsh8/+lHoWklCuFdCLXgRkX1491049lh47DE48cS4qwnUghcRicDMmTB+fHLCvRJqwYuItLNlCwwb\nBosWwXHHxV3NXjVpwZtZnZktMbOGwvMbzKzFzN41swYzG1a07zQz21R4/fzy/woiIvG64w4499xk\nhXslSmrBm9nVwEnAEHc/rRDoG4A/AV8HRrn7BWY2FGgATgcOAx4GjnH37e3Opxa8iCTS66/D8cfD\ns8/CoEFxV/NBkbfgzaweOA+YDRiAu69y9y2FXXYBbxZ+ngzc7+6r3X0x0AhMLKN+EZFY3X47XHZZ\n8sK9EqXMZL0TuAnoUbzRzH4IXAY0A6cUNtcD64p2awYGVl+miEjtvfIKPPAArFwZdyXR6LAFb2aT\ngK3u3kSh9d7G3S8H+gD3AnfXrEIRkS5y661w/fXQv3/clUSjsxb8eOAKM7uibYOZ/c7dTwJw9x1m\n9gOgqfByCzC46PhjgCf2deLp06fv+TmXy5HL5cqtXUQkMsuXw8KF8IMfxF3JXvl8nnw+X/HxJQ+T\nNLOTgRnu/mkz+0vgd8Bu4GvASHefbGYfB54Bcuy9yDrE3Xe0O5cusopIonzuc+Fx9dVxV7J/5V5k\nLWc1SQPaUvkC4N8I/fLPAF8FcPdXzGwGsATYAUxtH+4iIknzzDPw8svwyCNxVxItTXQSkW7NHU49\nFf7u7+BLX4q7mo5pqQIRkTI8/jhs2wYXXxx3JdFTwItIt7V7N9x8M3zrW9CjR+f7p40CXkS6rfvu\ng4MOgi98Ie5KakN98CLSLb33HgwfDj/5CZx2WtzVlEZ98CIiJZg7NywmlpZwr4Ra8CKyTwsXwrhx\n0Lt33JVE7513ws08FiyA0aPjrqZ0asGLSNXWrIEzz4RPfjKMEc+au+6CXC5d4V4JteBF5EP+6Z9g\n3Tr47Gfh2mvDn3fckY01WjZtCl0zixeHVnyaqAUvIlWbNw+mTIHzzoOXXgojTUaMgJ/+NEwMSrNv\nfxsuvDB94V4JteBF5ANWroQJE+C11z44NrypCa68Eg45BGbPDre0S5uWFhg1Cp5/Hurr466mfGrB\ni0hV5s0LLdz2E38+9Sn47/8OY8bHjYPbboOdO+OpsVK33QaXX57OcK+EWvAi8gEjRoQlc8eP3/8+\nr70G11wDK1bA3XfD6ad3XX2VWrUq/J1WroR+/eKupjLltuAV8CKyxwsvwFlnwdq1UFfC/+8feSRc\nhD3jDPjOd5J9EXbKlDAq6Otfj7uSyqmLRkQqNm8efPGLpYU7wLnnwosvwsEHJ/si7LJl0NAQfhl1\nJ2rBiwgQgvm44+BnP4OTTir/+KVL4YorQtjPmZOsi7CTJsHkyXDVVXFXUh214EWkIsuXw/vvw5gx\nlR1/4onhIuw554SLsP/wD8m4CJvPw+rV4eJqd1POHZ1EEu3v/z5c7Dv77LgrSae27hkruX34YT17\nwnXXwQUXhO6Q/v3jX+pg+3a45x7o1SveOuKgLhrJjKOPhiOPhN/+Nu5K0scdjjkGHn442un7W7ZA\na2t056tEjx5h7H4W1PKerCKJtXEjbN4cgqqxsfJuhu6qsREOOCCMMonSoYdGez4pj/rgJROWLg19\nwFOnwsyZcVeTPm1LE1TTPSPJoxa8ZEJTU5hpefnloavhjTfgYx+Lu6p0aG2FX/wCfvWruCuRqKkF\nL5nQFvCHHhpaonPmxF1ReixeDH37wsiRcVciUVPASya0BTyEKfRz5iRjiF4atHXPSPYo4CX13ngD\n/vSnMIoGwozK448P3Q7Ssd274YEHFPBZpYCX1Fu6NLTeiy8QXnttuGuPRuN2rKEBBgwIM1glexTw\nknrF3TNtzj4btm4N/cuyf+qeyTYFvKTevgK+rg6uvlpDJjvy/vvw0EMK+CxTwEuque874AG+8hV4\n8klYv77r60qDhQth8OAwrFSySQEvqfb66+FC4Z//+YdfO/hg+Ou/DreXkw9T90z2lRTwZlZnZkvM\nrKHw/FYzW2tm28zsYTM7pGjfaWa2ycxazOz8WhUuAntb7/ubgXnNNTB3blhwSvZ67z345S/D4mKS\nXaW24K8CVgFtYxLeBCYAg4D3gZsBzGwocCMwFrgQmGVmB0ZZsEix/XXPtBk2LLx+331dV1Ma/PrX\nYeTMUUfFXYnUUqcBb2b1wHnAbMAA3P1ud3/V3bcBjwIDCrtPBu5399XuvhhoBCbWpHIROg94CEMm\nZ87UkMli6p7pHkppwd8J3AR8aNFPMzPgS8BDhU31wLqiXZqBgVXWKLJPHV1gLfbZz8KOHfDMM11T\nV9Lt2AGPPQYXXRR3JVJrHS42ZmaTgK3u3mRmY/exy7eBV9z9sXLfePr06Xt+zuVy5HK5ck8h3dy6\ndWGJ2/r6jverqwt98TNnhhuCdHdPPAGjRnX+uUn88vk8+Xy+4uM7vOGHmd0OfKPd5iZ3P8nMbgZO\ncPcpRftfDwx292mF5/OB2e6+oN15dcMPqdpDD8G994bWaGfeeScMCVy2LPzZnV18MZx6avrvT9od\nRXpPVne/xd3r3L0OOAX4TSHcrwZywCXtDpkPTDGzYWY2DhgDPF3W30CkRKV0z7T5sz+DSy+FWbNq\nW1PSvfsuLFgQbqkn2VfOOHhj7yiam4DPADvNrNXMngRw9zXADGAJ8CAw1d13RFivyB7lBDyEma0/\n/nFYmKy7WrAgfGYDBnS+r6Sf7skqqeQO/frBihXlhdU558DnPgdXXlm72pLsoovCRee//du4K5FK\nlNtFo4CXVFqzBiZMCBday/H00+GC6wsvdL/b073zDgwcCK++Cv37x12NVCLSPniRpGpqquzG2hMm\nhGB/6qnoa0q6+fNh3DiFe3eigJdUKrf/vY3Z3olP3Y0mN3U/CnhJpUoDHuCSS8I68WvWRFtTkv3x\nj6F76txz465EupICXlKntTWMZz/xxMqO79MHLrsMvve9aOtKskcfhVwODjmk010lQxTwkjqrV8Nh\nh4VRNJWaOhV++lN4++3o6koydc90Twp4SZ1qumfaHHVUuOB6773R1JRkmzfDokUweXLclUhXU8BL\n6kQR8ADXXQff/W7o8smyX/4SzjgDDjoo7kqkqyngJXWiCvhTTw398U88Uf25kmzePPirv4q7ComD\nJjpJquzeHS4Url8fbslXrZ/8JATgr35V/bmSaMMGOPbYcGvDPn3irkaqpYlOkmkrVoRlbqMIdwgt\n22XLwnmz6KGH4KyzFO7dlQJeUiWq7pk2H/lIWJclq0MmNXqme1PAS6pEHfAAX/0q/PznsHVrtOeN\n2x/+AMuXw6RJcVcicVHAS6rUIuAHDgwheM890Z43bg8+CF/4QvhfinRPCnhJjV274LnnYPTo6M99\n3XWhm2b37ujPHRd1z4gCXlLjpZdgyJBwd6aonXxyWGXx3/89+nPH4bXX4OWX4cwz465E4tThTbdF\nkqQW3TNtzEIr/l/+JdyQOu3uvTcsLNarV9yVSJwU8JIajY21C3gIdzuaMycsypV2PXuGC8fSvSng\nJTWamsKNs2ulVy9oaKjd+UW6mmaySirs3AmHHgobN2rSjnRfmskqmfT882HKvcJdpHQKeEmFWl5g\nFckqBbykggJepHwKeEkFBbxI+XSRVRJv+/YwCWnLFujdO+5qROKji6ySOc8+C5/4hMJdpFwKeEk8\ndc+IVEYBL4mngBepTEkBb2Z1ZrbEzBoKzy8xsxVmttvM/le7faeZ2SYzazGz82tRtHQvCniRypTa\ngr8KWAW0XRltBM4Cflu8k5kNBW4ExgIXArPM7MBoSpXu6J13oLkZRoyIuxKR9Ok04M2sHjgPmA0Y\ngLuvdPfmfew+Gbjf3Ve7+2LCL4KJEdYr3czy5TBypFZFFKlEKS34O4GbgNYS9q0H1hU9bwYGVlCX\nCKDuGZFqdLiapJlNAra6e5OZjY3yjadPn77n51wuRy4La7RK5Jqa4DOfibsKkXjk83ny+XzFx3c4\n0cnMbge+0W5zk7ufVHi9AbjO3ZcVnl8PDHb3aYXn84HZ7r6g3Xk10UlKMnw4PPAAHH983JWIxC/S\niU7ufou717l7HXAK8Ju2cC9+z6Kf5wNTzGyYmY0DxgBPl1qMSLFt22D9+jDJSUTKV844eKMwisbM\nLjezVmAc0GhmywDcfQ0wA1gCPAhMdfcd0ZYs3cWyZXDCCeHuRCJSPq1FI4k1Y0a4efRdd8VdiUgy\naC0ayQyNoBGpjgJeEksBL1IdddFIIm3ZAoMHhz979Ii7GpFkUBeNZMLSpTB6tMJdpBoKeEkkdc+I\nVE8BL4mkgBepngJeEkkBL1I9BbwkzoYNsHUrDB0adyUi6aaAl8RZuhROPBHq9O0UqYr+CUniqHtG\nJBoKeEkcBbxINBTwkjgKeJFoKOAlUf7wB9i+HYYMibsSkfRTwEuiLF0aWu9W8mRsEdkfBbwkirpn\nRKKjgJdEUcCLREcBL4nhroAXiZICXhKjpSWE/KBBcVcikg0KeEmMtta7LrCKREMBL4mh7hmRaCng\nJTEU8CLR0i37JBHc4fDD4bnnoL4+7mpEkkm37JNUWrsWevdWuItEqWfcBUh1nnwSnn467iqq19ys\n7hmRqKmLJuXGjIFx4+DII+OupHoTJ8JJJ8VdhUhyldtFo4BPsU2b4OijYeNG6NUr7mpEpNbUB9+N\nLFwIn/60wl1E9k0Bn2L/+Z9wxhlxVyEiSVVSwJtZnZktMbOGwvO+ZrbAzLabWYOZDSjad5qZbTKz\nFjM7v1aFSwj4M8+MuwoRSapSW/BXAauAto7zG4CNwJHAYuA2ADMbCtwIjAUuBGaZ2YFRFizBq6+G\nG2OMGBF3JSKSVJ0GvJnVA+cBs4G2zv3JwAx33wr8c+H1tu33u/tqd18MNAITI69a9nTPaN0WEdmf\nUlrwdwI3Aa1F2wYCawHcfTPQy8x6AfXAuqL9mgv7SsTUPSMinelwopOZTQK2unuTmY3t5FxltSWn\nT5++5+dcLkculyvn8G5t9+4wuemuu+KuRERqKZ/Pk8/nKz6+w3HwZnY78I12m5cSfjF82d2fNbP+\nwMvufoSZXQ8MdvdphePnA7PdfUG782ocfBUaG+HLX4YXX4y7EhHpSpGOg3f3W9y9zt3rgFOA37j7\nGGA+cIOZ9QO+BjxSOORxYIqZDTOzccAYIAMT6ZPl179W94yIdK6ccfDG3lE0M4D+wOuEETO3Arj7\nK4XXlgAPAlPdfUdk1Qqg8e8iUhotVZAy774LAwbA66/DQQfFXY2IdCUtVZBxDQ0werTCXUQ6p4BP\nGXXPiEipFPApo/HvIlIq9cGnyJtvwnHHheWBe+pWLSLdjvrgM+yppyCXU7iLSGkU8Cmi8e8iUg4F\nfEq46wKriJRHAZ8SK1eGlSOHDYu7EhFJCwV8SrR1z2h5YBEplQI+JdQ9IyLl0jDJFNi1Cw4/HFat\ngiOOiLsaEYmLhklmUGMjDBmicBeR8ijgU0CzV0WkEgr4FFDAi0gl1AefcNu2QX09vPUW9OkTdzUi\nEif1wWfMf/0XnHyywl1EyqeATzgtTyAilVLAJ5zGv4tIpRTwCbZ+feh7Hz067kpEJI0U8An21FMw\ncSL06BF3JSKSRgr4BFP3jIhUQ8MkE8odjjwSFi+Go4+OuxoRSQINk8yIF16Aj35U4S4ilVPAJ5Rm\nr4pItRTwCaWAF5FqqQ8+gXbuDMsD/8//QL9+cVcjIkmhPvgMWLwYhg9XuItIdRTwCaTlCUQkCh0G\nvAUNZva2mW00s5mFbceb2e/M7E9m9h9mdkjRMdPMbJOZtZjZ+bX/K2SPxr+LSBQ6DPhCR/k1wGHA\nSGA88Hngh8CPC9uXATcDmNlQ4EZgLHAhMMvMDqxV8Vm0ZQu89BKMGxd3JSKSdp120bj7cnffCbQC\nu4ANhLC/x923Az8DLijsPhm4391Xu/tioBGYWJPKM2rhQhg/Hnr3jrsSEUm7kvrgzex94A1gqbsv\nAV4GLjWzPsDFwMDCrvXAuqJDm4tekxJoeKSIRKWkgHf3nsAQ4FNmdg5wBXAV8BbwUWB3rQrsbhTw\nIhKVnqXu6O7rzOxhYIy7Pwp8EsDMTgUmFHZrAQYXHXYM8MS+zjd9+vQ9P+dyOXK5XDl1Z1JzM7z9\nNowcGXclIpIE+XyefD5f8fEdTnQys3pgEPBc4c95wLeAF4GNwEHAfcC97j6ncJG1AcgRLsA+DAxx\n9x3tzquJTvswdy7k8/Dzn8ddiYgkUdQTnQ4EZgGbgYXAo+7+MHAqsBL4PbAIuBvA3dcAM4AlwIPA\n1PbhLvun8e8iEiUtVZAQra1wxBGwfDkMGhR3NSKSRFqqIKV+//uw/ozCXUSiooBPCHXPiEjUFPAJ\noeUJRCRq6oNPgO3bQ/97Swv07Rt3NSKSVOqDT6FFi2DUKIW7iERLAZ8Amr0qIrWggE8ABbyI1IL6\n4GO2YQN8/OOwcSMccEDc1YhIkqkPPmWeegpOP13hLiLRU8DHTOPfRaRWFPAxctf4dxGpnZKXC47a\nm2/G9c7JsXYt7N4Nw4fHXYmIZFFsAT9qVFzvnCyXXgpW8iUTEZHSaRSNiEhKaBSNiIgACngRkcxS\nwIuIZJQCXkQkoxTwIiIZpYAXEckoBbyISEYp4EVEMkoBLyKSUQp4EZGMUsCLiGSUAl5EJKMU8CIi\nGaWAFxHJqA4D3oIGM3vbzDaa2czCthPMrMnM3jWzF8zs9KJjppnZJjNrMbPza/9XEBGRfekw4AuL\ntl8DHAaMBMYDnwe+BdwHHAz8I/D/AMxsKHAjMBa4EJhlZgfWqngJ8vl83CVkij7P6OizjFenXTTu\nvtzddwKtwC5gA/AaYIXjrfAcYDJwv7uvdvfFQCMwsRaFy176RxQtfZ7R0WcZr5Ju2Wdm7xPCfLa7\nLzGzV4HFwB2EwB9f2LUeWFd0aDMwMLpyRUSkVCVdZHX3nsAQ4FNmdg4wB7gb6APcQOiuERGRBCnr\nnqxmdhNwEHAZMMrdNxa27yhsnwoMdvdphe3zCa3+Be3OoxuyiohUoJx7snbYRWNm9cAg4LnCn18k\nXFRdAXzFzL4HXAC85u67zOxx4Bkzm024MDsGuKiaAkVEpDKd9cEfCMwCRgCbgLnu/pCZrQB+BEwH\nXgEuBXD3V8xsBrAE2AFMdfcdNapdREQ6UFYXjYiIpEeXz2Q1swlm1mxm28zs/3b1+2dNYQJaa+Gx\nLe560sTM7jOzt8zs+aJtfc1sgZltL0zyGxBnjWmyn89zRtH3s1WTH0tjZvVm9riZ/bEwafTawvay\nvp9dGvBmZsCPgWuB4cAFZnZKV9aQQbvcva7w6Bt3MSnzXeAv2227AdgIHEkYCnxbVxeVYvv6PB24\npOg7+nAMdaVRb+BfgaMIn+n/MbMRlPn97OoW/CeBre4+391fB+YC+o0usXD33wJvt9s8GZjh7luB\nfwbO6/LCUmo/nyeEyZBSBndvdvd57v5Hd38BWA4MoMzvZ1cH/EBgbdFzTYSqXg8z22pm683sxriL\nyYA931F33wz0MrMD4i0p9b5f6JJ9xMwOj7uYtCksATOY0GIv6/sZ92qS+s1evdOBw4FJwJVmlou3\nnEzS97Ry3ycMsT4KeAv4TrzlpIuZ9Qd+AfyNu2/f3277O76rA3494TdRm6ML26RC7v6yu+8q/Dfu\nEeCEuGtKmfbDyFoIs7bb/nG95+7vdXVRKfaBz7PQ1fB2oUvhu+j7WTIz6ws8Dtzm7osKm8v6fnZ1\nwD8HHGJmk81sIHA5IZSkAmZ2RGHp5o+Y2fHAuYS+Oild+9bPY8ANZtYP+Br6fpbrA5+nmZ1mZgeb\n2aGElWn1/SxBYRXe+YS5R48WvVTe99Pdu/RBWF2ymXAx5h+7+v2z9ACGAc8D2wn9cv877prS9AAW\nEVZJbXtcC/QFFhAm6jUAH4u7zrQ89vF5XkcYCbIV2AI8CBwWd51peABntPssW4GLy/1+aqKTiEhG\nxX2RVUREakQBLyKSUQp4EZGMUsCLiGSUAl5EJKMU8CIiGaWAFxHJKAW8iEhG/X+fsU14pzGCEQAA\nAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results = [evolve() for _ in range(20)]\n", + "plt.plot(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After resetting the archipelago and ZeroMQ island, I'll start another instance of this code on a different window (because IPython can't execute two processes simultaneously) and see how the performance compares." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "del zmqisl\n", + "\n", + "zmqisl2 = zmq_island(algo, prob, 20)\n", + "zmqisl2.set_broker_details(\"127.0.0.1\", 6379)\n", + "zmqisl2.set_token(\"schwefel10_de10_pop20\")\n", + "zmqisl2.set_ip(\"127.0.0.1\")\n", + "\n", + "archi2 = archipelago(algo, prob, 3, 20)\n", + "archi2.push_back(zmqisl2)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEACAYAAAC57G0KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGMNJREFUeJzt3X2wVPV9x/H3B5AHjSAIMUIDKI5NYxybaUKMVrsxbbUt\nkkrVZBrjQ2OMU0eUFGrS2vaWaSONhHRihGlIa2OeiDHUFILGitkGRmgxGUhsTcRIZIJFnsVE7gW5\n3/5x9spyvdy7D2fvOXv4vGZ2vPfsefix2Xzud38PZxURmJlZ8QzJugFmZtYaDngzs4JywJuZFZQD\n3sysoBzwZmYF5YA3MyuomgJe0hBJ6yWtqfy+UFJ31WNW1b5zJO2WtK16u5mZDa5aK/g/BZ4BeibN\nB3BNRAypPJYDSJoGzAPOB64E7pU0KuU2m5lZDQYMeEkTgSuAJYCqn+pj95nAsojYHBHrgA3AJWk0\n1MzM6lNLBb8IuAPo7rV9saT9kh6SNKGybSKwtWqfLcCk5ptpZmb16jfgJV0G7IuIJzm6Yl8M/Aow\nGdgB3N2yFpqZWUOGDfD8hcBNkm7q2SDpvyNietXv9wD3V37dBkypOv5M4Du9TyrJN8AxM2tARPTV\nPd4n1XqzMUnvAhZGxEWSLgY2kXwC+AfghIi4QdJZwPeAEjAeWA5MjYjOXueKZm5y9sEPwuHDcN55\nDZ8iFf/0T3DfffCe92Tbjo6ODjo6OrJtRIH49UyPX8t0Saor4Aeq4I86N0dm0XwEuLzy+2rgZoCI\neFbSQmA90Anc0jvc07BjB8ydC5demvaZ6zNtWtKODRtgiFcUmFnO1BxLEbE+Ii6u/PyhiDglIsZG\nxJURsatqv0URMS4iJkbEN1vR6B074LTTWnHm+lx1FZxwAnzlK1m3xMzs9dqy7nzxRXjjG7NuBUjw\n6U/DX/4lHDiQXTtKpVJ2Fy8gv57p8WuZrZr74FO9aBN98N3dMGIEvPJKUj3nwZVXwm/8BnziE1m3\nxMyKrN4++LYL+J074S1vgd27U25UE559Fs4/H/73f/PxycLMiqnegG+7LpodO/IXomedBR/6EHiy\ngB3vImDfPnj6aXj8cfj+97Nu0fGtnlk0uZCXAdbe/uqvkk8Wt94Kv/ZrWbfGLF09wf3CC/B//5c8\nen7u/d9hw+D002HiRPjJT5KZZnPmJGNWNrjaLuDzMsDa27hx8PGPw5//OaxYkXVrzJqzaxfMmwfP\nPHMk0EeMOBLcp5+ePKZMSbonq7e94Q1HzrN1K8ycmVT0994Lw4dn9286HrVdH/xnPwubN8M996Tc\nqBR0dSXV+xe+AJf4FmvWpp57Dn7v9+AP/gBmzToS3Cee2Nj5fvGLZHHi/v3wzW8mxZA1pvB98Hmt\n4CGpcBYsSD6Sdve+NZtZG/jBD+Cii2D2bFi0CH7zN5MFfY2GOyQV/fLl8I53JNX+M8+k117rX9sF\nfF774HtcdVUS9F78ZO3m0UeT1eGf+xzccku65x46FO6+G+64I/kDsnp1uue3vrVdwOe5goejFz+9\n8krWrTGrzf33JzPB/u3f4IorWnedD38Yvv71pMvm859v3XUs0XYBn8dpkr1dcAG8613wj/+YdUvM\n+hcBd90Ff/3XUC4nXTKtVirB2rVJF9CcOcmNA6012m6Q9cwz4T/+I+kXzLOf/jQJ+f/5n3x3Kdnx\n6/DhpK997Vp4+OFkJsxg2rs36dIcORK++lUYPXpwr9+OCr+S9aSTYPt2OPnklBvVAh/7WHKPmiVL\nsm6J2dEOHEi6SfbtS7plxozJph2HDh35I7NiBUydmk07+vPqq/CjH8HBg82dZ/To5tfIFDrgf/lL\nmDAh+W87LJrYswd+9VfhP/8T3vrWrFtjltizJ5mbPnly8n0GI0Zk256IZNrzggXw4INJF2fW9uyB\nRx6BlSuT//ae39+I6dObn95d6IB/7rlkfvnPfpZ+m1pl0SL47ne9+Mny4fnnkznuv//78KlP5et7\nDB5+GK67Dj7zmeTTxWCKSBZjrVyZPDZuTL7IZ8aM5LWalJNvli50wK9fD7fdBv/1Xy1oVIt0dSXV\n+9KlXvxk2dq0KVm8NHcu3H571q3p21NPweWXwzXXwN/+bWv/AHV1JZ+ue0L91VeTa8+YkQwEjxrV\nums3qpXf6JS5vE+R7EvP4qc/+7Pkxkt5qpjs+PH44/CBDyRz3K++OuvWHNvb3pYUcFdcAT/+MXzx\ni80tsupt+3ZYtSoJ9NWrk+vNmAHf+lbyczt0/dajrSr4pUuTKv6f/7kFjWqhCLjwQrj5Zrj22qxb\nY8ebr30t+eT7wANJZdoOOjvhIx9JQv6uu5r77odDh+CJJ5JQ37wZfvd3k1C/7LJkTK+dFLqL5u//\nPhlg/eQnW9CoFlu3LqmcfvKTdCsSs/58+tPJeoxVq+Dcc7NuTX0iYOHCJJibISVfyDNjRjLPPy9f\nFNSIQgf87NnJPPi89h8O5Oqr4bzzklWuZq3U3Z10Cz76aDIL5M1vzrpFloZC98Hv2AHvfnfWrWjc\nXXclU6VuvNGLn+z1IpKbff3rvybzrpuxd28yt33tWhg7NpXmWRtqq4Bvx0HWatOmJdPAOjq8+MmO\n2LEjuTndffclt9a9/nr4m79pbsBvyJBkJXXWc9wtW23VRXPOOcmNit72thY0apDs2ZN881O57MVP\nx7NDh5J+8fvuS94L73sf3HADXHyxZ1rZsRW6D378+GJ8sfVnPpNM0Wp28Mjaz1NPJaH+la8k3+V7\nww3J2Ew73HrDslfYgH/11eSmRF1dyb2l21nP4qfPfx7e+96sW2OttndvMlXxvvuSr7679tqkG+bs\ns7NumbWbwgb89u3JDJQXX2xRowbZgw8m0z6ffLL9/2DZ6x0+DI89loT6I48kc66vvx5+53f8v7c1\nrrABv2lTsny52dkFedGz+Gn69OSGZFYcW7fCl78Mb3pT0gXzgQ/4e0gtHYWdJpn3r+qrl5R8Offn\nPgc//GHWrbE0jR3bnguLrHhqCnhJQ4AngEMRcZGk0cAy4D3Ak8CVEfFiZd85wJ1AJ3BrRCxPo6Ht\nPkWyL299KyxenHUrzKyoap2Q9afAM0BPv8pcYBdwOrAOmA8gaRowDzgfuBK4V1Iq92QrWgVvZtZq\nAwa8pInAFcASoKfvZyawMCL2AQsqz/dsXxYRmyNiHbABSOUmuUWs4M3MWqmWCn4RcAfQXbVtEvA8\nQETsAYZLGg5MBLZW7belsm/T2uHLts3M8qTfgJd0GbAvIp7kSPV+zN1Ta1Uf3EVjZlafgQZZLwRu\nknRTzwZJG4CfA1OBTZJOBQ5GRJekbcCUquPPBL7T14k7Ojpe+7lUKlEa4EbV7qIxs+NNuVymXC43\nfHzN8+AlvYuk3/0iSfOBM4DbgE8AYyLiJklnAd8DSsB4YDkwNSI6e52r7nnwkyfDmjUwZcrA+5qZ\nFVEr58GLI7NoFpJMk3yBZCD1KoCIeFbSQmA9yTTJW3qHeyMi3AdvZlavtljJ+tJLyRcW7N/fwkaZ\nmeVcvRV8W9yY1P3vZmb1a4uAd/eMmVn92ibgPUXSzKw+bRHw7qIxM6tfWwS8K3gzs/q1RcC7gjcz\nq19bBLwHWc3M6tcWAf/ii+6iMTOrV1sEvCt4M7P6tU3Au4I3M6tP7gO+qwt++Us45ZSsW2Jm1l5y\nH/A7d8KECTAk9y01M8uX3Memp0iamTUm9wHvAVYzs8bkPuA9RdLMrDG5D3hX8GZmjWmLgHcFb2ZW\nv9wHvAdZzcwak/uAdwVvZtaY3Ae8K3gzs8bkPuA9yGpm1hhFxOBfVIpartvdDSNGJLcqGD58EBpm\nZpZjkogI1bp/riv4vXvhDW9wuJuZNSLXAe8BVjOzxuU64D3AambWuFwHvCt4M7PG5TrgXcGbmTUu\n1wHvKZJmZo3LdcD7TpJmZo3rN+CVWCPpZUm7JH22sm2hpO6qx6yqY+ZI2i1pW/X2RriCNzNr3LD+\nnoyIkHQr8DQwFvg2MAMI4JqI+Gr1/pKmAfOA84HxwHJJD0fEgUYa50FWM7PGDdhFExEbI6IL6AYO\nATsrT/W1mmomsCwiNkfEOmADcEmjjfMgq5lZ42rqg5f0KrAd+H5ErK9sXixpv6SHJE2obJsIbK06\ndAswqdHGuYI3M2tcv100PSJimKTJwDckvQ9YDMwHhgKfAu4Grq/nwh0dHa/9XCqVKJVKRz3/yitw\n6BCcfHI9ZzUzK45yuUy5XG74+LpuNibpDuDkiLizatu5wP0R8XZJtwNTImJO5bkVwJKIWNXrPAPe\nbOxnP4OLL4atW/vdzczsuJHqzcYkTZQ0XdJISWcBVwM/kHSRpDGSxgK3Ahsrh6wE3i/pbEkXAO8E\nHm/kH+IpkmZmzRmoi2YUcC9wDrAbWBoRyyV9CbicZDbNauBmgIh4VtJCYD3QCdwSEZ2NNMxTJM3M\nmjPQNMmfklThvbd/qJ9jFgGLmm2YB1jNzJqT25WsniJpZtac3Aa8K3gzs+bkNuBdwZuZNSe3Ae9B\nVjOz5uQ24D1N0sysObkNeFfwZmbNqWsla2oXHWAl6+HDMHIkHDgAw2q6mYKZWfGlupI1K7t2wSmn\nONzNzJqRy4D3FEkzs+blMuA9RdLMrHm5DHgPsJqZNS+XAe8pkmZmzctlwLuCNzNrXm4D3hW8mVlz\nchnwHmQ1M2teLgPeFbyZWfNyGfCu4M3Mmpe7gI/wIKuZWRpyF/AvvwxDh8JJJ2XdEjOz9pa7gHf1\nbmaWjlwGvAdYzcyal7uA9wCrmVk6chfwruDNzNKRu4B3BW9mlo7cBbwHWc3M0pG7gPedJM3M0pG7\ngHcFb2aWjlwGvCt4M7Pm9RvwSqyR9LKkXZI+W9k2WtIqSQcqz59WdcwcSbslbZM0q94GeZDVzCwd\nioj+d5B+HXgaGAt8G+gA3glMBWYDfwGMiYiPSpoGrAF+CxgPLAfOjIgDvc4ZfV334MHkFgVdXTAk\nd58tzMyyJYmIUK37DxijEbExIrqAbuAQsBOYCSyMiH3AAuCKyu4zgWURsTki1gEbgEtqbczOnTB+\nvMPdzCwNNUWppFeB7cD3I2I9MAl4HiAi9gDDJQ0HJgJbqw7dUtm3Jh5gNTNLz7BadoqIYZImA9+Q\n9IfH2K3mjw0AHR0dr/1cKpUolUqeImlmVqVcLlMulxs+fsA++KN2lu4ATgZmANdFxCZJpwJPR8Qb\nJd0OTImIOZX9VwBLImJVr/P02Qd///3w6KPw5S83/O8xMyusVPvgJU2UNF3SSElnAVcDPwD+HZgr\naRzwceChyiErgfdLOlvSBSSDsY/X2hhPkTQzS89AXTSjgHuBc4DdwNKIWC7pMWAZ8ALJQOpVABHx\nrKSFwHqgE7glIjprbYynSJqZpaeuLprULnqMLprrroNSCW64YdCbZGaWe6lPkxxMHmQ1M0tPrgLe\n0yTNzNKTu4B3BW9mlo7c9MFHwIgRsH8/jBw56E0yM8u9tu2D37cPTjzR4W5mlpbcBLynSJqZpSs3\nAe8BVjOzdOUm4D1F0swsXbkJeFfwZmbpylXAu4I3M0tPbgLeg6xmZunKTcC7gjczS1duAt4VvJlZ\nunIT8B5kNTNLV24C3tMkzczSlYuAP3AAurpgzJisW2JmVhy5CPidO5PuGdX1td1mZtafXAS8B1jN\nzNKXi4D3FEkzs/TlIuBdwZuZpS8XAe8pkmZm6ctFwHuKpJlZ+nIR8K7gzczSl5uAdwVvZpauXAS8\nB1nNzNKXi4B3BW9mlj5FxOBfVIqe6x4+DCNHwiuvwAknDHpTzMzahiQiouY1/5lX8Hv2wOjRDncz\ns7T1G/CSJkpaKeklSdskza5sXyipu+oxq+qYOZJ2V/afdeyzJzxF0sysNYYN8PwI4EvAB4E3A49L\nWg0EcE1EfLV6Z0nTgHnA+cB4YLmkhyPiwLEu4CmSZmat0W/AR8QWYEvl15ckbQR66u2++oFmAssi\nYjOwWdIG4BLg28e6hgdYzcxao+Y++Ep1PhV4orJpsaT9kh6SNKGybSKwteqwLcCk/s7rKZJmZq0x\nUBcNAJJOBR4A/iQiOiUtBuYDQ4FPAXcD19dz4Y6ODgBWr4azzy4BpXoONzMrvHK5TLlcbvj4AadJ\nShoNfAdYEBHf6uP5c4H7I+Ltkm4HpkTEnMpzK4AlEbGq1zGvTZO88UaYPh1uuqnhf4OZ2XEh1WmS\nkkYBK4Cl1eEu6WJJYySNBW4FNlaeWgm8X9LZki4A3gk83t81PMhqZtYaA/XBXwhcBHyhakrkB4GP\nAM8DzwHjSGbOEBHPAguB9cCDwC0R0dnfBTxN0sysNTJfyXrGGfDYYzBt2qA3w8ysrbTdSlZPkzQz\na41MA/4Xv4AIOOmkLFthZlZMmQZ8T/Wumj9wmJlZrTINeC9yMjNrncwreAe8mVlrZF7Be4DVzKw1\nXMGbmRVU5gHvCt7MrDUy76JxBW9m1hqZV/AOeDOz1si8gncXjZlZa7iCNzMrqMxuNnbwYHDiidDZ\nCUOHDnoTzMzaTtvcbGzXLhg3zuFuZtYqmQW8p0iambVWZgHvKZJmZq2VaQXvgDcza51MK3h30ZiZ\ntY4reDOzgnIFb2ZWUK7gzcwKytMkzcwKytMkzcwKKrNbFQwfHuzbB6NGDfrlzczaUtvcqmDECIe7\nmVkrZRbw7p4xM2utzALeA6xmZq3lCt7MrKD6DXhJEyWtlPSSpG2SZle2j5a0StIBSWsknVZ1zBxJ\nuyv7zzrWuV3Bm5m11kAV/AjgS8Bk4FLgTknnAHOBXcDpwDpgPoCkacA84HzgSuBeSX0OpbqCNzNr\nrWH9PRkRW4AtlV9fkrQROA2YCVwbEfskLQB+DHy0sn1ZRGwGNkvaAFwCfLv3uR3wZmatVXMffKU6\nn0JSsU8CngeIiD3AcEnDgYnA1qrDtlT2fR130ZiZtVZNAS/pVOAB4MMRceBYu9VzYVfwZmat1W8X\nDSQDqsBKYH5ErK1s3gZMBTZVwv9gRHRJ2kZS5fc4E/hOX+f9xjc6+O53k59LpRKlUqnBf4KZWTGV\ny2XK5XLDx/d7q4LKAOkjwBcj4l+qts8HzgBuAz4BjImImySdBXwPKAHjgeXA1Ijo7HXe2L07GDeu\n4XabmR136r1VwUAB/9vAo702X0NS0S8jGUDdAFwVEdsrx3wMuBPoBG6NiG/2cd7o7g5UV6eOmdnx\nLdWAbxVJkcV1zczaWdvcbMzMzFrLAW9mVlAOeDOzgnLAm5kVlAPezKygHPBmZgXlgDczKygHvJlZ\nQTngzcwKygFvZlZQDngzs4JywJuZFZQD3sysoBzwZmYF5YA3MysoB7yZWUE54M3MCsoBb2ZWUA54\nM7OCcsCbmRWUA97MrKAc8GZmBeWANzMrKAe8mVlBOeDNzArKAW9mVlAOeDOzgnLAm5kV1IABL+lr\nknZI+lHVtoWSuqses6qemyNpt6Rt1dvNzGxw1VLB3wNc2mtbANdExJDKYzmApGnAPOB84ErgXkmj\n0mywvV65XM66CYXi1zM9fi2zNWDAR8QTwMt9PKU+ts0ElkXE5ohYB2wALmmuiTYQ/58oXX490+PX\nMlvN9MEvlrRf0kOSJlS2TQS2Vu2zBZjUxDXMzKxBjQb8YuBXgMnADuDu1FpkZmapUEQMvFPSt/5Q\nRJzbx3PnAvdHxNsl3Q5MiYg5ledWAEsiYlWvYwa+qJmZvU5E9NU93qdhNe531AklXQxsIvkEcCuw\nsfLUSuB7kpYA44F3Alc100AzM2vMgAEvaS1wQeXnbmAO8A7gcpLZNKuBmwEi4llJC4H1QCdwS0R0\ntqbpZmbWn5q6aMzMrP0M+kpWSe+RtKUyA+fvBvv6RSNpV9WCs/1Zt6edHGMR32hJqyQdkLRG0mlZ\ntrGd1Lso0o5N0kRJKyW9VFk0Oruyva7356AGvCQB/wLMBt4C/JGkdw9mGwroUNWCs9FZN6bN9LWI\nby6wCzgdWAfMH+xGtbGaF0XagEYAXyKZqXgpcKekc6jz/TnYFfyvA/siYkVEvAAsBfwX3TJxjEV8\nM4GFEbEPWABcMegNa1N1Loq0fkTEloj4ekS8FBFPkUxkOY0635+DHfCTgOerfvdCqOYNlbRP0s8l\nzcu6MQXw2ns0IvYAwyWdkG2T2l5fiyKtRpVp6lNIKva63p9Z303Sf9mb91vABOAy4KOSStk2p5D8\nPm2cF0U2QdKpwAPAhyPiwLF2O9bxgx3wPyf5S9TjjMo2a1BEPB0Rhyof4x4Czsu6TW2m9zSybcBU\neO3/XAcj4uBgN6qNHfV6VroaXq50KdyD3581kzSaZG3R/IhYW9lc1/tzsAP+h8ApkmZKmgTcSBJK\n1gBJb5R0nqSRlRXFf8iRRWdWm97Vz78DcyWNAz6O35/1et2iSEljJI3l6EWR1o/KXXhXAEsj4ltV\nT9X3/oyIQX2Q3F1yC8lgzCcH+/pFegBnAz8CDpD0y30s6za10wNYC3RXPWYDo4FVJAv11gBvyrqd\n7fLo4/W8jWQmyD5gL/AgMD7rdrbDA/jtXq9lN/DH9b4/vdDJzKygsh5kNTOzFnHAm5kVlAPezKyg\nHPBmZgXlgDczKygHvJlZQTngzcwKygFvZlZQ/w8SMFNUOXD3CwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results2 = [evolve() for _ in range(20)]\n", + "plt.plot(results2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, we get to the result much more quickly because the off-screen instance of this program had been working for a few seconds and was very close to the answer already." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From e29556ed5075bf84b35d1ee77cc91ae0cbc1dde5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20D=C3=ADez?= Date: Wed, 19 Aug 2015 21:19:02 +0200 Subject: [PATCH 19/25] Removed unnecessary commented code --- PyGMO/core/__init__.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/PyGMO/core/__init__.py b/PyGMO/core/__init__.py index 9778a3f6..04f77ec2 100644 --- a/PyGMO/core/__init__.py +++ b/PyGMO/core/__init__.py @@ -209,24 +209,6 @@ def __init__(self, *args, **kwargs): raise TypeError("r_policy must be a migration replacement policy.") super(type(self), self).__init__(*ctor_args) - #if isinstance(self, zmq_island): - # super(type(self), self).__init__(*ctor_args) - #elif isinstance(self, _zmq_island): - # self.__original_init__(*ctor_args) - #else: - # assert(self is None) - # n_pythonic_items = 0 - # if isinstance(args[0], base_algorithm): - # n_pythonic_items += 1 - # if isinstance(args[1], base_problem) or isinstance(args[1], base_problem_stochastic): - # n_pythonic_items += 1 - # elif isinstance(args[1], population) and (isinstance(args[1].problem, base_problem) or isinstance(args[1], base_problem_stochastic)): - # n_pythonic_items += 1 - # if n_pythonic_items > 0: - # return py_island(*args, **kwargs) - # else: - # return local_island(*args, **kwargs) - # _core._base_island.__init__(self, *args) def get_name(self): return str(type(self)) From 20e7ca61f4d053b3a7c5735214abc15821990de0 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 22:21:12 +0200 Subject: [PATCH 20/25] Added tutorial about monitoring --- PyGMO/core/core.cpp | 1 + doc/sphinx/images/zmq_monitoring.png | Bin 0 -> 20852 bytes .../distributed_computation_with_zeromq.rst | 2 +- doc/sphinx/tutorials/zmq_monitor.rst | 122 ++++++++++++++++++ 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 doc/sphinx/images/zmq_monitoring.png create mode 100644 doc/sphinx/tutorials/zmq_monitor.rst diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index 3d8a3f32..f9e49784 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -393,6 +393,7 @@ BOOST_PYTHON_MODULE(_core) .def("set_broker_details", &zmq_island::set_broker_details) .def("set_token", &zmq_island::set_token) .def("set_ip", &zmq_island::set_ip) + .def("set_evolve", &zmq_island::set_evolve) .def("connect", &zmq_island::connect); #endif diff --git a/doc/sphinx/images/zmq_monitoring.png b/doc/sphinx/images/zmq_monitoring.png new file mode 100644 index 0000000000000000000000000000000000000000..9cac0a9e4ceb2befed785a38689527442b5a0c05 GIT binary patch literal 20852 zcmeIaby!qw_clC&qJoWxsDKKHw9*X*!jMWgA_4-^C5^2ZC_SXXfG~9Tpolcc&@q4_ z-QDo5J>YZS&+q--_rLG2_uI#D-_C6IT)E<0=UUf(c~e>TAmu4a6bf|^Eq7fNh1!in zp~z_WQNSIwIARg}v)l11T74f}p8M{;fbaY5!_syiIQyhU_570)M_1LRUu2tRmsg)ZPZnbEP;QUNzT@IA_J+Fe ze)@sz6U~t$9AE6zk60Ugao7EQ;x@x~vMbW$N6DU@yUmN2n!IrGSV(EDc#UUIs+Qql zjN5Fjz`G~4LTz5elA9NgAA=#m)rYcoq22z7I?j|tM*8S$azt|%=~LO1*F zWBZfcfp6^FpM)ulX|_L|d7OqU23&8iXz$tnRB`tIZ*F@f{y$y2!x3VxDZhXJR?oE| zs^-SW$8&IVr_}0MT3Y6}>15q|ubXYjg6~ajZdR9=`A*$*`uw z)A0p`gGwnbsoLQ$~qL;R#jQ~^T~edET<9Gh0zuT+v0_g)upM_^>t6BC{cl9 zSaPHl-;+1YgJp@Xy3l z#}n-Pe^|68=~UO%y}3R@8pqjZM>XAUYHEg@;L?rl$TU|_&5zO;GAVm@u)3y3ZK|hO zf!LF4YqaO_m1Mo#$xl|>vbB8q8~fpcwaLPkcojzH;h(==51olmg~Q;$xoWJCOQ5G8U#Ya5=ST)=$v7GgkVn-4Dl< zl2yxv;l^w9VlF|<$vT-gs4pn~+{@_s;mv6wc1cOYfGnRaZ!|mK2Z_sKg_L-I`Dyo_wj3 z2danW5*!AqO;c`R@+v~JyCI?5yg}%p!M}IwM)O>SFK%VF-DYmt;;oXv`p&%k6!jda+@27M? zP+;KaX9rJ*bbF7tUxwAk#>NKa*RL*&!Cya6d7UCHU}UaoO^8Kb=|h*$hnJX=W?x8n zEDoZ{T2Rgks^-c+~lPe%bl;^cI#U#+p+Peh&o(% zA7vX!#Eqv=Dp$0z_9?>en?785EN}Sun;xv*C+n^pcDNip{23&IFc_UglSJ_bsjVXB z1if4vmR#qFP7O2~t($44l+dM>YRCop5iQ{nlCL~Nz(e;528AxP^ZKak%Brfzj3Ru= zXR_(sdc9Z=(w*O!ejJ2hTNq_2UWs!WigMASi0>bxwiEo(@IWn+xaaJwchQ0AE?d0D z*5<~-=Gqj8l$6ol!wk)Q4dS2Cffo|g;+3(TtT+u^9EQ)_a^`orZoUKd(bo^HDF!&V zxj~ESAU1hr<(z&*JJc&1KND=dnI^heLH(MG@`IwelMOf~I!#@j!5}+1X}U!=31SKJ~Uaq{)2ClgvJKw8MdZfn#8y^$G;N9MHY8Oh{ z0wjFt$7$Ojcf46$cxRgNwWleUEt@Zb9I;25g8W|Zt_oDedlO_C(PM22TMQL!=tM(< zQ=;nbgLEVKO@%u&`%$R+ue{Qz7w5<+4x|^Y^R9)Wefs+gMT88Nt(hKjPg-5&1H;ZkqCGtk~o`LwB@pzUpXC@%o%@);st4 zcY}3&jO0PjQ&mteBuPH%!HtW|N?~WY3^^s1BAeqIIvP||gbK;M*X^A~!GaClAo;3> zye;pMnF{Rd10#!F4PM^<3F%y@G)gBXJS6lRoyeGl3H{T5?gM=()kCsgfv^oiWLA_lYuk-@BSP zmKqwuBg7Zat?6Dc-{V0TiZNQxhDM%}Jiaf^b2VW;qE~LB6OGeS*uVc*zRgsR>SGGR z`pU0-1{5mq()REV)&vwd2ZjrUwx$Fhqb_rus1C0CuxMHnd+qr*SA&11`P85=mfIk< zz!@vl8x#=mZR_Oc8yNasTkAv9MyFAzNJi4;2={DmT=-^_M z^XKRD6S)_@6pUw7N|{x>HGAN388edN z-ha@d;_a~b1=!A~+pRhNe$t$qI^W5UM~& z=b}Z1k{&1FGT-EAPt|bo4kt~e$a}5=*z$5T$Qf+E$5N-kK@=+eA!!|QbPAfET=7|c z?6kGvqNP^SyVWy(DDU<|VlWK)G-*V)UxCtoyoNO{0|(9$(=h2*t(6()v^H@eiTz?) zA-Ia7?H#(6Yy8-0ELk;1lJM=RS$0KzV=rU-`&$cvumdtmL9_3p}T1@HYyJqtEjbx`54T90;d5wfSt};VB5(cZ&*DU7D zkfkW?B8}|`4VW`x0h&us-l{G9< zaN}ShM`E6QaBSme>_82+TfYzo?GSg?2k9tnDsx+dUfG0T6?am7)Cc27wYgXZLX;{#kpu5XBZg)@lY6R zMGFz{x`OCDI1Hw-RI|cgp(_U+?p2%7hWu(7V{MGjB43ltfPiES#$GKqW#qfC!Ig(u935bG zu(R`CQF?Xa!s}B~o9emhO>u*ga{Kq2xv9G;#12H>AW_B>oi*{U^6x)XQoSnRFO zv8>VW%<6ty9YFcood;7L8M^Mt#(}$^t8N&bkJsd)zV1B1PIPQYuu1rS-DAv!uOOrE z!pCL$4O67kP~hKHLVb^ik~l z{BO#2X5;Q(%r<$R&%<5sw9CN$DOk7t*p$3)yBRzW-O!3H$lxxlEjBo>kthd}_$~i; zt^)Ua_8DrG>rbx|{!|gvRL~9LObb)Ct}=~{U*2Qtg5RHeLszyW4YunkBN2tEe`i=+ z`-(2tnf5phhQ1=DK)ztfa8A5i{B`F@Mn)`)f}*TKOX48cxnJErl3iZG99>IfL&J?~ zL{%xnAFlIX^0xvM6pt)K^hQ!*2?S<*Z?wDpCU+Lc*pyFb2p3PCGwE7lE++!OYrC@Ew6 zE9&%SVpJmY(kg^wZjO1DrSe6EzB@ZX^k9Fty6!lFORep%s8Hb23ztqbSD}{AwqU}> zPTwKo>RW>c%ve>zuWaV$oA!Epu7M!?<}7anvep>l90q0|Cucr22-QF4(d#i(7r41B zJF>ZkEu6}M85B;6X8sXjLZV<_JMiMnd&mZdf6!V>#_0I9fKzIey^wWEQomqIGEb{dB7Geq+-x@l9!B1cM+I5XF$qCH|qn#$&EhXoP$cZ% z>EIJbF(*0Qv{EOk@h$7o%7~Hy!kgP*WYQuq;gZ#@sdQ0z_2#f9x{q%3S1>KSX>HzNM5&)xK9#6TDGc7@-d>ux_NnW(+UWnU=2Xo$$S;zT)We%I~Sf{K+P7#)`DYcakLn@PH z#9XpnuuO2Vo+Br;r_<=&*o_*hnG(X6+cZ~B${$fS`4MLHplJCM|J@g0$0k+P?|P%d zVqoV3z=;uQcBP&6cy?}(=7hYM%L82Is8dzdpipt4YsBno}*IU{mRU&v7E*I%75^ZrxhW+o!&JGb=n8X>|qyyL@e4HWY?c4=q^ zBWZXD=cj=)Gt-)AKgXqWexj4bdGsz>g!5Q^(LiD7pP`7GiKYq(DvX&y1HoE~uEZ!; z`Ij$UwDzIwaPv-Vd`;ur2m02FHgZ<5qw^ydruU#;rcqw(*q>&q;4$_ht~yu=w=fi= zkXmf!P43zMCa%kJgfN?H=UQl6@;*+951)KmNZGQjz9{U$lmsAM#d~`otcCD=wMr%S z^#Snn-NsXfBRO5M^-)T<_n^ujxIGIq*NGo|{yjWTm|E4kleu`Sgs#YYZGxcUV>zfd zFw|x6tGuQ<;MqYwF;9t^CvM(GUZ!S#`p&_`mR+bKX33sfasBM-akBk=waU>!HpMOL z#}=N_*0vo+*s-g%RJ&uF)kUrRLdm=rF*>J&jUVlQU{=pxn3;#Z$ZXwxk!bgr?mXW3 z@qF)=boW69h*f*Ys0Y_Sos%KW!C2EYbQ+`M~>Em`NtUgg&m+S$z zb!XVG5HXi919C?9$|suC7sApis+1!yW#*+6sg8EGaBk=osIRu+W4G2-5;VCAId-ES z7)*2$K3pL*sdzZ*NE|t2)hf3&Ck2Wl-==Q@*dX#n8s+7Nj5{WI?lJ;aW-k1l#d@ct z4BaP1_@0{#-?ZowEW~k4{5*%))f(p1dF-^A?{*MiI0_CS8`K(w5{R5 zBcq&JC+~?KE{VC<`w1vinjS*Wrrw*IHAb=F0uw0|zJW(I&&AO^ z*GNcMZ2ffsyuNF3vK}_nDC8|YAH9;6AzQLO+s5>@y9~K$KWl~Z7IhP@?d97i$y8R` z=-#(0!F!`poEwFr{p4z!aE07r(-T>oU-?6|JmN9#)wObad^67_T~!2Ox$!1;v}GdY zyVvi2tfZG>XcGV+m&sk^wE@-fF@n}TzwE}t=Klo7ttG24HgT7jy}?8XxMu~0os6yk zI3LNb6*5`WbbsD{_#No~A^PWE zm*PIiaNp&P+l&k_GUs(6_B7VOm$fP_br7O$DQj1C+&s=k)MK)ajm?(Z;xWoWdB$RVrTaVT);~ z%o-g89ta55GQwz=kDRfoXv&TtO1_@!y4l8tY5H^+& zBePjzn_tdJCvN|(b~bNT9)ijyuk5=oJM8TU%!o?GKwwQ9Zz9{|7Wv-Tl||agy7R7g zbcArDQ#ZM^46KS7xOF*g8nD=0ED>FeXY9IUMI-D8}( zS#rs`*#W&%T9%fH(GPyVlnZ6BAK$$Xg&Gft<{tPX$AbsOD}B13L}a;4@WxamX`+tU z0KXO-5rL!QGrI@`b4inT-TSDh4Bq z69JPS2k3zT&opmnZB9@_2;LiWBTqDpKh6(?B0+8`L$WT$c#K2==!XGSMQeDeLNj?B z)SVU>_-3yqVtfdf-e{*)s>a3R?9RR9hsDpvG{noiI+5Tq(P`P5go*ZA)G;+R9qBNS z;S%8iUb87)rR~L0re?g44+}H%7xRYbQ$jY8Kn!BLySwj6C8xC)cV!VX4;(?E+FAYy z&R-VMi^ZP17OQ2HU3lzC*-W{SH+N3k%3=XjECB7QBDc&@kG4Y0_YH)oX2014!lq=dttaiswH+ zV3ihy5Z2egeK9DU|NUS?Zec(r8)p!MhoWWEk8$62Txc^M4}_76)-x*S;ddCdmbf?b zQ=z9&M~=cq01@MgQ;Crr1!5Jm<}%rxQ1<*V&UviWbW%YFeRu#C z0Yay+JACr2HIdzU3NhdXHH+ivg?A!TWf_B8lliWgWSSwWH1;5W^4y+1d%}h7KmB-f zIv(;7As)*1M>MeV!MsZ>>uYUH^#)JW0Kug%v;RA&=uuqNAuXcTI#C7FZh zjsp*8Ru$Q`(YuA7=u*fu1LLU><9Vk5=bdC@&}6*OX+mU7QyeF zIA0+pk6(EM=JLfJ*bZLv`ZM$r?hvReiS>(k)rwZPE0OP|ro{jyvB9 zE}O&}ZB&o99vJGt_pXZ>_!N`)F7mUQ>KTQoC$cs+DGZ*|S0vV_%Mx#WzLBuzU3B{P zV~T^!Xh@lurr7tD?qXqK34U@@N;22D>>0}tB#;!p-oDAk#%7*mzP>yoFE7vT+~vEA zod0&IFQsJW^tPF-X9srT0Dk$Y(?ScO$c-@NZSE>Sv($!{XSxxFQ9mpJ^!4pQ`zZfd zW{Y1}sh_+)WAV(T*T<(kF)z!E0+{NnS+-vNGGDDS9(MoH}vM`%$H`-!1a8Dq%oDB(XTYd+r znb~lfo-$r(|9f3I$$ox*)kUCSO3tIrw{G2fAHu0~9&C`L*Xq5E)dj4XGK^8_p|mu7 z;mA+6YDuD$xF13w%`Vq)>^>+eUZQ;5jYB^+>{!lo*IoY7@!zy#ptNjghqsK zE#^`kPV{5BAnz}6VPj*8Fw<|g*y6Q>-ilV;b-HLVe7=$iAVY-oG1RA*Y3vkFl)Gpj ztj>=F#J*>e9W9(HX-Y9D1Y)fRvL-J0`17&|}0SMN) z7&kSD1u$x#WuLn~A3&HmzPhth&lz|NDZS@!EvNX;&87OF@ms~sxKO7C2~g9sC%}a7 zq=KCFKtUEJ3#Ud)ACg-VVqZcEFn+KuVr(L}mpfCT&%l+4ef|1CUD)Mo*;Z|>kmKdI zYE8;KWbexBLbe-L4SJlmgq zj2df|^8SZ^=(aKk5ag(eQlE>Av&FQ&0C4xn(k$Yi_I$ z5uDr5bAU#G@RhazIH03LXicu)zprGo(@7;+kCo`~ir>}7$($AvZS&u$&0G)+_en-| zsJ_u-T^q)RaN}=tGGBk$bAM>QEjmxw31Hdr6X9I%M@X9<6m3&b+Ur&^DX%Eh0yMK= z3n+WC4pZ^+o%!eYwG!`!eDs=mI`U_fP(FUtB@%g5h@FPSBmA+`$yl67uH(Y3`Y6=Rekqi6h)-y?@EBF!;= zzEzLF*C3Z|C5sQ4{a%hiS&wP*Jt)@V8G(dD)+r*6JVX1e1zNNwYlg7|Ghk zgIZp*FX{Ge%{QsudQS|_FMG}yzH70+A_Bl--*IHH4^c?%iCqcA4-68D6$;EDIrL&? zA#2XVXIYrTh?@)#O4mD*y_du+YEJN#S#KvONeVFeE}GlAhtq4_{z2+mE*=fKNWYiSU2b)lX~d7?+uBK&1PukM4(+YWq*f{y;8s9Ps(R%__YH$=4guv!fWq`M(N;Y{{fV5Eja{Q2+ zn;B~{@Y?d|_drC4^P|CIinqB27;SZm86RI#b6fw!3dYF)+spLL>d1T5*sD_~{H*|9h zANz=`kR!7fk`{bsVYw*=5ayw(Wue8}I;v&Q%%0PO3<;eSZFZ96(seRs^y%~MxIFda zG&W|dyVmIx--R5P8&Da+#-|o8s^LC;C3EbTEMtd3u-D@F`f#7)xZ%UC=&(getykujx4^h#bYMqbpH( z*Hot_C4%oRGs%|LWJ2H9fCkzQQQ`;Kma@ZGsSPv~3e0VaO|=_n%y)aFcC6U9uqPQH@s~V#s2k#0)&Lupmfno=Y~gzWP#u=Z$id&8>t0 ze)~^!c|Bj=H|J;aJH$FIrxAl?S|R-;v6+YIZ~6EbkVnVK{QD8;c^Xn& zQzaZ;GcqenJ`2J74Q{*%uqo+qQd^j$C!Qk4$xw}?`Uuc#@|4`8_=#T(Cu+$2SeziF z856j2!x~sncI4mKA4HpCm}*ck^V18HTmtuQyUebof4ge2c%M()Ye+e>rt5dPSu0IF z7q@c3oHE~o5^KFo&)FT@K^0?Cq3DYVI4?1GpytZ1yuHRW;td(KXRCg^nMeV1|KT^g zmf*)~0yaJbXKISM@{5`^{B0t5?*gavT;*KLZf)J%`UCX6d5xS{&tH2ovj}xp&t
    $yZz>rixx1Jkll->-4~{5>mm-ZmW_aAP{&cbUsp3Tp^Sp?o(bZ=FrDiyxGsZ_ zg)5$f(uZdIO+&fOXpuSs1axLl%dm7cr4C_-%g7ykHVl`eq-l&qDUC z+}cu;4%GB?^KdD-kFXTtUeH6pkAzOUQ0Xt(B&V}}N1=G|-AKw0uF{orNsr=z83+rv zv_%vNI$MXHn>ZCL+utR-!l^H&E`51*0e|S+m{o zLBk(qnVhd!d`XYw9fYoGGESx(BK-hap}@Ci-Y-tEBHj}^wuX+;B-#z61HKy|-^-0P zEE9`CtKmYb6dtkeAybw2Q~&(zZ_j>b;buDZS<)kuz<{gVL9ilR-9Y(9t7%`f?o~FY zA>FZoKQ~m>hfq=u?{tbRyvRdekz?CshkcSyG^Wqg1SDz(hx4Zw9wYtg=B@3Us%lta z$-?cq(Hqv6Iqt+7uS=S7x})ZaCvfytKxbpqfoBrVdHj69s9rq zP6tU#gITt0PTT3z?i>t^?SK|k8NgBykuM?T$D~Q)>3$8(-y+S6R9b-YjR4)}%gDq& zA`3y>JtA|^=7GOkx&c3@d+4j4orsY4J3B^&>{uEnn}CL2e!LeV^8 z-A(4CL2A7Z0;}oWHGfyE$Ty8LMN9jl*9yzdiapq6Ory^urYGthxwB?K#ylE>n5zo`lQ`q$FDV4z8$|Do) z>DTzo>p4Yom7_$ftExnX1o(J)`HjEr<}xTyFLqym41Y0pb7Kvu;DTYj`O_p~+m#!^ z^mk>*qcj-oTQsyRhJSv9qjWQNUyIzSlb7v=Ed$ z;aAyn4Hy_g(ZIrmOL`ShQc}uC1!6F4XiFdx61tiody9_oUbWbblky?4RG{X$molqO zC=Q+idm5?`kf@&UX%E$j;hZiTj{L1rT&gCN)-|V_$fA(|3|Tg>>92b`^XxzW?(b(o zuPsf5GPgjwS2QQ<2`K^GT3J~!7hhTg#hbJtv47-}DoLWM`mdLOc-G0cbv`l~i-k z`^yGcwx=0aO;o(4SG#mlU0%Lghtc(a#{X1a?55fwA_T(^E24^TjX>qphO1gJITL3d_AzAcTNgF_*pImfyyQ*sh2B%2{9 zrAk@z)D9sT7xZ;Tqz)Z*`}6E6MqWNX1;gBL9@-Sc_O>M}aic&FnrYz~JvLW{SkSQj z{~bw36}m52LN(~Zcn2Tv9xds!XS45}E1%JUax9Pu^J(5*CcnK2Q@wa2V1+1mlbsU0 zABI%ViZ~8lV%hieT-nCMITImh;|#KL)JY?)ERs~xk3sonLu5nh}tV! zp4RGM4w?fL*)9{hbrC`;YH^CU92_#XJ zkWy7d$Hb?i*PeX=p+K~6vSiIhEcyJ9r*)V<|`XrR|`tROs32pPogb=WkG2`_KHG zVSamRcx6fDTQEJSc}UqV{X|LMhdFS-^8NS3U?d!AJ8%anl}jPTc`&c<;DUt>Z@tg* z65H_<-^q|WE;&x?!vDU*K;i^o+r?l44<7pd_pAuW&Hp(o7}i^J9nM`O2|B9@f;L08 zYiG=Nz@eU*fB?N$atCmb9%yXePePo&?N_z;$nG5QIGY1MZ3KAwQ#w(nX$Kle^CLHR zfT8q#kfhH#q<{-1^@0T!DCyIAi!~R7QUA{!NcJw<{~Z8;*fTf)s5{_?G%xKC7Krfv z0R}Hlz*=Z-8}N_s+H?S^O1~aMDc~t!-?S4yyF_yYfy07oY)UL=@|HH0=)5 z)SY1^!JZw7zjGAsRQv}z`nF%w-cymz$#(9FWoPr>70d#8wTHx$3>BMX&wt(IeD!N| zd$6Ef!f#D4Ljo5rUu}SYvLf#4orR&>D!hHC;~#q>eIG>a^FQ8V`}=M()s5AEIsS2w zR#_4|3$5Nc4P#|Fa(n+-qz_a&0M)m*)R_U*?I1(AQ_a-lF(m;!D*y;K1V||>Yl6?p zzFQT*J7n{}No#TcFKMk863-<(VsF3uyA?>v;)`x}BKXxf#0NY-%-9NUYQC#?Q}f!h z1C5Yo(sD3_JQkQ^(H5MRN=HXWCy;|BtD_vwBLo?6D%cdv`}ZHyO)4yZyrvWNTuw7< zh)#yO`O&X?8DCdA?%Q4a*9zSUs?gmU0nOm5u~@7Hcq^7%IJ$8Ky!$9*3lqi`Cc3mBIWX3mOy!Xr!jgUO%1sgy zKuU#YzjSx&*F{U{G5V~ho7G3Mp*!;(;}0=PCLxDdM5G|o6a$5%Rcpw>7WKhysAEDC z;$yr|Erxe&t2&C5vS@?z5Luop0F(e6W!?TtMs(@24p7fHo@PTVFqKS!|BFI!1Xyz2 zdyB3?Nl{V#&Ye#HZDIro=D`)5dO3(&GS9T77-A zCm{CrrHfThwBr@zM31 zV+}aG#2hS|EABX0oA7h-BSd)k5+~gf((1jihoaTHW@aHpHcyw*VDLSjb4o1%f32k;y_zzrbg*_?|(O=_l=aFAX#PHxH=PTYq2 zPc))$iOYFGK*4_T;?1*1enOaoBns;=e6sfTgE8*^r>sT3VYX!pzvG}Hmtm1MhgOO@ zWKAJY0B1(Tl`yqIzDqJ`{5-~}asrDDi26kGg;3&wzKs$h)U^zPkzpw@{7g*g`# z!#==?AB)Z`3*+SCFO>(h<9FQr^3{(wD<{wQ(U*@s6SD2`?BT?eJ7?6Z@AwWK!msuu79gc@r#M4piMd*_~jaMq=;bwhK7_3Gj7|R}qG2n@hMY z^o_lp`#%=)(mvd6u^ObJ7rEe2<9+A(FIlds%^BDoB!glF6p)-_DFCz`A`h3#WW*QpIuCc$9R|cy2OBXUZ64we(nL-LbJqrKFo71)eUFPvi7 z;gWo3p{GP-=XP+3FAVlv;=fR3(DO9PTc$b$Y^XdM{TD$ZrE|B+cK|})eHe(w-^tWN zamy_=^}7^FB$g2Rt$zY{<`&viFUjptT*7=Xtb;p{#@F#&((2X((e}RP!p7E&))hTU z6LqeW&X#Yk9uFzsw?9wGmeZ2nN*^U;c7zyGfA4PnRM^(Jb4LU)bofiBZC}6HLoO`$ zp^Eu3Qy2Y96JzqIkSj{BrOznte)!g1=gDm0*L&lB>vUOd>EaCNrg{CCvEN?hXH?uA z>hp*Bl9SP4U(l+VP6=Ox1)w{~h<=Wt7!D-0kBkP_%>t93_!zNfvan%eg2HIS$K84e7NLVKknM{vU$$;mb(9yL-F{?ODhvOUsyScdSW z9tC-p?XI1n>%n%`T?I%X3`$r7zM9p9#Afg&;5e3;k;aCeYRjZrSS#Ilt4MK(^I!Km zGRij+;R)X+Ce21G&tqrdvt0p9yIhu$euH0Hh{Q}PKZl>U=S=)(D%X-mS)rIGjT3T80nN|ng2coL00LqK*miq3JD7o?Glyi5RL7ts4;IRUAEiLorFSm5k zZhvK~V5P(An8(ja;H)VufDFDOCx> z%!CBC!;;Il2Wv5b!1$O3^fgnV3?iw z{LgD{t0VD{_K&8P+#cq#Br zqd8u1)RHY&UI(bY{V{m!2$Jzae#tmCJX{Gth|g*Gr%l(=L{}(KPqB%K>`bz6YsWh> zn;IJ%t?Qr~CXm_W$LoVR;!ntQxy|(uDFsM};dbpRN zGQ+f{fPp7iKHt72K?gu;XaJzqSm!2idRTS# z0fGXL#4jsN%}t{VN~c<_yP!@X7-VwHsRtoZhVnuU-Pxes(@M_C%|m zZ5;qDP=#*m2>|`7uq+^l)FpA-_H*m0c8dz9K>@$xP<*;>^om>bm$;Upn`SSZ$wb+`wWGT&)d(U6d>5x-QQnwhf(fX3CPXZVCd1izjG(oy z%T!NPeSN*Y*Fp^_}8(*(eMW>A z-r~>zFEf}O&unl>ju5gngy28J?$>K$nyB^LdfA0=M2)V3k@d_*vG!-j9Z(AP3%9aRJO@0~F*H6b}MhuF4*q;l40x z=*}u_J4+>WJkX2UuV!=q&!z8)-brA^yK-$Cpu9DZc?yVdSfS9%CO_u2#oXtIi@;nO zNZEGhT}HbnLEWWOz@IllJ;TZA+ooQ-#UETNwzyb=npIgbV7xn@56$bjJk4MQlutCk z1h8JzDfIC23kq;xPz^ahI1I~KDk7gW8vj4MR)EvfFR-i(W4Oe`f@JW?e|?pSH3Zyk zNJ7sybEL@OaA`*_ESF+9KO6bpL$%0^C(Sj$0yB#|n7-{lbY_jY)Wiw|#Ss`FPXHYf zk2GXuW!nn2TsZGo_Y|#+7t!@40Y~K1W?CX`-!AkqZ#?FvV#Q|t$aTL>bke9fygMNb zYDmrDeG)qP4yNGa+ZNhcLiGcfJF=~`{D+Vn7afobT2EHzJ8QTLldOocj9x*i2|)_r zoAP`%fujjzh7fp3$4#oVL&#;k-N7Gq=K1!jmuEw5qvwQ`5y#TWNK?QEh(rLvH29R( zu#(=jyKy2Nmr}qE>x#H7|Do)xtUs$`sZ*;f?YrEds*iX_0}77=ob=W#VEc74?xF#e zNzRRPzyb+VV^!JU;TK` zaQG`mufWM7P9-MVW2#sPzcFDemc<0ys! zaTi>K;&`da_+>KliiF7+c;|-jWV#c8Ytyh(t9amzupd93)6%z>fdp$1plC%5FAG?a z$@JFrOrK*oVGc-4O>!EQi1Xmr9RT(Q0?6+}cHDboRUfLx(E$Jp`cn$QXz<y?)s-f5aXPbIPY>g+~k8cr93YY@5={A`^glu(eL4hd1@cKOa zekfJ<5AqO4UBvBJt&T!hTQIF#m@!DEg(7eTj5@8p-4&4EjC+lrva+)7LdCzA@a0G` zWIztx9%W?3Ch&F?3kW3Pc!Qdb4h9Nv%~oayM4X0i!byaV6z8{58=>0ebb3(oTo2Rg zwy&Tk=BR*p`)rIEWdcpdt3+|8pZ5O67#`1GR4cQg_{cw*OXdH=fG+;afUe%v6mE;G z-rPT5{MYa{(S%V~)Nd*${eEo?wh(YOadYo`1o)!^ie`SC9ym$09K^WZw>Kd%Q63_p zN$@WKthO#L$(6BVN-P5e!P z!p(=E&2f49mxKS_p(k;1aaj;%L0LT-cDZn_S^^d019l}iI9MV_OGCq1;!se9K%0JG z3>?cckWv9Rn_4t0pOQZqzCIr>RTr~4AJ412_l)DXwp&KO1uRxf{hl-M9;kU6DIr0@ zL70hK-^0BbfCCu($<5oK!Xg+bK*WKAhRFSH@8}iD^y5Qf4G1F^+r7uL?rk~x-l3E9 zd|-p$SS(r{#c>q;O{Y+&q_l>15GI9%RcnB{_h~hiP%sL+Jr1)sJ;Wb z$7gGUp-XDH?=f=~Hq@}QjcFI_43@`>4IEbeb4G9N$y>f>BtkAAQHs0xB4iuxLMGq) z*Rw|nDJjZIN_F;2dVS-<;A!BMI*}k=reHehglzQNOh6JNKoU=jx!ecC*V^xZ2kXRK zwa{i{^fuj>dU;4{3-geC?;wOJoJ$v_1O9F0D|B^r&nSHKmB*?sP7*!3*9K7LvL+@b z@Is)7p%|Z%iueXGmqpt + +Example: + +.. code-block:: bash + + $ redis-cli smembers pagmo.islands.zeromq_test + 1) "192.168.1.39:2286" + 2) "192.168.1.39:1568" + 3) "192.168.1.39:2639" + +In this example all three connected islands are running on one machine, but any device that can reach the broker could advertise its IP. + +Monitoring control events on a channel +-------------------------------------- + +.. code-block:: bash + + redis-cli subscribe pagmo.islands..control + +At the moment the only control messages sent are "connected" and "disconnected": the rest of the communication happens directly on the ZeroMQ sockets. + +.. code-block:: bash + + $ redis-cli subscribe pagmo.islands.zeromq_test.control + Reading messages... (press Ctrl-C to quit) + 1) "subscribe" + 2) "pagmo.islands.zeromq_test.control" + 3) (integer) 1 + 1) "message" + 2) "pagmo.islands.zeromq_test.control" + 3) "disconnected/192.168.1.39:2639" + 1) "message" + 2) "pagmo.islands.zeromq_test.control" + 3) "connected/192.168.1.39:1905" + +In this monitoring session we see two events: 192.168.1.39:2639 disconnects from the swarm and then 192.168.1.39:1905 connects to the swarm. From 52168c9a68d3ec19bd05d598f6cbd2cdaf582ed4 Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Wed, 19 Aug 2015 22:42:26 +0200 Subject: [PATCH 21/25] Updated PyGMO documentation --- doc/sphinx/documentation/island.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/sphinx/documentation/island.rst b/doc/sphinx/documentation/island.rst index 8ac7b784..07c25043 100644 --- a/doc/sphinx/documentation/island.rst +++ b/doc/sphinx/documentation/island.rst @@ -265,7 +265,15 @@ island and that, in reality, is a helper function returning automatically the co The token can be any string, but describing the algorithm and problem in the token is useful to monitor the progress of the computation. A way of monitoring the progress is described in the tutorial (TODO: link), where a user connects to the ZeroMQ network but does not perform evolution tasks. - .. method:: PyGMO.zmq_island.initialise((string) ip) + .. method:: PyGMO.zmq_island.set_ip((string) ip) + + This method sets the IP address to be used for incoming connections. + + .. method:: PyGMO.zmq_island.set_evolve((bool) evolve) + + If `set_evolve(false)` is called, then this island will not perform any evolution operations. + + .. method:: PyGMO.zmq_island.connect() This method will initialise the communication capabilities of the ZeroMQ island by communicating with the network, connecting to its peers and broadcasting the new connection. @@ -283,7 +291,8 @@ island and that, in reality, is a helper function returning automatically the co isl = zmq_island(algo,pop) isl.set_broker_details("127.0.0.1", 6379) isl.set_token("schwefel2_de10_pop10") - isl.initialise("127.0.0.1") + isl.set_ip("127.0.0.1") + isl.connect() while True: isl.evolve(10) From b318bfc28c2a74f4b3263f4e53f97c866efbc7bc Mon Sep 17 00:00:00 2001 From: jdiez17 Date: Thu, 20 Aug 2015 14:03:14 +0200 Subject: [PATCH 22/25] Publisher disconnects by default --- src/zmq_island.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index 8ff57755..f2f0b36a 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -257,8 +257,6 @@ void zmq_island::disconnect() { m_brokerConn.disconnect(); m_brokerSubscriber.disconnect(); - m_publisherSocket.disconnect(("tcp://" + m_IP).c_str()); - std::cout << "DEBUG: Closed" << std::endl; } } From 8e51bf5c7664aa8566fe170a6444a4fbc9279429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20D=C3=ADez?= Date: Thu, 20 Aug 2015 14:28:14 +0200 Subject: [PATCH 23/25] pygmo/core: fixed typo --- PyGMO/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyGMO/core/__init__.py b/PyGMO/core/__init__.py index 04f77ec2..6ad83060 100644 --- a/PyGMO/core/__init__.py +++ b/PyGMO/core/__init__.py @@ -145,7 +145,7 @@ def _generic_island_ctor(self, *args, **kwargs): class zmq_island(_core._zmq_island): def __init__(self, *args, **kwargs): - """ZMQ Island, yo. Unnamed arguments: + """ZMQ Island. Unnamed arguments: #. algorithm #. problem or population From e2bb5606668088cd545283dfb4508a914cb44d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20D=C3=ADez?= Date: Thu, 20 Aug 2015 14:39:30 +0200 Subject: [PATCH 24/25] Add ldconfig after redox for good measure --- doc/sphinx/tutorials/zmq_setup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/tutorials/zmq_setup.rst b/doc/sphinx/tutorials/zmq_setup.rst index 5c12d003..35ab76fb 100644 --- a/doc/sphinx/tutorials/zmq_setup.rst +++ b/doc/sphinx/tutorials/zmq_setup.rst @@ -34,7 +34,7 @@ We also need to compile and install `redox` because Debian does not provide a bi root@dbc834f0d0aa:/tmp/redox# mkdir build && cd build root@dbc834f0d0aa:/tmp/redox/build# cmake .. -DLIB_SUFFIX="/" (output omitted...) - root@dbc834f0d0aa:/tmp/redox/build# make && make install + root@dbc834f0d0aa:/tmp/redox/build# make && make install && ldconfig Next, we'll clone the PaGMO source tree and compile it with PyGMO and ZeroMQ enabled. The `make` command takes around 15-20 minutes to complete (could take longer), so this is the perfect time to grab a caffeinated beverage. From 4dca6a826faff40209579b765b57de8b0cb2b6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20D=C3=ADez?= Date: Fri, 21 Aug 2015 12:03:14 +0200 Subject: [PATCH 25/25] zmq_island: add doxygen docs --- src/zmq_island.cpp | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp index f2f0b36a..1ecfbdf3 100644 --- a/src/zmq_island.cpp +++ b/src/zmq_island.cpp @@ -113,6 +113,15 @@ base_island_ptr zmq_island::clone() const } // This method performs the local evolution for this island's population. +/** + * This method will evolve the population if m_evolve is true, + * will broadcast its population to the ZeroMQ swarm if this island is connected, + * and it will replace the population in this island if an alternative one is received from the swarm. + * + * @param[in] algo The algorithm used to evolve the population. + * @param[in, out] pop The population of the island, that can be changed if a new one is received. + * + */ void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) const { if(m_evolve) { @@ -177,11 +186,26 @@ std::string zmq_island::get_name() const return "ZMQ island"; } +/** + * Sets the details of the broker host. + * + * @param[in] host The host or IP address. + * @param[in] port The port. + * + */ void zmq_island::set_broker_details(std::string host, int port) { m_brokerHost = host; m_brokerPort = port; } +/** + * Sets the channel token. It can be any string, but it's often useful to use a string + * that describes the algorithm, problem and population used in the island. For example, + * a good channel token would be schwefel10_de10_pop20. + * + * @param[in] token The channel token. + * + */ void zmq_island::set_token(std::string token) { m_token = token; } @@ -192,6 +216,19 @@ void zmq_island::connect_host(std::string host) { m_subscriptionSocket.connect(("tcp://" + host).c_str()); } +/** + * Connects this island to the ZeroMQ swarm. The initialisation protocol is as follows: + * 1. Connection with the broker is established. + * 2. The island acquires a list of peers that are already in the swarm with the same channel token. + * 3. The subscription socket is connected to every peer received from the broker. + * 4. The island adds itself to the set of peers in the ZeroMQ swarm and announces its presence via a realtime control channel. + * 5. The island binds the publisher socket (i.e used for outgoing communication) to the IP address given by `set_ip()`. + * 6. A callback is registered such that whenever a new peer sends a messsage over the control channel, the island can process it correctly. + * + * + * @returns true if the initialisation succeeded, false otherwise. + * + */ bool zmq_island::connect() { if(m_brokerHost == "" || m_brokerPort == -1 || m_token == "" || m_IP == "") { return false; // Can't initialise if we're missing those parameters @@ -247,6 +284,7 @@ bool zmq_island::connect() { return true; } +/// Disconnects the island from the swarm. void zmq_island::disconnect() { if(m_initialised) { std::string brokerKey = "pagmo.islands." + m_token; @@ -261,22 +299,45 @@ void zmq_island::disconnect() { } } +/** + * Sets whether the island should perform evolutionary functions. + * It's useful to disable evolution if this island were to be used for monitoring the swarm + * instead of actively participating in it. + * + * @param[bool] evolve Determines whether this island will evolve its population. + */ void zmq_island::set_evolve(bool e) { m_evolve = e; } +/// Returns whether the island evolves its population. bool zmq_island::get_evolve() { return m_evolve; } +/** + * Sets a callback on network activity. + * Every time a message is received, this callback will receive a reference to the message. + * + * The callback is of type zmq_island::callback, which is void (*callback)(zmq::message_t&). + * Note: most users won't need to use this function in normal use. + * + * @param[in] callback The callback to be used. + */ void zmq_island::set_callback(zmq_island::callback c) { m_callback = c; } +/// Disables the low-level network callback. void zmq_island::disable_callback() { m_callback = NULL; } +/** + * Sets the IP address and port used for peer to peer communication. + * + * @param[in] ip The IP to be used. + */ void zmq_island::set_ip(std::string ip) { srand(time(0)); m_localPort = rand() % 2000 + 1000;