diff --git a/CMakeLists.txt b/CMakeLists.txt index 37dc41d9..8fc863b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,9 @@ OPTION(ENABLE_WORHP "Enable support for WORHP minimiser." OFF) # Build Option: support for MPI clustering. OPTION(ENABLE_MPI "Enable support for the Message Passage Interface (MPI)." OFF) +# Build Option: ZeroMQ islands. +OPTION(ENABLE_ZMQ "Enable support for ZeroMQ islands." OFF) + # Build option: enable test set. OPTION(ENABLE_TESTS "Build test set." OFF) @@ -199,6 +202,12 @@ IF(ENABLE_MPI) ADD_DEFINITIONS(-DPAGMO_ENABLE_MPI) ENDIF(ENABLE_MPI) +IF(ENABLE_ZMQ) + ADD_DEFINITIONS(-DPAGMO_ENABLE_ZMQ) + SET(MANDATORY_LIBRARIES ${MANDATORY_LIBRARIES} zmq redox ev hiredis) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") # workaround for void perform_evolution() const +ENDIF(ENABLE_ZMQ) + # If GSL support is requested, look for the library. IF(ENABLE_GSL) FIND_LIBRARY(GSL_GSL_LIBRARY NAMES gsl) diff --git a/PyGMO/core/__init__.py b/PyGMO/core/__init__.py index 12f2eefc..6ad83060 100644 --- a/PyGMO/core/__init__.py +++ b/PyGMO/core/__init__.py @@ -18,6 +18,9 @@ 'population', 'py_island'] +if '_zmq_island' in dir(_core): + __all__.append('zmq_island') + _orig_signal = _signal.getsignal(_signal.SIGINT) _main_pid = _os.getpid() @@ -136,6 +139,84 @@ def _generic_island_ctor(self, *args, **kwargs): local_island.__original_init__ = local_island.__init__ local_island.__init__ = _generic_island_ctor +if '_zmq_island' in dir(_core): +# Raw C++ ZMQ island class. + _zmq_island = _core._zmq_island + + class zmq_island(_core._zmq_island): + def __init__(self, *args, **kwargs): + """ZMQ Island. Unnamed arguments: + + #. algorithm + #. problem or population + #. number of individuals (optional and valid only if the second argument is a problem, defaults to 0 if not specified) + + Keyword arguments: + + * *s_policy* -- migration selection policy (defaults to 'best selection' policy) + * *r_policy* -- migration replacement policy (defaults to 'fair replacement' policy) + + """ + if len(args) == 0: + raise ValueError( + "Cannot initialise ZeroMQ island without parameters for the constructor.") + + from PyGMO.algorithm._algorithm import _base as _base_algorithm + from PyGMO.algorithm import base as base_algorithm + from PyGMO.problem._problem import _base as _base_problem + from PyGMO.problem._problem import _base_stochastic as _base_problem_stochastic + from PyGMO.problem import base as base_problem + from PyGMO.problem import base_stochastic as base_problem_stochastic + from PyGMO.migration._migration import best_s_policy, fair_r_policy, _base_s_policy, _base_r_policy + + if len(args) < 2 or len(args) > 3: + raise ValueError( + "Unnamed arguments list must have either 2 or three elements, but %d elements were found instead." % + (len(args),)) + if not isinstance(args[0], _base_algorithm): + raise TypeError("The first unnamed argument must be an algorithm.") + ctor_args = [args[0]] + if isinstance(args[1], _base_problem) or isinstance(args[1], _base_problem_stochastic): + ctor_args.append(args[1]) + if len(args) == 3: + if not isinstance(args[2], int): + raise TypeError( + "Please provide an integer for the number of individuals in the island.") + ctor_args.append(args[2]) + else: + ctor_args.append(0) + elif isinstance(args[1], population): + if len(args) == 3: + raise ValueError( + "When the second unnamed argument is a population, there cannot be a third unnamed argument.") + ctor_args.append(args[1]) + else: + raise TypeError( + "The second unnamed argument must be either a problem or a population.") + + if 's_policy' in kwargs: + ctor_args.append(kwargs['s_policy']) + else: + ctor_args.append(best_s_policy()) + if not isinstance(ctor_args[-1], _base_s_policy): + raise TypeError("s_policy must be a migration selection policy.") + + if 'r_policy' in kwargs: + ctor_args.append(kwargs['r_policy']) + else: + ctor_args.append(fair_r_policy()) + if not isinstance(ctor_args[-1], _base_r_policy): + raise TypeError("r_policy must be a migration replacement policy.") + + super(type(self), self).__init__(*ctor_args) + + def get_name(self): + return str(type(self)) + + def __get_deepcopy__(self): + from copy import deepcopy + return deepcopy(self) + # This is the function that will be called by the separate process # spawned from py_island. diff --git a/PyGMO/core/core.cpp b/PyGMO/core/core.cpp index 3a189bef..f9e49784 100644 --- a/PyGMO/core/core.cpp +++ b/PyGMO/core/core.cpp @@ -49,6 +49,9 @@ #include "../../src/algorithm/base.h" #include "../../src/archipelago.h" #include "../../src/base_island.h" +#ifdef PAGMO_ENABLE_ZMQ + #include "../../src/zmq_island.h" +#endif #include "../../src/config.h" #include "../../src/exceptions.h" #include "../../src/migration/base_r_policy.h" @@ -380,6 +383,20 @@ BOOST_PYTHON_MODULE(_core) // Register to_python conversion from smart pointer. register_ptr_to_python(); +#ifdef PAGMO_ENABLE_ZMQ + // ZMQ island class. + class_ >("_zmq_island", "ZMQ island class.",init >()) + .def(init >()) + .def(init()) + .def("__copy__", &Py_copy_from_ctor) + .def("__deepcopy__", &Py_deepcopy_from_ctor) + .def("set_broker_details", &zmq_island::set_broker_details) + .def("set_token", &zmq_island::set_token) + .def("set_ip", &zmq_island::set_ip) + .def("set_evolve", &zmq_island::set_evolve) + .def("connect", &zmq_island::connect); +#endif + // Expose archipelago class. class_("archipelago", "Archipelago class.", init >()) diff --git a/PyGMO/utils.h b/PyGMO/utils.h index 80bd84ec..7cbeda58 100644 --- a/PyGMO/utils.h +++ b/PyGMO/utils.h @@ -90,7 +90,7 @@ struct python_class_pickle_suite: boost::python::pickle_suite throw_error_already_set(); } // Restore the object's __dict__. - dict d = extract(obj.attr("__dict__"))(); + boost::python::dict d = extract(obj.attr("__dict__"))(); d.update(state[0]); // Restore the internal state of the C++ object. const std::string str = extract(state[1]); diff --git a/doc/notebooks/tutorials/first_contact_zmq.ipynb b/doc/notebooks/tutorials/first_contact_zmq.ipynb new file mode 100644 index 00000000..af01b744 --- /dev/null +++ b/doc/notebooks/tutorials/first_contact_zmq.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First contact\n", + "===\n", + "\n", + "Now that we have a Redis server running and we have compiled PaGMO and PyGMO with ZeroMQ support (see The Setup (TODO: link)), we can run an example program that leverages the functionality of this type of island.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from PyGMO import *\n", + "import time\n", + "\n", + "prob = problem.schwefel(10)\n", + "algo = algorithm.de(10)\n", + "pop = population(prob, 20)\n", + "\n", + "isl = zmq_island(algo, pop)\n", + "isl.set_broker_details(\"127.0.0.1\", 6379)\n", + "isl.set_token(\"schwefel10_de10_pop20\")\n", + "isl.set_ip(\"127.0.0.1\")\n", + "\n", + "def evolve():\n", + " isl.evolve(10)\n", + " print(\"Best: \", isl.population.champion.x)\n", + "\n", + " time.sleep(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This code will set up a single ZeroMQ island working on the 10-dimensional Schwefel problem and using the Differential Evolution algorithm. The population of the island will be 20. Lines 4-6 set up the problem, and lines 8-12 set up the island itself. \n", + "\n", + "In this tutorial we assume that the broker is at 127.0.0.1:6379, and we will be binding the receiving port only on the loopback inteface, but in a networked setup you would use IPs that are accessible beyond localhost.\n", + "\n", + "The channel token is set to `schwefel10_de10_pop20`, which can be any string, but choosing a token relevant to the actual properties of the computation will help you determine which group of islands is working on what problem. If we start this program, it will connect to the broker and advertise itself as an island working on that token, but because there are no other peers it will simply perform the evolution locally. This is a typical output:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best: (424.4920569537634, 440.7206390429582, 399.43102290788795, -299.52078183929916, 424.4274999225018, 395.96019766970727, 396.61431073790675, 414.7987356103403, -293.15767979233385, -152.01178644903854)\n" + ] + } + ], + "source": [ + "evolve()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that because the island is not actually connected to the ZeroMQ network yet, it's not exchanging solutions with peers." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, if we call the `connect()` function on this island, we'll be ready to accept migrants and send our own as well." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "isl.connect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Because IPython doesn't have an easy way of executing two threads and displaying the result easily, I'll start up another program just like this outside IPython and call evolve() here -- it should receive the population from the other program, and display it." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best: (420.9687728816528, 420.9687569400687, 420.9687642651889, 420.968743361018, 420.96875116870063, 420.968771074309, 420.96871292112354, 420.9687127857178, 420.96875246113143, 420.9687562620334)\n" + ] + } + ], + "source": [ + "evolve()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, our current solution is much better than the original and because the program in the background had been running for a little while before that, and it had already converged on the optimum for the Schwefel function." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's a snippet of the output on the other side: \n", + "\n", + " Best: (420.96880347940544, 420.9687194811639, 420.96880839006843, 420.9687540217262, 420.9687810359755, 420.9687609702579, 420.968730572192, 420.96875630500364, 420.9687648447646, 420.96880903640675)\n", + " Best: (420.9687351908159, 420.96872163933637, 420.9687399639167, 420.9687069931382, 420.9686997709129, 420.9687976850297, 420.9687312413354, 420.9687104977846, 420.9687392971252, 420.9687677879039)\n", + " DEBUG: Opening connection to 127.0.0.1:2609\n", + " Best: (420.9687701356386, 420.9687781629428, 420.9687691217583, 420.9687483448487, 420.96878827364174, 420.96875668743473, 420.96870707941497, 420.9687079359776, 420.96874266075383, 420.96872310305565)\n", + " Best: (420.9687728816528, 420.9687569400687, 420.9687642651889, 420.968743361018, 420.96875116870063, 420.968771074309, 420.96871292112354, 420.9687127857178, 420.96875246113143, 420.9687562620334)\n", + " Best: (420.9687533680281, 420.96876243183203, 420.9687590126924, 420.9687372408334, 420.96875017480545, 420.96877891439186, 420.9687558066539, 420.9687460272027, 420.9687420838824, 420.9687647092545)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note how it opens a connection to our island as soon as we call `isl.connect()`. Then it sent its population, which had already converged on the answer and was better than our original value." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/doc/notebooks/tutorials/index.ipynb b/doc/notebooks/tutorials/index.ipynb index c083b52b..5107b30a 100644 --- a/doc/notebooks/tutorials/index.ipynb +++ b/doc/notebooks/tutorials/index.ipynb @@ -1,47 +1,66 @@ { - "metadata": { - "name": "", - "signature": "sha256:4e3c0951d9d81090c3c1c2520fd6e4170f76fedd5b98af109335e7e59a85e1d4" - }, - "nbformat": 3, - "nbformat_minor": 0, - "worksheets": [ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PyGMO Tutorials" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this section you can get yourself acquainted with the very basics of using PyGMO." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Content" + ] + }, { - "cells": [ - { - "cell_type": "heading", - "level": 1, - "metadata": {}, - "source": [ - "PyGMO Tutorials" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this section you can get yourself acquainted with the very basics of using PyGMO." - ] - }, - { - "cell_type": "heading", - "level": 1, - "metadata": {}, - "source": [ - "Content" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "* [Adding new problems and algorithms to PyGMO](adding_new_problems_and_algorithms_to_pygmo.ipynb)\n", - "* [Meta problems](meta_problems.ipynb)\n", - "* [Multi objective optimization](multi_objective_optimization.ipynb)" - ] - } - ], - "metadata": {} + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* [Adding new problems and algorithms to PyGMO](adding_new_problems_and_algorithms_to_pygmo.ipynb)\n", + "* [Meta problems](meta_problems.ipynb)\n", + "* [Multi objective optimization](multi_objective_optimization.ipynb)\n", + "* [First contact with ZMQ Islands](first_contact_zmq.ipynb)\n", + "* [Using ZMQ Islands in an archipelago](zmq_islands_archipelago.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/doc/notebooks/tutorials/zmq_islands_archipelago.ipynb b/doc/notebooks/tutorials/zmq_islands_archipelago.ipynb new file mode 100644 index 00000000..1430546e --- /dev/null +++ b/doc/notebooks/tutorials/zmq_islands_archipelago.ipynb @@ -0,0 +1,206 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using ZeroMQ islands in an archipelago\n", + "====================\n", + "\n", + "While ZeroMQ islands can be used standalone (without being part of an archipelago), they are used most effectively inside one. The idea is that through archipelago migration strategies, local islands exchange individuals with a single ZeroMQ inside it, which acts as a nexus to send solutions to other archipelagos with ZeroMQ islands inside them. This also works in reverse; a ZeroMQ island is constantly receiving population broadcasts from its peers, so the archipelago can migrate them from there to other islands.\n", + "\n", + "\n", + "\n", + "In this image there are three archipelagos, each with a different topology. The dotted lines represent the ZeroMQ network, which is always fully connected.\n", + "\n", + "To simplify the example, the following program instantiates an archipelago with three local islands and one ZeroMQ island." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from PyGMO import *\n", + "import time\n", + "\n", + "prob = problem.schwefel(10)\n", + "algo = algorithm.de(10)\n", + "\n", + "zmqisl = zmq_island(algo, prob, 20)\n", + "zmqisl.set_broker_details(\"127.0.0.1\", 6379)\n", + "zmqisl.set_token(\"schwefel10_de10_pop20\")\n", + "zmqisl.set_ip(\"127.0.0.1\")\n", + "\n", + "archi = archipelago(algo, prob, 3, 20)\n", + "archi.push_back(zmqisl)\n", + "\n", + "def evolve():\n", + " archi.evolve(1)\n", + " for isl in archi:\n", + " if isl.get_name().startswith(\"ZMQ\"):\n", + " return(isl.population.champion.x[0])\n", + "\n", + " time.sleep(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following demonstrations will use graphics and plots, so we need to load matplotlib:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we will evolve the archipelago 20 times and record the value of the best individual in the ZeroMQ island over time, and plot the results." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEACAYAAAC57G0KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGitJREFUeJzt3XuUVNWZ9/Hv0yBEMqKCSmwYQYlIBsTwGhTBaEE0gzHB\ne5jlmGXiODoRb7xLx8S8GkbfzEwMy3klIWBIYswko8RLVBwyOlFqbBKYdEPwzkXsAWmjcg0aAZF+\n3j92NZQtdNflVJ9L/z5r1aLr1DmnHmoVv97ss/c+5u6IiEj21MVdgIiI1IYCXkQkoxTwIiIZpYAX\nEckoBbyISEYp4EVEMqqkgDezOjNbYmYN7bZfbWatZnZM0bZpZrbJzFrM7PyoCxYRkdKU2oK/ClgF\n7Bk0b2b1wOTC9rZtQ4EbgbHAhcAsMzswsmpFRKRknQZ8IcjPA2YDVvTSncBNFIU+IfDvd/fV7r4Y\naAQmRleuiIiUqpQWfFuQt7ZtMLOzgbfc/fft9q0H1hU9bwYGVlukiIiUr8OAN7NJwFZ3b6LQei90\nudwK3GJmbS16288pREQkJj07eX08cIWZXVG0bR3QD9hStG2VmY0CWoDBRduPAZ5of1Iz0wI4IiIV\ncPeSG9QdtuDd/RZ3r3P3OuAU4Dfufri79yjavhI41t1fBOYDU8xsmJmNA8YAT+/n3HpE9PjmN78Z\new1Zeujz1GeZ1Ee5OmvBFzM+eEF1X6G9xsxmAEuAHcBUd99RdlUiIlK1kgPe3ZcAp+1j+yfaPb+T\ncGFWRERipJmsGZDL5eIuIVP0eUZHn2W8rJJ+narf1MzjeF8RkTQzMzyqi6wiIpJeCngRkYxSwIuI\nZJQCXkQkoxTwIiIZpYAXEckoBbyISEYp4EVEMkoBLyKSUQp4EZGMUsCLiGSUAl5EJKMU8CIiGaWA\nFxHJKAW8iEhGKeBFRDJKAS8iklEKeBGRjFLAi4hkVM+4CxARSbLWVpg5EzZvru48Q4fCpZdGU1Op\n1IIXEenAvHnwwx9Cz57VPepiSFtz965/UzOP431FRMrx3nvwF38Bc+fChAlxVwNmhrtbqfurBS8i\nsh8/+lHoWklCuFdCLXgRkX1491049lh47DE48cS4qwnUghcRicDMmTB+fHLCvRJqwYuItLNlCwwb\nBosWwXHHxV3NXjVpwZtZnZktMbOGwvMbzKzFzN41swYzG1a07zQz21R4/fzy/woiIvG64w4499xk\nhXslSmrBm9nVwEnAEHc/rRDoG4A/AV8HRrn7BWY2FGgATgcOAx4GjnH37e3Opxa8iCTS66/D8cfD\ns8/CoEFxV/NBkbfgzaweOA+YDRiAu69y9y2FXXYBbxZ+ngzc7+6r3X0x0AhMLKN+EZFY3X47XHZZ\n8sK9EqXMZL0TuAnoUbzRzH4IXAY0A6cUNtcD64p2awYGVl+miEjtvfIKPPAArFwZdyXR6LAFb2aT\ngK3u3kSh9d7G3S8H+gD3AnfXrEIRkS5y661w/fXQv3/clUSjsxb8eOAKM7uibYOZ/c7dTwJw9x1m\n9gOgqfByCzC46PhjgCf2deLp06fv+TmXy5HL5cqtXUQkMsuXw8KF8IMfxF3JXvl8nnw+X/HxJQ+T\nNLOTgRnu/mkz+0vgd8Bu4GvASHefbGYfB54Bcuy9yDrE3Xe0O5cusopIonzuc+Fx9dVxV7J/5V5k\nLWc1SQPaUvkC4N8I/fLPAF8FcPdXzGwGsATYAUxtH+4iIknzzDPw8svwyCNxVxItTXQSkW7NHU49\nFf7u7+BLX4q7mo5pqQIRkTI8/jhs2wYXXxx3JdFTwItIt7V7N9x8M3zrW9CjR+f7p40CXkS6rfvu\ng4MOgi98Ie5KakN98CLSLb33HgwfDj/5CZx2WtzVlEZ98CIiJZg7NywmlpZwr4Ra8CKyTwsXwrhx\n0Lt33JVE7513ws08FiyA0aPjrqZ0asGLSNXWrIEzz4RPfjKMEc+au+6CXC5d4V4JteBF5EP+6Z9g\n3Tr47Gfh2mvDn3fckY01WjZtCl0zixeHVnyaqAUvIlWbNw+mTIHzzoOXXgojTUaMgJ/+NEwMSrNv\nfxsuvDB94V4JteBF5ANWroQJE+C11z44NrypCa68Eg45BGbPDre0S5uWFhg1Cp5/Hurr466mfGrB\ni0hV5s0LLdz2E38+9Sn47/8OY8bHjYPbboOdO+OpsVK33QaXX57OcK+EWvAi8gEjRoQlc8eP3/8+\nr70G11wDK1bA3XfD6ad3XX2VWrUq/J1WroR+/eKupjLltuAV8CKyxwsvwFlnwdq1UFfC/+8feSRc\nhD3jDPjOd5J9EXbKlDAq6Otfj7uSyqmLRkQqNm8efPGLpYU7wLnnwosvwsEHJ/si7LJl0NAQfhl1\nJ2rBiwgQgvm44+BnP4OTTir/+KVL4YorQtjPmZOsi7CTJsHkyXDVVXFXUh214EWkIsuXw/vvw5gx\nlR1/4onhIuw554SLsP/wD8m4CJvPw+rV4eJqd1POHZ1EEu3v/z5c7Dv77LgrSae27hkruX34YT17\nwnXXwQUXhO6Q/v3jX+pg+3a45x7o1SveOuKgLhrJjKOPhiOPhN/+Nu5K0scdjjkGHn442un7W7ZA\na2t056tEjx5h7H4W1PKerCKJtXEjbN4cgqqxsfJuhu6qsREOOCCMMonSoYdGez4pj/rgJROWLg19\nwFOnwsyZcVeTPm1LE1TTPSPJoxa8ZEJTU5hpefnloavhjTfgYx+Lu6p0aG2FX/wCfvWruCuRqKkF\nL5nQFvCHHhpaonPmxF1ReixeDH37wsiRcVciUVPASya0BTyEKfRz5iRjiF4atHXPSPYo4CX13ngD\n/vSnMIoGwozK448P3Q7Ssd274YEHFPBZpYCX1Fu6NLTeiy8QXnttuGuPRuN2rKEBBgwIM1glexTw\nknrF3TNtzj4btm4N/cuyf+qeyTYFvKTevgK+rg6uvlpDJjvy/vvw0EMK+CxTwEuque874AG+8hV4\n8klYv77r60qDhQth8OAwrFSySQEvqfb66+FC4Z//+YdfO/hg+Ou/DreXkw9T90z2lRTwZlZnZkvM\nrKHw/FYzW2tm28zsYTM7pGjfaWa2ycxazOz8WhUuAntb7/ubgXnNNTB3blhwSvZ67z345S/D4mKS\nXaW24K8CVgFtYxLeBCYAg4D3gZsBzGwocCMwFrgQmGVmB0ZZsEix/XXPtBk2LLx+331dV1Ma/PrX\nYeTMUUfFXYnUUqcBb2b1wHnAbMAA3P1ud3/V3bcBjwIDCrtPBu5399XuvhhoBCbWpHIROg94CEMm\nZ87UkMli6p7pHkppwd8J3AR8aNFPMzPgS8BDhU31wLqiXZqBgVXWKLJPHV1gLfbZz8KOHfDMM11T\nV9Lt2AGPPQYXXRR3JVJrHS42ZmaTgK3u3mRmY/exy7eBV9z9sXLfePr06Xt+zuVy5HK5ck8h3dy6\ndWGJ2/r6jverqwt98TNnhhuCdHdPPAGjRnX+uUn88vk8+Xy+4uM7vOGHmd0OfKPd5iZ3P8nMbgZO\ncPcpRftfDwx292mF5/OB2e6+oN15dcMPqdpDD8G994bWaGfeeScMCVy2LPzZnV18MZx6avrvT9od\nRXpPVne/xd3r3L0OOAX4TSHcrwZywCXtDpkPTDGzYWY2DhgDPF3W30CkRKV0z7T5sz+DSy+FWbNq\nW1PSvfsuLFgQbqkn2VfOOHhj7yiam4DPADvNrNXMngRw9zXADGAJ8CAw1d13RFivyB7lBDyEma0/\n/nFYmKy7WrAgfGYDBnS+r6Sf7skqqeQO/frBihXlhdU558DnPgdXXlm72pLsoovCRee//du4K5FK\nlNtFo4CXVFqzBiZMCBday/H00+GC6wsvdL/b073zDgwcCK++Cv37x12NVCLSPniRpGpqquzG2hMm\nhGB/6qnoa0q6+fNh3DiFe3eigJdUKrf/vY3Z3olP3Y0mN3U/CnhJpUoDHuCSS8I68WvWRFtTkv3x\nj6F76txz465EupICXlKntTWMZz/xxMqO79MHLrsMvve9aOtKskcfhVwODjmk010lQxTwkjqrV8Nh\nh4VRNJWaOhV++lN4++3o6koydc90Twp4SZ1qumfaHHVUuOB6773R1JRkmzfDokUweXLclUhXU8BL\n6kQR8ADXXQff/W7o8smyX/4SzjgDDjoo7kqkqyngJXWiCvhTTw398U88Uf25kmzePPirv4q7ComD\nJjpJquzeHS4Url8fbslXrZ/8JATgr35V/bmSaMMGOPbYcGvDPn3irkaqpYlOkmkrVoRlbqMIdwgt\n22XLwnmz6KGH4KyzFO7dlQJeUiWq7pk2H/lIWJclq0MmNXqme1PAS6pEHfAAX/0q/PznsHVrtOeN\n2x/+AMuXw6RJcVcicVHAS6rUIuAHDgwheM890Z43bg8+CF/4QvhfinRPCnhJjV274LnnYPTo6M99\n3XWhm2b37ujPHRd1z4gCXlLjpZdgyJBwd6aonXxyWGXx3/89+nPH4bXX4OWX4cwz465E4tThTbdF\nkqQW3TNtzEIr/l/+JdyQOu3uvTcsLNarV9yVSJwU8JIajY21C3gIdzuaMycsypV2PXuGC8fSvSng\nJTWamsKNs2ulVy9oaKjd+UW6mmaySirs3AmHHgobN2rSjnRfmskqmfT882HKvcJdpHQKeEmFWl5g\nFckqBbykggJepHwKeEkFBbxI+XSRVRJv+/YwCWnLFujdO+5qROKji6ySOc8+C5/4hMJdpFwKeEk8\ndc+IVEYBL4mngBepTEkBb2Z1ZrbEzBoKzy8xsxVmttvM/le7faeZ2SYzazGz82tRtHQvCniRypTa\ngr8KWAW0XRltBM4Cflu8k5kNBW4ExgIXArPM7MBoSpXu6J13oLkZRoyIuxKR9Ok04M2sHjgPmA0Y\ngLuvdPfmfew+Gbjf3Ve7+2LCL4KJEdYr3czy5TBypFZFFKlEKS34O4GbgNYS9q0H1hU9bwYGVlCX\nCKDuGZFqdLiapJlNAra6e5OZjY3yjadPn77n51wuRy4La7RK5Jqa4DOfibsKkXjk83ny+XzFx3c4\n0cnMbge+0W5zk7ufVHi9AbjO3ZcVnl8PDHb3aYXn84HZ7r6g3Xk10UlKMnw4PPAAHH983JWIxC/S\niU7ufou717l7HXAK8Ju2cC9+z6Kf5wNTzGyYmY0DxgBPl1qMSLFt22D9+jDJSUTKV844eKMwisbM\nLjezVmAc0GhmywDcfQ0wA1gCPAhMdfcd0ZYs3cWyZXDCCeHuRCJSPq1FI4k1Y0a4efRdd8VdiUgy\naC0ayQyNoBGpjgJeEksBL1IdddFIIm3ZAoMHhz979Ii7GpFkUBeNZMLSpTB6tMJdpBoKeEkkdc+I\nVE8BL4mkgBepngJeEkkBL1I9BbwkzoYNsHUrDB0adyUi6aaAl8RZuhROPBHq9O0UqYr+CUniqHtG\nJBoKeEkcBbxINBTwkjgKeJFoKOAlUf7wB9i+HYYMibsSkfRTwEuiLF0aWu9W8mRsEdkfBbwkirpn\nRKKjgJdEUcCLREcBL4nhroAXiZICXhKjpSWE/KBBcVcikg0KeEmMtta7LrCKREMBL4mh7hmRaCng\nJTEU8CLR0i37JBHc4fDD4bnnoL4+7mpEkkm37JNUWrsWevdWuItEqWfcBUh1nnwSnn467iqq19ys\n7hmRqKmLJuXGjIFx4+DII+OupHoTJ8JJJ8VdhUhyldtFo4BPsU2b4OijYeNG6NUr7mpEpNbUB9+N\nLFwIn/60wl1E9k0Bn2L/+Z9wxhlxVyEiSVVSwJtZnZktMbOGwvO+ZrbAzLabWYOZDSjad5qZbTKz\nFjM7v1aFSwj4M8+MuwoRSapSW/BXAauAto7zG4CNwJHAYuA2ADMbCtwIjAUuBGaZ2YFRFizBq6+G\nG2OMGBF3JSKSVJ0GvJnVA+cBs4G2zv3JwAx33wr8c+H1tu33u/tqd18MNAITI69a9nTPaN0WEdmf\nUlrwdwI3Aa1F2wYCawHcfTPQy8x6AfXAuqL9mgv7SsTUPSMinelwopOZTQK2unuTmY3t5FxltSWn\nT5++5+dcLkculyvn8G5t9+4wuemuu+KuRERqKZ/Pk8/nKz6+w3HwZnY78I12m5cSfjF82d2fNbP+\nwMvufoSZXQ8MdvdphePnA7PdfUG782ocfBUaG+HLX4YXX4y7EhHpSpGOg3f3W9y9zt3rgFOA37j7\nGGA+cIOZ9QO+BjxSOORxYIqZDTOzccAYIAMT6ZPl179W94yIdK6ccfDG3lE0M4D+wOuEETO3Arj7\nK4XXlgAPAlPdfUdk1Qqg8e8iUhotVZAy774LAwbA66/DQQfFXY2IdCUtVZBxDQ0werTCXUQ6p4BP\nGXXPiEipFPApo/HvIlIq9cGnyJtvwnHHheWBe+pWLSLdjvrgM+yppyCXU7iLSGkU8Cmi8e8iUg4F\nfEq46wKriJRHAZ8SK1eGlSOHDYu7EhFJCwV8SrR1z2h5YBEplQI+JdQ9IyLl0jDJFNi1Cw4/HFat\ngiOOiLsaEYmLhklmUGMjDBmicBeR8ijgU0CzV0WkEgr4FFDAi0gl1AefcNu2QX09vPUW9OkTdzUi\nEif1wWfMf/0XnHyywl1EyqeATzgtTyAilVLAJ5zGv4tIpRTwCbZ+feh7Hz067kpEJI0U8An21FMw\ncSL06BF3JSKSRgr4BFP3jIhUQ8MkE8odjjwSFi+Go4+OuxoRSQINk8yIF16Aj35U4S4ilVPAJ5Rm\nr4pItRTwCaWAF5FqqQ8+gXbuDMsD/8//QL9+cVcjIkmhPvgMWLwYhg9XuItIdRTwCaTlCUQkCh0G\nvAUNZva2mW00s5mFbceb2e/M7E9m9h9mdkjRMdPMbJOZtZjZ+bX/K2SPxr+LSBQ6DPhCR/k1wGHA\nSGA88Hngh8CPC9uXATcDmNlQ4EZgLHAhMMvMDqxV8Vm0ZQu89BKMGxd3JSKSdp120bj7cnffCbQC\nu4ANhLC/x923Az8DLijsPhm4391Xu/tioBGYWJPKM2rhQhg/Hnr3jrsSEUm7kvrgzex94A1gqbsv\nAV4GLjWzPsDFwMDCrvXAuqJDm4tekxJoeKSIRKWkgHf3nsAQ4FNmdg5wBXAV8BbwUWB3rQrsbhTw\nIhKVnqXu6O7rzOxhYIy7Pwp8EsDMTgUmFHZrAQYXHXYM8MS+zjd9+vQ9P+dyOXK5XDl1Z1JzM7z9\nNowcGXclIpIE+XyefD5f8fEdTnQys3pgEPBc4c95wLeAF4GNwEHAfcC97j6ncJG1AcgRLsA+DAxx\n9x3tzquJTvswdy7k8/Dzn8ddiYgkUdQTnQ4EZgGbgYXAo+7+MHAqsBL4PbAIuBvA3dcAM4AlwIPA\n1PbhLvun8e8iEiUtVZAQra1wxBGwfDkMGhR3NSKSRFqqIKV+//uw/ozCXUSiooBPCHXPiEjUFPAJ\noeUJRCRq6oNPgO3bQ/97Swv07Rt3NSKSVOqDT6FFi2DUKIW7iERLAZ8Amr0qIrWggE8ABbyI1IL6\n4GO2YQN8/OOwcSMccEDc1YhIkqkPPmWeegpOP13hLiLRU8DHTOPfRaRWFPAxctf4dxGpnZKXC47a\nm2/G9c7JsXYt7N4Nw4fHXYmIZFFsAT9qVFzvnCyXXgpW8iUTEZHSaRSNiEhKaBSNiIgACngRkcxS\nwIuIZJQCXkQkoxTwIiIZpYAXEckoBbyISEYp4EVEMkoBLyKSUQp4EZGMUsCLiGSUAl5EJKMU8CIi\nGaWAFxHJqA4D3oIGM3vbzDaa2czCthPMrMnM3jWzF8zs9KJjppnZJjNrMbPza/9XEBGRfekw4AuL\ntl8DHAaMBMYDnwe+BdwHHAz8I/D/AMxsKHAjMBa4EJhlZgfWqngJ8vl83CVkij7P6OizjFenXTTu\nvtzddwKtwC5gA/AaYIXjrfAcYDJwv7uvdvfFQCMwsRaFy176RxQtfZ7R0WcZr5Ju2Wdm7xPCfLa7\nLzGzV4HFwB2EwB9f2LUeWFd0aDMwMLpyRUSkVCVdZHX3nsAQ4FNmdg4wB7gb6APcQOiuERGRBCnr\nnqxmdhNwEHAZMMrdNxa27yhsnwoMdvdphe3zCa3+Be3OoxuyiohUoJx7snbYRWNm9cAg4LnCn18k\nXFRdAXzFzL4HXAC85u67zOxx4Bkzm024MDsGuKiaAkVEpDKd9cEfCMwCRgCbgLnu/pCZrQB+BEwH\nXgEuBXD3V8xsBrAE2AFMdfcdNapdREQ6UFYXjYiIpEeXz2Q1swlm1mxm28zs/3b1+2dNYQJaa+Gx\nLe560sTM7jOzt8zs+aJtfc1sgZltL0zyGxBnjWmyn89zRtH3s1WTH0tjZvVm9riZ/bEwafTawvay\nvp9dGvBmZsCPgWuB4cAFZnZKV9aQQbvcva7w6Bt3MSnzXeAv2227AdgIHEkYCnxbVxeVYvv6PB24\npOg7+nAMdaVRb+BfgaMIn+n/MbMRlPn97OoW/CeBre4+391fB+YC+o0usXD33wJvt9s8GZjh7luB\nfwbO6/LCUmo/nyeEyZBSBndvdvd57v5Hd38BWA4MoMzvZ1cH/EBgbdFzTYSqXg8z22pm683sxriL\nyYA931F33wz0MrMD4i0p9b5f6JJ9xMwOj7uYtCksATOY0GIv6/sZ92qS+s1evdOBw4FJwJVmlou3\nnEzS97Ry3ycMsT4KeAv4TrzlpIuZ9Qd+AfyNu2/f3277O76rA3494TdRm6ML26RC7v6yu+8q/Dfu\nEeCEuGtKmfbDyFoIs7bb/nG95+7vdXVRKfaBz7PQ1fB2oUvhu+j7WTIz6ws8Dtzm7osKm8v6fnZ1\nwD8HHGJmk81sIHA5IZSkAmZ2RGHp5o+Y2fHAuYS+Oild+9bPY8ANZtYP+Br6fpbrA5+nmZ1mZgeb\n2aGElWn1/SxBYRXe+YS5R48WvVTe99Pdu/RBWF2ymXAx5h+7+v2z9ACGAc8D2wn9cv877prS9AAW\nEVZJbXtcC/QFFhAm6jUAH4u7zrQ89vF5XkcYCbIV2AI8CBwWd51peABntPssW4GLy/1+aqKTiEhG\nxX2RVUREakQBLyKSUQp4EZGMUsCLiGSUAl5EJKMU8CIiGaWAFxHJKAW8iEhG/X+fsU14pzGCEQAA\nAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results = [evolve() for _ in range(20)]\n", + "plt.plot(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After resetting the archipelago and ZeroMQ island, I'll start another instance of this code on a different window (because IPython can't execute two processes simultaneously) and see how the performance compares." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "del zmqisl\n", + "\n", + "zmqisl2 = zmq_island(algo, prob, 20)\n", + "zmqisl2.set_broker_details(\"127.0.0.1\", 6379)\n", + "zmqisl2.set_token(\"schwefel10_de10_pop20\")\n", + "zmqisl2.set_ip(\"127.0.0.1\")\n", + "\n", + "archi2 = archipelago(algo, prob, 3, 20)\n", + "archi2.push_back(zmqisl2)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEACAYAAAC57G0KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGMNJREFUeJzt3X2wVPV9x/H3B5AHjSAIMUIDKI5NYxybaUKMVrsxbbUt\nkkrVZBrjQ2OMU0eUFGrS2vaWaSONhHRihGlIa2OeiDHUFILGitkGRmgxGUhsTcRIZIJFnsVE7gW5\n3/5x9spyvdy7D2fvOXv4vGZ2vPfsefix2Xzud38PZxURmJlZ8QzJugFmZtYaDngzs4JywJuZFZQD\n3sysoBzwZmYF5YA3MyuomgJe0hBJ6yWtqfy+UFJ31WNW1b5zJO2WtK16u5mZDa5aK/g/BZ4BeibN\nB3BNRAypPJYDSJoGzAPOB64E7pU0KuU2m5lZDQYMeEkTgSuAJYCqn+pj95nAsojYHBHrgA3AJWk0\n1MzM6lNLBb8IuAPo7rV9saT9kh6SNKGybSKwtWqfLcCk5ptpZmb16jfgJV0G7IuIJzm6Yl8M/Aow\nGdgB3N2yFpqZWUOGDfD8hcBNkm7q2SDpvyNietXv9wD3V37dBkypOv5M4Du9TyrJN8AxM2tARPTV\nPd4n1XqzMUnvAhZGxEWSLgY2kXwC+AfghIi4QdJZwPeAEjAeWA5MjYjOXueKZm5y9sEPwuHDcN55\nDZ8iFf/0T3DfffCe92Tbjo6ODjo6OrJtRIH49UyPX8t0Saor4Aeq4I86N0dm0XwEuLzy+2rgZoCI\neFbSQmA90Anc0jvc07BjB8ydC5demvaZ6zNtWtKODRtgiFcUmFnO1BxLEbE+Ii6u/PyhiDglIsZG\nxJURsatqv0URMS4iJkbEN1vR6B074LTTWnHm+lx1FZxwAnzlK1m3xMzs9dqy7nzxRXjjG7NuBUjw\n6U/DX/4lHDiQXTtKpVJ2Fy8gv57p8WuZrZr74FO9aBN98N3dMGIEvPJKUj3nwZVXwm/8BnziE1m3\nxMyKrN4++LYL+J074S1vgd27U25UE559Fs4/H/73f/PxycLMiqnegG+7LpodO/IXomedBR/6EHiy\ngB3vImDfPnj6aXj8cfj+97Nu0fGtnlk0uZCXAdbe/uqvkk8Wt94Kv/ZrWbfGLF09wf3CC/B//5c8\nen7u/d9hw+D002HiRPjJT5KZZnPmJGNWNrjaLuDzMsDa27hx8PGPw5//OaxYkXVrzJqzaxfMmwfP\nPHMk0EeMOBLcp5+ePKZMSbonq7e94Q1HzrN1K8ycmVT0994Lw4dn9286HrVdH/xnPwubN8M996Tc\nqBR0dSXV+xe+AJf4FmvWpp57Dn7v9+AP/gBmzToS3Cee2Nj5fvGLZHHi/v3wzW8mxZA1pvB98Hmt\n4CGpcBYsSD6Sdve+NZtZG/jBD+Cii2D2bFi0CH7zN5MFfY2GOyQV/fLl8I53JNX+M8+k117rX9sF\nfF774HtcdVUS9F78ZO3m0UeT1eGf+xzccku65x46FO6+G+64I/kDsnp1uue3vrVdwOe5goejFz+9\n8krWrTGrzf33JzPB/u3f4IorWnedD38Yvv71pMvm859v3XUs0XYBn8dpkr1dcAG8613wj/+YdUvM\n+hcBd90Ff/3XUC4nXTKtVirB2rVJF9CcOcmNA6012m6Q9cwz4T/+I+kXzLOf/jQJ+f/5n3x3Kdnx\n6/DhpK997Vp4+OFkJsxg2rs36dIcORK++lUYPXpwr9+OCr+S9aSTYPt2OPnklBvVAh/7WHKPmiVL\nsm6J2dEOHEi6SfbtS7plxozJph2HDh35I7NiBUydmk07+vPqq/CjH8HBg82dZ/To5tfIFDrgf/lL\nmDAh+W87LJrYswd+9VfhP/8T3vrWrFtjltizJ5mbPnly8n0GI0Zk256IZNrzggXw4INJF2fW9uyB\nRx6BlSuT//ae39+I6dObn95d6IB/7rlkfvnPfpZ+m1pl0SL47ne9+Mny4fnnkznuv//78KlP5et7\nDB5+GK67Dj7zmeTTxWCKSBZjrVyZPDZuTL7IZ8aM5LWalJNvli50wK9fD7fdBv/1Xy1oVIt0dSXV\n+9KlXvxk2dq0KVm8NHcu3H571q3p21NPweWXwzXXwN/+bWv/AHV1JZ+ue0L91VeTa8+YkQwEjxrV\nums3qpXf6JS5vE+R7EvP4qc/+7Pkxkt5qpjs+PH44/CBDyRz3K++OuvWHNvb3pYUcFdcAT/+MXzx\ni80tsupt+3ZYtSoJ9NWrk+vNmAHf+lbyczt0/dajrSr4pUuTKv6f/7kFjWqhCLjwQrj5Zrj22qxb\nY8ebr30t+eT7wANJZdoOOjvhIx9JQv6uu5r77odDh+CJJ5JQ37wZfvd3k1C/7LJkTK+dFLqL5u//\nPhlg/eQnW9CoFlu3LqmcfvKTdCsSs/58+tPJeoxVq+Dcc7NuTX0iYOHCJJibISVfyDNjRjLPPy9f\nFNSIQgf87NnJPPi89h8O5Oqr4bzzklWuZq3U3Z10Cz76aDIL5M1vzrpFloZC98Hv2AHvfnfWrWjc\nXXclU6VuvNGLn+z1IpKbff3rvybzrpuxd28yt33tWhg7NpXmWRtqq4Bvx0HWatOmJdPAOjq8+MmO\n2LEjuTndffclt9a9/nr4m79pbsBvyJBkJXXWc9wtW23VRXPOOcmNit72thY0apDs2ZN881O57MVP\nx7NDh5J+8fvuS94L73sf3HADXHyxZ1rZsRW6D378+GJ8sfVnPpNM0Wp28Mjaz1NPJaH+la8k3+V7\nww3J2Ew73HrDslfYgH/11eSmRF1dyb2l21nP4qfPfx7e+96sW2OttndvMlXxvvuSr7679tqkG+bs\ns7NumbWbwgb89u3JDJQXX2xRowbZgw8m0z6ffLL9/2DZ6x0+DI89loT6I48kc66vvx5+53f8v7c1\nrrABv2lTsny52dkFedGz+Gn69OSGZFYcW7fCl78Mb3pT0gXzgQ/4e0gtHYWdJpn3r+qrl5R8Offn\nPgc//GHWrbE0jR3bnguLrHhqCnhJQ4AngEMRcZGk0cAy4D3Ak8CVEfFiZd85wJ1AJ3BrRCxPo6Ht\nPkWyL299KyxenHUrzKyoap2Q9afAM0BPv8pcYBdwOrAOmA8gaRowDzgfuBK4V1Iq92QrWgVvZtZq\nAwa8pInAFcASoKfvZyawMCL2AQsqz/dsXxYRmyNiHbABSOUmuUWs4M3MWqmWCn4RcAfQXbVtEvA8\nQETsAYZLGg5MBLZW7belsm/T2uHLts3M8qTfgJd0GbAvIp7kSPV+zN1Ta1Uf3EVjZlafgQZZLwRu\nknRTzwZJG4CfA1OBTZJOBQ5GRJekbcCUquPPBL7T14k7Ojpe+7lUKlEa4EbV7qIxs+NNuVymXC43\nfHzN8+AlvYuk3/0iSfOBM4DbgE8AYyLiJklnAd8DSsB4YDkwNSI6e52r7nnwkyfDmjUwZcrA+5qZ\nFVEr58GLI7NoFpJMk3yBZCD1KoCIeFbSQmA9yTTJW3qHeyMi3AdvZlavtljJ+tJLyRcW7N/fwkaZ\nmeVcvRV8W9yY1P3vZmb1a4uAd/eMmVn92ibgPUXSzKw+bRHw7qIxM6tfWwS8K3gzs/q1RcC7gjcz\nq19bBLwHWc3M6tcWAf/ii+6iMTOrV1sEvCt4M7P6tU3Au4I3M6tP7gO+qwt++Us45ZSsW2Jm1l5y\nH/A7d8KECTAk9y01M8uX3Memp0iamTUm9wHvAVYzs8bkPuA9RdLMrDG5D3hX8GZmjWmLgHcFb2ZW\nv9wHvAdZzcwak/uAdwVvZtaY3Ae8K3gzs8bkPuA9yGpm1hhFxOBfVIpartvdDSNGJLcqGD58EBpm\nZpZjkogI1bp/riv4vXvhDW9wuJuZNSLXAe8BVjOzxuU64D3AambWuFwHvCt4M7PG5TrgXcGbmTUu\n1wHvKZJmZo3LdcD7TpJmZo3rN+CVWCPpZUm7JH22sm2hpO6qx6yqY+ZI2i1pW/X2RriCNzNr3LD+\nnoyIkHQr8DQwFvg2MAMI4JqI+Gr1/pKmAfOA84HxwHJJD0fEgUYa50FWM7PGDdhFExEbI6IL6AYO\nATsrT/W1mmomsCwiNkfEOmADcEmjjfMgq5lZ42rqg5f0KrAd+H5ErK9sXixpv6SHJE2obJsIbK06\ndAswqdHGuYI3M2tcv100PSJimKTJwDckvQ9YDMwHhgKfAu4Grq/nwh0dHa/9XCqVKJVKRz3/yitw\n6BCcfHI9ZzUzK45yuUy5XG74+LpuNibpDuDkiLizatu5wP0R8XZJtwNTImJO5bkVwJKIWNXrPAPe\nbOxnP4OLL4atW/vdzczsuJHqzcYkTZQ0XdJISWcBVwM/kHSRpDGSxgK3Ahsrh6wE3i/pbEkXAO8E\nHm/kH+IpkmZmzRmoi2YUcC9wDrAbWBoRyyV9CbicZDbNauBmgIh4VtJCYD3QCdwSEZ2NNMxTJM3M\nmjPQNMmfklThvbd/qJ9jFgGLmm2YB1jNzJqT25WsniJpZtac3Aa8K3gzs+bkNuBdwZuZNSe3Ae9B\nVjOz5uQ24D1N0sysObkNeFfwZmbNqWsla2oXHWAl6+HDMHIkHDgAw2q6mYKZWfGlupI1K7t2wSmn\nONzNzJqRy4D3FEkzs+blMuA9RdLMrHm5DHgPsJqZNS+XAe8pkmZmzctlwLuCNzNrXm4D3hW8mVlz\nchnwHmQ1M2teLgPeFbyZWfNyGfCu4M3Mmpe7gI/wIKuZWRpyF/AvvwxDh8JJJ2XdEjOz9pa7gHf1\nbmaWjlwGvAdYzcyal7uA9wCrmVk6chfwruDNzNKRu4B3BW9mlo7cBbwHWc3M0pG7gPedJM3M0pG7\ngHcFb2aWjlwGvCt4M7Pm9RvwSqyR9LKkXZI+W9k2WtIqSQcqz59WdcwcSbslbZM0q94GeZDVzCwd\nioj+d5B+HXgaGAt8G+gA3glMBWYDfwGMiYiPSpoGrAF+CxgPLAfOjIgDvc4ZfV334MHkFgVdXTAk\nd58tzMyyJYmIUK37DxijEbExIrqAbuAQsBOYCSyMiH3AAuCKyu4zgWURsTki1gEbgEtqbczOnTB+\nvMPdzCwNNUWppFeB7cD3I2I9MAl4HiAi9gDDJQ0HJgJbqw7dUtm3Jh5gNTNLz7BadoqIYZImA9+Q\n9IfH2K3mjw0AHR0dr/1cKpUolUqeImlmVqVcLlMulxs+fsA++KN2lu4ATgZmANdFxCZJpwJPR8Qb\nJd0OTImIOZX9VwBLImJVr/P02Qd///3w6KPw5S83/O8xMyusVPvgJU2UNF3SSElnAVcDPwD+HZgr\naRzwceChyiErgfdLOlvSBSSDsY/X2hhPkTQzS89AXTSjgHuBc4DdwNKIWC7pMWAZ8ALJQOpVABHx\nrKSFwHqgE7glIjprbYynSJqZpaeuLprULnqMLprrroNSCW64YdCbZGaWe6lPkxxMHmQ1M0tPrgLe\n0yTNzNKTu4B3BW9mlo7c9MFHwIgRsH8/jBw56E0yM8u9tu2D37cPTjzR4W5mlpbcBLynSJqZpSs3\nAe8BVjOzdOUm4D1F0swsXbkJeFfwZmbpylXAu4I3M0tPbgLeg6xmZunKTcC7gjczS1duAt4VvJlZ\nunIT8B5kNTNLV24C3tMkzczSlYuAP3AAurpgzJisW2JmVhy5CPidO5PuGdX1td1mZtafXAS8B1jN\nzNKXi4D3FEkzs/TlIuBdwZuZpS8XAe8pkmZm6ctFwHuKpJlZ+nIR8K7gzczSl5uAdwVvZpauXAS8\nB1nNzNKXi4B3BW9mlj5FxOBfVIqe6x4+DCNHwiuvwAknDHpTzMzahiQiouY1/5lX8Hv2wOjRDncz\ns7T1G/CSJkpaKeklSdskza5sXyipu+oxq+qYOZJ2V/afdeyzJzxF0sysNYYN8PwI4EvAB4E3A49L\nWg0EcE1EfLV6Z0nTgHnA+cB4YLmkhyPiwLEu4CmSZmat0W/AR8QWYEvl15ckbQR66u2++oFmAssi\nYjOwWdIG4BLg28e6hgdYzcxao+Y++Ep1PhV4orJpsaT9kh6SNKGybSKwteqwLcCk/s7rKZJmZq0x\nUBcNAJJOBR4A/iQiOiUtBuYDQ4FPAXcD19dz4Y6ODgBWr4azzy4BpXoONzMrvHK5TLlcbvj4AadJ\nShoNfAdYEBHf6uP5c4H7I+Ltkm4HpkTEnMpzK4AlEbGq1zGvTZO88UaYPh1uuqnhf4OZ2XEh1WmS\nkkYBK4Cl1eEu6WJJYySNBW4FNlaeWgm8X9LZki4A3gk83t81PMhqZtYaA/XBXwhcBHyhakrkB4GP\nAM8DzwHjSGbOEBHPAguB9cCDwC0R0dnfBTxN0sysNTJfyXrGGfDYYzBt2qA3w8ysrbTdSlZPkzQz\na41MA/4Xv4AIOOmkLFthZlZMmQZ8T/Wumj9wmJlZrTINeC9yMjNrncwreAe8mVlrZF7Be4DVzKw1\nXMGbmRVU5gHvCt7MrDUy76JxBW9m1hqZV/AOeDOz1si8gncXjZlZa7iCNzMrqMxuNnbwYHDiidDZ\nCUOHDnoTzMzaTtvcbGzXLhg3zuFuZtYqmQW8p0iambVWZgHvKZJmZq2VaQXvgDcza51MK3h30ZiZ\ntY4reDOzgnIFb2ZWUK7gzcwKytMkzcwKytMkzcwKKrNbFQwfHuzbB6NGDfrlzczaUtvcqmDECIe7\nmVkrZRbw7p4xM2utzALeA6xmZq3lCt7MrKD6DXhJEyWtlPSSpG2SZle2j5a0StIBSWsknVZ1zBxJ\nuyv7zzrWuV3Bm5m11kAV/AjgS8Bk4FLgTknnAHOBXcDpwDpgPoCkacA84HzgSuBeSX0OpbqCNzNr\nrWH9PRkRW4AtlV9fkrQROA2YCVwbEfskLQB+DHy0sn1ZRGwGNkvaAFwCfLv3uR3wZmatVXMffKU6\nn0JSsU8CngeIiD3AcEnDgYnA1qrDtlT2fR130ZiZtVZNAS/pVOAB4MMRceBYu9VzYVfwZmat1W8X\nDSQDqsBKYH5ErK1s3gZMBTZVwv9gRHRJ2kZS5fc4E/hOX+f9xjc6+O53k59LpRKlUqnBf4KZWTGV\ny2XK5XLDx/d7q4LKAOkjwBcj4l+qts8HzgBuAz4BjImImySdBXwPKAHjgeXA1Ijo7HXe2L07GDeu\n4XabmR136r1VwUAB/9vAo702X0NS0S8jGUDdAFwVEdsrx3wMuBPoBG6NiG/2cd7o7g5UV6eOmdnx\nLdWAbxVJkcV1zczaWdvcbMzMzFrLAW9mVlAOeDOzgnLAm5kVlAPezKygHPBmZgXlgDczKygHvJlZ\nQTngzcwKygFvZlZQDngzs4JywJuZFZQD3sysoBzwZmYF5YA3MysoB7yZWUE54M3MCsoBb2ZWUA54\nM7OCcsCbmRWUA97MrKAc8GZmBeWANzMrKAe8mVlBOeDNzArKAW9mVlAOeDOzgnLAm5kV1IABL+lr\nknZI+lHVtoWSuqses6qemyNpt6Rt1dvNzGxw1VLB3wNc2mtbANdExJDKYzmApGnAPOB84ErgXkmj\n0mywvV65XM66CYXi1zM9fi2zNWDAR8QTwMt9PKU+ts0ElkXE5ohYB2wALmmuiTYQ/58oXX490+PX\nMlvN9MEvlrRf0kOSJlS2TQS2Vu2zBZjUxDXMzKxBjQb8YuBXgMnADuDu1FpkZmapUEQMvFPSt/5Q\nRJzbx3PnAvdHxNsl3Q5MiYg5ledWAEsiYlWvYwa+qJmZvU5E9NU93qdhNe531AklXQxsIvkEcCuw\nsfLUSuB7kpYA44F3Alc100AzM2vMgAEvaS1wQeXnbmAO8A7gcpLZNKuBmwEi4llJC4H1QCdwS0R0\ntqbpZmbWn5q6aMzMrP0M+kpWSe+RtKUyA+fvBvv6RSNpV9WCs/1Zt6edHGMR32hJqyQdkLRG0mlZ\ntrGd1Lso0o5N0kRJKyW9VFk0Oruyva7356AGvCQB/wLMBt4C/JGkdw9mGwroUNWCs9FZN6bN9LWI\nby6wCzgdWAfMH+xGtbGaF0XagEYAXyKZqXgpcKekc6jz/TnYFfyvA/siYkVEvAAsBfwX3TJxjEV8\nM4GFEbEPWABcMegNa1N1Loq0fkTEloj4ekS8FBFPkUxkOY0635+DHfCTgOerfvdCqOYNlbRP0s8l\nzcu6MQXw2ns0IvYAwyWdkG2T2l5fiyKtRpVp6lNIKva63p9Z303Sf9mb91vABOAy4KOSStk2p5D8\nPm2cF0U2QdKpwAPAhyPiwLF2O9bxgx3wPyf5S9TjjMo2a1BEPB0Rhyof4x4Czsu6TW2m9zSybcBU\neO3/XAcj4uBgN6qNHfV6VroaXq50KdyD3581kzSaZG3R/IhYW9lc1/tzsAP+h8ApkmZKmgTcSBJK\n1gBJb5R0nqSRlRXFf8iRRWdWm97Vz78DcyWNAz6O35/1et2iSEljJI3l6EWR1o/KXXhXAEsj4ltV\nT9X3/oyIQX2Q3F1yC8lgzCcH+/pFegBnAz8CDpD0y30s6za10wNYC3RXPWYDo4FVJAv11gBvyrqd\n7fLo4/W8jWQmyD5gL/AgMD7rdrbDA/jtXq9lN/DH9b4/vdDJzKygsh5kNTOzFnHAm5kVlAPezKyg\nHPBmZgXlgDczKygHvJlZQTngzcwKygFvZlZQ/w8SMFNUOXD3CwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results2 = [evolve() for _ in range(20)]\n", + "plt.plot(results2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, we get to the result much more quickly because the off-screen instance of this program had been working for a few seconds and was very close to the answer already." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/doc/sphinx/documentation/island.rst b/doc/sphinx/documentation/island.rst index 6b39412e..07c25043 100644 --- a/doc/sphinx/documentation/island.rst +++ b/doc/sphinx/documentation/island.rst @@ -4,7 +4,7 @@ Island and Archipelago NOTE: There are two different types of islands in PyGMO. The PyGMO.local_island and the PyGMO.py_island. The need for two different types of island is purely technical (it is necessary to give the user the possibility of implementing the problem or the algorithm in python directly) and the user needs not to know any details. We will thus here document only one class that we call -island and that, in reality, is a helper function returning automatically the correct object type. +island and that, in reality, is a helper function returning automatically the correct object type. .. class:: PyGMO.island @@ -27,7 +27,7 @@ island and that, in reality, is a helper function returning automatically the co .. method:: PyGMO.island.__init__((PyGMO.algorithm)algo, (PyGMO.problem)prob, (int)N=0 [, s_policy = best_s_policy(1), r_policy=fair_r_policy(1)]) - Constructs an island directly from a problem. The resulting population (of size N) will be evolved by algo. + Constructs an island directly from a problem. The resulting population (of size N) will be evolved by algo. Migration occurs at the end of each evolution if the *island* belongs to an :class:`PyGMO.archipelago` .. code-block:: python @@ -38,17 +38,17 @@ island and that, in reality, is a helper function returning automatically the co isl = island(algo,prob,20) .. method:: PyGMO.island.evolve((int)n) - + Evolves the :class:`PyGMO.population` in the *island* performing n calls to :class:`PyGMO.algorithm`. At the end of each call migration occurs if the *island* belongs to a :class:`PyGMO.archipelago`. Evolution happens in the background on a dedicated thread while the program flows continues. - + .. code-block:: python from PyGMO import * prob = problem.schwefel(1500) algo = algorithm.de(100) #instantiates differential evolution with default params and 100 generations - pop = population(prob,20) + pop = population(prob,20) isl = island(algo,pop) isl.evolve(1) #calls algo to evolve pop. The program flow continues @@ -62,7 +62,7 @@ island and that, in reality, is a helper function returning automatically the co from PyGMO import * prob = problem.schwefel(1500) algo = algorithm.de(100) #instantiates differential evolution with default params and 100 generations - pop = population(prob,20) + pop = population(prob,20) isl = island(algo,pop) isl.evolve(1) #calls algo to evolve pop. The program flow continues isl.join() #Waits for the evolve to finish (i.e. waits for completion of the 100 generations of differential evolution) @@ -73,7 +73,7 @@ island and that, in reality, is a helper function returning automatically the co .. method:: PyGMO.island.set_x((int)idx,(list) x) - Sets a new chromosome for the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` + Sets a new chromosome for the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` of the *island* to x. .. code-block:: python @@ -82,14 +82,14 @@ island and that, in reality, is a helper function returning automatically the co prob = problem.ackley(5) algo = algorithm.de(10) #instantiates differential evolution with default params and 10 generations isl = island(algo,prob,10) - isl.population.set_x(0,[1,2,3,4,5]) # This is completely uneffective + isl.population.set_x(0,[1,2,3,4,5]) # This is completely uneffective # as the 'attribute' population returns a copy isl.set_x(0,[1,2,3,4,5]) # This works!! - + .. method:: PyGMO.island.set_v((int)idx,(list) v) - Sets the velocity of the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` + Sets the velocity of the idx-th :class:`PyGMO.individual` in the :class:`PyGMO.population` of the *island* to v. .. code-block:: python @@ -98,7 +98,7 @@ island and that, in reality, is a helper function returning automatically the co prob = problem.ackley(5) algo = algorithm.de(10) #instantiates differential evolution with default params and 10 generations isl = island(algo,prob,10) - isl.population.set_v(0,[0.02,0.03,-0.3,0.12,0.1]) # This is completely uneffective + isl.population.set_v(0,[0.02,0.03,-0.3,0.12,0.1]) # This is completely uneffective # as the 'attribute' population returns a copy isl.set_v(0,[0.02,0.03,-0.3,0.12,0.1]) # This works!! @@ -129,7 +129,7 @@ island and that, in reality, is a helper function returning automatically the co .. class:: PyGMO.archipelago - Probably the most important object in all PyGMO. An *Archipelago* is a container of :class:`PyGMO.island` + Probably the most important object in all PyGMO. An *Archipelago* is a container of :class:`PyGMO.island` and is responsible to start the asynchronous island model. The solutions exchange among :class:`PyGMO.island` is done following the routes allowed by the underlying topology @@ -149,7 +149,7 @@ island and that, in reality, is a helper function returning automatically the co Constructs an empty *archipelago* from a topology (defaults to :class:`PyGMO.topology.unconnected()`) a distribution type (defaults to :class:`PyGMO.distribution_type.point_to_point`) and a migration - direction (defaults to :class:`PyGMO.migration_direction.destination`). + direction (defaults to :class:`PyGMO.migration_direction.destination`). It then pushes back into the archipelago n_isl :class:`PyGMO.island` constructed using defaults values for the kwargs and (algo,prob,n_ind) as args. @@ -163,10 +163,10 @@ island and that, in reality, is a helper function returning automatically the co #and a copy of the griewank(30) problem .. method:: PyGMO.archipelago.evolve((int)n) - - Calls the method :class:`PyGMO.island.evolve` (n) on all the :class:`PyGMO.island` of the *archipelago*. + + Calls the method :class:`PyGMO.island.evolve` (n) on all the :class:`PyGMO.island` of the *archipelago*. In other words, it starts the asynchronous generalized island model that is at the core of PyGMO. - + .. code-block:: python from PyGMO import * @@ -174,9 +174,9 @@ island and that, in reality, is a helper function returning automatically the co algo = algorithm.de(500) #instantiates differential evolution with default #params and 100 generations archi = archipelago(algo,prob,8,20) - archi.evolve(10) #starts the asynchronous generalized island model. + archi.evolve(10) #starts the asynchronous generalized island model. #each of the 8 islands will call algo 10 times and try to migrate in between calls - + .. method:: PyGMO.archipelago.push_back((PyGMO.island) isl) Pushes back isl in the archipelago taking also to also update the topological links between islands. @@ -195,17 +195,17 @@ island and that, in reality, is a helper function returning automatically the co algo = algorithm.de(500) #instantiates differential evolution with default #params and 100 generations archi = archipelago(algo,prob,8,20) - archi.evolve(10) #starts the asynchronous generalized island model. + archi.evolve(10) #starts the asynchronous generalized island model. archi.join() #waits for it to finish [isl.population.champion.f for isl in archi] #builds a list with the best fittnesses found .. method:: PyGMO.archipelago.busy() - Returns True if evolution is ongoing in the *archipelago*. + Returns True if evolution is ongoing in the *archipelago*. .. method:: PyGMO.archipelago.interrupt() - Halts evolution at the first occasion in all islands. + Halts evolution at the first occasion in all islands. .. automethod:: PyGMO.archipelago.draw() @@ -219,8 +219,85 @@ island and that, in reality, is a helper function returning automatically the co .. method:: PyGMO.archipelago.dump_migration_history() - Returns a temporal history of all the archipelago migrations in one string. Each entry is in the form + Returns a temporal history of all the archipelago migrations in one string. Each entry is in the form (n,src_isl,dest_isl) and logs that n individuals, from the *island* having the index src_isl, successfully replaced n individuals in the *population* of the *island* having the index dest_isl. - + +.. class:: PyGMO.zmq_island + + The ZeroMQ island behaves like any :class:`PyGMO.island` because it manages the algorithm, population and migration strategies, + but it also communicates with other ZeroMQ islands that are in the same configuration. This is used to distribute the computation + workload between networked machines. If it is not initialised, it will behave exactly like a local island. + + .. method:: PyGMO.zmq_island.__init__((PyGMO.algorithm)algo, (PyGMO.population)pop [, s_policy = best_s_policy(1), r_policy=fair_r_policy(1)]) + + Instantiates a :class:`PyGMO.zmq_island`, with the given population and algorithm. After every evolution, the island will broadcast a copy of the population to the ZeroMQ network. + + .. code-block:: python + + from PyGMO import * + prob = problem.schwefel(2) + algo = algorithm.de(10) + pop = population(prob,20) + + isl = zmq_island(algo,pop) + + Note that because the parameters specific to the ZMQ island are not set yet, this island cannot communicate and will work exactly like a local island. + + .. method:: PyGMO.zmq_island.set_broker_details((string) host, (int) port) + + Stores the broker host and port. At the moment, the broker software is a Redis server. + + .. code-block:: python + + from PyGMO import * + isl = zmq_island(a, p) + isl.set_broker_details("127.0.0.1", 6379) + + .. method:: PyGMO.zmq_island.set_token((string) token) + + Sets the token (also known as channel topic), which is used to determine which nodes are working on a problem with similar configurations. + + .. code-block:: python + + isl.set_token("schwefel2_de10_pop20") + + The token can be any string, but describing the algorithm and problem in the token is useful to monitor the progress of the computation. A way of monitoring the progress is described in the tutorial (TODO: link), where a user connects to the ZeroMQ network but does not perform evolution tasks. + + .. method:: PyGMO.zmq_island.set_ip((string) ip) + + This method sets the IP address to be used for incoming connections. + + .. method:: PyGMO.zmq_island.set_evolve((bool) evolve) + + If `set_evolve(false)` is called, then this island will not perform any evolution operations. + + .. method:: PyGMO.zmq_island.connect() + + This method will initialise the communication capabilities of the ZeroMQ island by communicating with the network, connecting to its peers and broadcasting the new connection. + + The IP address given as an argument will be used to bind the receiving socket, so it must be accessible by all of the nodes in the network. A receiving port will be chosen at random, from 1000 to 2000. + + .. code-block:: python + + from PyGMO import * + import time + + prob = problem.schwefel(10) + algo = algorithm.de(10) + pop = population(prob,20) + + isl = zmq_island(algo,pop) + isl.set_broker_details("127.0.0.1", 6379) + isl.set_token("schwefel2_de10_pop10") + isl.set_ip("127.0.0.1") + isl.connect() + + while True: + isl.evolve(10) + print("Best: ", isl.population.champion.x) + + time.sleep(1) + + The given example will solve the 10-dimensional Schwefel problem locally, but if another instance of this program is started up in the same computer, they will exchange solutions until they converge on a solution. diff --git a/doc/sphinx/images/zmq_monitoring.png b/doc/sphinx/images/zmq_monitoring.png new file mode 100644 index 00000000..9cac0a9e Binary files /dev/null and b/doc/sphinx/images/zmq_monitoring.png differ diff --git a/doc/sphinx/tutorials/distributed_computation_with_zeromq.rst b/doc/sphinx/tutorials/distributed_computation_with_zeromq.rst new file mode 100644 index 00000000..601efb94 --- /dev/null +++ b/doc/sphinx/tutorials/distributed_computation_with_zeromq.rst @@ -0,0 +1,12 @@ +.. _zeromq: + +=================================== +Distributed computation with ZeroMQ +=================================== + +.. toctree:: + :maxdepth: 2 + + zmq_setup + zmq_first_contact + zmq_monitor diff --git a/doc/sphinx/tutorials/index.rst b/doc/sphinx/tutorials/index.rst index 5d6af46f..554db478 100644 --- a/doc/sphinx/tutorials/index.rst +++ b/doc/sphinx/tutorials/index.rst @@ -15,3 +15,4 @@ This is a collection of tutorials on PyGMO's basics: racing tsp landscape_analysis_with_DrPyGMO + distributed_computation_with_zeromq diff --git a/doc/sphinx/tutorials/zmq_first_contact.rst b/doc/sphinx/tutorials/zmq_first_contact.rst new file mode 100644 index 00000000..b196e303 --- /dev/null +++ b/doc/sphinx/tutorials/zmq_first_contact.rst @@ -0,0 +1,80 @@ +.. _zeromq_first_contact: + +============= +First contact +============= + +Noe that we have a Redis server running and we have compiled PaGMO and PyGMO with ZeroMQ support (see :ref:`zeromq_setup`), we can run an example program that leverages the functionality of this type of island. + +.. code-block:: python + :linenos: + + from PyGMO import * + import time + + prob = problem.schwefel(10) + algo = algorithm.de(10) + pop = population(prob, 20) + + isl = zmq_island(algo, pop) + isl.set_broker_details("127.0.0.1", 6379) + isl.set_token("schwefel10_de10_pop20") + isl.set_ip("127.0.0.1") + isl.connect() + + while True: + isl.evolve(10) + print("Best: ", isl.population.champion.x) + + time.sleep(1) + +This code will set up a single ZeroMQ island working on the 10-dimensional Schwefel problem and using the Differential Evolution algorithm. The population of the island will be 20. Lines 4-6 set up the problem, and lines 8-12 set up the island itself. + +In this tutorial we assume that the broker is at 127.0.0.1:6379, and we will be binding the receiving port only on the loopback inteface, but in a networked setup you would use IPs that are accessible beyond localhost. + +The channel token is set to `schwefel10_de10_pop20`, which can be any string, but choosing a token relevant to the actual properties of the computation will help you determine which group of islands is working on what problem. If we start this program, it will connect to the broker and advertise itself as an island working on that token, but because there are no other peers it will simply perform the evolution locally. This is a typical output: + +.. code-block:: bash + + root@f3428b4e74bd:/# python /tmp/zmq.py + DEBUG: IP: '127.0.0.1:2401' + ('Best: ', (401.9594173543314, 424.7239031087063, -489.08965281305683, 192.79310987286283, 422.3702354040747, -280.2352091575768, -304.1105751377624, 419.61971508325234, -304.4306512922853, 412.3797020751874)) + ('Best: ', (401.9594173543314, 422.9252917377888, 397.6750493053389, -308.787292270248, 417.6016092014828, 415.24467267670747, -304.1105751377624, 419.61971508325234, 443.02530229342995, 412.3797020751874)) + ('Best: ', (417.7624473989758, 419.38998223162383, 423.9121066735651, 420.9581335823198, 423.1917112361371, 416.75217338872466, 416.5656801251844, 423.65078535816195, 427.8092589385355, 417.23729521493794)) + ('Best: ', (421.19930412300704, 420.80770338293433, 420.95461980350495, 421.1387381497945, 420.9882620455983, 420.7632718115494, 420.9945625003906, 420.68798717006837, 421.2449946368809, 420.9255606400046)) + ('Best: ', (420.97784637271286, 420.95030001558206, 420.97695438787514, 420.9925460783401, 420.9588231827722, 420.9747030586656, 420.96113438989653, 420.9744491052427, 420.9570307864317, 420.9708116286286)) + ('Best: ', (420.96796502949985, 420.9692843203323, 420.9674981715592, 420.96888513790157, 420.96902668827914, 420.96711830078385, 420.96698080790026, 420.9675721284126, 420.9690259384316, 420.9693847247998)) + +However, if we run two instances of this program simultaneously, they will connect to each other and start exchanging solutions. + + +.. code-block:: bash + :linenos: + + root@f3428b4e74bd:/# python /tmp/zmq.py + DEBUG: IP: '127.0.0.1:2839' + Best: (446.330415191082, -318.57151650616345, -319.141745784881, 409.37490624414585, 434.9522671118881, 436.11092724710943, 412.9042329474612, -327.3902497012161, 433.12563296150233, 405.2335813101326) + Best: (406.4818160886365, 429.4589205156737, 411.6918195936836, 412.4947146394673, 439.92422758511657, 413.4096775345739, 402.7337709186839, 410.8803588738704, 458.24230339934326, 436.5535606636335) + Best: (416.6611528308124, 424.2092721283991, 418.8522866953207, 422.8026613418712, 423.3550577222186, 420.2964268628981, 422.0640382301092, 422.3657526856015, 422.8089197437483, 421.1248410730861) + DEBUG: Opening connection to 127.0.0.1:2758 + Best: (420.8118964319755, 420.97105998402435, 420.8675085594526, 421.0028607982087, 420.95935903890796, 420.991509807542, 420.84368415020265, 421.08197535480474, 420.81508001828564, 421.10958460178466) + Best: (421.07749509648613, 420.8546192606146, 420.7665760480669, 421.17780038893375, 421.0733383225067, 421.0745532974556, 421.1227771446935, 421.23088854902136, 420.9196459791927, 421.0675596920381) + Best: (421.02737839902096, 421.37184378009175, 421.25678912369284, 421.0148941900929, 421.1551021658338, 421.0769746858749, 421.14611943756057, 420.829339492172, 421.08317952008963, 420.76028074583166) + Best: (420.98011516631004, 420.7954519611739, 420.99948716750913, 420.83980802618726, 420.9522553458674, 421.28867861574423, 420.7846133663843, 421.114115625814, 421.0752711472328, 421.1199157748332) + Best: (420.8864797873097, 420.93967731273574, 420.80247869292685, 420.8796939441704, 421.04690058491394, 420.9704882856439, 420.9380577878827, 420.98689425859016, 420.9460027704123, 421.2146020009918) + Best: (420.6410077561692, 420.8774213590491, 420.9770237342764, 420.8169234318866, 421.0771442746173, 420.7741545848519, 421.2015181934143, 420.75764749753006, 421.08444174574305, 421.2578651544008) + +And on the other side: + +.. code-block:: bash + :linenos: + + root@f3428b4e74bd:/# python /tmp/zmq.py + DEBUG: IP: '127.0.0.1:2758' + DEBUG: Opening connection to 127.0.0.1:2839 + Best: (170.02156365996962, 432.0808563954199, -299.9044328266113, 449.57827719040404, 414.32650241479433, 442.74256832933975, -497.56415018442954, 409.9770554543145, -128.49045206283606, 437.2655093302242) + Best: (420.8118964319755, 420.97105998402435, 420.8675085594526, 421.0028607982087, 420.95935903890796, 420.991509807542, 420.84368415020265, 421.08197535480474, 420.81508001828564, 421.10958460178466) + Best: (421.1237678412085, 420.9910361828901, 420.80333273611603, 421.0852572932901, 421.0638265999589, 420.97221532173234, 420.84697129401894, 421.1221559791297, 420.94034652647935, 421.01529447856126) + Best: (421.01592546566957, 421.2374973637021, 420.932682470239, 420.9163032282556, 421.02010123092407, 421.16968911726184, 421.09670372915707, 420.9404867930009, 420.94383616409635, 420.9957624510262) + +Here we can see that the last best value from the first instance is 416.6611528308124 (line 5), and the second program produces a better candidate, 420.8118964319755 (line 5 on the second program), which is sent over the ZeroMQ sockets and appears on the output of the first program after it opens a connection to the new peer (lines 6 and 7). diff --git a/doc/sphinx/tutorials/zmq_monitor.rst b/doc/sphinx/tutorials/zmq_monitor.rst new file mode 100644 index 00000000..7f22af16 --- /dev/null +++ b/doc/sphinx/tutorials/zmq_monitor.rst @@ -0,0 +1,122 @@ +.. _zeromq_monitoring: + +========================= +Monitoring a ZeroMQ swarm +========================= + +The ZeroMQ swarm is brokered by a Redis server, which contains a list of IP addresses and port numbers associated with any given channel token and hosts the realtime communication channel where nodes advertise when they become connected to or disconected from the network. + +Along with the Redis package in most distributions is a commandline utility called `redis-cli` which allows us to query the server. + +Observing the progress of the swarm +----------------------------------- + +The preferred way of obtaining results from the computation is to use a ZeroMQ island to join the swarm and receive the population by the same mechanism that all of the other peers receive it. The default behaviour of the island is to evolve the population locally, which can be disabled by calling `set_evolve(false)`: this way, the monitoring peer does not consume resources. + +In this example, we'll be monitoring the progress of an archipelago solving the Schwefel problem using differential evolution. The code for our "worker cluster" is the following: + +.. code-block:: python + + from PyGMO import * + import time + + prob = problem.schwefel(10) + algo = algorithm.de(10) + + zmqisl = zmq_island(algo, prob, 20) + zmqisl.set_broker_details("127.0.0.1", 6379) + zmqisl.set_token("schwefel10_de10_pop20") + zmqisl.set_ip("127.0.0.1") + + archi = archipelago(algo, prob, 3, 20) + archi.push_back(zmqisl) + + while True: + archi.evolve(1) + for isl in archi: + if isl.get_name().startswith("ZMQ"): + print(isl.population.champion.x[0]) + + time.sleep(1) + +We can imagine that this code would be deployed on a supercomputer, or on a remote computing cluster, but for demonstration purposes both programs will be running on the same computer. + +Now we'll create a program that instantiates a ZeroMQ island with the same problem, algorithm and population configuration, but that will perform no evolution and will instead display the best individual in the population, as computed by the worker code above. + +.. code-block:: python + + from PyGMO import * + import matplotlib.pyplot as plt + import time + + prob = problem.schwefel(10) + algo = algorithm.de(10) + + isl = zmq_island(algo, prob, 20) + isl.set_broker_details("127.0.0.1", 6379) + isl.set_token("schwefel10_de10_pop20") + isl.set_ip("127.0.0.1") + isl.set_evolve(False) # Don't evolve, "observer mode" + isl.connect() + + best_x = [] + for _ in range(20): + isl.evolve(1) + + # Get best individual from received population + champion_x = isl.population.champion.x[0] + best_x.append(champion_x) + print(champion_x) + + time.sleep(1) + + # Plot consensus value over time + plt.plot(best_x) + plt.show() + +And the result of running this program is: + +.. image:: ../images/zmq_monitoring.png + +Listing peers active on a channel +--------------------------------- + +.. code-block:: bash + + redis-cli smembers pagmo.islands. + +Example: + +.. code-block:: bash + + $ redis-cli smembers pagmo.islands.zeromq_test + 1) "192.168.1.39:2286" + 2) "192.168.1.39:1568" + 3) "192.168.1.39:2639" + +In this example all three connected islands are running on one machine, but any device that can reach the broker could advertise its IP. + +Monitoring control events on a channel +-------------------------------------- + +.. code-block:: bash + + redis-cli subscribe pagmo.islands..control + +At the moment the only control messages sent are "connected" and "disconnected": the rest of the communication happens directly on the ZeroMQ sockets. + +.. code-block:: bash + + $ redis-cli subscribe pagmo.islands.zeromq_test.control + Reading messages... (press Ctrl-C to quit) + 1) "subscribe" + 2) "pagmo.islands.zeromq_test.control" + 3) (integer) 1 + 1) "message" + 2) "pagmo.islands.zeromq_test.control" + 3) "disconnected/192.168.1.39:2639" + 1) "message" + 2) "pagmo.islands.zeromq_test.control" + 3) "connected/192.168.1.39:1905" + +In this monitoring session we see two events: 192.168.1.39:2639 disconnects from the swarm and then 192.168.1.39:1905 connects to the swarm. diff --git a/doc/sphinx/tutorials/zmq_setup.rst b/doc/sphinx/tutorials/zmq_setup.rst new file mode 100644 index 00000000..35ab76fb --- /dev/null +++ b/doc/sphinx/tutorials/zmq_setup.rst @@ -0,0 +1,68 @@ +.. _zeromq_setup: + +========= +The Setup +========= + +In order for ZeroMQ islands to find their peers and announce when they connect to and disconnect from the network, we must first set up the broker server. Our broker software of choice is Redis. This server acts as a bootstrap and communication channel. + +There are packages available in most distributions of Linux, but in this tutorial we will use the `debian:latest` Docker image. It's not necessary to install it in a container, but it makes it easier to reproduce the setup steps. + +First we will update the package list and install the Redis server: + +.. code-block:: bash + + root@dbc834f0d0aa:/# apt-get -qq update + root@dbc834f0d0aa:/# apt-get -qq install redis-server -y + (output omitted...) + root@dbc834f0d0aa:/# + +Now we'll install the compilation tools and necessary libraries (this will take a while) + +.. code-block:: bash + + root@dbc834f0d0aa:/# apt-get -qq install build-essential cmake git libboost-dev libboost-system-dev libboost-serialization-dev libboost-thread-dev libboost-python-dev libhiredis-dev libev-dev libzmq5-dev -y + (output omitted...) + +We also need to compile and install `redox` because Debian does not provide a binary package. + +.. code-block:: bash + + root@dbc834f0d0aa:~# git clone https://github.com/hmartiro/redox /tmp/redox + (output omitted...) + root@dbc834f0d0aa:~# cd /tmp/redox/ + root@dbc834f0d0aa:/tmp/redox# mkdir build && cd build + root@dbc834f0d0aa:/tmp/redox/build# cmake .. -DLIB_SUFFIX="/" + (output omitted...) + root@dbc834f0d0aa:/tmp/redox/build# make && make install && ldconfig + +Next, we'll clone the PaGMO source tree and compile it with PyGMO and ZeroMQ enabled. The `make` command takes around 15-20 minutes to complete (could take longer), so this is the perfect time to grab a caffeinated beverage. + +.. code-block:: bash + + root@dbc834f0d0aa:/# git clone https://github.com/jdiez17/pagmo /tmp/pagmo + root@dbc834f0d0aa:/# cd /tmp/pagmo + root@dbc834f0d0aa:/tmp/pagmo# mkdir build && cd build + root@dbc834f0d0aa:/tmp/pagmo/build# cmake .. -DENABLE_ZMQ=ON -DBUILD_PYGMO=ON + root@dbc834f0d0aa:/tmp/pagmo/build# make && make install && ldconfig + +If everything went well, you should be able to import PyGMO and zmq_island from Python! + +.. code-block:: bash + + root@dbc834f0d0aa:/tmp/pagmo/build# python + Python 2.7.9 (default, Mar 1 2015, 12:57:24) + [GCC 4.9.2] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import PyGMO + >>> 'zmq_island' in dir(PyGMO) + True + >>> + +Finally, we'll start a Redis server and daemonize it so it runs in the background. + +.. code-block:: bash + + root@dbc834f0d0aa:/# redis-server --daemonize yes + +At this point, you are ready to run ZMQ islands! diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 582921a9..68957b88 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,3 +1,8 @@ +IF(ENABLE_ZMQ) + ADD_EXECUTABLE(zeromq zeromq.cpp) + TARGET_LINK_LIBRARIES(zeromq ${MANDATORY_LIBRARIES} pagmo_static) +ENDIF(ENABLE_ZMQ) + IF(ENABLE_GTOP_DATABASE) ADD_EXECUTABLE(hm_2_asteroids hm_2_asteroids.cpp) diff --git a/examples/zeromq.cpp b/examples/zeromq.cpp new file mode 100644 index 00000000..d0e3b9b0 --- /dev/null +++ b/examples/zeromq.cpp @@ -0,0 +1,46 @@ +#include "../src/pagmo.h" +#include +#include +#include +#include + +using namespace pagmo; + +std::atomic quit; + +void sig(int) { + quit.store(true); +} + +void callback(zmq::message_t& msg) { + std::cout << "Got a " << msg.size() << "byte message from the network." << std::endl; +} + +int main() { + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = &sig; + sigfillset(&sa.sa_mask); + sigaction(SIGINT, &sa, NULL); // register signal handler + + quit.store(false); + + problem::dejong p(10); + algorithm::monte_carlo a(100); + zmq_island i(a, p, 100); + + i.set_broker_details("192.168.1.39", 6379); + i.set_token("zeromq_test"); + i.set_ip("192.168.1.39"); + i.connect(); + + i.set_callback(&callback); + + while(!quit) { + i.evolve(1); + std::cout << "Best: " << i.get_population().champion().x << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + return 0; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0a7727dd..5a775c13 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -242,6 +242,14 @@ IF(ENABLE_MPI) ) ENDIF(ENABLE_MPI) +# ZeroMQ support + +IF(ENABLE_ZMQ) + SET(PAGMO_LIB_SRC_LIST ${PAGMO_LIB_SRC_LIST} + ${CMAKE_CURRENT_SOURCE_DIR}/zmq_island.cpp + ) +ENDIF(ENABLE_ZMQ) + # Create a pagmo_static library if main or tests or examples are requested. IF(BUILD_MAIN OR ENABLE_TESTS OR BUILD_EXAMPLES) ADD_LIBRARY(pagmo_static STATIC ${PAGMO_LIB_SRC_LIST}) diff --git a/src/archipelago.cpp b/src/archipelago.cpp index 69679fab..c9eab88a 100644 --- a/src/archipelago.cpp +++ b/src/archipelago.cpp @@ -207,6 +207,18 @@ archipelago::size_type archipelago::locate_island(const base_island &isl) const return std::distance(m_container.begin(),it); } +/// Determine whether we should return a copy of the island, or a pointer to it. +/** + * @param[in] isl island to be checked + */ +bool archipelago::should_clone(base_island_ptr isl) const { + if(isl->get_name().find("ZMQ") != std::string::npos) { + // we don't want to clone ZMQ islands + return false; + } + return true; +} + /// Add an island to the archipelago. /** * Both the island and the archipelago will be synchronised before any operation takes place. The island will then @@ -711,9 +723,16 @@ base_island_ptr archipelago::get_island(const size_type &idx) const if (idx >= m_container.size()) { pagmo_throw(index_error,"invalid island index"); } - base_island_ptr retval = m_container[idx]->clone(); - // The island is no more in an archipelago. - retval->m_archi = 0; + base_island_ptr retval; + + if(should_clone(m_container[idx])) { + retval = m_container[idx]->clone(); + // The island is no more in an archipelago. + retval->m_archi = 0; + } else { + retval = m_container[idx]; + } + return retval; } diff --git a/src/archipelago.h b/src/archipelago.h index 50a5b8d5..7b817cfd 100644 --- a/src/archipelago.h +++ b/src/archipelago.h @@ -162,6 +162,8 @@ class __PAGMO_VISIBLE archipelago bool destruction_checks() const; void reevaluate_immigrants(std::vector > &, const base_island &) const; + + bool should_clone(base_island_ptr) const; private: friend class boost::serialization::access; template diff --git a/src/pagmo.h b/src/pagmo.h index c03a734a..9d00d942 100644 --- a/src/pagmo.h +++ b/src/pagmo.h @@ -44,4 +44,8 @@ #include "mpi_island.h" #endif +#ifdef PAGMO_ENABLE_ZMQ + #include "zmq_island.h" +#endif + #endif diff --git a/src/zmq_island.cpp b/src/zmq_island.cpp new file mode 100644 index 00000000..1ecfbdf3 --- /dev/null +++ b/src/zmq_island.cpp @@ -0,0 +1,349 @@ +/***************************************************************************** + * Copyright (C) 2004-2015 The PaGMO development team, * + * Advanced Concepts Team (ACT), European Space Agency (ESA) * + * * + * https://github.com/esa/pagmo * + * * + * act@esa.int * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the * + * Free Software Foundation, Inc., * + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * + *****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "algorithm/base.h" +#include "base_island.h" +#include "exceptions.h" +#include "zmq_island.h" +#include "migration/base_r_policy.h" +#include "migration/base_s_policy.h" +#include "population.h" +#include "problem/base.h" + +namespace pagmo +{ + +/// Constructor from problem::base, algorithm::base, number of individuals, migration probability and selection/replacement policies. +/** + * @see pagmo::base_island constructors. + */ +zmq_island::zmq_island(const algorithm::base &a, const problem::base &p, int n, + const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): + base_island(a,p,n,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), + m_initialised(false), m_evolve(true), m_callback(NULL), + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) +{} + +/// Constructor from population. +/** + * @see pagmo::base_island constructors. + */ +zmq_island::zmq_island(const algorithm::base &a, const population &pop, + const migration::base_s_policy &s_policy, const migration::base_r_policy &r_policy): + base_island(a,pop,s_policy,r_policy), m_brokerHost(""), m_brokerPort(-1), m_token(""), + m_initialised(false), m_evolve(true), m_callback(NULL), + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) + +{} + +/// Copy constructor. +/** + * @see pagmo::base_island constructors. + */ +zmq_island::zmq_island(const zmq_island &isl):base_island(isl), + m_initialised(false), m_evolve(true), m_callback(NULL), // TODO: does this make sense? + m_zmqContext(1), m_publisherSocket(m_zmqContext, ZMQ_PUB), m_subscriptionSocket(m_zmqContext, ZMQ_SUB) +{ + m_brokerHost = isl.m_brokerHost; + m_brokerPort = isl.m_brokerPort; + m_token = isl.m_token; + m_IP = isl.m_IP; + m_localPort = isl.m_localPort; + m_evolve = isl.m_evolve; + m_callback = isl.m_callback; + + connect(); +} + +/// Destructor. +zmq_island::~zmq_island() { + disconnect(); +} + +/// Assignment operator. +zmq_island &zmq_island::operator=(const zmq_island &isl) +{ + base_island::operator=(isl); + return *this; +} + +base_island_ptr zmq_island::clone() const +{ + disconnect(); + return base_island_ptr(new zmq_island(*this)); +} + +// This method performs the local evolution for this island's population. +/** + * This method will evolve the population if m_evolve is true, + * will broadcast its population to the ZeroMQ swarm if this island is connected, + * and it will replace the population in this island if an alternative one is received from the swarm. + * + * @param[in] algo The algorithm used to evolve the population. + * @param[in, out] pop The population of the island, that can be changed if a new one is received. + * + */ +void zmq_island::perform_evolution(const algorithm::base &algo, population &pop) const +{ + if(m_evolve) { + algo.evolve(pop); + } + + if(m_initialised) { + const boost::shared_ptr pop_copy(new population(pop)); + const algorithm::base_ptr algo_copy = algo.clone(); + //const std::pair,algorithm::base_ptr> out(pop_copy,algo_copy); + const boost::shared_ptr out = pop_copy; + + // First, we send a copy of our population and algorithm + std::stringstream ss; + boost::archive::text_oarchive oa(ss); + oa << out; + std::string buffer(ss.str()); + zmq::message_t msg(buffer.size()); + memcpy((void *) msg.data(), buffer.c_str(), buffer.size() - 1); + m_publisherSocket.send(msg); + + + // See if there is any data available + zmq::message_t incoming; +#if ZMQ_VERSION >= ZMQ_MAKE_VERSION(3, 0, 0) + // https://github.com/zeromq/libzmq/blob/master/NEWS#L548 + if(m_subscriptionSocket.recv(&incoming, ZMQ_DONTWAIT) > 0) { +#else + if(m_subscriptionSocket.recv(&incoming, ZMQ_NOBLOCK) > 0) { +#endif + if(incoming.size()) { + if(m_callback != NULL) { + m_callback(incoming); + } + + try { + std::string bytes_in((char *) incoming.data(), incoming.size()); + + std::stringstream incoming_ss(bytes_in); + boost::archive::text_iarchive ia(incoming_ss); + boost::shared_ptr in; + + ia >> in; + pop = *in; + } catch (const boost::archive::archive_exception &e) { + std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ": " << e.what() << std::endl; + } catch (...) { + std::cout << "ZMQ Recv Error during island evolution using " << algo.get_name() << ", unknown exception caught. :(" << std::endl; + } + } + } + } +} + +/// Return a string identifying the island's type. +// TODO: Add topic string +/** + * @return the string "ZMQ island". + */ +std::string zmq_island::get_name() const +{ + return "ZMQ island"; +} + +/** + * Sets the details of the broker host. + * + * @param[in] host The host or IP address. + * @param[in] port The port. + * + */ +void zmq_island::set_broker_details(std::string host, int port) { + m_brokerHost = host; + m_brokerPort = port; +} + +/** + * Sets the channel token. It can be any string, but it's often useful to use a string + * that describes the algorithm, problem and population used in the island. For example, + * a good channel token would be schwefel10_de10_pop20. + * + * @param[in] token The channel token. + * + */ +void zmq_island::set_token(std::string token) { + m_token = token; +} + +void zmq_island::connect_host(std::string host) { + std::cout << "DEBUG: Opening connection to " << host << std::endl; + + m_subscriptionSocket.connect(("tcp://" + host).c_str()); +} + +/** + * Connects this island to the ZeroMQ swarm. The initialisation protocol is as follows: + * 1. Connection with the broker is established. + * 2. The island acquires a list of peers that are already in the swarm with the same channel token. + * 3. The subscription socket is connected to every peer received from the broker. + * 4. The island adds itself to the set of peers in the ZeroMQ swarm and announces its presence via a realtime control channel. + * 5. The island binds the publisher socket (i.e used for outgoing communication) to the IP address given by `set_ip()`. + * 6. A callback is registered such that whenever a new peer sends a messsage over the control channel, the island can process it correctly. + * + * + * @returns true if the initialisation succeeded, false otherwise. + * + */ +bool zmq_island::connect() { + if(m_brokerHost == "" || m_brokerPort == -1 || m_token == "" || m_IP == "") { + return false; // Can't initialise if we're missing those parameters + } + + // Connect to the broker + if(!m_brokerConn.connect(m_brokerHost, m_brokerPort) || !m_brokerSubscriber.connect(m_brokerHost, m_brokerPort)) { + std::cout << "ERROR: Can't connect to broker" << std::endl; // TODO: better error reporting + return false; // can't connect to broker + } + + // Initialise subscription socket + m_subscriptionSocket.setsockopt(ZMQ_SUBSCRIBE, "", 0); + + std::cout << "DEBUG: IP: '" << m_IP << "'" << std::endl; + + std::string brokerKey = "pagmo.islands." + m_token; + // Get list of peers + redox::Command >& result = + m_brokerConn.commandSync >({"SMEMBERS", brokerKey}); + + if(!result.ok()) { + std::cout << "ERROR: Unable to get list of peers" << std::endl; + return false; + } + + // Connect to peers + auto peers = result.reply(); + for(auto it = peers.begin(); it != peers.end(); ++it) { + connect_host(*it); + } + + // Add ourselves to the list of islands on the chosen topic. + m_brokerConn.commandSync({"SADD", brokerKey, m_IP}); + + // Broadcast that we've added ourselves to the list + m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "connected/" + m_IP}); + + // Open incoming socket + m_publisherSocket.bind(("tcp://" + m_IP).c_str()); + + // Connect to new peers when they advertise on the control channel + m_brokerSubscriber.subscribe(brokerKey + ".control", [&](const std::string&, const std::string& msg) { + std::vector data; + boost::split(data, msg, boost::is_any_of("/")); + if(data[0] == "connected") { + connect_host(data[1]); + } else { /* disconnect */ } + }); + + m_initialised = true; + + return true; +} + +/// Disconnects the island from the swarm. +void zmq_island::disconnect() { + if(m_initialised) { + std::string brokerKey = "pagmo.islands." + m_token; + + m_brokerConn.commandSync({"SREM", brokerKey, m_IP}); + m_brokerConn.commandSync({"PUBLISH", brokerKey + ".control", "disconnected/" + m_IP}); + + m_brokerConn.disconnect(); + m_brokerSubscriber.disconnect(); + + std::cout << "DEBUG: Closed" << std::endl; + } +} + +/** + * Sets whether the island should perform evolutionary functions. + * It's useful to disable evolution if this island were to be used for monitoring the swarm + * instead of actively participating in it. + * + * @param[bool] evolve Determines whether this island will evolve its population. + */ +void zmq_island::set_evolve(bool e) { + m_evolve = e; +} + +/// Returns whether the island evolves its population. +bool zmq_island::get_evolve() { + return m_evolve; +} + +/** + * Sets a callback on network activity. + * Every time a message is received, this callback will receive a reference to the message. + * + * The callback is of type zmq_island::callback, which is void (*callback)(zmq::message_t&). + * Note: most users won't need to use this function in normal use. + * + * @param[in] callback The callback to be used. + */ +void zmq_island::set_callback(zmq_island::callback c) { + m_callback = c; +} + +/// Disables the low-level network callback. +void zmq_island::disable_callback() { + m_callback = NULL; +} + +/** + * Sets the IP address and port used for peer to peer communication. + * + * @param[in] ip The IP to be used. + */ +void zmq_island::set_ip(std::string ip) { + srand(time(0)); + m_localPort = rand() % 2000 + 1000; + m_IP += ip + ":" + std::to_string(m_localPort); +} + +} + +BOOST_CLASS_EXPORT_IMPLEMENT(pagmo::zmq_island) diff --git a/src/zmq_island.h b/src/zmq_island.h new file mode 100644 index 00000000..f24e4cba --- /dev/null +++ b/src/zmq_island.h @@ -0,0 +1,176 @@ +/***************************************************************************** + * Copyright (C) 2004-2015 The PaGMO development team, * + * Advanced Concepts Team (ACT), European Space Agency (ESA) * + * * + * https://github.com/esa/pagmo * + * * + * act@esa.int * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the * + * Free Software Foundation, Inc., * + * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * + *****************************************************************************/ + +#ifndef PAGMO_ZMQ_ISLAND_H +#define PAGMO_ZMQ_ISLAND_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base_island.h" +#include "config.h" +#include "algorithm/base.h" +#include "migration/base_r_policy.h" +#include "migration/base_s_policy.h" +#include "migration/best_s_policy.h" +#include "migration/fair_r_policy.h" +#include "population.h" +#include "problem/base.h" +#include "serialization.h" + +// Forward declarations. +namespace pagmo { + +class zmq_island; + +} + +namespace boost { namespace serialization { + +template +void save_construct_data(Archive &, const pagmo::zmq_island *, const unsigned int); + +template +inline void load_construct_data(Archive &, pagmo::zmq_island *, const unsigned int); + +}} + +namespace pagmo +{ + +/// ZMQ island class. +/** + * This island can communicate with other ZeroMQ islands active on as well as local islands. + * The intended use of this island class is to exchange solutions with remote archipelagos + * through a ZeroMQ socket. The functionality is similar to the MPI island (TODO: add ref), + * but using a different transport protocol. + * + * NOTE: this class is available only if PaGMO was compiled with ZeroMQ support. + * + * @author Jose Diez (me@jdiez.me) + */ +class __PAGMO_VISIBLE zmq_island: public base_island +{ + template + friend void boost::serialization::save_construct_data(Archive &, const pagmo::zmq_island *, const unsigned int); + template + friend void boost::serialization::load_construct_data(Archive &, pagmo::zmq_island *, const unsigned int); + public: + zmq_island(const zmq_island &); + ~zmq_island(); + explicit zmq_island(const algorithm::base &, const problem::base &, int = 0, + const migration::base_s_policy & = migration::best_s_policy(), + const migration::base_r_policy & = migration::fair_r_policy()); + explicit zmq_island(const algorithm::base &, const population &, + const migration::base_s_policy & = migration::best_s_policy(), + const migration::base_r_policy & = migration::fair_r_policy()); + zmq_island &operator=(const zmq_island &); + base_island_ptr clone() const; + protected: + void perform_evolution(const algorithm::base &, population &) const; + public: + typedef void (*callback)(zmq::message_t&); + + std::string get_name() const; + + void set_broker_details(std::string, int); + void set_token(std::string); + bool connect(); + void set_evolve(bool); + bool get_evolve(); + + void set_callback(zmq_island::callback); + void disable_callback(); + void set_ip(std::string ip); + void close(); + + protected: + std::string m_brokerHost; + int m_brokerPort; + std::string m_token; + std::string m_IP; + int m_localPort; + + bool m_initialised; + bool m_evolve; + + zmq_island::callback m_callback; + + private: + redox::Redox m_brokerConn; + redox::Subscriber m_brokerSubscriber; + + zmq::context_t m_zmqContext; + zmq::socket_t m_publisherSocket; + zmq::socket_t m_subscriptionSocket; + + void connect_host(std::string); + void disconnect(); + + friend class boost::serialization::access; + template + void serialize(Archive &ar, const unsigned int) + { + // Join is already done in base_island. + ar & boost::serialization::base_object(*this); + } +}; + +} + +namespace boost { namespace serialization { + +template +inline void save_construct_data(Archive &ar, const pagmo::zmq_island *isl, const unsigned int) +{ + // Save data required to construct instance. + pagmo::algorithm::base_ptr algo = isl->m_algo->clone(); + pagmo::problem::base_ptr prob = isl->m_pop.problem().clone(); + ar << algo; + ar << prob; +} + +template +inline void load_construct_data(Archive &ar, pagmo::zmq_island *isl, const unsigned int) +{ + // Retrieve data from archive required to construct new instance. + pagmo::algorithm::base_ptr algo; + pagmo::problem::base_ptr prob; + ar >> algo; + ar >> prob; + // Invoke inplace constructor to initialize instance of the algorithm. + ::new(isl)pagmo::zmq_island(*algo,*prob); +} + +}} //namespaces + +BOOST_CLASS_EXPORT_KEY(pagmo::zmq_island) + +#endif