diff --git a/Codes_for_Neural_Networks_and_Deep_Learning_Michael_Nielsen.ipynb b/Codes_for_Neural_Networks_and_Deep_Learning_Michael_Nielsen.ipynb new file mode 100644 index 0000000..d1aaab9 --- /dev/null +++ b/Codes_for_Neural_Networks_and_Deep_Learning_Michael_Nielsen.ipynb @@ -0,0 +1,6063 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "yx0rEFhlVvrp" + }, + "source": [ + "# Amirhossein Heydarian Ardakani 400123174\n", + "## Codes for Neural Networks and Deep Learning - Michael Nielsen" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Yqw2eywEsrCV", + "outputId": "6d1b7e20-32de-46fa-a947-3d7e82309f49" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Cloning into 'neural-networks-and-deep-learning'...\n", + "remote: Enumerating objects: 104, done.\u001b[K\n", + "remote: Counting objects: 100% (48/48), done.\u001b[K\n", + "remote: Compressing objects: 100% (34/34), done.\u001b[K\n", + "remote: Total 104 (delta 24), reused 28 (delta 14), pack-reused 56\u001b[K\n", + "Receiving objects: 100% (104/104), 16.64 MiB | 29.08 MiB/s, done.\n", + "Resolving deltas: 100% (52/52), done.\n", + "/content/neural-networks-and-deep-learning\n", + "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", + "Collecting theano\n", + " Downloading Theano-1.0.5.tar.gz (2.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.8/2.8 MB\u001b[0m \u001b[31m46.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.8/dist-packages (from theano) (1.21.6)\n", + "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.8/dist-packages (from theano) (1.7.3)\n", + "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.8/dist-packages (from theano) (1.15.0)\n", + "Building wheels for collected packages: theano\n", + " Building wheel for theano (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for theano: filename=Theano-1.0.5-py3-none-any.whl size=2668112 sha256=9067e19e73c976f81c9a41f6af305953be30b3bdbee4eff4a919acd5fee0ac93\n", + " Stored in directory: /root/.cache/pip/wheels/84/cb/19/235b5b10d89b4621f685112f8762681570a9fa14dc1ce904d9\n", + "Successfully built theano\n", + "Installing collected packages: theano\n", + "Successfully installed theano-1.0.5\n" + ] + } + ], + "source": [ + "!git clone https://github.com/amirhosseinh77/neural-networks-and-deep-learning.git\n", + "%cd neural-networks-and-deep-learning\n", + "!pip install theano\n", + "\n", + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wg_GCuyR2BjD" + }, + "source": [ + "### chapter 1 - read the input data." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 302 + }, + "id": "xsWZ0O1F2OlS", + "outputId": "2ae92cc6-394b-40d8-bd33-bff127aa885a" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2cAAAEdCAYAAACIbR/rAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd7hU1dn+8ft5AbGgKEIQBUQUFSxRIZgottiworHHAtYEYy/BmPywv9EYC9hRURSiEuzR1xKsoKIHghURCwoEBMSGHV2/P86wXHs8c5hzpu018/1cl5f3nj179prznJk5m3n22uacEwAAAACgsv6n0gMAAAAAAHBwBgAAAACpwMEZAAAAAKQAB2cAAAAAkAIcnAEAAABACnBwBgAAAAApUNDBmZn1N7PpZva2mZ1VrEGh+KhVPKhVPKhVPKhVHKhTPKhVPKhVXKy51zkzsxaS3pK0s6TZkl6SdIhz7o3iDQ/FQK3iQa3iQa3iQa3iQJ3iQa3iQa3iU8g3Z30lve2ce9c5962kOyUNKM6wUGTUKh7UKh7UKh7UKg7UKR7UKh7UKjItC9h2LUmzguXZkrZsbIP27du7bt26FbBLLMvMmTO1cOFCy7qZWqUQtYpHMWpFnUovR50kapU6vP/Fg1rFg1rFo5HPq4IOzvJiZsdJOk6Sunbtqrq6ulLvsqb16dOn2dtSq/KiVvFobq2oU3nxmooHtYoHtYoHtYpHY7UqpK1xjqQuwXLnzG0JzrkRzrk+zrk+HTp0KGB3KAC1ige1iscya0WdUoNaxYH3v3hQq3hQq8gUcnD2kqQeZraOmS0n6WBJDxRnWCgyahUPahUPahUPahUH6hQPahUPahWZZrc1OueWmNkJkh6V1ELSSOfc60UbGYqGWsWDWsWDWsWDWsWBOsWDWsWDWsWnoHPOnHMPS3q4SGNBCVGreFCreFCreFCrOFCneFCreFCruBR0EWoAAAAAQHFwcAYAAAAAKcDBGQAAAACkAAdnAAAAAJACHJwBAAAAQApwcAYAAAAAKVDQVPq1ZtasWT4PGzYsse6KK67w+dRTT/X55JNP9rlLly4CAADV74ILLvB56NChPvft29fnxx57zOe2bduWZ2AAUo1vzgAAAAAgBTg4AwAAAIAUoK1xGebMmePz5ptv7vMnn3ySuJ+Z+XzllVf6PGrUKJ8XLFhQiiGiBKZNm+bzTjvtlFg3depUnzt06FC2MdW6G2+80eff//73Pv/www8+T58+PbHN+uuvX/qBASnzzTff+Pzdd98l1k2YMMHn8PNt4MCBPrdsyZ8GzZH9d8Hw4cN9/p//+fHfwidPnuzzBx984PMmm2xSwtEhtHDhwsTykiVLfH7xxRd9HjBggM9hDZvjyCOP9PmGG25IrGvRokVBj11Lvv/+e5/feecdn0855RSfH3744bKOqdj45gwAAAAAUoCDMwAAAABIAXoXGvD+++/7vP322/v88ccf+xy2MUrJWZZat27t8/z5831+9913fV577bUT21fDV9ozZszwOfxZhTNTxWLSpEk+77jjjhUcSW0bP368z6eddprPudpLsl+XQDUL2+guu+wyn5944gmfw/eyxoQtjuHMgsjfiiuumFjee++9fb711lvLPBpI0rx583y+7bbbfB4xYkTifmF7fNhqGn7WFPr5Ev4OrLbaaol1F154oc/h35D4qbBte8MNN/S5c+fOPi9evNjnNm3alGdgRcQ3ZwAAAACQAhycAQAAAEAK1GxbY/YMVmErY//+/X0OLzzdmM0228zniy66yOd+/fr53KNHD5+zv1I/+uij89pPmoUtaG+++abPsbQ1Oud8Dls033rrrUoMB0r+7L/++usKjqS6zZw50+ew9eaRRx7x+aWXXsq5/ZgxY3zu0qWLz48//rjPgwYN8rlbt27NG2gNyp7ld9iwYQ3mr776yufwvWydddZJbL/66qv7HM4aGM4eN3jwYJ+ZkTZ/yy23XGI5+2eP8jvrrLN8Hj16dAVHknTFFVcklsMZiNddd91yD6cqzJ492+dPP/3UZ9oaAQAAAADNwsEZAAAAAKQAB2cAAAAAkAI1e87ZmWeemVi++uqrC3q8p59+2ucvvvjC53333dfne+65x+f//Oc/Be0vjYYPH+7zLrvsUsGRNE849epf//pXn08++eTE/TgHo3TeeOONxPK5557b4P222GILnx977DGfV1pppZKMq9pMnDgxsXzggQf6/OGHH/ocnrv0m9/8xufsc3EPO+ywBvcTbh+eO3XNNdc0ccTVLzynMpxW+7rrrkvcLzyXIpdNNtnE5/CzSZKWLFnic8eOHX0O6x7ug/e7/GWfF1uNn/Ox2WuvvXxu7JyzNddc0+czzjjD53CK/VyXcJGkZ5991ud77723yeNE4cLPm9jxzRkAAAAApAAHZwAAAACQAjXV1hi24mR/vZ3r69CwLXG//fbzObuNJ5w+umfPnj4PGTLE53Hjxi1zfzH7/vvvKz2EgoRT2YbCeqL43n77bZ933333xLpFixY1uM3FF1/sc9u2bUszsCoQtuSE0+XvscceifuFLb377LOPz2F7XXgpkOzX+lFHHeXznXfe2eBYttpqqzxHXZvCVtPw9ztfvXr18vmZZ57xeZVVVknc76OPPmrG6JCP7Ev0ZLdpN+SFF17wuWvXrol1vLcVLvwbLtfniZRsWWzO1Ou/+93vfA7/Zvjggw8avH/4nilJa6+9dpP3iSQz8/mbb76p4EgKt8xvzsxspJnNN7PXgtvamdnjZjYj8//VSjtM5OOoo47Syy+/LGqVftQqHtQqHjNnzhSfV3GgVnHg/S8e1Kp65NPWeKuk/lm3nSVpvHOuh6TxmWVU2KBBgxL/up1BrVKIWsWDWsUjc4FlPq8iQK3iwPtfPKhV9VhmW6Nz7hkz65Z18wBJ22fyKElPSRqiFJozZ47Pm2++uc+ffPJJ4n7h16GHHnqozzfeeKPPYYtCeLskHXzwwT6vuOKKPoczAIVfm99+++2J7cOr2Ictkk2x7bbbqkWLFtk3l7RW//3vf30Of9YxytXysPPOOxd9X5WoVVrddNNNPmfPAhgKZwvcYYcdSjqmUMy1evLJJ33eddddc97voIMO8nnkyJE+t27dusH7T5gwIbGcq5WxW7duPoftRaWy8sorS1L2CzmKWt1666153W/99df3+de//rXPF110kc/ZrYyh999/v+mDK4GYa5VL5jl5p556qs+DBw9ucJvw9swBqxe+51VKzO9/UvLvrsZeF4WaMmWKzwsXLlzm/bNbWFu2LPwso9hrVUxTp071uXv37hUcSfM0d0KQjs65uZk8T1LHxu6MiqJW8aBW8aBW8aBW8aBWcaBO8aBWESp4tkZXP7NFztktzOw4M6szs7rwOjcoP2oVD2oVj8ZqRZ3ShVrFg1rFgc+qeFCreDT3e9QPzayTc26umXWSND/XHZ1zIySNkKQ+ffqUZYrC8CvlSy65xOePP/7Y5/Dim5K0zjrr+By2GSy33HI+b7bZZg3m5vjyyy8Ty5deeqnP4cWci6CktQovAJz9nGIQXjD81VdfbfA+2a0mJZTq11Uxhb8r4e9+9kU+w5/9BRdcUPqB5S+vWlWiTuH7R9hWFbZuDx06NLFNOKtsrlbG0CmnnJLXWO666y6fw3bvMkttrULXXnutz7/61a987t8/eVpW+NnVnIuuz5+f820lDaKoVb6OO+44n3O1NUaqZj6rcslu7R42bJjP+fwtdOaZZxZ9TDlUXa3CvxNWW+3H+U3Cv/GnTZtW1jEVW3O/OXtA0sBMHijp/uIMByVAreJBreJBreJBreJBreJAneJBrSKUz1T6d0h6XtIGZjbbzI6WdLGknc1shqSdMsuosEMOOURvvvmmRK1Sj1rFg1rF491335X4vIoCtYoD73/xoFbVI5/ZGg/JsWrHIo8FBbrjjjs0Y8YM1dXVtcpaRa1ShlrFg1rFo3v37qqrq+vUwCpqlTLUKg68/8WDWlWPwufuTIElS5Ykls844wyfR48e7XPbtm19fvTRRxPbrLfeej5/9913xR7iMr333ntl32cxvPbaaw3eXug5eeXy5z//2efwsgCbbrqpz+F5h2i+8PIVAwYMyGubc8891+cNN9yw2EOqCtdff31iOTzPLDx/LLzcx5/+9KfENq1aZX+W1wvfW19++WWfZ8yYkbhf/Xnm9cJz3vr06dPo2PGjcBr2448/vmT7eeKJJ0r22Mjthx9+8Dn73Fqk1zPPPOPz6aef7vPrr7+euN+33367zMfaZpttfOZ3oPmWX355n/faay+fb7vttkoMpyT47QAAAACAFODgDAAAAABSoCraGj/44IPEctjKGHrhhRd8Xn/99XM+3gorrFCcgdWwLbfcsqL7/+abb3yePHlyYt2IESN8Dqf6DoWtWeFX6Gi+Z5991ufnnnuuwfsccMABieVBgwaVckjR+vrrr33OvsRAOGV+2Mo4cuTIvB570aJFPh900EE+P/nkkzm3+d3vfufzsccem9d+ULhx48b5/Nlnn/kctpmGvw/ST98Pl9pjjz187t69e7GGiIywjS27JiidsJ1+7NixPj/88MN5bf/ggw/6nG/dVl11VZ/DVrt+/fr5nKuVHJD45gwAAAAAUoGDMwAAAABIgapoa/zDH/6QWA5bOvbdd1+fG2tlLIfGZmsKx1wNwlaCfIWzJYY/q6effjpxv3Bmy3CGpKuuusrn77//3ueVVlopsf0uu+zic9iyGM7S2bNnzyaNHQ176aWXfB44cGCD9wlnW7rxxhsT62gpbVj4+/3hhx/mvN8VV1zh8xdffOFz2A4nJdt7n3/+eZ/DVrmwpSe7veeYY47xmdlNCxe+F4Xvi5I0dOhQn3O18Oc7M2CXLl18vuWWW/LaBki7uXPn+rz99tv7/M4775Rl/+Fn2u67716WfSJp4cKFlR5CQXgHBgAAAIAU4OAMAAAAAFIg2rbG//znPz6HFwmUki032bO/VVJjszXFerHWFVdc0efwOe29994+b7DBBnk9VthOFbZ5tmyZ/DVt06aNz+GskOHFx8OLPWZfEDtscwzbesK2rw4dOuQ1ZiRlt7P+8pe/XOY24QXgs1tQ0bAWLVr4vMYaayTWzZs3z+d27dr5nO9MY127dvU5nHVs1qxZPnfs2DGxzRZbbJHXYyMpbE+dPXu2z2ErVvhzl5LvueH712677ebzHXfc4fPixYtz7j+8yPhDDz3k829/+1ufw981IDbh3xLNOX2kORcPD2doPPnkk33O/lsEpTNq1Cifw/b+WPDNGQAAAACkAAdnAAAAAJAC0bY1hhdhDS84LElrrrmmz+GFNcslbBUJL2Yc2n///RPLZ599dknHVCrnn3++z+uuu67PTz31VJMfq0ePHj6HbTVh25skrbPOOk1+7FB48cmwBWzDDTcs6HEhXXbZZYnlfNpAhgwZUqrhVK1wFssJEyYk1oWtpAsWLPC5V69ePh9++OGJbY444gifw9bS8H5he93gwYObM+yaF7YxStLUqVN9Dlu0Q9dee21ieccdd/Q5fM/96quvfH7llVd8njRpUs7xhO9/Rx55pM/hRaizx5XdZo785NMe9/jjjyeWf/Ob35R0TNWqU6dOPoczBv/zn//0OZy1WWr6LLM333xzYvmcc85p0vYojv79+/sctpPGjm/OAAAAACAFODgDAAAAgBTg4AwAAAAAUqAqm8fD8zHCaddLKTzP7LrrrvP5j3/8o8/dunXz+c9//nNi+6b2O6fRwIEDG8xp869//avB24866qgyj6Q6zJkzx+dx48bltU14fguXLShM+L4iJc8jao4ZM2b4fN999/kcnifD+Zn5C88zGzZsWGJd+PkQCs+5Dc8HlJKfb19++aXPe+65p88vvPCCz61bt05sf+mll/ocnvN2yy23+Lzddtv5fOCBBya2Hzp0qM+5Pl87d+7c4O21rLFL6Sx14403JpbPPfdcn7MvX4H8tG3b1udjjjmmaI97+umnJ5Y556wycs1B8O233/r86aefJtaFvxNpxTdnAAAAAJACHJwBAAAAQApUZVtj9jTRpRC2cknSJZdc4nM49XHYvpXdsoB0Ydri5unTp4/PCxcuzHm/XXfd1eerr766pGNC84WXKcnVirXbbruVdUyxCadNv/LKK33OvmzEyiuv7POtt97qc/haCdsYJen999/3+dhjj/X5mWee8XmTTTbx+c4770xsH7akhpehOfHEE30eOXKkz6NGjUpsP3bsWDUknH7/rbfeavA+tewvf/mLzxdddFFe24R/M4Tbo/KmTJlS6SFAUosWLRq83Tnn83fffVeu4RQN35wBAAAAQApwcAYAAAAAKRBtW2P4lWWYpWR7yP/7f/+vaPu84447fA5bQCTp448/9vmkk07y+Yorrija/oE0mj9/vs9hG1y2sKWrGmYnrVZhSxyaJ5wRNvy9z57d8MEHH/S5d+/ePk+fPt3n66+/PrHN6NGjff7qq698DluFw9keV1lllZzjDGdy3HTTTX0OWzH322+/xDa52vP5rGtc+PNF4cJZUF999dXEuo022sjnVq1aFW2fjz/+uM8HHHBA0R4XzReeVrHZZpv5HM5EO3z48MQ2559/fukHVqBlfnNmZl3M7Ekze8PMXjezkzO3tzOzx81sRub/q5V+uGjMrFmzNH36dFGr9KNW8aBWcaBO8aBW8aBW8aBW1SOftsYlkk53zvWS9EtJfzCzXpLOkjTeOddD0vjMMiqoZcuW6tKli6hV+lGreFCrOFCneFCreFCreFCr6rHMtkbn3FxJczP5czObJmktSQMkbZ+52yhJT0ka0sBDlEQ4c1j2BR1nz57tc/j15dFHH+1zOEuWJL3++us+33DDDT4/++yzPs+cOdPnddddN7H9wQcf7HPY1lhOnTp10oorrigpXbVKs7AlNpwFLZx5rBRir9UZZ5zhczgzXWNibeuJvVZNld0iFIs01en4449v8PYlS5Yklv/85z/7HF4o9bXXXstrP9ddd53P4edbY+3FTbXNNts0utwcaapVuYTtoT179vT5jTfeyLlNeFpG+DvVrl27Io8utzTVasaMGT6HF+i+6667EvdbtGiRz81pawzbhV988UWfw7/zFi9enHP7pT8v6aezrZZSmmpVCeGM2++9957PQ4cOrcRwCtKkd3Az6yZpc0mTJHXMHLhJ0jxJXL4+RahVPKhVPKhVHKhTPKhVPKhVPKhV3PI+ODOzNpLulnSKc+6zcJ2r//rB5djuODOrM7O6BQsWFDRY5IdaxYNaxaM5taJO5cdrKh7UKh7UKh7UKn55zdZoZq1UX+gxzrl7Mjd/aGadnHNzzayTpPkNbeucGyFphCT16dOnwV+IYgtn8QnbGm+++Wafs9sC8mnlCS+82r9//8S6E044ocnjLIVMm140taq0sCU23/a8YomtVuGF18eNG+dz2EIVzv4mSeecc47PK620UglHV1rNrVWMr6l333230kNotrS8prp16+bzvHnzfA4v8C1JEydObHD7ww47zOedd945sS78HFp11VV9LmYrYzmkpVaV0LdvX5+nTZuW835pqWlaajVo0CCfJ02alPN+4cyhjc1Wmks4i+rTTz/tc/YpNKGwpe7000/3ObzoezmkpVaVFtYq14Wq0yyf2RpN0s2SpjnnLg9WPSBpYCYPlHR/8YeHpnDOLT0vjlqlHLWKB7WKA3WKB7WKB7WKB7WqHvl8c7a1pMMlvWpmSy8ccLakiyWNNbOjJb0v6cDSDBH5mjhx4tITYX9NrdKNWsWDWsWBOsWDWsWDWsWDWlWPfGZrnCAp13e5OxZ3OChEv3791Lt3b9XV1TU0LR61ShFqFQ9qFQfqFA9qFQ9qFQ9qVT3yOucsjcIrwO+0006Jdf/+978b3CacYj88fybbz372M58HDx7sczitLarPE0884fOOO/I+li2cOjjX6yc810aShgyputl6q154Pkx4HmZazn+Jwfjx431+/vnnfc4+x6xTp04+H3TQQT6H02/HeL4EGhdebmfUqFEVHEl1uuCCC0ryuGuuuabPhx9+eGLdeeed53PLltH+aV01PvnkE5/DyyFI0pZbblnu4TQZn7YAAAAAkAIcnAEAAABACkT73Ws4PWo4rbck3XbbbT6H7QONufDCC30+9thjfV599dWbO0REIDPtLICMsNVu44039jmc8vvDDz9MbLPOOuuUfmARCS8psf322zeYUbvC9u/evXv7PHny5AqMJh533XWXz8OHD/f58ssvb+juTdKrVy+fw78vd9llF5/Dvw3D90mkw4gRI3wOW8O7d+9eieEUhG/OAAAAACAFODgDAAAAgBSItq0x1KZNm8Ty8ccf32AGJGm//fbz+frrr6/gSOKy1lpr+bzHHnv4/OCDD1ZiOCiDK6+80uddd93V5z/+8Y+J+1199dU+d+zYsfQDAyLWtm1bnydNmlTBkcSlc+fOPv/v//6vz9tuu23ifsccc4zPCxcu9Pmoo47yee+9905sE7YcZ/9NiTjstddePk+ZMsXn5ZZbrhLDKQjfnAEAAABACnBwBgAAAAApUBVtjUBThBeYDi+yi8aFrR733XdfBUeCcunXr5/PBx54oM9jx45N3K99+/Y+Dxs2zOcY20kApF94oec999wzsW7evHnlHg5S4Jprrqn0EIqGb84AAAAAIAU4OAMAAACAFODgDAAAAABSgHPOAAANat26tc+33HKLzxtssEHifhdccIHP5557rs9Mqw8AQNPwzRkAAAAApAAHZwAAAACQArQ1AgCWKWxxPOeccxLrspcBAEDz8M0ZAAAAAKQAB2cAAAAAkALmnCvfzswWSPpC0sKy7TR92qu0z39t51yHQh+EWkmiVjFJfa0ydXpfpR9r2pXy+RfzNVXrtUr9a0qiVhnUKh4x1Yq/KypUq7IenEmSmdU55/qUdacpEtPzj2mspRDT849prKUQ0/OPaaylENPzj2msxRbbc49tvMUU23OPbbzFFNNzj2mspVDJ509bIwAAAACkAAdnAAAAAJAClTg4G1GBfaZJTM8/prGWQkzPP6axlkJMzz+msZZCTM8/prEWW2zPPbbxFlNszz228RZTTM89prGWQsWef9nPOQMAAAAA/BRtjQAAAACQAmU9ODOz/mY23czeNrOzyrnvcjOzLmb2pJm9YWavm9nJmdvbmdnjZjYj8//VKj3WhlArapVGMdeqluokUauYUKt4UKt4UKs4pLFOZWtrNLMWkt6StLOk2ZJeknSIc+6NsgygzMysk6ROzrkpZraypMmS9pE0SNIi59zFmV/41ZxzQyo41J+gVtQqrWKtVa3VSaJWMaFW8aBW8aBWcUhjncr5zVlfSW875951zn0r6U5JA8q4/7Jyzs11zk3J5M8lTZO0luqf86jM3Uap/hcgbagVtUqliGtVU3WSqFVMqFU8qFU8qFUc0linch6crSVpVrA8O3Nb1TOzbpI2lzRJUkfn3NzMqnmSOlZoWI2hVtQq9SKrVc3WSaJWMaFW8aBW8aBWcUhLnZgQpMTMrI2kuyWd4pz7LFzn6ntKmS4zJahVPKhVPKhVPKhVPKhVPKhVHNJUp3IenM2R1CVY7py5rWqZWSvVF3qMc+6ezM0fZvpbl/a5zq/U+BpBrepRqxSKtFY1VyeJWsWEWsWDWsWDWsUhbXUq58HZS5J6mNk6ZracpIMlPVDG/ZeVmZmkmyVNc85dHqx6QNLATB4o6f5yjy0P1KoetUqZiGtVU3WSqFVMqFU8qFU8qFUc0linsl6E2sx2l3SlpBaSRjrnLirbzsvMzPpJelbSq5J+yNx8tur7WMdK6irpfUkHOucWVWSQjaBW1CqNYq5VLdVJolYxoVbxoFbxoFZxSGOdynpwBgAAAABoGBOCAAAAAEAKcHAGAAAAACnAwRkAAAAApAAHZwAAAACQAhycAQAAAEAKVO3BmZktXsb6bmb2WhMf81Yz238Z9zEzG25mb5vZK2a2RVP2UYsqVavgvr8wsyX53r+WVfB1taGZPW9m35jZGU15/FpUwTqtZmb3Zt77XjSzjZuyj1pUwVodmqnTq2b2nJn9vCn7qEW8/8WjgrUakHldTTWzusw07WgEr6ufalnpAVSh3ST1yPy3paTrMv9HCplZC0mXSHqs0mNBoxZJOknSPpUeCBp1tqSpzrl9zWxDSddI2rHCY0LD3pO0nXPuYzPbTdII8VmVVrz/xWO8pAecc87MNlX9dbI2rPCY0LDUvq6q9puzpcysjZmNN7MpmX8hHBCsbmlmY8xsmpmNM7MVM9v0NrOnzWyymT1qZp2asMsBkm5z9V6QtGoTt69ZFaiVJJ0o6W5J84v1PGpBuWvlnJvvnHtJ0nfFfi7VrAKvqV6SnpAk59ybkrqZWcfiPaPqVYHX1HPOuY8ziy9I6lzEp1PVeP+LRwVqtdj9eAHhlSRxMeE88br6UdUfnEn6WtK+zrktJO0g6TIzs8y6DSRd65zrKekzScebWStJV0na3znXW9JIST+5MrqZnW9mezewv7UkzQqWZ2duw7KVtVZmtpakfVX/7SaaptyvKzRPuev0sqTfZO7TV9La4o/+fFXyNXW0pP8r0vOoBbz/xaPstTKzfc3sTUkPSTqq6M+oevG6yqiFtkaT9L9mtq2kH1R/oLT0X3JnOecmZvJo1X+9+YikjSU9nvmdaCFpbvaDOueGlnjctajctbpS0hDn3A8/vv6RJ15XcSh3nS6WNMzMpkp6VdJ/JH1fnKdS9SrymjKzHVR/cMa5Mfnj/S8eZa+Vc+5eSfdm9nmBpJ2K81SqHq+rjFo4ODtUUgdJvZ1z35nZTEnLZ9Zlf93sVP/L8bpz7lfN3N8cSV2C5c6Z27Bs5a5VH0l3Zl7U7SXtbmZLnHP3NfPxakm5a4XmKWudnHOfSTpSqp8cSfXnNb3bnMeqQWV/TVn9OTE3SdrNOfdRcx+nBvH+F4+K1co594yZdTez9s65hYU+Xg3gdZVRC22NbSXNzxR6B9W32SzV1cyWFvW3kiZImi6pw9LbzayVmW3UhP09IOkIq/dLSZ86535yJI8GlbVWzrl1nHPdnHPdJI2TdDwHZn4sXvEAACAASURBVHkr9+sKzVPWOpnZqma2XGbxGEnPZA7YsGzlrlVXSfdIOtw591ZRnkHt4P0vHuV+Xa23tBXP6mfrbi2Jf/jID6+rjFo4OBsjqY+ZvSrpCElvBuumS/qDmU2TtJqk65xz30raX9IlZvaypKmStsp+0EZ6WB9W/b8Uvy3pRknHF/PJVLly1wrNV9ZamdkaZjZb0mmS/mJms81slaI/q+pT7tdUT0mvmdl01c9ce3JRn011K3ethkpaXdK1lpn2u7hPp6rx/hePcr+u9lP9e+BU1c9We1AwQQgax+sqw/idAQAAAIDKq4VvzgAAAAAg9Tg4AwAAAIAU4OAMAAAAAFKAgzMAAAAASAEOzgAAAAAgBTg4AwAAAIAU4OAMAAAAAFKAgzMAAAAASAEOzgAAAAAgBTg4AwAAAIAU4OAMAAAAAFKAgzMAAAAASAEOzgAAAAAgBTg4AwAAAIAU4OAMAAAAAFKAgzMAAAAASAEOzgAAAAAgBTg4AwAAAIAU4OAMAAAAAFKAgzMAAAAASAEOzgAAAAAgBTg4AwAAAIAU4OAMAAAAAFKAgzMAAAAASAEOzgAAAAAgBTg4AwAAAIAUKOjgzMz6m9l0M3vbzM4q1qBQfNQqHtQqHtQqHtQqDtQpHtQqHtQqLuaca96GZi0kvSVpZ0mzJb0k6RDn3BvFGx6KgVrFg1rFg1rFg1rFgTrFg1rFg1rFp5BvzvpKets5965z7ltJd0oaUJxhocioVTyoVTyoVTyoVRyoUzyoVTyoVWRaFrDtWpJmBcuzJW3Z2Abt27d33bp1K2CXWJaZM2dq4cKFlnUztUohahWPYtSKOpVejjpJ1Cp1eP+LB7WKB7WKRyOfVwUdnOXFzI6TdJwkde3aVXV1daXeZU3r06dPs7elVuVFreLR3FpRp/LiNRUPahUPahUPahWPxmpVSFvjHEldguXOmdsSnHMjnHN9nHN9OnToUMDuUABqFQ9qFY9l1oo6pQa1igPvf/GgVvGgVpEp5ODsJUk9zGwdM1tO0sGSHijOsFBk1Coe1Coe1Coe1CoO1Cke1Coe1CoyzW5rdM4tMbMTJD0qqYWkkc6514s2MhQNtYoHtYoHtYoHtYoDdYoHtYoHtYpPQeecOecelvRwkcaCEqJW8aBW8aBW8ajVWi1cuDCxvPXWW/u8ZMkSn995552yjakxtVqnGFGreFCruBR0EWoAAAAAQHFwcAYAAAAAKcDBGQAAAACkQMmvcwYAAMrnvPPO8/n6669PrFuwYIHPRxxxRNnGBADID9+cAQAAAEAKcHAGAAAAAClAW2NKHHDAAT4753weN25cJYaTKh9++GFi+dFHH/X54osv9vnXv/61z3379s35eIceeqjPLVq0KMYQAaDsvvjiC5/Dz5DwPdLMEttsueWWPl9zzTUlHB0AoDn45gwAAAAAUoCDMwAAAABIAdoaK+Siiy5KLD/00EM+n3rqqeUeTur861//8vm3v/1tYt3nn3/e4DbTpk3zubF2nbDlccMNN2zuEIGS+Pbbb31+8sknfV5hhRV8njhxos+ffvppYvurrrrK53333dfnzp07N3ksa621ls8DBgzwuWvXrk1+LBTHwoULfT7jjDN8fuyxxxq8/y233JJY/sUvfuFz+DuFwoWnJEjSCSec4POYMWN8/uCDD3xeZZVVSj8wIGJPPfVUYvnee+/1+YUXXvD5P//5T4Pb77bbbonlu+66y+fll1++CCMsPr45AwAAAIAU4OAMAAAAAFKAtsYyuuyyy3zObmtcbrnlfN5jjz3KNqa02nHHHX1u06ZNYl2utsZ8bb311j4//fTTPm+88cYFPS5QDMOGDfN5yJAhBT3WP/7xj0KH44Xt1n369EmsO/bYY33eb7/9fF511VWLtn/U++yzz3wePXr0Mu/frVu3xDKt3KWzZMmSxHJ4ukJYt+eee87n/v37l35gQAS+/PJLn8OW4FGjRiXu165dO5/DlsUNNtjA5/vuu8/nBx98MLH9Vltt5fOUKVMKGHHp8M0ZAAAAAKQAB2cAAAAAkAK0NZbRhAkTfA5nZJOkvfbay+fwK9daFc4idsMNNyTWHXLIIT6HF2Ht3r27z++++27Ox160aJHP4dfdtDXGKZytMPt1NXbsWJ8vvPDCBrcPL0ouSX//+9+LOLqmy55db1l+9rOfJZa32WabJu+zZ8+ePoezns6fP9/nZ5991ucXX3wxsX243Lt3b58322yzJo8FSeHsjFKyjSd7dsClJk2a5HN2CypKp1WrVonl8GcfztA4Z86cso0JxRO2iX/99deJda+++qrPw4cPb3D7zTff3Oe6uroijy5+4Sk9b7zxhs+XXnpp4n6DBw/2OdeMswsWLPC5R48eiXWvvPKKz9dff73Pv//975s44tLhmzMAAAAASAEOzgAAAAAgBTg4AwAAAIAU4JyzjBkzZvg8dOhQn0eOHOlzrt7WxoTnaYTT5/bq1StxvyuuuKLJj10rwvPxJOnnP/+5z+HPtH379j43ds5ZKE09xmhc2IN+5513+nzNNdf4/PHHHye2MbNlPu748eOLMLriCc9NDc9T6dq1a4P3Dy/DIf300hOF+Oabb3zeaKONfG7s9fXPf/7TZ845K9wdd9yRWH7nnXd8Puyww3y++uqrfV555ZVLPzAs05lnnunzPffc4/Nrr71WieGgEW+99ZbP4WfNo48+6vNNN93kc67zPaXcnzvhuU5bbLFFYl1ap3Qvtddff93nZ555xufwvLLTTjutyY/boUMHn7PPNz/ppJN8vvzyy31O09+DfHMGAAAAACnAwRkAAAAApABtjRkHHHCAz+GUqBdccIHP6623XpMfN/w6NpyWOvuK5WuuuWaTH7tWXXbZZT6fccYZPk+cOLHJj/Xdd98VZUwojiFDhiSWw1aPfNoP27Ztm1g+8cQTfQ6nmN9hhx18btkyXW+D7dq1azBXQjgle2OtjMsvv7zPxx13XEnHVAvC6fLDVh9JWn/99X0OW3JoZUyf8BIVoXD67vBvjGK2JNeyxYsX+3z44Yf7/PLLL+fcJmyJ//zzz30O2xe33357n59++ukmj+uHH37wObwETC0L/wYLW+cHDRpUtH385je/SSyHbY1ffvmlz2Ebf+vWrYu2/+ZY5jdnZjbSzOab2WvBbe3M7HEzm5H5/2qlHSbycdRRR+nll18WtUo/ahUPahWPmTNnis+rOFCrOPD+Fw9qVT3yaWu8VVL/rNvOkjTeOddD0vjMMips0KBBP7nYnqhVKlGreFCreKy++uoSn1dRoFZx4P0vHtSqeiyzn8c594yZdcu6eYCk7TN5lKSnJA1RxFZZZRWfw5l2vv322yY/1pw5c3wOZ4H8n//58Vg4/Pq0WLbddlu1aNEi++aqq9Uvf/lLnx955BGfd9ppJ5/DdqzG/OUvf/F5xIgRRRhdfmqlVrl89dVXPp9//vk+X3rppYn7hTMuhS0lf/3rX33u3r27z9kzF2a3OTZHLdTq+++/9/mcc87xOd9ZZMOZzjp37ly8gTVRprVvUdbNUdSqrq7O58cee8zn7JnfjjnmGJ9btWpV+oGVSMy1ao6wPS78/H/qqad83nPPPcs5pLzE8P4Xzq4oSfvss4/P+c7cnMu8efN8DttOw9ZJSfroo498Dus4c+bMBh83/DumWGKoVbaNN97Y58mTJ/tczPe2xh7rv//9r8/h+272LOHl1twJQTo65+Zm8jxJHYs0HhQftYoHtYoHtYoHtYoHtYoDdYoHtYpQwbM1uvp/Dsp5wQczO87M6sysbsGCBYXuDgWgVvGgVvForFbUKV2oVTyoVRz4rIoHtYpHc6cp+9DMOjnn5ppZJ0nzc93ROTdC0ghJ6tOnT+6r9pXZVVddlVh+/vnnfd5888197tatW16PF7Y/hi1X4Vffu+66q89bbbVV3mMtUPS1yhbOXha2L7744otNfqwdd9yxKGMqkqqrVS7hjJt/+9vffD7vvPMS9wtnb8xuWaywvGqV1jq9+eabieWbb77Z57A2obA15O67706sW2ONNYo4uqJLba2+/vprn/O9GHr79u19Dtvx8xVeJDxXy1f2rKlllNpaFSrXhYlLcYpDGaTqsypsjZfya2UMZ5iVpNtuu83n3r17+xy21odWWGGFxHL4N2WuVsZwptUbb7xxmWMsklTVKls5ZkvOnvU4rG/YShleEDvWtsYHJA3M5IGS7i/OcFAC1Coe1Coe1Coe1Coe1CoO1Cke1CpC+Uylf4ek5yVtYGazzexoSRdL2tnMZkjaKbOMCjvkkEOW/os4tUo5ahUPahWPzL+Y83kVAWoVB97/4kGtqkc+szUekmNVqvrBIN1xxx2aMWOG6urqsqemoVYpQ63iQa3i0b17d9XV1XVqYBW1ShlqFQfe/+JBrapH6Zs9U+Szzz7z+eKLk/94EJ5PMWbMGJ9XXHHFvB47PFfm+uuv97lr164+P/zww/kPtsaFJ6PusssuiXWvveavr6glS5YUtJ/sx0Zhvvvuu8RyeHmC4cOH+/yPf/zD5/79f7zU0WabbZbYvhz96LXi/fff93mTTTZJrAun0s8lvBRI9nT5uc6nQePCn1t4zuwPP/zgc/hzl6RtttlmmY97xx13NLgPKXmphLfffrvB7c8668dLIYWfm5KfAh+oqPDvgPCSOo1Zd911fc7+eyxc1xwffPDBMu9zxBFH+Jzv35YoXPblBVq3bl2hkeSv4NkaAQAAAACF4+AMAAAAAFKg6nuG5s6d6/NOO+3k84cffpi4X9iWGE53mkvY+ihJf//73xu8X9jKhfy99957PmdP+11oK2MorE/Y7oPmufrqqxPLZ5xxhs+DBw/2+ec//7nPtC6Wx5133ulzPm2M2cIpv7fYYovEuh122MHngw46yOdwOuJOnRo6vai2vfHGGz7ff/+Pk6iFrYzZ7Va5ps+fM2eOz+G0/LfeemvO/Yctit27d/f55Zdf9vmAAw5IbHPXXXf53LZt25yPDZTSRRdd5HN4yaJse+yxh8/h6SzNaWMML32RfemeBx54YJn7HzBgQJP3icJl/834xRdfNHi/5lyapFT45gwAAAAAUoCDMwAAAABIgaroJwpntpKkJ5980udwNr7GZsB6+umnfV5jjTV8HjhwoM/hV9rZrSLO/Xgx9VNPPdXnPffcc5njx0/17dvX59tvvz2xLpzx6KuvvipoP2ErEAp32mmnJZbDmeKOPPJIn2llLL+wPS2c6UyS/v3vf/uc3fKdj/A9N8zHH3+8z2Eb0gknnJDYfqWVVmryPmMUtoZK/lpfP9GlSxefTzrppMS61Vdf3eeFCxf6fMkll/h8yy23+NyxY8fE9uHvwZlnnunzl19+6XPPnj19nj9/foNjRNOEfyMwu2nhTjnlFJ//+9//JtZ16NDB5/BvtTZt2hS0z3CW4eOOOy7n/X7xi1/4HJ4CU+j+0Twff/xxYjls2w7ttttueT1e+F4ZzoI8YcIEn/fZZ5/ENuHvZD745gwAAAAAUoCDMwAAAABIgaroLXr22WcTy7vuuqvPYftA2Mq40UYbJbZ54oknGszhzFQzZszwedasWYntw1bISy+9NO+xY9n233//xHKPHj18zr5A6lLhbHT77rtvYt0nn3xSxNEhFM6IKiVfS2E71YMPPuhz9msRpRHOxpfdKvzpp5/6HL6mFi1a5HN4YePs97iwZSsUtpL/6U9/8jl7prNx48b5XM0tX9kzz4YzW4bCi0D//ve/T6wLZxoLZ0MdPXq0z+EsitntV3/5y198Dtsiw7GE2++9996J7ZmhsXmq+fe6Erbcckufw9NSim3KlCk+Z7djh1q1auVz+PqllbF8wlkZw8+xqVOn5rX9YYcd5vPWW2/t83PPPZe4XzgLfDizePjemP1ef9lll+U1hqX45gwAAAAAUoCDMwAAAABIgWjbGidOnOhzdivVcsst53O7du18DmckCy++KSVn/rn33nt9DtuyGpttKZzhbJ111vF58uTJDY4FzRdewDiXsFYXXnhhYl3YmhDOrhO2dtG681MzZ870OZxNrkWLFj5nX4gznDXuxBNP9Llfv34+T58+3eef/exnRRkrmib8fQ9zWOfwdbf77rsntg9fY+H7bC7he6yUbLMMZ2OtNvm212S3MobC9uDHHnuswfu88MILPq+//vqJdeEMkdnrlgpn1hwyZEjjg0VBNtlkk0oPAY0IZ15srDX17rvv9jn7/RG5fffddz6HsyCGbYHZpy49/PDDDT7W559/7nP4t3e+wnb7BQsW5Lxf2E4eXmQ8vIj1qquu2uT9h/jmDAAAAABSgIMzAAAAAEgBDs4AAAAAIAWiPefsiiuu8Hm99dZLrBs+fLjPO++8c16Pd/XVV/v81Vdf+fzII4/ktX14jlN4ZXDOM6uMcCr9xqa/bd26tc9MdSwtXrzY57CXWkqeGxZeYmK77bbzeYUVVkhsM2jQIJ/Dc87CaW7DfXLOWRy23XbbxHL4Prnffvv5fP/99+f1eNOmTSvOwFLuo48+SiyHnxtHHnlkg9vMmTMnsfz66683uP0//vEPn8NzycLp8iVpt912W+b2uab4R/GFl+FBOoR/X4aXAwkvx5QtPDcNSeF5ZcOGDUusC9938j0nNxSe2xWe89Wy5Y+HN+EU+9nCc2qPP/54n8PzrSuBb84AAAAAIAU4OAMAAACAFIi2rTFsu9h1110T68KvNvMVtlk9//zzDd4nnM5z3XXXzflYhU6hicJdfvnled0vnBK1Ob831WbDDTf0+ZNPPkmsu+2223wOWxkbc9NNNzV4+4EHHujzWmut1ZQhIoXCdp8tt9zS53zbGjfeeOOijykGYSt1vm3V4c863Kaurs7nP/3pTz6HbfpS8mcdbhO2eAO1JjwVInxd5Hq9jRs3LrF9+/btSzi6uIWXRwlPiZCk5Zdf3ufDDz/c5/Bv7PDyIZLUqlUrn8NTIcK/4TbbbDOfX3nllcT24d855513ns/hZbgqjW/OAAAAACAFODgDAAAAgBSItq0x+2vOpvr6668Ty2PGjPE5bOfq1auXz1tttVVB+6xVYVvN4MGDE+uOOuoon7NngGuqcNa/v/71r3lts/vuuxe0z2pz/vnn+3zSSScl1u2///7L3D67Pe21117zOZxV9W9/+5vPtFMVV/g6GD16tM+bbrqpz8V+LwtnNJsyZcoy7x/OpCVJffv2Lep40iqcyVeS/vjHP/p8yy23+By2JYazM0rSp59+2uBjhzPMhbMwduzYMXG/Sy+91OeVV145n2GjhBqbSQ6lE84gKEmPP/64z9mtd0uFMz/3798/sY7ZnnO78847fQ5nkpWk8ePH+9y5c+cmP3b42fP3v//d53feecfnTp06JbZ59NFHfU5TK2Nomd+cmVkXM3vSzN4ws9fN7OTM7e3M7HEzm5H5/2qlHy4aM2vWLE2fPl3UKv2oVTyoVRyoUzyoVTyoVTyoVfXIp61xiaTTnXO9JP1S0h/MrJeksySNd871kDQ+s4wKatmypbp06SJqlX7UKh7UKg7UKR7UKh7UKh7Uqnoss63ROTdX0txM/tzMpklaS9IASdtn7jZK0lOShjTwEKkUXvhOki688EKfw69AJ06cWLYxFapTp05accUVJaWrVuFF/kaNGpVYF150cOzYsT6HMx9lX8h71qxZPs+cOdPnsBUoe6bB0MUXX+xzpdp60lqrsM00u91w0qRJPmfPVLXUggULEsuHHXaYz5dddpnPq6++ekHjLKe01mqpsI1RknbeeWefw5p9+eWXRdvnF198kVi+7rrrfM71uxHq3bt3YrlHjx4FjyntdZKSs4xJUps2bXwO6xj+PJrTLtW2bVufjzvuuMS6cBazSomhVuXy5JNP+hxewD0tqqlW33zzjc+nnXZaYt0NN9zQ4DZhi2NYnzS2Maa1VuHPKrvNes0112zy44WtwEcffbTPt99+u8/hLJBPPPFEYvtKX2A6H02aEMTMuknaXNIkSR0zB26SNE9SxxyboQKoVTyoVTyoVRyoUzyoVTyoVTyoVdzyPjgzszaS7pZ0inPus3Cdqz/72OXY7jgzqzOzuux/VUdpUKt4UKt4NKdW1Kn8eE3Fg1rFg1rFg1rFL6/ZGs2sleoLPcY5d0/m5g/NrJNzbq6ZdZI0v6FtnXMjJI2QpD59+jT4C1Eu4SxX4YxVUvJr17A9LrYLE2dm6UpVrU455RSfZ8yYkVj3yCOP+LzBBhv4HLb1hBe1laQHH3zQ51wzl4X1zG7jOfXUU33OnjWunNJYq9Chhx6ac3n48OGl2GVqNbdW5ahT2DYsJVsZQx999JHP4YU7pZ+22y0VzmgWXlD87LPPTtwv1+swnDUwbLXLbm8ulrS/prLbaZ566imfL7roIp/vuece5SNszQpbRTfffHOfs2dHS4u016oYVlppJZ/D+kyePLkSw2m2aqlV+D6Vq41RSs7Snc8sxWmSxlqFMwVPmDAhsS78/Fq4cKHP4eulZ8+eObcJT43ZZZddfA4/r5ozC2Sl5TNbo0m6WdI059zlwaoHJA3M5IGS7i/+8NAUzrml52BRq5SjVvGgVnGgTvGgVvGgVvGgVtUjn68NtpZ0uKRXzWzpIerZki6WNNbMjpb0vqQDSzNE5GvixIlatGiRJP2aWqUbtYoHtYoDdYoHtYoHtYoHtaoe+czWOEFSrmlpdizucFCIfv36qXfv3qqrq9u0gdXUKkWoVTyoVRyoUzyoVTyoVTyoVfWo3Ak3FdCvXz+fs899Ovnkk33+wx/+ULYx1YLu3bv7vN122yXWDR482OcBAwb4HNYnu1b5CKdqnzJlSpO3B2Kx5557JpbDae1D4flO22yzTWJdhw4dGtwmPCn82WefbfLYwvPMnn/+eZ/Teh5UuYXnw/7zn/+s4EhQCi1atPA512VbwnOo0ziVfuzC97DLL7885/3C86LCyxugcOF5YVdddVViXXj+2Pfff+9z9uWuQoMGDfJ55MiRPoc1jF2TptIHAAAAAJQGB2cAAAAAkAI11dYYTun+u9/9LrHuwAM5P7IczjrrrMRyeKX32267rcFtXnzxxcTy1Vdf3eD9VlttNZ9pZUSt+NWvfpVYDtuyr7nmmga3aU6LYmPCS1KEU8IfcMABPnfr1q2o+wRi0rdvX5/DSygsXry4AqOpHRdccIHP1157bc77nXPOOT6H7dgorhNPPLHRZdTjmzMAAAAASAEOzgAAAAAgBWqqrfHoo49uMKNywnaoww8/vMH7ZN+ePdsPUMtWXXXVxPIVV1zh80EHHeTzQw895PPGG2+c2Gbs2LENPnavXr0avH2PPfZILIcti507d258wEANClv6w9nrBg4c2NDdUYB58+b5/OmnnzZ4n7PPPjuxvNVWW5V0TEBT8M0ZAAAAAKQAB2cAAAAAkAI11dYIANUubBXu169fgznboYceWtIxAbUubD9+5JFHKjiS6jd69Gifx4wZ43OPHj18zp4lsEOHDqUfGJAnvjkDAAAAgBTg4AwAAAAAUoCDMwAAAABIAc45AwAAQFUIL/URXsLg9ttv95lzzJBmfHMGAAAAACnAwRkAAAAApABtjQAAAKgKPXv29HnJkiUVHAnQPHxzBgAAAAApwMEZAAAAAKSAOefKtzOzBZK+kLSwbDtNn/Yq7fNf2zlX8DRE1EoStYpJ6muVqdP7Kv1Y066Uz7+Yr6lar1XqX1MStcqgVvGIqVb8XVGhWpX14EySzKzOOdenrDtNkZief0xjLYWYnn9MYy2FmJ5/TGMthZief0xjLbbYnnts4y2m2J57bOMtppiee0xjLYVKPn/aGgEAAAAgBTg4AwAAAIAUqMTB2YgK7DNNYnr+MY21FGJ6/jGNtRRiev4xjbUUYnr+MY212GJ77rGNt5hie+6xjbeYYnruMY21FCr2/Mt+zhkAAAAA4KdoawQAAACAFCjrwZmZ9Tez6Wb2tpmdVc59l5uZdTGzJ83sDTN73cxOztzezsweN7MZmf+vVumxNoRaUas0irlWtVQniVrFhFrFg1rFg1rFIY11Kltbo5m1kPSWpJ0lzZb0kqRDnHNvlGUAZWZmnSR1cs5NMbOVJU2WtI+kQZIWOecuzvzCr+acG1LBof4EtaJWaRVrrWqtThK1igm1ige1ige1ikMa61TOb876SnrbOfeuc+5bSXdKGlDG/ZeVc26uc25KJn8uaZqktVT/nEdl7jZK9b8AaUOtqFUqRVyrmqqTRK1iQq3iQa3iQa3ikMY6lfPgbC1Js4Ll2Znbqp6ZdZO0uaRJkjo65+ZmVs2T1LFCw2oMtaJWqRdZrWq2ThK1igm1ige1ige1ikNa6sSEICVmZm0k3S3pFOfcZ+E6V99TynSZKUGt4kGt4kGt4kGt4kGt4kGt4pCmOpXz4GyOpC7BcufMbVXLzFqpvtBjnHP3ZG7+MNPfurTPdX6lxtcIalWPWqVQpLWquTpJ1Com1Coe1Coe1CoOaatTOQ/OXpLUw8zWMbPlJB0s6YEy7r+szMwk3SxpmnPu8mDVA5IGZvJASfeXe2x5oFb1qFXKRFyrmqqTRK1iQq3iQa3iQa3ikMY6lfUi1Ga2u6QrJbWQNNI5d1HZdl5mZtZP0rOSXpX0Q+bms1XfxzpWUldJ70s60Dm3qCKDbAS1olZpFHOtaqlOErWKCbWKB7WKB7WKQxrrVNaDMwAAAABAw5gQBAAAAABSgIMzAAAAAEgBDs4AAAAAIAU4OAMAAACAFODgDAAAAABSoGoPzsxs8TLWdzOz15r4mLea2f553vcXZrYk3/vXskrVyswGmNkrZjbVzOoy06miERWs1fZm9mmmVlPNbGhT9lFreE3Fo5KfVZnX1VQze93Mnm7KPmpRBV9XG5rZ82b2jZmd0ZTHr1UVrNWZwefUa2b2vZm1a8p+ak0Fa3Vo5vPqVTN7zsx+3pR9lFLLSg+g1UpmOwAABEFJREFUGplZC0mXSHqs0mNBo8ZLesA558xsU9Vfz2LDCo8JuT3rnNuz0oNAo3hNRcLMVpV0raT+zrkPzOxnlR4Tclok6SRJ+1R6IGicc+5SSZdKkpntJenUtF3DDN57krZzzn1sZrtJGiFpywqPSVIVf3O2lJm1MbPxZjYlc3Q8IFjd0szGmNk0MxtnZitmtultZk+b2WQze9TMOjVxtydKulvS/GI9j1pQ7lo55xa7Hy/0t5IkLvqXpwq9rtBEvKbiUYHX1G8l3eOc+0CSnHN8XuWpAq+r+c65lyR9V+znUu0q/Fl1iKQ7Cn4SNaICr6vnnHMfZxZfkNS5iE+nIFV/cCbpa0n7Oue2kLSDpMvMzDLrNpB0rXOup6TPJB1vZq0kXSVpf+dcb0kjJf3kyuhmdr6Z7d3A7WtJ2lfSdSV5NtWtrLXKrNvXzN6U9JCko4r+jKpX2Wsl6Vdm9rKZ/Z+ZbVTsJ1SleE3Fo9y1Wl/Samb2VOYPmyNK8JyqVSXe/9A8FalV5uChv+r/oR75qeTr6mhJ/1ek51GwWmhrNEn/a2bbSvpB0lqSOmbWzXLOTczk0apvG3hE0saSHs/8TrSQNDf7QZ1zuc55uVLSEOfcDz/+TiFP5a6VnHP3Sro3s88LJO1UnKdS9cpdqymS1nbOLTaz3SXdJ6lHkZ5LNeM1FY9y16qlpN6SdpS0gqTnzewF59xbxXk6Va3srys0W6VqtZekibQ0NklFamVmO6j+4Cw150jXwsHZoZI6SOrtnPvOzGZKWj6zLrvlxqn+l+N159yvmrm/PpLuzPyitJe0u5ktcc7d18zHqyXlrtWPD+bcM2bW3czaO+cWFvp4NaCstXLOfRbkh83sWmqVF15T8Sh3rWZL+sg594WkL8zsGUk/l8TB2bJV7HWFJqtUrQ4WLY1NVfZaWf250TdJ2s0591FzH6fYaqGtsa2k+ZlC7yBp7WBdVzNbWtTfSpogabqkDktvN7NWTWmhcs6t45zr5pzrJmmcpOM5MMtbWWtlZust/crczLaQ1FpSal6cKVfuWq0R1Kqv6t+7qNWy8ZqKR1lrJel+Sf3MrGWmBWtLSdMKfha1ody1QvOVvVZm1lbSdqp/jSF/5f686irpHkmHp61joBYOzsZI6mNmr0o6QtKbwbrpkv5gZtMkrSbpOufct5L2l3SJmb0saaqkrbIflN7wkih3rfaT9JqZTZV0jaSDgskM0Lhy12p/1dfqZUnDJR1MrfLCayoeZa2Vc26a6tuCXpH0oqSbnHNNmq66hpW1Vpl/nJot6TRJfzGz2Wa2StGfVXWqxN+A+0p6LPOtNPJX7loNlbS6pGstc/mX4j6d5jM+NwEAAACg8mrhmzMAAAAASD0OzgAAAAAgBTg4AwAAAIAU4OAMAAAAAFKAgzMAAAAASAEOzgAAAAAgBTg4AwAAAIAU4OAMAAAAAFLg/wNGwAENMrIg8QAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "from elements import mnist_loader\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()\n", + "training_data = list(training_data)\n", + "test_data = list(test_data)\n", + "\n", + "plt.figure(figsize=(15,5))\n", + "for i in range(1,17):\n", + " plt.subplot(2,8,i)\n", + " plt.imshow(training_data[i][0].reshape(28,28), cmap='Greys')\n", + " plt.xlabel(f'label: {np.argmax(training_data[i][1])}')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JdLBjP6_3UPH" + }, + "source": [ + "### chapter 1 - network.py example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wF1RgHd02UAd", + "outputId": "823e24a0-4203-4aea-947d-5e8d22b61ed7" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 0 : 8987 / 10000\n", + "Epoch 1 : 9200 / 10000\n", + "Epoch 2 : 9322 / 10000\n", + "Epoch 3 : 9376 / 10000\n", + "Epoch 4 : 9410 / 10000\n", + "Epoch 5 : 9412 / 10000\n", + "Epoch 6 : 9425 / 10000\n", + "Epoch 7 : 9425 / 10000\n", + "Epoch 8 : 9470 / 10000\n", + "Epoch 9 : 9469 / 10000\n", + "Epoch 10 : 9433 / 10000\n", + "Epoch 11 : 9470 / 10000\n", + "Epoch 12 : 9479 / 10000\n", + "Epoch 13 : 9466 / 10000\n", + "Epoch 14 : 9510 / 10000\n", + "Epoch 15 : 9487 / 10000\n", + "Epoch 16 : 9484 / 10000\n", + "Epoch 17 : 9502 / 10000\n", + "Epoch 18 : 9512 / 10000\n", + "Epoch 19 : 9514 / 10000\n", + "Epoch 20 : 9505 / 10000\n", + "Epoch 21 : 9545 / 10000\n", + "Epoch 22 : 9524 / 10000\n", + "Epoch 23 : 9518 / 10000\n", + "Epoch 24 : 9511 / 10000\n", + "Epoch 25 : 9527 / 10000\n", + "Epoch 26 : 9528 / 10000\n", + "Epoch 27 : 9519 / 10000\n", + "Epoch 28 : 9501 / 10000\n", + "Epoch 29 : 9515 / 10000\n" + ] + } + ], + "source": [ + "from elements import network\n", + "from elements import mnist_loader\n", + "\n", + "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()\n", + "training_data = list(training_data)\n", + "test_data = list(test_data)\n", + "\n", + "net = network.Network([784, 30, 10])\n", + "net.SGD(training_data, 30, 10, 3.0, test_data=test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WpuED0lS3l6-" + }, + "source": [ + "### chapter 2 - network2.py example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "cLAKSN8l3e7k", + "outputId": "6c468ef3-8973-42d8-8cc3-d54e774b42a7" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 0 training complete\n", + "Accuracy on evaluation data: 9271 / 10000\n", + "Epoch 1 training complete\n", + "Accuracy on evaluation data: 9423 / 10000\n", + "Epoch 2 training complete\n", + "Accuracy on evaluation data: 9489 / 10000\n", + "Epoch 3 training complete\n", + "Accuracy on evaluation data: 9515 / 10000\n", + "Epoch 4 training complete\n", + "Accuracy on evaluation data: 9552 / 10000\n", + "Epoch 5 training complete\n", + "Accuracy on evaluation data: 9569 / 10000\n", + "Epoch 6 training complete\n", + "Accuracy on evaluation data: 9564 / 10000\n", + "Epoch 7 training complete\n", + "Accuracy on evaluation data: 9573 / 10000\n", + "Epoch 8 training complete\n", + "Accuracy on evaluation data: 9584 / 10000\n", + "Epoch 9 training complete\n", + "Accuracy on evaluation data: 9597 / 10000\n", + "Epoch 10 training complete\n", + "Accuracy on evaluation data: 9598 / 10000\n", + "Epoch 11 training complete\n", + "Accuracy on evaluation data: 9609 / 10000\n", + "Epoch 12 training complete\n", + "Accuracy on evaluation data: 9616 / 10000\n", + "Epoch 13 training complete\n", + "Accuracy on evaluation data: 9621 / 10000\n", + "Epoch 14 training complete\n", + "Accuracy on evaluation data: 9627 / 10000\n", + "Epoch 15 training complete\n", + "Accuracy on evaluation data: 9613 / 10000\n", + "Epoch 16 training complete\n", + "Accuracy on evaluation data: 9647 / 10000\n", + "Epoch 17 training complete\n", + "Accuracy on evaluation data: 9623 / 10000\n", + "Epoch 18 training complete\n", + "Accuracy on evaluation data: 9625 / 10000\n", + "Epoch 19 training complete\n", + "Accuracy on evaluation data: 9636 / 10000\n", + "Epoch 20 training complete\n", + "Accuracy on evaluation data: 9633 / 10000\n", + "Epoch 21 training complete\n", + "Accuracy on evaluation data: 9635 / 10000\n", + "Epoch 22 training complete\n", + "Accuracy on evaluation data: 9639 / 10000\n", + "Epoch 23 training complete\n", + "Accuracy on evaluation data: 9635 / 10000\n", + "Epoch 24 training complete\n", + "Accuracy on evaluation data: 9635 / 10000\n", + "Epoch 25 training complete\n", + "Accuracy on evaluation data: 9648 / 10000\n", + "Epoch 26 training complete\n", + "Accuracy on evaluation data: 9652 / 10000\n", + "Epoch 27 training complete\n", + "Accuracy on evaluation data: 9653 / 10000\n", + "Epoch 28 training complete\n", + "Accuracy on evaluation data: 9643 / 10000\n", + "Epoch 29 training complete\n", + "Accuracy on evaluation data: 9656 / 10000\n" + ] + } + ], + "source": [ + "from elements import network2\n", + "from elements import mnist_loader\n", + "\n", + "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()\n", + "training_data = list(training_data)\n", + "test_data = list(test_data)\n", + "\n", + "net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)\n", + "#net.large_weight_initializer()\n", + "evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \\\n", + " net.SGD(training_data, 30, 10, 0.1, lmbda = 5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "q7BtRKij440U" + }, + "source": [ + "### chapter 3 - Overfitting example - too many epochs of learning applied on small (1k samples) amount of data. Overfitting is treating noise as a signal." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "1H4XSDZ043ib", + "outputId": "9caa2fe0-7d9b-407e-c0b6-33d6a6484949" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 0 training complete\n", + "Cost on training data: 1.8190217461210199\n", + "Accuracy on training data: 653 / 1000\n", + "Cost on evaluation data: 2.20515439650697\n", + "Accuracy on evaluation data: 5423 / 10000\n", + "Epoch 1 training complete\n", + "Cost on training data: 1.2952284866159138\n", + "Accuracy on training data: 801 / 1000\n", + "Cost on evaluation data: 1.7512553282067156\n", + "Accuracy on evaluation data: 6662 / 10000\n", + "Epoch 2 training complete\n", + "Cost on training data: 1.06319139272246\n", + "Accuracy on training data: 853 / 1000\n", + "Cost on evaluation data: 1.57554801111598\n", + "Accuracy on evaluation data: 7134 / 10000\n", + "Epoch 3 training complete\n", + "Cost on training data: 0.914761385830091\n", + "Accuracy on training data: 882 / 1000\n", + "Cost on evaluation data: 1.5015445471270783\n", + "Accuracy on evaluation data: 7361 / 10000\n", + "Epoch 4 training complete\n", + "Cost on training data: 0.7740499254722\n", + "Accuracy on training data: 898 / 1000\n", + "Cost on evaluation data: 1.3816325998905399\n", + "Accuracy on evaluation data: 7591 / 10000\n", + "Epoch 5 training complete\n", + "Cost on training data: 0.6729182162932685\n", + "Accuracy on training data: 922 / 1000\n", + "Cost on evaluation data: 1.3499490462176436\n", + "Accuracy on evaluation data: 7664 / 10000\n", + "Epoch 6 training complete\n", + "Cost on training data: 0.6342086340245277\n", + "Accuracy on training data: 922 / 1000\n", + "Cost on evaluation data: 1.3171817812788404\n", + "Accuracy on evaluation data: 7735 / 10000\n", + "Epoch 7 training complete\n", + "Cost on training data: 0.5545793789220899\n", + "Accuracy on training data: 927 / 1000\n", + "Cost on evaluation data: 1.2746332499652857\n", + "Accuracy on evaluation data: 7845 / 10000\n", + "Epoch 8 training complete\n", + "Cost on training data: 0.5196991595676717\n", + "Accuracy on training data: 942 / 1000\n", + "Cost on evaluation data: 1.2992338431082353\n", + "Accuracy on evaluation data: 7806 / 10000\n", + "Epoch 9 training complete\n", + "Cost on training data: 0.44647566279306766\n", + "Accuracy on training data: 949 / 1000\n", + "Cost on evaluation data: 1.2280160184618512\n", + "Accuracy on evaluation data: 7980 / 10000\n", + "Epoch 10 training complete\n", + "Cost on training data: 0.430666742219667\n", + "Accuracy on training data: 949 / 1000\n", + "Cost on evaluation data: 1.2574084594281811\n", + "Accuracy on evaluation data: 7917 / 10000\n", + "Epoch 11 training complete\n", + "Cost on training data: 0.38387678370817585\n", + "Accuracy on training data: 958 / 1000\n", + "Cost on evaluation data: 1.21032585301388\n", + "Accuracy on evaluation data: 8033 / 10000\n", + "Epoch 12 training complete\n", + "Cost on training data: 0.3484593114520667\n", + "Accuracy on training data: 967 / 1000\n", + "Cost on evaluation data: 1.217695951841451\n", + "Accuracy on evaluation data: 7978 / 10000\n", + "Epoch 13 training complete\n", + "Cost on training data: 0.3195578919532098\n", + "Accuracy on training data: 973 / 1000\n", + "Cost on evaluation data: 1.2177074854360348\n", + "Accuracy on evaluation data: 8031 / 10000\n", + "Epoch 14 training complete\n", + "Cost on training data: 0.2945137405508818\n", + "Accuracy on training data: 967 / 1000\n", + "Cost on evaluation data: 1.1892922624590674\n", + "Accuracy on evaluation data: 8093 / 10000\n", + "Epoch 15 training complete\n", + "Cost on training data: 0.28001218123947746\n", + "Accuracy on training data: 976 / 1000\n", + "Cost on evaluation data: 1.197268137755829\n", + "Accuracy on evaluation data: 8093 / 10000\n", + "Epoch 16 training complete\n", + "Cost on training data: 0.2504416527323233\n", + "Accuracy on training data: 979 / 1000\n", + "Cost on evaluation data: 1.1925521358374176\n", + "Accuracy on evaluation data: 8109 / 10000\n", + "Epoch 17 training complete\n", + "Cost on training data: 0.24139629143324323\n", + "Accuracy on training data: 981 / 1000\n", + "Cost on evaluation data: 1.1955119047427971\n", + "Accuracy on evaluation data: 8116 / 10000\n", + "Epoch 18 training complete\n", + "Cost on training data: 0.2256396805504697\n", + "Accuracy on training data: 981 / 1000\n", + "Cost on evaluation data: 1.1999682873657025\n", + "Accuracy on evaluation data: 8150 / 10000\n", + "Epoch 19 training complete\n", + "Cost on training data: 0.21340410003567364\n", + "Accuracy on training data: 980 / 1000\n", + "Cost on evaluation data: 1.2208915177774942\n", + "Accuracy on evaluation data: 8115 / 10000\n", + "Epoch 20 training complete\n", + "Cost on training data: 0.19860318512839306\n", + "Accuracy on training data: 982 / 1000\n", + "Cost on evaluation data: 1.202448546568788\n", + "Accuracy on evaluation data: 8138 / 10000\n", + "Epoch 21 training complete\n", + "Cost on training data: 0.18507093860538082\n", + "Accuracy on training data: 981 / 1000\n", + "Cost on evaluation data: 1.2062719422683896\n", + "Accuracy on evaluation data: 8149 / 10000\n", + "Epoch 22 training complete\n", + "Cost on training data: 0.17809517689647422\n", + "Accuracy on training data: 985 / 1000\n", + "Cost on evaluation data: 1.2170795638928877\n", + "Accuracy on evaluation data: 8142 / 10000\n", + "Epoch 23 training complete\n", + "Cost on training data: 0.1662709451762902\n", + "Accuracy on training data: 984 / 1000\n", + "Cost on evaluation data: 1.1959673372846107\n", + "Accuracy on evaluation data: 8185 / 10000\n", + "Epoch 24 training complete\n", + "Cost on training data: 0.16047510505960833\n", + "Accuracy on training data: 987 / 1000\n", + "Cost on evaluation data: 1.2072620078162155\n", + "Accuracy on evaluation data: 8175 / 10000\n", + "Epoch 25 training complete\n", + "Cost on training data: 0.1547811455080219\n", + "Accuracy on training data: 986 / 1000\n", + "Cost on evaluation data: 1.2029371048588935\n", + "Accuracy on evaluation data: 8178 / 10000\n", + "Epoch 26 training complete\n", + "Cost on training data: 0.14812337421509558\n", + "Accuracy on training data: 987 / 1000\n", + "Cost on evaluation data: 1.203532872529958\n", + "Accuracy on evaluation data: 8203 / 10000\n", + "Epoch 27 training complete\n", + "Cost on training data: 0.13873855840783622\n", + "Accuracy on training data: 988 / 1000\n", + "Cost on evaluation data: 1.2071228787753379\n", + "Accuracy on evaluation data: 8211 / 10000\n", + "Epoch 28 training complete\n", + "Cost on training data: 0.13304700870363378\n", + "Accuracy on training data: 987 / 1000\n", + "Cost on evaluation data: 1.2060367291049054\n", + "Accuracy on evaluation data: 8192 / 10000\n", + "Epoch 29 training complete\n", + "Cost on training data: 0.12701496171187182\n", + "Accuracy on training data: 987 / 1000\n", + "Cost on evaluation data: 1.2276471430541558\n", + "Accuracy on evaluation data: 8182 / 10000\n", + "Epoch 30 training complete\n", + "Cost on training data: 0.12215577099387842\n", + "Accuracy on training data: 989 / 1000\n", + "Cost on evaluation data: 1.231831199094082\n", + "Accuracy on evaluation data: 8191 / 10000\n", + "Epoch 31 training complete\n", + "Cost on training data: 0.11870974061064307\n", + "Accuracy on training data: 991 / 1000\n", + "Cost on evaluation data: 1.2398147968420437\n", + "Accuracy on evaluation data: 8176 / 10000\n", + "Epoch 32 training complete\n", + "Cost on training data: 0.11209023730789419\n", + "Accuracy on training data: 990 / 1000\n", + "Cost on evaluation data: 1.2325114607907561\n", + "Accuracy on evaluation data: 8203 / 10000\n", + "Epoch 33 training complete\n", + "Cost on training data: 0.10917944912550458\n", + "Accuracy on training data: 991 / 1000\n", + "Cost on evaluation data: 1.2366223602725728\n", + "Accuracy on evaluation data: 8203 / 10000\n", + "Epoch 34 training complete\n", + "Cost on training data: 0.10432941419571955\n", + "Accuracy on training data: 991 / 1000\n", + "Cost on evaluation data: 1.234667248725637\n", + "Accuracy on evaluation data: 8203 / 10000\n", + "Epoch 35 training complete\n", + "Cost on training data: 0.10010380717004927\n", + "Accuracy on training data: 993 / 1000\n", + "Cost on evaluation data: 1.2428871254849807\n", + "Accuracy on evaluation data: 8208 / 10000\n", + "Epoch 36 training complete\n", + "Cost on training data: 0.09683580352477932\n", + "Accuracy on training data: 994 / 1000\n", + "Cost on evaluation data: 1.253354941716545\n", + "Accuracy on evaluation data: 8182 / 10000\n", + "Epoch 37 training complete\n", + "Cost on training data: 0.09383430828300757\n", + "Accuracy on training data: 995 / 1000\n", + "Cost on evaluation data: 1.2713844338536768\n", + "Accuracy on evaluation data: 8175 / 10000\n", + "Epoch 38 training complete\n", + "Cost on training data: 0.08899136488388808\n", + "Accuracy on training data: 994 / 1000\n", + "Cost on evaluation data: 1.2543401659609548\n", + "Accuracy on evaluation data: 8194 / 10000\n", + "Epoch 39 training complete\n", + "Cost on training data: 0.08688568380452226\n", + "Accuracy on training data: 997 / 1000\n", + "Cost on evaluation data: 1.2664433128342079\n", + "Accuracy on evaluation data: 8194 / 10000\n", + "Epoch 40 training complete\n", + "Cost on training data: 0.08225990024808638\n", + "Accuracy on training data: 996 / 1000\n", + "Cost on evaluation data: 1.266300863265206\n", + "Accuracy on evaluation data: 8212 / 10000\n", + "Epoch 41 training complete\n", + "Cost on training data: 0.08069900066033274\n", + "Accuracy on training data: 996 / 1000\n", + "Cost on evaluation data: 1.2749342553172758\n", + "Accuracy on evaluation data: 8196 / 10000\n", + "Epoch 42 training complete\n", + "Cost on training data: 0.07779943322628868\n", + "Accuracy on training data: 997 / 1000\n", + "Cost on evaluation data: 1.266149620469757\n", + "Accuracy on evaluation data: 8199 / 10000\n", + "Epoch 43 training complete\n", + "Cost on training data: 0.0746640839743523\n", + "Accuracy on training data: 997 / 1000\n", + "Cost on evaluation data: 1.2747231557900125\n", + "Accuracy on evaluation data: 8205 / 10000\n", + "Epoch 44 training complete\n", + "Cost on training data: 0.07344696139468176\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.2769717284999926\n", + "Accuracy on evaluation data: 8197 / 10000\n", + "Epoch 45 training complete\n", + "Cost on training data: 0.06990440560515732\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.270198172550871\n", + "Accuracy on evaluation data: 8205 / 10000\n", + "Epoch 46 training complete\n", + "Cost on training data: 0.06768190719335276\n", + "Accuracy on training data: 997 / 1000\n", + "Cost on evaluation data: 1.2756674784661721\n", + "Accuracy on evaluation data: 8215 / 10000\n", + "Epoch 47 training complete\n", + "Cost on training data: 0.06539741952424255\n", + "Accuracy on training data: 997 / 1000\n", + "Cost on evaluation data: 1.287981638663985\n", + "Accuracy on evaluation data: 8198 / 10000\n", + "Epoch 48 training complete\n", + "Cost on training data: 0.06288011722315233\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.2950539977025026\n", + "Accuracy on evaluation data: 8196 / 10000\n", + "Epoch 49 training complete\n", + "Cost on training data: 0.060962755044427386\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.2962547103297726\n", + "Accuracy on evaluation data: 8198 / 10000\n", + "Epoch 50 training complete\n", + "Cost on training data: 0.059313394569255294\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.2931806203943674\n", + "Accuracy on evaluation data: 8198 / 10000\n", + "Epoch 51 training complete\n", + "Cost on training data: 0.057754311772527316\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.295027398274516\n", + "Accuracy on evaluation data: 8216 / 10000\n", + "Epoch 52 training complete\n", + "Cost on training data: 0.05605083208252143\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.3067080115203575\n", + "Accuracy on evaluation data: 8192 / 10000\n", + "Epoch 53 training complete\n", + "Cost on training data: 0.05452741019437828\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.301623196775712\n", + "Accuracy on evaluation data: 8209 / 10000\n", + "Epoch 54 training complete\n", + "Cost on training data: 0.05302349796679658\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.3102661450816484\n", + "Accuracy on evaluation data: 8203 / 10000\n", + "Epoch 55 training complete\n", + "Cost on training data: 0.05183517623059908\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 1.317102971511828\n", + "Accuracy on evaluation data: 8197 / 10000\n", + "Epoch 56 training complete\n", + "Cost on training data: 0.05065876789288546\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 1.3101413075430852\n", + "Accuracy on evaluation data: 8215 / 10000\n", + "Epoch 57 training complete\n", + "Cost on training data: 0.04931316532439156\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 1.3182059078509132\n", + "Accuracy on evaluation data: 8210 / 10000\n", + "Epoch 58 training complete\n", + "Cost on training data: 0.04808201147554886\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 1.3219914017224528\n", + "Accuracy on evaluation data: 8205 / 10000\n", + "Epoch 59 training complete\n", + "Cost on training data: 0.04694599921758915\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 1.3220168930418552\n", + "Accuracy on evaluation data: 8214 / 10000\n", + "Epoch 60 training complete\n", + "Cost on training data: 0.04586907126383279\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 1.3288195315653581\n", + "Accuracy on evaluation data: 8212 / 10000\n", + "Epoch 61 training complete\n", + "Cost on training data: 0.04501465781679831\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3309329727814414\n", + "Accuracy on evaluation data: 8209 / 10000\n", + "Epoch 62 training complete\n", + "Cost on training data: 0.044062922149367266\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3377398814506263\n", + "Accuracy on evaluation data: 8199 / 10000\n", + "Epoch 63 training complete\n", + "Cost on training data: 0.042864655186375375\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3410402890574369\n", + "Accuracy on evaluation data: 8207 / 10000\n", + "Epoch 64 training complete\n", + "Cost on training data: 0.04237331595616183\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3413560628190784\n", + "Accuracy on evaluation data: 8204 / 10000\n", + "Epoch 65 training complete\n", + "Cost on training data: 0.04113688040734209\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3387939608069348\n", + "Accuracy on evaluation data: 8202 / 10000\n", + "Epoch 66 training complete\n", + "Cost on training data: 0.04031110051258729\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3481975882928705\n", + "Accuracy on evaluation data: 8195 / 10000\n", + "Epoch 67 training complete\n", + "Cost on training data: 0.03943277505626922\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3496508626383046\n", + "Accuracy on evaluation data: 8204 / 10000\n", + "Epoch 68 training complete\n", + "Cost on training data: 0.03864212004622959\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.35748984543882\n", + "Accuracy on evaluation data: 8201 / 10000\n", + "Epoch 69 training complete\n", + "Cost on training data: 0.038010181275447516\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3576343756543063\n", + "Accuracy on evaluation data: 8183 / 10000\n", + "Epoch 70 training complete\n", + "Cost on training data: 0.03730032081667362\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3651495545396548\n", + "Accuracy on evaluation data: 8194 / 10000\n", + "Epoch 71 training complete\n", + "Cost on training data: 0.03644099597149351\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3605378855609154\n", + "Accuracy on evaluation data: 8206 / 10000\n", + "Epoch 72 training complete\n", + "Cost on training data: 0.03576282857667165\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.36831602035525\n", + "Accuracy on evaluation data: 8195 / 10000\n", + "Epoch 73 training complete\n", + "Cost on training data: 0.03522387788586625\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3697112210327866\n", + "Accuracy on evaluation data: 8210 / 10000\n", + "Epoch 74 training complete\n", + "Cost on training data: 0.03446312021481\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3710765577006996\n", + "Accuracy on evaluation data: 8198 / 10000\n", + "Epoch 75 training complete\n", + "Cost on training data: 0.03387647325113247\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3781921461506643\n", + "Accuracy on evaluation data: 8193 / 10000\n", + "Epoch 76 training complete\n", + "Cost on training data: 0.03330670862326028\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3792077144819084\n", + "Accuracy on evaluation data: 8200 / 10000\n", + "Epoch 77 training complete\n", + "Cost on training data: 0.03271599005830556\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3834590240489502\n", + "Accuracy on evaluation data: 8200 / 10000\n", + "Epoch 78 training complete\n", + "Cost on training data: 0.03217764200563641\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3783618991638205\n", + "Accuracy on evaluation data: 8211 / 10000\n", + "Epoch 79 training complete\n", + "Cost on training data: 0.03157999774153734\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.382546586547743\n", + "Accuracy on evaluation data: 8205 / 10000\n", + "Epoch 80 training complete\n", + "Cost on training data: 0.031179003172666947\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3894644019949025\n", + "Accuracy on evaluation data: 8200 / 10000\n", + "Epoch 81 training complete\n", + "Cost on training data: 0.030572208841224944\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3893881907065881\n", + "Accuracy on evaluation data: 8217 / 10000\n", + "Epoch 82 training complete\n", + "Cost on training data: 0.030085994430996237\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3922900676560082\n", + "Accuracy on evaluation data: 8214 / 10000\n", + "Epoch 83 training complete\n", + "Cost on training data: 0.02960066312073094\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.39349540112667\n", + "Accuracy on evaluation data: 8210 / 10000\n", + "Epoch 84 training complete\n", + "Cost on training data: 0.029107644388579147\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.397521888744422\n", + "Accuracy on evaluation data: 8205 / 10000\n", + "Epoch 85 training complete\n", + "Cost on training data: 0.028684292973173934\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.3959370955842871\n", + "Accuracy on evaluation data: 8215 / 10000\n", + "Epoch 86 training complete\n", + "Cost on training data: 0.028231901490790634\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.400024209711782\n", + "Accuracy on evaluation data: 8208 / 10000\n", + "Epoch 87 training complete\n", + "Cost on training data: 0.027757609130062477\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4040656765800947\n", + "Accuracy on evaluation data: 8209 / 10000\n", + "Epoch 88 training complete\n", + "Cost on training data: 0.027368445810644258\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4074845098971687\n", + "Accuracy on evaluation data: 8213 / 10000\n", + "Epoch 89 training complete\n", + "Cost on training data: 0.02695504102934996\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4054809209342527\n", + "Accuracy on evaluation data: 8213 / 10000\n", + "Epoch 90 training complete\n", + "Cost on training data: 0.026513622514702254\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4121669855346197\n", + "Accuracy on evaluation data: 8211 / 10000\n", + "Epoch 91 training complete\n", + "Cost on training data: 0.02611663820395111\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4140175338533816\n", + "Accuracy on evaluation data: 8211 / 10000\n", + "Epoch 92 training complete\n", + "Cost on training data: 0.025807948376968332\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4149255230327944\n", + "Accuracy on evaluation data: 8219 / 10000\n", + "Epoch 93 training complete\n", + "Cost on training data: 0.025454316039523618\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.420811063340669\n", + "Accuracy on evaluation data: 8214 / 10000\n", + "Epoch 94 training complete\n", + "Cost on training data: 0.02506843889856931\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4235729903942418\n", + "Accuracy on evaluation data: 8212 / 10000\n", + "Epoch 95 training complete\n", + "Cost on training data: 0.024692125980303802\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.423270556868474\n", + "Accuracy on evaluation data: 8210 / 10000\n", + "Epoch 96 training complete\n", + "Cost on training data: 0.02434940941283731\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4225790397958038\n", + "Accuracy on evaluation data: 8214 / 10000\n", + "Epoch 97 training complete\n", + "Cost on training data: 0.024035830277295753\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.426261150778658\n", + "Accuracy on evaluation data: 8226 / 10000\n", + "Epoch 98 training complete\n", + "Cost on training data: 0.023722457438395943\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.428368681068149\n", + "Accuracy on evaluation data: 8220 / 10000\n", + "Epoch 99 training complete\n", + "Cost on training data: 0.02341213092361567\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4326207694949253\n", + "Accuracy on evaluation data: 8217 / 10000\n", + "Epoch 100 training complete\n", + "Cost on training data: 0.023098537240013827\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4332303401692348\n", + "Accuracy on evaluation data: 8220 / 10000\n", + "Epoch 101 training complete\n", + "Cost on training data: 0.022817151016843056\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4376528618941582\n", + "Accuracy on evaluation data: 8222 / 10000\n", + "Epoch 102 training complete\n", + "Cost on training data: 0.022523199499019108\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4352667825412915\n", + "Accuracy on evaluation data: 8220 / 10000\n", + "Epoch 103 training complete\n", + "Cost on training data: 0.0222404394004425\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4390918080206805\n", + "Accuracy on evaluation data: 8222 / 10000\n", + "Epoch 104 training complete\n", + "Cost on training data: 0.022004442644593614\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.439276849599714\n", + "Accuracy on evaluation data: 8221 / 10000\n", + "Epoch 105 training complete\n", + "Cost on training data: 0.02169944556001633\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4437729928841094\n", + "Accuracy on evaluation data: 8216 / 10000\n", + "Epoch 106 training complete\n", + "Cost on training data: 0.02146282067262799\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4487354622635429\n", + "Accuracy on evaluation data: 8216 / 10000\n", + "Epoch 107 training complete\n", + "Cost on training data: 0.02120841717961023\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4466606304585425\n", + "Accuracy on evaluation data: 8224 / 10000\n", + "Epoch 108 training complete\n", + "Cost on training data: 0.02093161019103893\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4504603595368637\n", + "Accuracy on evaluation data: 8220 / 10000\n", + "Epoch 109 training complete\n", + "Cost on training data: 0.020699579524513075\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4536277965804805\n", + "Accuracy on evaluation data: 8213 / 10000\n", + "Epoch 110 training complete\n", + "Cost on training data: 0.02045880033849954\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4530603777273063\n", + "Accuracy on evaluation data: 8219 / 10000\n", + "Epoch 111 training complete\n", + "Cost on training data: 0.020234713128084246\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4562429027783668\n", + "Accuracy on evaluation data: 8214 / 10000\n", + "Epoch 112 training complete\n", + "Cost on training data: 0.020024149093450464\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4551093569695415\n", + "Accuracy on evaluation data: 8219 / 10000\n", + "Epoch 113 training complete\n", + "Cost on training data: 0.019814460996070486\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.457550101778239\n", + "Accuracy on evaluation data: 8224 / 10000\n", + "Epoch 114 training complete\n", + "Cost on training data: 0.01955988459431243\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4608060881204445\n", + "Accuracy on evaluation data: 8217 / 10000\n", + "Epoch 115 training complete\n", + "Cost on training data: 0.01936409452274061\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4612312688527824\n", + "Accuracy on evaluation data: 8219 / 10000\n", + "Epoch 116 training complete\n", + "Cost on training data: 0.01916388931384982\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.464551838095764\n", + "Accuracy on evaluation data: 8217 / 10000\n", + "Epoch 117 training complete\n", + "Cost on training data: 0.018933590567148967\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4677958223654286\n", + "Accuracy on evaluation data: 8221 / 10000\n", + "Epoch 118 training complete\n", + "Cost on training data: 0.018729390083159346\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4689921494085334\n", + "Accuracy on evaluation data: 8224 / 10000\n", + "Epoch 119 training complete\n", + "Cost on training data: 0.018525749142230194\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4700814569969478\n", + "Accuracy on evaluation data: 8219 / 10000\n", + "Epoch 120 training complete\n", + "Cost on training data: 0.018339461850728867\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4715316261535325\n", + "Accuracy on evaluation data: 8223 / 10000\n", + "Epoch 121 training complete\n", + "Cost on training data: 0.018159367735947188\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4727635802262093\n", + "Accuracy on evaluation data: 8216 / 10000\n", + "Epoch 122 training complete\n", + "Cost on training data: 0.01796040030799893\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4750908027308307\n", + "Accuracy on evaluation data: 8220 / 10000\n", + "Epoch 123 training complete\n", + "Cost on training data: 0.017790972983079512\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4769727566117825\n", + "Accuracy on evaluation data: 8221 / 10000\n", + "Epoch 124 training complete\n", + "Cost on training data: 0.01757811022456622\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4762209204256154\n", + "Accuracy on evaluation data: 8223 / 10000\n", + "Epoch 125 training complete\n", + "Cost on training data: 0.017416367243459144\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4818646093531178\n", + "Accuracy on evaluation data: 8219 / 10000\n", + "Epoch 126 training complete\n", + "Cost on training data: 0.01722513967049368\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4798721867741391\n", + "Accuracy on evaluation data: 8219 / 10000\n", + "Epoch 127 training complete\n", + "Cost on training data: 0.017038005623370794\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.483668047996717\n", + "Accuracy on evaluation data: 8221 / 10000\n", + "Epoch 128 training complete\n", + "Cost on training data: 0.016875901711556916\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4841369937772184\n", + "Accuracy on evaluation data: 8218 / 10000\n", + "Epoch 129 training complete\n", + "Cost on training data: 0.01670295763927375\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4868322941159384\n", + "Accuracy on evaluation data: 8216 / 10000\n", + "Epoch 130 training complete\n", + "Cost on training data: 0.01653414525930512\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4894895132330574\n", + "Accuracy on evaluation data: 8215 / 10000\n", + "Epoch 131 training complete\n", + "Cost on training data: 0.016368627579098096\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.490645630141911\n", + "Accuracy on evaluation data: 8218 / 10000\n", + "Epoch 132 training complete\n", + "Cost on training data: 0.01620489532559457\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4935179414778919\n", + "Accuracy on evaluation data: 8225 / 10000\n", + "Epoch 133 training complete\n", + "Cost on training data: 0.0160550699394903\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4931559047420946\n", + "Accuracy on evaluation data: 8221 / 10000\n", + "Epoch 134 training complete\n", + "Cost on training data: 0.015894962059204494\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.494821165242315\n", + "Accuracy on evaluation data: 8226 / 10000\n", + "Epoch 135 training complete\n", + "Cost on training data: 0.01574436368426297\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.4982987830100258\n", + "Accuracy on evaluation data: 8224 / 10000\n", + "Epoch 136 training complete\n", + "Cost on training data: 0.015578465821747413\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.498983373342035\n", + "Accuracy on evaluation data: 8223 / 10000\n", + "Epoch 137 training complete\n", + "Cost on training data: 0.015427546615023003\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5024046412827112\n", + "Accuracy on evaluation data: 8225 / 10000\n", + "Epoch 138 training complete\n", + "Cost on training data: 0.015268064813783552\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5021758832252254\n", + "Accuracy on evaluation data: 8228 / 10000\n", + "Epoch 139 training complete\n", + "Cost on training data: 0.015103504506514301\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5034903114302511\n", + "Accuracy on evaluation data: 8227 / 10000\n", + "Epoch 140 training complete\n", + "Cost on training data: 0.014934826715961831\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.506221174763389\n", + "Accuracy on evaluation data: 8230 / 10000\n", + "Epoch 141 training complete\n", + "Cost on training data: 0.014770536938262864\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5081859311873334\n", + "Accuracy on evaluation data: 8230 / 10000\n", + "Epoch 142 training complete\n", + "Cost on training data: 0.0146299853105846\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5068049737772495\n", + "Accuracy on evaluation data: 8234 / 10000\n", + "Epoch 143 training complete\n", + "Cost on training data: 0.014477448717495065\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5099669764920043\n", + "Accuracy on evaluation data: 8230 / 10000\n", + "Epoch 144 training complete\n", + "Cost on training data: 0.014330075658197803\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5106095649051596\n", + "Accuracy on evaluation data: 8236 / 10000\n", + "Epoch 145 training complete\n", + "Cost on training data: 0.014191433847530778\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5113200173728714\n", + "Accuracy on evaluation data: 8241 / 10000\n", + "Epoch 146 training complete\n", + "Cost on training data: 0.014062532482983479\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5122063170071816\n", + "Accuracy on evaluation data: 8239 / 10000\n", + "Epoch 147 training complete\n", + "Cost on training data: 0.013929671467982666\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.515155506532603\n", + "Accuracy on evaluation data: 8234 / 10000\n", + "Epoch 148 training complete\n", + "Cost on training data: 0.013808595438217752\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5160198756216068\n", + "Accuracy on evaluation data: 8242 / 10000\n", + "Epoch 149 training complete\n", + "Cost on training data: 0.013677575394746105\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5193383158292824\n", + "Accuracy on evaluation data: 8240 / 10000\n", + "Epoch 150 training complete\n", + "Cost on training data: 0.01356197371507288\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5222955908440372\n", + "Accuracy on evaluation data: 8233 / 10000\n", + "Epoch 151 training complete\n", + "Cost on training data: 0.013435214796044476\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.521238157477064\n", + "Accuracy on evaluation data: 8234 / 10000\n", + "Epoch 152 training complete\n", + "Cost on training data: 0.013320338943232096\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5211165723793136\n", + "Accuracy on evaluation data: 8235 / 10000\n", + "Epoch 153 training complete\n", + "Cost on training data: 0.0132160865996535\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.52314659664655\n", + "Accuracy on evaluation data: 8232 / 10000\n", + "Epoch 154 training complete\n", + "Cost on training data: 0.01308122729565835\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5257559485093612\n", + "Accuracy on evaluation data: 8235 / 10000\n", + "Epoch 155 training complete\n", + "Cost on training data: 0.012973813867916196\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5283583402669176\n", + "Accuracy on evaluation data: 8233 / 10000\n", + "Epoch 156 training complete\n", + "Cost on training data: 0.012857676107700585\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.528796192317271\n", + "Accuracy on evaluation data: 8240 / 10000\n", + "Epoch 157 training complete\n", + "Cost on training data: 0.012748464294141526\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.531003226996045\n", + "Accuracy on evaluation data: 8240 / 10000\n", + "Epoch 158 training complete\n", + "Cost on training data: 0.012637999185439681\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.53086365071215\n", + "Accuracy on evaluation data: 8239 / 10000\n", + "Epoch 159 training complete\n", + "Cost on training data: 0.01253917600244282\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5335835590847948\n", + "Accuracy on evaluation data: 8239 / 10000\n", + "Epoch 160 training complete\n", + "Cost on training data: 0.012426312791981805\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5332154355594414\n", + "Accuracy on evaluation data: 8242 / 10000\n", + "Epoch 161 training complete\n", + "Cost on training data: 0.01233115898703347\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5343559839610852\n", + "Accuracy on evaluation data: 8235 / 10000\n", + "Epoch 162 training complete\n", + "Cost on training data: 0.012231580170086466\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5358457412906548\n", + "Accuracy on evaluation data: 8242 / 10000\n", + "Epoch 163 training complete\n", + "Cost on training data: 0.012132977473173169\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.535897960937641\n", + "Accuracy on evaluation data: 8243 / 10000\n", + "Epoch 164 training complete\n", + "Cost on training data: 0.012037281975717308\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5379522375264936\n", + "Accuracy on evaluation data: 8246 / 10000\n", + "Epoch 165 training complete\n", + "Cost on training data: 0.01194839607425753\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5387319727955346\n", + "Accuracy on evaluation data: 8242 / 10000\n", + "Epoch 166 training complete\n", + "Cost on training data: 0.011856252031422097\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5404425014271417\n", + "Accuracy on evaluation data: 8245 / 10000\n", + "Epoch 167 training complete\n", + "Cost on training data: 0.011764630017164277\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5415652159293862\n", + "Accuracy on evaluation data: 8239 / 10000\n", + "Epoch 168 training complete\n", + "Cost on training data: 0.011674903664946534\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.542339220279419\n", + "Accuracy on evaluation data: 8250 / 10000\n", + "Epoch 169 training complete\n", + "Cost on training data: 0.011589566947431152\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5435281772875544\n", + "Accuracy on evaluation data: 8253 / 10000\n", + "Epoch 170 training complete\n", + "Cost on training data: 0.011501007482879019\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5452941322923461\n", + "Accuracy on evaluation data: 8248 / 10000\n", + "Epoch 171 training complete\n", + "Cost on training data: 0.011417269687122752\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5472745754372332\n", + "Accuracy on evaluation data: 8246 / 10000\n", + "Epoch 172 training complete\n", + "Cost on training data: 0.011334434709263212\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.547238926959627\n", + "Accuracy on evaluation data: 8252 / 10000\n", + "Epoch 173 training complete\n", + "Cost on training data: 0.011255226856906195\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5489111857321223\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 174 training complete\n", + "Cost on training data: 0.011172222806953567\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5510675394769664\n", + "Accuracy on evaluation data: 8252 / 10000\n", + "Epoch 175 training complete\n", + "Cost on training data: 0.01109307925054622\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5516230263707744\n", + "Accuracy on evaluation data: 8251 / 10000\n", + "Epoch 176 training complete\n", + "Cost on training data: 0.011013904975749476\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.552493081633442\n", + "Accuracy on evaluation data: 8243 / 10000\n", + "Epoch 177 training complete\n", + "Cost on training data: 0.010938711593987552\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.553381667995361\n", + "Accuracy on evaluation data: 8245 / 10000\n", + "Epoch 178 training complete\n", + "Cost on training data: 0.01086145842247982\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5539812918566476\n", + "Accuracy on evaluation data: 8245 / 10000\n", + "Epoch 179 training complete\n", + "Cost on training data: 0.01078413752110269\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.554704075224815\n", + "Accuracy on evaluation data: 8249 / 10000\n", + "Epoch 180 training complete\n", + "Cost on training data: 0.010711799767546146\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5560847479533892\n", + "Accuracy on evaluation data: 8246 / 10000\n", + "Epoch 181 training complete\n", + "Cost on training data: 0.01063691151131155\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5566143387679001\n", + "Accuracy on evaluation data: 8246 / 10000\n", + "Epoch 182 training complete\n", + "Cost on training data: 0.010565925186662488\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5578021206100057\n", + "Accuracy on evaluation data: 8245 / 10000\n", + "Epoch 183 training complete\n", + "Cost on training data: 0.010496857027637\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5595682446942714\n", + "Accuracy on evaluation data: 8248 / 10000\n", + "Epoch 184 training complete\n", + "Cost on training data: 0.01042094715309233\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5605737148664311\n", + "Accuracy on evaluation data: 8250 / 10000\n", + "Epoch 185 training complete\n", + "Cost on training data: 0.010350871101044991\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.561525631681385\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 186 training complete\n", + "Cost on training data: 0.01028380367783822\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5615933851867305\n", + "Accuracy on evaluation data: 8248 / 10000\n", + "Epoch 187 training complete\n", + "Cost on training data: 0.010214413748011075\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.563565335711899\n", + "Accuracy on evaluation data: 8251 / 10000\n", + "Epoch 188 training complete\n", + "Cost on training data: 0.01014722924997424\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5643793049751595\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 189 training complete\n", + "Cost on training data: 0.010081283763745444\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5670653232518992\n", + "Accuracy on evaluation data: 8240 / 10000\n", + "Epoch 190 training complete\n", + "Cost on training data: 0.010017977235901135\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5687535566213113\n", + "Accuracy on evaluation data: 8241 / 10000\n", + "Epoch 191 training complete\n", + "Cost on training data: 0.009952750262860877\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5680076945638508\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 192 training complete\n", + "Cost on training data: 0.00988743316022295\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.568962686682649\n", + "Accuracy on evaluation data: 8245 / 10000\n", + "Epoch 193 training complete\n", + "Cost on training data: 0.009824493684716372\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5705078456430996\n", + "Accuracy on evaluation data: 8248 / 10000\n", + "Epoch 194 training complete\n", + "Cost on training data: 0.009761498957291571\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5707022552489367\n", + "Accuracy on evaluation data: 8246 / 10000\n", + "Epoch 195 training complete\n", + "Cost on training data: 0.009702107081864487\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.571925410236475\n", + "Accuracy on evaluation data: 8246 / 10000\n", + "Epoch 196 training complete\n", + "Cost on training data: 0.009639153988079349\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5729906857422657\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 197 training complete\n", + "Cost on training data: 0.009580395070749082\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.573808540823732\n", + "Accuracy on evaluation data: 8250 / 10000\n", + "Epoch 198 training complete\n", + "Cost on training data: 0.00951972545081291\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5751813545355047\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 199 training complete\n", + "Cost on training data: 0.009461494808678769\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5762370604358307\n", + "Accuracy on evaluation data: 8250 / 10000\n", + "Epoch 200 training complete\n", + "Cost on training data: 0.009404008428627307\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5771404395450144\n", + "Accuracy on evaluation data: 8252 / 10000\n", + "Epoch 201 training complete\n", + "Cost on training data: 0.009346364014697569\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.579238555188802\n", + "Accuracy on evaluation data: 8251 / 10000\n", + "Epoch 202 training complete\n", + "Cost on training data: 0.009289604906728512\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5797176320272455\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 203 training complete\n", + "Cost on training data: 0.009236213216646163\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5803834379144446\n", + "Accuracy on evaluation data: 8244 / 10000\n", + "Epoch 204 training complete\n", + "Cost on training data: 0.009181120069335098\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5801338531380646\n", + "Accuracy on evaluation data: 8247 / 10000\n", + "Epoch 205 training complete\n", + "Cost on training data: 0.009124044272123344\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5816886645929218\n", + "Accuracy on evaluation data: 8248 / 10000\n", + "Epoch 206 training complete\n", + "Cost on training data: 0.009069037561121103\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5825150635201113\n", + "Accuracy on evaluation data: 8250 / 10000\n", + "Epoch 207 training complete\n", + "Cost on training data: 0.009015787967924594\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5837476374182395\n", + "Accuracy on evaluation data: 8248 / 10000\n", + "Epoch 208 training complete\n", + "Cost on training data: 0.008962348906204846\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5845912855677802\n", + "Accuracy on evaluation data: 8249 / 10000\n", + "Epoch 209 training complete\n", + "Cost on training data: 0.008909330346897117\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5861070413192737\n", + "Accuracy on evaluation data: 8248 / 10000\n", + "Epoch 210 training complete\n", + "Cost on training data: 0.0088578503570985\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5862293894286048\n", + "Accuracy on evaluation data: 8254 / 10000\n", + "Epoch 211 training complete\n", + "Cost on training data: 0.008805395298068718\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5875894256803806\n", + "Accuracy on evaluation data: 8250 / 10000\n", + "Epoch 212 training complete\n", + "Cost on training data: 0.008756162374926822\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5890041586758294\n", + "Accuracy on evaluation data: 8251 / 10000\n", + "Epoch 213 training complete\n", + "Cost on training data: 0.008704774313529161\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5905536950944812\n", + "Accuracy on evaluation data: 8252 / 10000\n", + "Epoch 214 training complete\n", + "Cost on training data: 0.008655116969098696\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5906104188035737\n", + "Accuracy on evaluation data: 8251 / 10000\n", + "Epoch 215 training complete\n", + "Cost on training data: 0.00860622006861436\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.591899503370647\n", + "Accuracy on evaluation data: 8255 / 10000\n", + "Epoch 216 training complete\n", + "Cost on training data: 0.008556648616859492\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.592984854787386\n", + "Accuracy on evaluation data: 8254 / 10000\n", + "Epoch 217 training complete\n", + "Cost on training data: 0.008507934583225968\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5930018770885517\n", + "Accuracy on evaluation data: 8254 / 10000\n", + "Epoch 218 training complete\n", + "Cost on training data: 0.008460195380096708\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.593093083320141\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 219 training complete\n", + "Cost on training data: 0.00841415827981467\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5941457626432558\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 220 training complete\n", + "Cost on training data: 0.008366023246569062\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5958548820763452\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 221 training complete\n", + "Cost on training data: 0.008319358255204188\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5963544806142091\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 222 training complete\n", + "Cost on training data: 0.008273008931560948\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5977850066725487\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 223 training complete\n", + "Cost on training data: 0.008228660000918632\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5982923375115397\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 224 training complete\n", + "Cost on training data: 0.008181725042985326\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.5996178530036418\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 225 training complete\n", + "Cost on training data: 0.008136863651526004\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.600734060270051\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 226 training complete\n", + "Cost on training data: 0.008094377669737892\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6016651807848425\n", + "Accuracy on evaluation data: 8257 / 10000\n", + "Epoch 227 training complete\n", + "Cost on training data: 0.00804813607280009\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6013758046147637\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 228 training complete\n", + "Cost on training data: 0.00800672659661432\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.602354670872372\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 229 training complete\n", + "Cost on training data: 0.00796153505530008\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.603617158578109\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 230 training complete\n", + "Cost on training data: 0.007917234348583506\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6042379803410476\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 231 training complete\n", + "Cost on training data: 0.007875293994275593\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.60491873701571\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 232 training complete\n", + "Cost on training data: 0.007832319483670584\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6054881692429004\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 233 training complete\n", + "Cost on training data: 0.007790706070193431\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.606277329885692\n", + "Accuracy on evaluation data: 8267 / 10000\n", + "Epoch 234 training complete\n", + "Cost on training data: 0.007749011070199978\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6077228729963486\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 235 training complete\n", + "Cost on training data: 0.007706369894621162\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6087583178473122\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 236 training complete\n", + "Cost on training data: 0.007665709696913906\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6091552844759875\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 237 training complete\n", + "Cost on training data: 0.007625233750307463\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6106943981305684\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 238 training complete\n", + "Cost on training data: 0.007583735529223643\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6110209479594637\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 239 training complete\n", + "Cost on training data: 0.0075434225650592894\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6120964197267988\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 240 training complete\n", + "Cost on training data: 0.007503923612725445\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.612536767386138\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 241 training complete\n", + "Cost on training data: 0.007463604505461669\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6130865236821745\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 242 training complete\n", + "Cost on training data: 0.007424279495715489\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6155033425683878\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 243 training complete\n", + "Cost on training data: 0.007384343579617704\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6153007697642516\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 244 training complete\n", + "Cost on training data: 0.007345821666857079\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6155547223874642\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 245 training complete\n", + "Cost on training data: 0.007306525171442447\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6162727841392313\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 246 training complete\n", + "Cost on training data: 0.007268397972046537\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6171320716891022\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 247 training complete\n", + "Cost on training data: 0.007229574059114321\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6182869531255215\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 248 training complete\n", + "Cost on training data: 0.007191360576969794\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6193693150795545\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 249 training complete\n", + "Cost on training data: 0.007153416387507835\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6202257076154774\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 250 training complete\n", + "Cost on training data: 0.007116273075116602\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.621018109475594\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 251 training complete\n", + "Cost on training data: 0.007079448197868202\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6222043232042254\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 252 training complete\n", + "Cost on training data: 0.007043844085072149\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.622665085175885\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 253 training complete\n", + "Cost on training data: 0.007007076398652079\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6233609917847345\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 254 training complete\n", + "Cost on training data: 0.0069700031324625225\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6250647833218514\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 255 training complete\n", + "Cost on training data: 0.0069344116152204125\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6251958255148409\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 256 training complete\n", + "Cost on training data: 0.006899215395444137\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6256890901710934\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 257 training complete\n", + "Cost on training data: 0.0068643682854109\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6268078724794777\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 258 training complete\n", + "Cost on training data: 0.006829852079409992\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6282695132070089\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 259 training complete\n", + "Cost on training data: 0.006795345880340338\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.628365182192767\n", + "Accuracy on evaluation data: 8257 / 10000\n", + "Epoch 260 training complete\n", + "Cost on training data: 0.00676252131053845\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6288900725771003\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 261 training complete\n", + "Cost on training data: 0.006728999291564674\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6301185286120188\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 262 training complete\n", + "Cost on training data: 0.006696904635645604\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.63121273884661\n", + "Accuracy on evaluation data: 8256 / 10000\n", + "Epoch 263 training complete\n", + "Cost on training data: 0.006663305064515444\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6321270262678036\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 264 training complete\n", + "Cost on training data: 0.006631686240391007\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.631832530579785\n", + "Accuracy on evaluation data: 8257 / 10000\n", + "Epoch 265 training complete\n", + "Cost on training data: 0.006599603059553482\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6332499883063964\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 266 training complete\n", + "Cost on training data: 0.006567987991887161\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6335770602443906\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 267 training complete\n", + "Cost on training data: 0.006537058437640403\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6340649452970486\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 268 training complete\n", + "Cost on training data: 0.006506815411209714\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6349323481733453\n", + "Accuracy on evaluation data: 8257 / 10000\n", + "Epoch 269 training complete\n", + "Cost on training data: 0.00647566623928613\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.635756338403813\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 270 training complete\n", + "Cost on training data: 0.006446070412089933\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.63705649237236\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 271 training complete\n", + "Cost on training data: 0.006416127651882635\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6369032795134182\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 272 training complete\n", + "Cost on training data: 0.00638669842350974\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6380542002910468\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 273 training complete\n", + "Cost on training data: 0.006357451676923163\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6390724351824668\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 274 training complete\n", + "Cost on training data: 0.006328955931002924\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6399045816529796\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 275 training complete\n", + "Cost on training data: 0.006300362201816859\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6401675333314898\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 276 training complete\n", + "Cost on training data: 0.006272487490410543\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6404131827274355\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 277 training complete\n", + "Cost on training data: 0.006244710524048656\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6415504861202117\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 278 training complete\n", + "Cost on training data: 0.006216829935219213\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6424981238647989\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 279 training complete\n", + "Cost on training data: 0.006189506991397907\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.643186632053072\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 280 training complete\n", + "Cost on training data: 0.00616246405676362\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6439712299081704\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 281 training complete\n", + "Cost on training data: 0.0061355832587149444\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.644873997748866\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 282 training complete\n", + "Cost on training data: 0.006109404203449112\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6455129428825017\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 283 training complete\n", + "Cost on training data: 0.006083064848133412\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6457854349325536\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 284 training complete\n", + "Cost on training data: 0.00605705041164134\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6466444056502463\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 285 training complete\n", + "Cost on training data: 0.00603124563686737\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.647327707456011\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 286 training complete\n", + "Cost on training data: 0.006005446738726149\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6481434366981265\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 287 training complete\n", + "Cost on training data: 0.005980504395016348\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6486872369606032\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 288 training complete\n", + "Cost on training data: 0.005955223909777157\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6497310205137445\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 289 training complete\n", + "Cost on training data: 0.005930267170141301\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.65018934777546\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 290 training complete\n", + "Cost on training data: 0.00590601627456752\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.65060537744573\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 291 training complete\n", + "Cost on training data: 0.0058814554409195455\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.651454144674819\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 292 training complete\n", + "Cost on training data: 0.005857039055755918\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6519202612550719\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 293 training complete\n", + "Cost on training data: 0.005833117741047484\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6528771137511697\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 294 training complete\n", + "Cost on training data: 0.005809151255610471\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.653243596666332\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 295 training complete\n", + "Cost on training data: 0.0057857524497790394\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6540473621858318\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 296 training complete\n", + "Cost on training data: 0.0057626614653838466\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6548327730642833\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 297 training complete\n", + "Cost on training data: 0.00573894201980642\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6558875085910072\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 298 training complete\n", + "Cost on training data: 0.005715846914915137\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6563279866978224\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 299 training complete\n", + "Cost on training data: 0.005693194514775354\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6570123113691686\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 300 training complete\n", + "Cost on training data: 0.005670603215700334\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6574322117093276\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 301 training complete\n", + "Cost on training data: 0.0056482602534283655\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6580054134403943\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 302 training complete\n", + "Cost on training data: 0.0056254520676485035\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6585890718527607\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 303 training complete\n", + "Cost on training data: 0.005603420245050404\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6593421338624457\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 304 training complete\n", + "Cost on training data: 0.005581445506785208\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6598425005269104\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 305 training complete\n", + "Cost on training data: 0.005559558212762005\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6604453975491615\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 306 training complete\n", + "Cost on training data: 0.005537925986503345\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.661066976658005\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 307 training complete\n", + "Cost on training data: 0.00551636484867353\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6617929269331915\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 308 training complete\n", + "Cost on training data: 0.005495360384284129\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6619725568713157\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 309 training complete\n", + "Cost on training data: 0.00547377265071863\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6629110253339092\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 310 training complete\n", + "Cost on training data: 0.005452945144169912\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6636071870009699\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 311 training complete\n", + "Cost on training data: 0.005432313258222344\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.664130346325899\n", + "Accuracy on evaluation data: 8268 / 10000\n", + "Epoch 312 training complete\n", + "Cost on training data: 0.005411256173184737\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6644397769094854\n", + "Accuracy on evaluation data: 8267 / 10000\n", + "Epoch 313 training complete\n", + "Cost on training data: 0.005390542324592822\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6655005941384036\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 314 training complete\n", + "Cost on training data: 0.00537003501709931\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6661116634455302\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 315 training complete\n", + "Cost on training data: 0.005349676803802429\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6665968593034226\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 316 training complete\n", + "Cost on training data: 0.0053294734768183916\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6672165384376592\n", + "Accuracy on evaluation data: 8268 / 10000\n", + "Epoch 317 training complete\n", + "Cost on training data: 0.00530956620136094\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.668002559215349\n", + "Accuracy on evaluation data: 8268 / 10000\n", + "Epoch 318 training complete\n", + "Cost on training data: 0.005289663985422463\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6688553495346934\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 319 training complete\n", + "Cost on training data: 0.005269946086041686\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6693075494599285\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 320 training complete\n", + "Cost on training data: 0.0052505229085850124\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.669853236382765\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 321 training complete\n", + "Cost on training data: 0.005230977936130353\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6707829452258167\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 322 training complete\n", + "Cost on training data: 0.005211853681500548\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6715867289604964\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 323 training complete\n", + "Cost on training data: 0.005192217680047017\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6720985593167006\n", + "Accuracy on evaluation data: 8266 / 10000\n", + "Epoch 324 training complete\n", + "Cost on training data: 0.005173285940357145\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6729252115648292\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 325 training complete\n", + "Cost on training data: 0.0051542636361430775\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6733978376453427\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 326 training complete\n", + "Cost on training data: 0.005135412326276922\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6739437961551715\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 327 training complete\n", + "Cost on training data: 0.005116875476029445\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.674052188363298\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 328 training complete\n", + "Cost on training data: 0.005098514320420585\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6745783604683062\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 329 training complete\n", + "Cost on training data: 0.005079909416105223\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6754445097909576\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 330 training complete\n", + "Cost on training data: 0.005061511072394871\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6757368261352805\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 331 training complete\n", + "Cost on training data: 0.005043474906331434\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6766746708213132\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 332 training complete\n", + "Cost on training data: 0.005025305108634019\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.677331323553987\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 333 training complete\n", + "Cost on training data: 0.005007500816247154\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6780402069284128\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 334 training complete\n", + "Cost on training data: 0.004989561798409312\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6784907963557851\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 335 training complete\n", + "Cost on training data: 0.004971890395930867\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.679159773406658\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 336 training complete\n", + "Cost on training data: 0.004954523308164781\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6796148608953612\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 337 training complete\n", + "Cost on training data: 0.004937009931754602\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6803997323954925\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 338 training complete\n", + "Cost on training data: 0.004919631843866032\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6810197261232134\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 339 training complete\n", + "Cost on training data: 0.0049024057291484355\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.681230947136724\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 340 training complete\n", + "Cost on training data: 0.004885336701014626\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6820730040790024\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 341 training complete\n", + "Cost on training data: 0.004868412714072125\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.682746212929891\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 342 training complete\n", + "Cost on training data: 0.004851504004786531\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6832860180101294\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 343 training complete\n", + "Cost on training data: 0.0048348582176695075\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6837122798176372\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 344 training complete\n", + "Cost on training data: 0.0048182128654199975\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6842629328056888\n", + "Accuracy on evaluation data: 8265 / 10000\n", + "Epoch 345 training complete\n", + "Cost on training data: 0.004801637346051215\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.684807116810647\n", + "Accuracy on evaluation data: 8264 / 10000\n", + "Epoch 346 training complete\n", + "Cost on training data: 0.0047853669573391594\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6851517384158974\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 347 training complete\n", + "Cost on training data: 0.004769128045135498\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6855400710775272\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 348 training complete\n", + "Cost on training data: 0.004752905567987157\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6860911053868033\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 349 training complete\n", + "Cost on training data: 0.004737014629575726\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6865487857201666\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 350 training complete\n", + "Cost on training data: 0.004720894082042677\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6876098710636076\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 351 training complete\n", + "Cost on training data: 0.004705083941506755\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6882700371768171\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 352 training complete\n", + "Cost on training data: 0.0046893465166420985\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6886416630272116\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 353 training complete\n", + "Cost on training data: 0.004673728225116529\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6891524386241665\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 354 training complete\n", + "Cost on training data: 0.004658333224470052\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6895467797528891\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 355 training complete\n", + "Cost on training data: 0.0046428848307164565\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.690101776165049\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 356 training complete\n", + "Cost on training data: 0.004627592249922394\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6908809028191816\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 357 training complete\n", + "Cost on training data: 0.004612758064794903\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6917622121452638\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 358 training complete\n", + "Cost on training data: 0.004597619682318191\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.69194461698808\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 359 training complete\n", + "Cost on training data: 0.00458240558472866\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6922757276705014\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 360 training complete\n", + "Cost on training data: 0.0045674661189958705\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.69290929114963\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 361 training complete\n", + "Cost on training data: 0.00455260047778786\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.693535789653878\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 362 training complete\n", + "Cost on training data: 0.004537901598695279\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6939093622299455\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 363 training complete\n", + "Cost on training data: 0.004523475601230529\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6941812838283166\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 364 training complete\n", + "Cost on training data: 0.004509024438086374\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.694708634016367\n", + "Accuracy on evaluation data: 8256 / 10000\n", + "Epoch 365 training complete\n", + "Cost on training data: 0.004494448458313414\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6956616442888162\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 366 training complete\n", + "Cost on training data: 0.004480082875996692\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6959008986227975\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 367 training complete\n", + "Cost on training data: 0.004465894367302122\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6965972540880916\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 368 training complete\n", + "Cost on training data: 0.004451774401480516\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.697045460803176\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 369 training complete\n", + "Cost on training data: 0.004437679059832115\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6976740373091415\n", + "Accuracy on evaluation data: 8258 / 10000\n", + "Epoch 370 training complete\n", + "Cost on training data: 0.004423795123569889\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6979160617745577\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 371 training complete\n", + "Cost on training data: 0.004410010642978251\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6984418220118542\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 372 training complete\n", + "Cost on training data: 0.004396204282922317\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.69901336575197\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 373 training complete\n", + "Cost on training data: 0.004382503899635278\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6993922789057865\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 374 training complete\n", + "Cost on training data: 0.004368948605163991\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.6996481374258623\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 375 training complete\n", + "Cost on training data: 0.004355313860951414\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7002910099604072\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 376 training complete\n", + "Cost on training data: 0.0043419399322806584\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.700878855047439\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 377 training complete\n", + "Cost on training data: 0.0043284167002218665\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7017612170590064\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 378 training complete\n", + "Cost on training data: 0.004315154144463605\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7023523312746343\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 379 training complete\n", + "Cost on training data: 0.004301952525829851\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.702780729400329\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 380 training complete\n", + "Cost on training data: 0.004288832827696747\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7032954834239875\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 381 training complete\n", + "Cost on training data: 0.004275740882980199\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7038400731693575\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 382 training complete\n", + "Cost on training data: 0.004262800640731024\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7042784920025298\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 383 training complete\n", + "Cost on training data: 0.004249898337732379\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7049437907310814\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 384 training complete\n", + "Cost on training data: 0.004237128580603489\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7054109983427048\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 385 training complete\n", + "Cost on training data: 0.004224358993102038\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.705834006906528\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 386 training complete\n", + "Cost on training data: 0.004211722801421006\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.706313224246325\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 387 training complete\n", + "Cost on training data: 0.004199144451630486\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7070334320566591\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 388 training complete\n", + "Cost on training data: 0.004186651895061263\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7071736277238458\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 389 training complete\n", + "Cost on training data: 0.0041741027537171675\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7076219598621805\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 390 training complete\n", + "Cost on training data: 0.004161768076698958\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7080054833451552\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 391 training complete\n", + "Cost on training data: 0.004149512540419438\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7086850899391604\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 392 training complete\n", + "Cost on training data: 0.004137216785004015\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7093693851619893\n", + "Accuracy on evaluation data: 8259 / 10000\n", + "Epoch 393 training complete\n", + "Cost on training data: 0.0041251104235614745\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.709951690585734\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 394 training complete\n", + "Cost on training data: 0.004112948228031242\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7103160318590973\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 395 training complete\n", + "Cost on training data: 0.004100945363309996\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.710905582289716\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 396 training complete\n", + "Cost on training data: 0.0040890163558027475\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7113677253423882\n", + "Accuracy on evaluation data: 8262 / 10000\n", + "Epoch 397 training complete\n", + "Cost on training data: 0.004077138135973765\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7118517323524272\n", + "Accuracy on evaluation data: 8261 / 10000\n", + "Epoch 398 training complete\n", + "Cost on training data: 0.004065283890559541\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7123626884394971\n", + "Accuracy on evaluation data: 8260 / 10000\n", + "Epoch 399 training complete\n", + "Cost on training data: 0.00405351314039449\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 1.7129584728715945\n", + "Accuracy on evaluation data: 8260 / 10000\n" + ] + } + ], + "source": [ + "from elements import network2\n", + "from elements import mnist_loader\n", + "\n", + "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()\n", + "training_data = list(training_data)\n", + "test_data = list(test_data)\n", + "\n", + "net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)\n", + "net.large_weight_initializer()\n", + "evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \\\n", + " net.SGD(training_data[:1000], 400, 10, 0.5, evaluation_data=test_data,\n", + " monitor_evaluation_cost=True,\n", + " monitor_evaluation_accuracy=True,\n", + " monitor_training_cost=True,\n", + " monitor_training_accuracy=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 368 + }, + "id": "STvKzz4M5TT5", + "outputId": "81dbe19c-a008-4cdb-aabf-b87e4830b364" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 0, 'Epoch')" + ] + }, + "metadata": {}, + "execution_count": 12 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAFNCAYAAAAJsbjVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wUdf7H8deHJCT0EiB0QhMEpAVBFBREDywnFlTsnnqcBcupv1POcp5nwzu7eJ4CHoqKXTlFbICISpeOSChKF6QZeuDz+2OHuzUSSEKS2WTfz8djH8x+5zuz75ks+9kpO2PujoiIxJ8yYQcQEZFwqACIiMQpFQARkTilAiAiEqdUAERE4pQKgIhInFIBkLhjZhPM7Kqwc+xnZs+a2V2F3fdwmdlyMzupOF5LwqECIAdlZhea2XQzyzKzNWb2oZl1O8x5FtsHi5ndY2Yji3D+h70s7n61u/+tsPsWJzNzM2sWdg7JHxUAyZWZ3Qw8DjwApAENgWeAvmHmKknMLDHsDCK5cnc99PjVA6gCZAHnHqRPMpECsTp4PA4kB+NqAO8Dm4GNwBdEvnC8BOwDdgTz/1Mu8/49kBlMOxqoGzXOgauBxcH8hwB2gHn0AXYDe4LXmh20TwD+BnwJ/Ax8DNSImu4Y4Ktg3rOBHrlk/NWyAOlBviuBH4CJQd83gLXAFmAi0DpqPv8G7guGewArgVuAH4E1wO8K2DcV+A+wFZgG3AdMOsjf8xLge+An4A5gOXBSMK4z8HWwTtYATwNlg3ETg2XeFqyH84Fqwd9/PbApGK4f9vtajxx/87AD6BGbj+DDMxtIPEife4HJQC2gZvCh+bdg3IPAs0BS8Oi+/0M6+oMll/meCGwAOhIpMk/t/yANxnvwgVKVyFbJeqBPLvO6BxiZo20CsAQ4AigXPH8oGFcv+AA8lUjBOjl4XjOX+f9iWfhfAXgRqACUC9qvACrxv6I5K2qanB/q2cG6TQpybAeqFaDvqOBRHmgFrCCXAhCMzwKODzI+Gsx7fwHIIFIYE4NlXAjclONv0izqeSpwTvDalYgUwHfDfl/r8cuHdgFJblKBDe6efZA+FwH3uvuP7r4e+CuRb5EQ+dZdB2jk7nvc/QsPPhny4CJguLvPdPddwCCgq5mlR/V5yN03u/sPwHigfZ6XLOIFd//O3XcAr0dNfzEwxt3HuPs+d/8EmE7kwzU/7nH3bcH8cffh7v5zsDz3AO3MrEou0+4hsl73uPsYIh/MLfLT18wSiHwA/8Xdt7v7AmDEQfL2A95394lBxruIbN0Q5J/h7pPdPdvdlwP/Ak7IbWbu/pO7vxW89s/A/QfrL+FQAZDc/ATUOMQ+7LpEdhns933QBvB3IrtwPjazpWZ2ez5e+xfzdfesIE+9qD5ro4a3AxXzMf+DTd8IONfMNu9/AN2IFLP8WLF/wMwSzOwhM1tiZluJbDVAZDfZgfyUo/AebPly61uTyLf1FVHjoodzqhs93t23EVnn+5fhCDN738zWBsvwwEHyY2blzexfZvZ90H8iUDUoTBIjVAAkN18Du4AzD9JnNZEPzP0aBm0E33ZvcfcmwBnAzWbWK+h3qC2BX8zXzCoQ2SJZla8lyNtr5bQCeMndq0Y9Krj7Q/mcf3T7hUQOnJ9E5NhKetBu+cyWH+uJ7MKpH9XW4CD910SPN7PyRNb5fv8EvgWau3tl4M8cPP8tRLZaugT9j98/67wugBQ9FQA5IHffAtwNDDGzM4NvdElmdoqZPRx0exW408xqmlmNoP9IADM73cyamZkROfC5l//tUlgHNDnIy78K/M7M2ptZMpFvm1OCXQ/5tQ5IN7O8vtdHAr81s97BN/cUM+thZvVz6X+oZYHIPvBdRL5RlyeyPEXK3fcCbwP3BH+7lsClB5nkTeB0M+tmZmWJHFeIXmeViBxMzgrmdU2O6XOuh0pEDo5vNrPqwF8Oa4GkSKgASK7c/RHgZuBOIt8oVwADgXeDLvcR2T8+B5gLzAzaAJoDnxLZJ/018Iy7jw/GPUikcGw2s1sP8LqfEtkH/RaRb6ZNgf4FXIw3gn9/MrOZh+rs7iuIfFv/M/9b5v8j9/8rB12WwItEdmmtAhYQOXBeHAYS2eJYS+SMpVeJFKJfcff5wHXAK0TW+SYiZxjtdyuRLZmfgeeB13LM4h5gRLAeziNyoLsckYP5k4GxhbJEUqj2n5UhIqWcmQ0Garv7ZWFnkdigLQCRUsrMWppZW4voTOS3Ce+EnUtih36lKFJ6VSKy26cukX30jwDvhZpIYop2AYmIxCntAhIRiVMqACIicapEHQOoUaOGp6enF2jabdu2UaFChcINVAiUK/9iNZty5Y9y5V9Bss2YMWODu9c84MiwL0aUn0dGRoYX1Pjx4ws8bVFSrvyL1WzKlT/KlX8FyQZMd10MTkREoqkAiIjEKRUAEZE4pQIgIhKnVABEROKUCoCISJxSARARiVMqACIicUoFQEQkTpWoS0EUhLszevZqtv6879CdRUTiSKnfAti6M5t7Rs/nlYW7cF36WkTkv0p9AahSLombf9OChRv38dH8tWHHERGJGaW+AABccHQD6lc07vtgITv37A07johITIiLApCYUIYLj0xm5aYdDP1iadhxRERiQlwUAIBWqQn0aV2bIeOXsHbLzrDjiIiELm4KAMAdpx3JXnceGLMw7CgiIqGLqwLQoHp5rj6hKaNnr+bLzA1hxxERCVVcFQCAa3s0pVFqee56bx67snVAWETiV9wVgJSkBO45ozVL129j6BfLwo4jIhKauCsAAD1b1OKUNrV58rPFrNi4Pew4IiKhiMsCAHD3b1uRUMb463/mhx1FRCQUcVsA6lQpxw29mvPpwh8Zv+jHsOOIiBS7uC0AAFcc15gmNSpw738W6ICwiMSduC4AZRPL8JczWrNswzaGTdIBYRGJL3FdAABOOKImJ7dK4+lxmfqFsIjElTwVADPrY2aLzCzTzG4/wPhkM3stGD/FzNKjxg0K2heZWe+o9hvNbJ6ZzTezmwpjYQrqrtNakb3Pue+DBWHGEBEpVocsAGaWAAwBTgFaAReYWasc3a4ENrl7M+AxYHAwbSugP9Aa6AM8Y2YJZtYG+D3QGWgHnG5mzQpnkfKvYWp5ru3RlPfnrGGCDgiLSJzIyxZAZyDT3Ze6+25gFNA3R5++wIhg+E2gl5lZ0D7K3Xe5+zIgM5jfkcAUd9/u7tnA58DZh784BXdNj6Y0rVmBu96bx47dOiAsIqVfXgpAPWBF1POVQdsB+wQf6FuA1INMOw/obmapZlYeOBVoUJAFKCzJiQk8cNZRrNi4g8c/+y7MKCIixSKUewK7+0IzGwx8DGwDZgEH/NptZgOAAQBpaWlMmDChQK+ZlZWVp2m710vk+YlLqbdnNQ0rJxTotYoiV3GL1VwQu9mUK3+UK/8KPZu7H/QBdAU+ino+CBiUo89HQNdgOBHYAFjOvtH9ckz/AHDtobJkZGR4QY0fPz5P/TZt2+Ud7/3Yz3h6kmfv3Vfg18urvOYqbrGayz12sylX/ihX/hUkGzDdc/lMzcsuoGlAczNrbGZliRzUHZ2jz2jgsmC4HzAueOHRQP/gLKHGQHNgKoCZ1Qr+bUhk//8reS9bRadq+bLcdXorZq/YzMjJ34cdR0SkyBxyF5C7Z5vZQCLf3hOA4e4+38zuJVJZRgPDgJfMLBPYSKRIEPR7HVgAZAPXufv+XT1vmVkqsCdo31zYC1dQfdvX5a2ZK/n7R4vo3bo2taukhB1JRKTQ5ekYgLuPAcbkaLs7angncG4u094P3H+A9u75SlqMzIz7zmzDbx6byD2j5/PsJRlhRxIRKXRx/0vg3DRKrcANvZozdv5aPlmwLuw4IiKFTgXgIAYc34QWaZW4+715ZO3KDjuOiEihUgE4iKSEMjxw9lGs3bqTRz5eFHYcEZFCpQJwCBmNqnFRl4aM+Go5s1fEzHFqEZHDpgKQB3/q05JalVK45Y3Z7Nyjy0SISOmgApAHlVOSGNyvLZk/ZvHYJ7pMhIiUDioAeXTCETW5oHNDnvtiKdOXbww7jojIYVMByIc7TjuSelXLccsbs9m+W2cFiUjJpgKQDxWTE/nHue34/qftDP7w27DjiIgcFhWAfDqmSSq/Oy6dEV9/z5eZG8KOIyJSYCoABfCn3i1pUqMCf3pzDj/v3BN2HBGRAlEBKIByZRP4x3ntWLNlB/e9vzDsOCIiBaICUEAdG1bjDyc05bXpKxj3ra4VJCIljwrAYbjppOa0SKvEbW/NZfP23WHHERHJFxWAw5CcmMAj57Vj07bd/GX0/LDjiIjkiwrAYWpTrwrXn9ic92atZszcNWHHERHJMxWAQnBtz6YcVa8Kd747jw1Zu8KOIyKSJyoAhSApoQyPnNeOrF3Z/PntuftvdC8iEtNUAArJEWmVuOXkI/h4wTrenbUq7DgiIoekAlCIrurehIxG1bj7vfms3bIz7DgiIgelAlCIEsoYj5zbjuy9zm1vzdGuIBGJaSoAhSy9RgVuP6Uln3+3nlHTVoQdR0QkVyoAReCSYxpxbNNU7nt/ASs2bg87jojIAakAFIEyZYyH+7XFzPi/N2ezb592BYlI7FEBKCL1q5Xn7tNbMXnpRoZOWhp2HBGRX1EBKELndqpPn9a1+ftHi5i7ckvYcUREfkEFoAiZGQ+dcxSpFZK5YdQ3bNul20iKSOxQAShiVcuX5bHz27P8p23c+58FYccREfkvFYBi0LVpKtf1aMZr01fwwRxdME5EYoMKQDG58aTmtG9QlUFvz2HV5h1hxxERUQEoLkkJZXiyfwf2Ofxx1Cz26tRQEQmZCkAxaphanr+d2ZqpyzfyzPjMsOOISJxLDDtAvDmrQ30+X7Sexz9bTNemqWHHEZE4pi2AEPztzDbUr1aOG179hqzd2hUkIuFQAQhBpZQknr6gI+uzdjF07i5dNVREQqECEJKj6lfhz6ceyaz1exk2aVnYcUQkDqkAhOjyY9PpWCuBwWO/ZdaKzWHHEZE4k6cCYGZ9zGyRmWWa2e0HGJ9sZq8F46eYWXrUuEFB+yIz6x3V/kczm29m88zsVTNLKYwFKknMjCuPSqZWpRQGvjKTLTv2hB1JROLIIQuAmSUAQ4BTgFbABWbWKke3K4FN7t4MeAwYHEzbCugPtAb6AM+YWYKZ1QNuADq5exsgIegXdyokGU9d2IG1W3Zy25u6i5iIFJ+8bAF0BjLdfam77wZGAX1z9OkLjAiG3wR6mZkF7aPcfZe7LwMyg/lB5BTUcmaWCJQHVh/eopRcHRtW4099WjB2/lqGf7k87DgiEifyUgDqAdH3NlwZtB2wj7tnA1uA1NymdfdVwD+AH4A1wBZ3/7ggC1Ba/L57E37TKo0Hxyxkxvebwo4jInEglB+CmVk1IlsHjYHNwBtmdrG7jzxA3wHAAIC0tDQmTJhQoNfMysoq8LRFKTpX3zrON8vg9y98zT3HlqNyWYuJXLEmVrMpV/4oV/4VejZ3P+gD6Ap8FPV8EDAoR5+PgK7BcCKwAbCcfff3A84FhkW1Xwo8c6gsGRkZXlDjx48v8LRFKWeuuSs3e/M7xvjFQyd79t594YTy2F1f7rGbTbnyR7nyryDZgOmey2dqXnYBTQOam1ljMytL5GDt6Bx9RgOXBcP9gHHBC48G+gdnCTUGmgNTiez6OcbMygfHCnoBC/NetkqvNvWq8NczWvPF4g08+dnisOOISCl2yF1A7p5tZgOJfHtPAIa7+3wzu5dIZRkNDANeMrNMYCPBGT1Bv9eBBUA2cJ277wWmmNmbwMyg/RvgucJfvJKp/9ENmL58E0+OW0yHhlXp0aJW2JFEpBTK0zEAdx8DjMnRdnfU8E4iu3UONO39wP0HaP8L8Jf8hI0XZsZ9Z7Zh/uot3PTaLD64oTv1qpYLO5aIlDL6JXCMKlc2gWcu6kj2Xufal2eyK3tv2JFEpJRRAYhhTWpW5B/ntmX2is088IEOkYhI4VIBiHF92tThqm6NGfH197w3a1XYcUSkFFEBKAFuO6UlndOrc9tbc5i/ekvYcUSklFABKAGSEsow5KKOVCtflgEvzmDjtt1hRxKRUkAFoISoWSmZf12SwfqsXQx8ZSbZe/eFHUlESjgVgBKkbf2qPHDWUXy15Cce/PDbsOOISAmnm8KXMP0y6jNv1RaGTVpG67qVObtj/bAjiUgJpS2AEuiO046kS+PqDHp7LvNW6aCwiBSMCkAJlJRQhmcu6kiNiskMeHE6G7J2hR1JREogFYASKrVi5KDwT9t2c93LM9mjg8Iikk8qACVYm3pVeOico5iybCP365fCIpJPOghcwp3VoT7zVm3970Hhczs1CDuSiJQQ2gIoBQad0pJjm6Zyx7vzmL1ic9hxRKSEUAEoBRITyvD0hR2pWTGZP7w0g/U/66CwiByaCkApUb1CWZ67NIPNO3Zz7csz2J2tg8IicnAqAKVI67pVeLhfO6Yt38Sd787df79lEZED0kHgUuaMdnXJXPczT47LpGnNivzhhKZhRxKRGKUCUAr98eQjWLphGw+N/Zb0GhXo3bp22JFEJAZpF1ApZGb849x2tKtflZtGzdLlIkTkgFQASqmUpASev7QT1SuU5coR01i7ZWfYkUQkxqgAlGI1KyUz7PJObNu1lytHTGP77uywI4lIDFEBKOVa1q7MUxd0YOGardw0ahb79unMIBGJUAGIAz1b1uKu01vx8YJ1DB6rG8mISITOAooTlx+bztL12/jXxKU0qVmB849uGHYkEQmZtgDihJnxl9+2onvzGtzxzjy+WrIh7EgiEjIVgDiSmFCGIRd1pHGNClwzciZL12eFHUlEQqQCEGcqpyQx/PKjSSxjXPHvaWzatjvsSCISEhWAONSgenmeuzSD1Vt2cvVIXThOJF6pAMSpjEbV+Xu/tkxZtpE73tGF40Tikc4CimN929djyfptPPnZYprUrMg1PXThOJF4ogIQ5/54UnOWbdjG4LHf0rhGefq0qRN2JBEpJtoFFOfMjL/3a0uHhlW56bVZzPxhU9iRRKSYqADIfy8cl1Y5hatGTNfpoSJxQgVAAKhRMZkRv+uMAZe9MJUtu3RQWKS0UwGQ/0qvUYFhlx/Nhp9389iMnWzbpauHipRmKgDyC+0bVOXpCzvw/dZ9XPvyTPbs1W8EREorFQD5lV5HpnFZ67J8/t16bntrji4hLVJK5akAmFkfM1tkZplmdvsBxieb2WvB+Clmlh41blDQvsjMegdtLcxsVtRjq5ndVFgLJYevR4Mkbj75CN6euYq//me+figmUgod8ncAZpYADAFOBlYC08xstLsviOp2JbDJ3ZuZWX9gMHC+mbUC+gOtgbrAp2Z2hLsvAtpHzX8V8E4hLpcUgutPbMbWHXsYOmkZlVKSuLV3i7AjiUghyssWQGcg092XuvtuYBTQN0efvsCIYPhNoJeZWdA+yt13ufsyIDOYX7RewBJ3/76gCyFFw8y447Qj6X90A54en8m/Pl8SdiQRKUR2qE17M+sH9HH3q4LnlwBd3H1gVJ95QZ+VwfMlQBfgHmCyu48M2ocBH7r7m1HTDgdmuvvTubz+AGAAQFpaWsaoUaMKtKBZWVlUrFixQNMWpZKQa587z87exdS1e7m8dVl6NEiKmWyxRLnyR7nyryDZevbsOcPdOx1wpLsf9AH0A4ZGPb8EeDpHn3lA/ajnS4AawNPAxVHtw4B+Uc/LAhuAtEPlcHcyMjK8oMaPH1/gaYtSScm1a89ev3z4FE+//X1/95uV4YQKlJR1FiuUK39iNZd7wbIB0z2Xz9S87AJaBTSIel4/aDtgHzNLBKoAP+Vh2lOIfPtfl4ccEqKyiWX458UZHJ1enVten81nC/UnEynp8lIApgHNzayxmZUlclB3dI4+o4HLguF+wLig8owG+gdnCTUGmgNTo6a7AHj1cBZAik9KUgLDLutEq7qVueblmXy95KewI4nIYThkAXD3bGAg8BGwEHjd3eeb2b1mdkbQbRiQamaZwM3A7cG084HXgQXAWOA6d98LYGYViJxZ9HbhLpIUpUopSYz4XWcaVS/PVSOmMWvF5rAjiUgB5el3AO4+xt2PcPem7n5/0Ha3u48Ohne6+7nu3szdO7v70qhp7w+ma+HuH0a1b3P3VHffUtgLJUWrWoWyjLyqC6kVk7ls+FQWrf057EgiUgD6JbAUSFrlFF6+qgspSWW4eNgUlm/YFnYkEcknFQApsAbVyzPyyi5k793HRUOnsGbLjrAjiUg+qADIYWmeVokXr+jClh17uHjoFH7K2hV2JBHJIxUAOWxH1a/CsMs6sXLTDi4dPpWtO/eEHUlE8kAFQApFlyapPHtJBt+t+5kr/z2N7bt1LwGRWKcCIIWmZ4taPH5+B2Z8v4krVAREYp4KgBSq09rW4bHz2zN12UYuHz5NdxUTiWEqAFLo+ravxxP9OzDjh01cNnwqWSoCIjFJBUCKxG/b1eXJ/h34ZsVmLh02RQeGRWKQCoAUmdPa1mHIhR2Ys3ILlw7T2UEisUYFQIpUnzZ1eOaijsxfvYVLhk5hy3YVAZFYoQIgRe43rWvzz4syWLjmZy54frJ+LCYSI1QApFic1CqN5y7NYMn6LM5/bjJrt+wMO5JI3FMBkGLTo0UtRlzRmTWbd3Dev75mxcbtYUcSiWsqAFKsjmmSysu/P4YtO/Zw7rNfs2R9VtiRROKWCoAUu/YNqjJqwDFk79vHec9+zdyVuiWESBhUACQUR9apzGt/6EpKUgL9n/uaSYs3hB1JJO6oAEhomtasyNvXHkuD6uX53b+nMnr26rAjicQVFQAJVVrlFF77Q1c6NKzGDa9+w/BJy8KOJBI3VAAkdFXKJfHiFZ3p3TqNe99fwOCx3+LuYccSKfVUACQmpCQl8MxFGVzYpSH/nLCE/3tzDnv27gs7lkiplhh2AJH9EsoY95/ZhlqVknn808Vs3LabIRd2pFzZhLCjiZRK2gKQmGJm3HTSEdx3ZhsmLPqRC4dOZtO23WHHEimVVAAkJl18TCOeuSiD+au30u/Zr/SrYZEioAIgMatPm9q8dEVn1v+8izOHfMmM7zeGHUmkVFEBkJjWpUkq7153HJVSErnguSm8+82qsCOJlBoqABLzmtSsyDvXHkeHhlW56bVZvL14N/v26TRRkcOlAiAlQrUKZXnpyi6c16k+o5fs4fpR37Bzz96wY4mUaDoNVEqMsollGHxOW/h5HW/MXcPKTTt4/tIMalVKCTuaSImkLQApUcyMUxuX5dmLM/hu7c+c+fSXLFyzNexYIiWSCoCUSL1b1+aNq7uy151+//yKcd+uCzuSSImjAiAlVpt6VXjvum40rlmBq0ZM51+fL9E1hETyQQVASrTaVVJ4/Q9d6dOmNg9++C03jJrF9t3ZYccSKRFUAKTEK182kSEXduRPfVrw/pzVnP2MfjkskhcqAFIqmBnX9mjGC5cfzerNO/jt05P4YvH6sGOJxDQVAClVerSoxeiB3UirlMJlw6fquIDIQagASKmTXqMCb197rI4LiBxCngqAmfUxs0Vmlmlmtx9gfLKZvRaMn2Jm6VHjBgXti8ysd1R7VTN708y+NbOFZta1MBZIBKBC8q+PCyzbsC3sWCIx5ZAFwMwSgCHAKUAr4AIza5Wj25XAJndvBjwGDA6mbQX0B1oDfYBngvkBPAGMdfeWQDtg4eEvjsj/RB8XWLt1J2c8NYmx89aEHUskZuRlC6AzkOnuS919NzAK6JujT19gRDD8JtDLzCxoH+Xuu9x9GZAJdDazKsDxwDAAd9/t7psPf3FEfq1Hi1q8f303mtSswNUjZ/K39xfodpMi5K0A1ANWRD1fGbQdsI+7ZwNbgNSDTNsYWA+8YGbfmNlQM6tQoCUQyYP61crz+tVduaxrI4ZNWkb/5yazdsvOsGOJhMoOdYaEmfUD+rj7VcHzS4Au7j4wqs+8oM/K4PkSoAtwDzDZ3UcG7cOAD4HlwGTgOHefYmZPAFvd/a4DvP4AYABAWlpaxqhRowq0oFlZWVSsWLFA0xYl5cq/w802eU02L8zbRdkEuLptCq1rFM49h2N1nSlX/sRqLihYtp49e85w904HHOnuB30AXYGPop4PAgbl6PMR0DUYTgQ2AJaz7/5+QG1geVR7d+CDQ2XJyMjwgho/fnyBpy1KypV/hZFt8bqtftIjEzz99vf98U++871798VErqKgXPkTq7ncC5YNmO65fKbmZRfQNKC5mTU2s7JEDuqOztFnNHBZMNwPGBe88Gigf3CWUGOgOTDV3dcCK8ysRTBNL2BBHrKIFIpmtSrx3sDj6NuuLo99+h2XvTCVH7dql5DEl0MWAI/s0x9I5Nv7QuB1d59vZvea2RlBt2FAqpllAjcDtwfTzgdeJ/LhPha4zt3338XjeuBlM5sDtAceKLzFEjm08mUTeez89jxw1lFMW76RU574QlcVlbiSpxvCuPsYYEyOtrujhncC5+Yy7f3A/QdonwUceL+USDExMy7s0pCj06tx/avfcMW/p3P5sencfkpLUpIK59iASKzSL4FFgOZplXj3uuO4/Nh0/v3Vcs565isyf/w57FgiRUoFQCSQkpTAPWe0ZvjlnVi3dSenPzWJV6b8oGsJSamlAiCSw4kt0xh7Y3c6NarOn9+ZyzUjZ7J5++6wY4kUOhUAkQOoVTmFF6/ozKBTWvLpwnWc8sQXTFn6U9ixRAqVCoBILsqUMf5wQlPevvZYkhPLcMHzk3n040Vk6zISUkqoAIgcQtv6VXn/hu6c3bE+T47L5Lx/fa07jkmpoAIgkgcVkxP5x7ntePKCDixel8UpT3zBG9NX6ACxlGgqACL5cEa7uoy5sTut6lbm/96cw+9fnMH6n3eFHUukQFQARPKpQfXyjPr9Mdx52pFMXLye3o9P1H0GpERSARApgDJljKu6N+GD67tRt2oKV4+cyXNzdrFlx56wo4nkmQqAyGFonlaJd649jht7NWfymmz6PD6RSYs3hB1LJE9UAEQOU1JCGf548hHceUwK5csmcPGwKdz57lyydulG9BLbVABECkmTKgl8cEN3rurWmJen/EDvxyby+Xfrw44lkvXCmuQAABVCSURBVCsVAJFClJKUwJ2nt+LNq4+lXNkELhs+lVten61LSUhMUgEQKQIZjarxwQ3dGNizGe/OWsXJj01k7Ly1YccS+QUVAJEikpyYwK29W/DedcdRs2IyV4+cwXWvzGRDln43ILFBBUCkiLWpV4X3Bh7Hrb85gk/mr+PkRz/nvVmr9CtiCZ0KgEgxSEoow8ATm/PBDd1Ir1GBG0fN4qoR01m7RfchlvCoAIgUo+ZplXjz6mO587Qj+XLJBk5+9HNe+no5e/dpa0CKnwqASDFLCH5FPPbG42nboAp3vTefs//5FfNXbwk7msQZFQCRkKTXqMDIK7vw+PntWbVpO799ahJ/e38B2/QDMikmKgAiITIzzuxQj89u7kH/zg0ZNmkZJz36OR/N1ymjUvRUAERiQJXySTxw1lG8dc2xVCmXxB9emsFVI6azcpNuPCNFRwVAJIZkNKrGf67vxp9PbcmXmRs4+dGJPDdxCXt0G0opAioAIjEmKaEMA45vyic3H89xzVJ5YMy3/PapScz4flPY0aSUUQEQiVH1q5Xn+Us78a9LMtiyYw/9nv2KP78zly3bdc8BKRwqACIxzMzo3bo2n9x8Alcc15hRU3+gxz/G8+rUH/TbATlsKgAiJUDF5ETuOr0V71/fnea1KjHo7bmcOeRLZv6g3UJScCoAIiVIq7qVee0Px/BE//b8+PNOzn7mK259Y7ZuTC8FogIgUsKYGX3b1+OzW3pw9QlNeW/WKk78xwSGfrFUZwtJvqgAiJRQFZMTuf2Ulnx00/FkpFfjvg8WcuoTX/Blpu5JLHmjAiBSwjWpWZEXLj+aoZd2Ylf2Pi4aOoWrX5rB8g3bwo4mMS4x7AAicvjMjJNapdGteQ2GfrGUZyYs4bNv13FZ13Q6JOtsITkwFQCRUiQlKYGBJzbnvE4NePST7xj25TLKJ8KP5Zdx8TGNSErQRr/8j94NIqVQrcopPHROWz64vjvplcvw1/8soPdjE/l4/lrdiUz+SwVApBRrVbcyt3ZK4YXLj8YMBrw0gwuen8y8Vbr3gKgAiJR6ZkbPlrUYe9Px/K1va75bl8Vvn57ELa/P1i0p41yeCoCZ9TGzRWaWaWa3H2B8spm9FoyfYmbpUeMGBe2LzKx3VPtyM5trZrPMbHphLIyI5C4poQyXdE1n/K09GNC9Cf+ZvZqe/5jAY598x/bduglNPDpkATCzBGAIcArQCrjAzFrl6HYlsMndmwGPAYODaVsB/YHWQB/gmWB++/V09/bu3umwl0RE8qRKuSQGnXokn91yAiceWYsnPlvMCX+fwMjJ3+uHZHEmL1sAnYFMd1/q7ruBUUDfHH36AiOC4TeBXmZmQfsod9/l7suAzGB+IhKyBtXLM+TCjrx1TVcaVS/Pne/O4zePTeSDOWt0oDhO5KUA1ANWRD1fGbQdsI+7ZwNbgNRDTOvAx2Y2w8wG5D+6iBSGjEbVeePqrgy9tBNJCcZ1r8yk75Av9YviOGCHqvRm1g/o4+5XBc8vAbq4+8CoPvOCPiuD50uALsA9wGR3Hxm0DwM+dPc3zayeu68ys1rAJ8D17j7xAK8/ABgAkJaWljFq1KgCLWhWVhYVK1Ys0LRFSbnyL1azlYZc+9z5clU272TuYeNOp3VqGc49oizpVRIOPXER5ipOsZoLCpatZ8+eM3Ldze7uB30AXYGPop4PAgbl6PMR0DUYTgQ2AJazb3S/HNPfA9x6qCwZGRleUOPHjy/wtEVJufIvVrOVplw7dmf78xOXeLu/fuSNbnvfr3t5hi9bnxV6ruIQq7ncC5YNmO65fKbmZRfQNKC5mTU2s7JEDuqOztFnNHBZMNwPGBe88Gigf3CWUGOgOTDVzCqYWSUAM6sA/AaYl4csIlIMUpISuKp7Eyb+qScDezbjs4U/ctKjnzPo7bms3rwj7HhSSA55KQh3zzazgUS+vScAw919vpndS6SyjAaGAS+ZWSawkUiRIOj3OrAAyAauc/e9ZpYGvBM5Tkwi8Iq7jy2C5RORw1A5JYlbe7fg0q6NGDI+k1em/sBbM1ZyYZeGXNujKbUqp4QdUQ5Dnq4F5O5jgDE52u6OGt4JnJvLtPcD9+doWwq0y29YEQlHrcop/LVvGwac0JSnxy3mpcnfM2raD1zaNZ0/HN+E1IrJYUeUAtAvgUUkz+pVLceDZ7fls5tP4NQ2dRj6xVKOf3g8//hokW5WXwKpAIhIvqXXqMCj57fn4z8eT4+WtXh6fCbdHh7Hk58tZutOFYKSQgVARAqsWa1KDLmwI2Nu6M4xTVJ59JPv6PbQOB7/9DttEZQAKgAictha1a3M85d24j8Du3FMk1Qe/3Qx3QaP45GPF7Fp2+6w40kuVABEpNAcVb8Kz13aiTE3dKf7ETV4alwm3QaPY/DYb/kpa1fY8SQH3RFMRApdq7qVeeaiDBat/Zmnx2fy7OdL+PeXy7mkayN+370JNSvprKFYoAIgIkWmRe1KPHVBB27s1Zxnxmcy9IuljPhqORd2aUjbJF15NGwqACJS5JrVqsij57fnhl7NGTI+kxe//h5zZ+r2OQw4vimNa1QIO2Jc0jEAESk26TUq8Pdz2zHh1h4cXz+Rt2eu4sRHJnDtyzOYs3Jz2PHijgqAiBS7BtXLc2nrZCbddiLX9mjKpMUbOOPpL7nw+cl8sXi97kdQTFQARCQ0NSsl83+9W/LVoF7cceqRLFmfxSXDpnL6U5P4z+zV7N2nQlCUVABEJHQVkxP5/fGRq48+fE5bduzZy/WvfsOJj0RuVblzz96wI5ZKKgAiEjOSExM47+gGfPrHE3j24gyqli/Lne/O47jg18X6LUHh0llAIhJzypQx+rSpTe/WaUxeupGhXyzl8U8X88yEJZzTsR5XdmtMs1qVwo5Z4qkAiEjMMjO6Nk2la9NUMn/MYviXy3hrxkpenbqCni1qclX3JhzbNJXg3iKST9oFJCIlQrNaFXngrKP46vYT+eNJRzB31RYuGjqF056cxNszV7I7Wz8syy8VABEpUVIrJnPjSc2ZdNuJDD7nKPbs3cfNr8+m+8PjeGZCJpu36+JzeaVdQCJSIqUkJXD+0Q05r1MDPv9uPcMmLePhsYt48rPFnNm+Hpcdm86RdSqHHTOmqQCISIlmZvRoUYseLWrx7dqtjPhqOe98s4pR01ZwTJPqXH5sOicdmUZignZ45KQ1IiKlRsvalXnw7LZ8fXsvbj+lJSs27uDqkTM54e8TePbzJdo9lIMKgIiUOtUqlOXqE5ry+f/14NmLM2hQvRwPffgtxzz4Gbe/NYcFq7eGHTEmaBeQiJRaiQll6NOmNn3a1Gbhml/uHurYsCoXdWnEaW3rkJKUEHbUUGgLQETiwpF1KvPQOW2Z8ude3HnakWzevodb3pjNMQ9+xv0fLGDZhm1hRyx22gIQkbhStXxZrurehCu7NearJT/x8pTveeHL5Tz/xTK6NatB+4rZdNu7Ly4OGqsAiEhcMjOOa1aD45rV4MetO3lt2gpenfoDkzJ38cbScfQ/uiH9OzegTpVyYUctMqW/xImIHEKtyilc36s5E//Ukxs7JnNknco8OW4x3QaPZ8CL05n43Xr2lcJLU2sLQEQkkJhQhg61Evljj8788NN2Xp32A69PW8HHC9bRKLU85x/dgH4d61OrckrYUQuFtgBERA6gYWp5buvTkq8GnciTF3SgduUUHh67iK4PjeOqEdP4eP5a9uwt2dcf0haAiMhBJCcmcEa7upzRri7LNmzj9ekreHPGSj5d+CM1KyVzTsf6nNepPk1qVgw7ar6pAIiI5FHjGhW4rU9Lbjn5CCYsWs+oaSt4/oulPPv5EjqnV+f8oxtw6lF1KFe2ZPyuQAVARCSfEhPKcFKrNE5qlcaPW3fy1sxVvD59Bbe8MZt7Rs/nt+3r0v/oBhxVr0pM36tABUBE5DDUqpzCNT2acvUJTZi6bCOvTV/B2zNX8sqUH2iRVomzO9bjrA71YvLAsQqAiEghMDO6NEmlS5NU7jmjNaNnreatmSt58MNvGTz2W7o3r8nZHevRu3XtmLn0hAqAiEghq5ySxMXHNOLiYxqxdH0Wb89cxTvfrOLGUbOolJzIaW3rcHbH+hydXi3UXUQqACIiRahJzYrc2rsFN598BJOX/cRbM1YxevZqRk1bQcPq5Tm7Yz3O7lCfhqnliz2bCoCISDEoU8Y4tmkNjm1ag7+d2Zqx89by1syVPPHZYh7/dDGd06tzdsd6nNq2DpVTkoolkwqAiEgxK182kbM71ufsjvVZvXkH73yzirdmruT2t+fyl9HzOalVGme1r8fxR9SkbGLR/V43TwXAzPoATwAJwFB3fyjH+GTgRSAD+Ak4392XB+MGAVcCe4Eb3P2jqOkSgOnAKnc//bCXRkSkhKlbtRzX9WzGtT2aMnvlFt6euZL356zhgzlrqFo+idOOqsNZHeqR0ahaob/2IQtA8CE9BDgZWAlMM7PR7r4gqtuVwCZ3b2Zm/YHBwPlm1groD7QG6gKfmtkR7r43mO5GYCGgOzeLSFwzM9o3qEr7BlW56/RWfLF4Pe9+EzmT6MN5a5ny516F/pp52QLoDGS6+9Ig5CigLxBdAPoC9wTDbwJPW+TQdl9glLvvApaZWWYwv6/NrD5wGnA/cHMhLIuISKmQlFCGE1umcWLLNLJ2ZZP5YxZJRXB/grzMsR6wIur5yqDtgH3cPRvYAqQeYtrHgT8BJftqSiIiRahiciLtG1QtknmHchDYzE4HfnT3GWbW4xB9BwADANLS0pgwYUKBXjMrK6vA0xYl5cq/WM2mXPmjXPlX6Nnc/aAPoCvwUdTzQcCgHH0+AroGw4nABsBy9t3fD3iQyNbAcmAtsB0YeagsGRkZXlDjx48v8LRFSbnyL1azKVf+KFf+FSQbMN1z+UzNyy6gaUBzM2tsZmWJHNQdnaPPaOCyYLgfMC544dFAfzNLNrPGQHNgqrsPcvf67p4ezG+cu1+c56olIiKH7ZC7gNw928wGEvn2ngAMd/f5ZnYvkcoyGhgGvBQc5N1I5EOdoN/rRA4YZwPX+f/OABIRkRDl6RiAu48BxuRouztqeCdwbi7T3k/kTJ/c5j0BmJCXHCIiUnh0S0gRkTilAiAiEqdUAERE4pQKgIhInFIBEBGJUxY5Xb9kMLP1wPcFnLwGkR+oxRrlyr9YzaZc+aNc+VeQbI3cveaBRpSoAnA4zGy6u3cKO0dOypV/sZpNufJHufKvsLNpF5CISJxSARARiVPxVACeCztALpQr/2I1m3Llj3LlX6Fmi5tjACIi8kvxtAUgIiJRSn0BMLM+ZrbIzDLN7PYQczQws/FmtsDM5pvZjUH7PWa2ysxmBY9TQ8q33MzmBhmmB23VzewTM1sc/Fv4d6U+eKYWUetllpltNbObwlhnZjbczH40s3lRbQdcPxbxZPCem2NmHUPI9ncz+zZ4/XfMrGrQnm5mO6LW3bPFnCvXv52ZDQrW2SIz613MuV6LyrTczGYF7cW5vnL7jCi691luNwooDQ8il69eAjQBygKzgVYhZakDdAyGKwHfAa2I3Ev51hhYV8uBGjnaHgZuD4ZvBwaH/LdcCzQKY50BxwMdgXmHWj/AqcCHRG6KdAwwJYRsvwESg+HBUdnSo/uFkOuAf7vg/8JsIBloHPy/TSiuXDnGPwLcHcL6yu0zosjeZ6V9C+C/N7R3993A/hvaFzt3X+PuM4Phn4GF/PreyrGmLzAiGB4BnBlill7AEncv6A8BD4u7TyRyr4toua2fvsCLHjEZqGpmdYozm7t/7JH7cwNMBuoX1evnJ9dB9AVGufsud18GZBL5/1usuczMgPOAV4vitQ/mIJ8RRfY+K+0FIC83tC92ZpYOdACmBE0Dg0244cW9myWKAx+b2QyL3IcZIM3d1wTDa4G0cKIBkZsMRf+njIV1ltv6ibX33RVEvinu19jMvjGzz82sewh5DvS3i5V11h1Y5+6Lo9qKfX3l+IwosvdZaS8AMcfMKgJvATe5+1bgn0BToD2whsjmZxi6uXtH4BTgOjM7PnqkR7Y5QzllzCK3Ij0DeCNoipV19l9hrp+DMbM7iNyN7+WgaQ3Q0N07ADcDr5hZ5WKMFHN/uxwu4JdfNIp9fR3gM+K/Cvt9VtoLwCqgQdTz+kFbKMwsicgf9mV3fxvA3de5+1533wc8TxFt9h6Ku68K/v0ReCfIsW7/JmXw749hZCNSlGa6+7ogY0ysM3JfPzHxvjOzy4HTgYuCDw6CXSw/BcMziOxrP6K4Mh3kbxf6OjOzROBs4LX9bcW9vg70GUERvs9KewHIyw3ti0Wwb3EYsNDdH41qj95ndxYwL+e0xZCtgplV2j9M5ADiPCLr6rKg22XAe8WdLfCLb2WxsM4Cua2f0cClwVkaxwBbojbhi4WZ9QH+BJzh7tuj2muaWUIw3ARoDiwtxly5/e1GA/3NLNnMGge5phZXrsBJwLfuvnJ/Q3Gur9w+IyjK91lxHN0O80HkSPl3RCr3HSHm6EZk020OMCt4nAq8BMwN2kcDdULI1oTIGRizgfn71xOQCnwGLAY+BaqHkK0C8BNQJaqt2NcZkQK0BthDZF/rlbmtHyJnZQwJ3nNzgU4hZMsksn94/3vt2aDvOcHfeBYwE/htMefK9W8H3BGss0XAKcWZK2j/N3B1jr7Fub5y+4wosveZfgksIhKnSvsuIBERyYUKgIhInFIBEBGJUyoAIiJxSgVARCROqQCIRDGzvfbLK5AW2hVkgytLhvWbBZFfSQw7gEiM2eHu7cMOIVIctAUgkgfBNeIftsg9E6aaWbOgPd3MxgUXN/vMzBoG7WkWuQ7/7OBxbDCrBDN7Prje+8dmVi60hZK4pwIg8kvlcuwCOj9q3BZ3Pwp4Gng8aHsKGOHubYlccO3JoP1J4HN3b0fk2vPzg/bmwBB3bw1sJvJLU5FQ6JfAIlHMLMvdKx6gfTlworsvDS7YtdbdU81sA5HLGewJ2te4ew0zWw/Ud/ddUfNIBz5x9+bB89uAJHe/r+iXTOTXtAUgkneey3B+7Ioa3ouOw0mIVABE8u78qH+/Doa/InKVWYCLgC+C4c+AawDMLMHMqhRXSJG80rcPkV8qZ8ENwQNj3X3/qaDVzGwOkW/xFwRt1wMvmNn/AeuB3wXtNwLPmdmVRL7pX0PkCpQiMUPHAETyIDgG0MndN4SdRaSwaBeQiEic0haAiEic0haAiEicUgEQEYlTKgAiInFKBUBEJE6pAIiIxCkVABGROPX/nilxLYJNTWAAAAAASUVORK5CYII=\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(training_cost[200:])\n", + "plt.grid()\n", + "plt.title('Cost on the training data')\n", + "plt.xlabel('Epoch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 368 + }, + "id": "FI1WqETM6fer", + "outputId": "ec22665d-0908-42ee-bbb4-53d6c70cbba9" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 0, 'Epoch')" + ] + }, + "metadata": {}, + "execution_count": 13 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAFNCAYAAADsA7/8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eXwkV3X3/T29L5JG0kgz49mlsY33bRxjgzEzNgmEBPBLgAABQoghkBB4eEOA5HkSCMkbEiAPEAgQIITNYAy2gYAxBtvyeF/GY894G48tzb5pl7pbvd/3j6pbXd2qVrf2luZ+Px991F3rraquX50699xzRCmFwWAwGE4NfIvdAIPBYDAsHEb0DQaD4RTCiL7BYDCcQhjRNxgMhlMII/oGg8FwCmFE32AwGE4hjOgbljwiEhaRp0XktBmse4GI3D8f7apz/9tE5PBi7X8uERElIqcvdjsMU2NEf5kjIj0iMiwi4cVuyzzyHmCHUuoYgIi8VUSOich+EdmuFxKRLSJyv4j49TSl1G5gREResxANnU9hFJF3isi9c7St/SLyirnYlse2N9vnITAf2zdMjRH9ZYyIbAZeBijgtQu874W8od8LfNe1338BLgHeD3zRtdy/Ax9SShUq1r8e+LMFaKfBsOgY0V/evAN4EPgW8MfuGSKyQURuFpF+ERkUkS+55r1bRJ4RkXHbbXKJPb3MShWRb4nIP9mft4nIYRH5qIgcB/5bRNpE5Of2Pobtz+td67eLyH+LyFF7/k/s6U+6LW8RCYrIgIhcXHmAIrIR6AYesietBI7YVv9v7HmIyBvs6Q9VbgPoAa6p9jYkImtF5GciMiQiz4vIu13zPiEiN4rId+zz9ZSIXFplOzvsj0+ISEJE/tA1769E5KT9hvInrulhEfmsiBwUkRMi8lURiXps+2zgq8AV9rZHaq0vIh32NRmxj+0eEfGJyHeBjcD/2Nv6SJXj+Wu7vUdF5F0V835PRHaJyJiIHBKRT7hm6/MwYm//Cvst7E77tzggIteLSKvXfg2zRCll/pbpH/A88OfAViAHrLan+4EngM8BcSACXGnPeyNwBPgtQIDTgU32PAWc7tr+t4B/sj9vA/LAvwJhIIolwH8AxIBm4EfAT1zr/wL4IdAGBIGX29M/AvzQtdzrgD1VjvH3gKdc333Ac8B64DXAI/a+HwdWTnGuxoALqszbAXzZPk8XAf3A1fa8TwBp4NX2ef0U8OAU+6k8h/q8fdI+B68GUkCbPf9zwM+Advs4/gf4VJVtvxO4t2Ja1fXttn7V3m8Q661Q7Hn7gVdMcRyvAk4A59m/oe+7j80+rvPt63GBvey19rzN9rIB1/ZOB37b/u102uf884t9Dy3Hv0VvgPmbpwsLV2IJfYf9/Vks1wbAFbZwBTzW+xXwwSrbrCX6WSAyRZsuAobtz6cBRS1uFcutBcaBFvv7j4GPVNnmH1WKLHAN1hvO3fY+/y/wp8B24C77GM+rWOcIcJXH9jcABaDZNe1TwLfsz58AfuOadw4wMcU58BL9iQoBPAlcjvXQTQJbXPOuAPqqbPuduES/1vpYD5qfutvjWm4/U4v+N4F/cX0/s/LYKpb/PPA5+/NmKkTfY/lrgV2LfR8txz/j3lm+/DFwu1JqwP7+fUoung3AAaVU3mO9DcALM9xnv1Iqrb+ISExE/lNEDojIGJb11mp3pG4AhpRSw5UbUUodBe4D/sB+xf9dLL+7F8NYFqx7/TuUUpcrpV6OJS6XYj2gvoMljP8IfKNiO83AiMf219rtHHdNOwCsc30/7vqcAiLT7NMYrLgWKaAJy+KNATttF8wIcJs9vR5qrf8ZrLfB20WkV0Q+No02rwUOub4fcM8UkReLyF22a28Uq9+lo9rGRGS1iNwgIkfs38r3plreMHNM7/kyxPbZvgnw2/51sF6bW0XkQqybdaOIBDyE/xCwpcqmU1giolkDuMMNK1O2/hXwIuDFSqnjInIRsAvLAj0EtItIq1LKS2y/DVyH9Rt9QCl1pEqbdgNdXsciIgJ8CfhLLAHxK6UO2OfkAtdy64AQsNdj+0ftdja7hH8j1pvBfDOA9RZw7hTH76by/E+5vn08fwX8lYicB9wpIo8ope7w2FYlx7Ae3JqNFfO/j3Xuf1cplRaRz1MSca9t/7M9/Xyl1JCIXGuvb5hjjKW/PLkWyyVxDpZ74yLgbOAerM7dh7Fu2n8RkbiIRETkpfa63wA+LCJbxeJ0Edlkz3sceKuI+EXkVcDLa7SjGUt0RkSkHfi4nqGsjtZfAl8Wq8M3KCJXudb9CVYEzgexLHRPlFKHsazVyzxmXwc8ppR6HBgEoiJyDpabp9e13MuBO5VSGY/tHwLuBz5ln6cLsFxF36tx7NU4gd25XAulVBH4OvA5EVkF1gNKRF45xbbXi0ionvVF5Pft6yvAKNZvplhnO28E3iki54hIDNe1tWnGekNKi8hlwFtd8/rt/XRXLJ8ARu2H8F9PsW/DbFhs/5L5m/s/rFf4f/OY/iYsV0QAyzL7CZYYDgD/7lruvVhWbwJ4ErjYnn4p8BSWv/27wA8o9+kfrtjfWqzImARW5+qf4fLlYnUufhtLYIaBmyvW/waWT7qpxvH+BfCVimkddttbXNP+yD7+/cB21/RfAK+dYvvrgZ8DQ1iur/e65n0C+J7r+2am8Ffb5/YYlivpTVXO235sfzpW5/E/Yz2kxoBngA9U2XbIPpYhYKDW+sCH7H0lsd7Y/s61rdcBB+12frjK/j5mn8+jwLso78h9A5bLZ9w+d1+qOE+fxBL/Eaz+i3OBnfZv5XGsN5DDXvs1f7P70z31BkPDISJ/D5yplHpbjeXCWG6ja5Q9QGsa+7gA+E+l1BUzb6nBsHQwom9oSGx30C7g7UqpHbWWNxgM9WF8+oaGQ6zBT4eAXxrBNxjmFmPpGwwGwymEsfQNBoPhFMKIvsFgMJxCNPzgrI6ODrV58+YZrZtMJonH43PboDnAtGv6NGrbTLumh2nX9JlJ23bu3DmglPIeub3YMaO1/rZu3apmyl133TXjdecT067p06htM+2aHqZd02cmbQMeVVU01bh3DAaD4RTCiL7BYDCcQhjRNxgMhlMII/oGg8FwCmFE32AwGE4hjOgbDAbDKYQRfYPBYDiFMKJvMBgMpxBG9A0Gg+EUwoi+wbCI7Hiun0LRO9PtQCLDDx85yI2PHGIklV3glhmWK0b0DYZF4rkT47zjmw9z+1PHPed/454+PnrTHj5y026uf+jgArfOsFwxom8wLBJDSct633cy4Tl/dCLLyniIUMDHeDq/kE0zLGMaPsumwbBcSWYsIe8bSFaZX6A5EiBXKJLOFRayaYZljBF9g2GRSGYtIe+tKvp54uEAE7mCEX3DnGHcOwbDIqEt/d7+BMqjbGkikyceChAJ+o3oG+YMI/oGwyKhRX88nWcwOTk6J5UtEA/7iQb9TBjRN8wRRvQNhkUimSkJuZdfX7t3wkE/E7niQjbNsIwxom8wLBLJbCkip69/suhr90406DPuHcOcYUTfYFgkEpk8bbEgQb94duZa7h3j0zfMLSZ6p8HRHXwissgtMcw1qUyelmiQlU1h+gbKY/WVUiSzeZpsn74RfcNcYSz9BufHOw9z+afuqDpU37B0SWQKxEIBNq+Mc2AwVTYvlS2gFMRsS9905BrmCiP6Dc7e4+OcGMsYS28ZksxYlnxbLMjoRK58nu3vL7l3TEeuYW4wot/gDKcsMTCW3vIjmc0TCwWIhwMkMuVpFnRkT1PYTyToI501198wNxjRb3B0dkVj6S8/LEs/QDzst905qmweQCwUsHz6eXP9DXODEf0GZ9iI/rIlmbEGX8XDAQpFRSZfdM2zRL/Jdu/kCop8wbh4DLPHiH6DM2K7d4xPd/mRzFjunaawFUTndvG4ffrRoB+AdN78Bgyzx4h+g6MtfePTX16UQjIDxEKW6KdcI3QTFT59gAnj1zfMAUb0G5hiUTlRHca9s7xI54oUlWXJN4UtS95t6adcPv2ItvTNb8AwBxjRb2DG0jl0eL5x7ywvtMBrnz6Up2UozTeib5hbjOg3MDpcE4x7Z7mhO2rjoZJ7J+n26dvunXjIX/Lpmwe/YQ4wor/IZPIF/uL7j5VlWezZe5K//+mTTjk9qM/K+4+7nuf3/v0e3vTVBzg5niaRyfPe7+7k+Gh6XtpumD7DySzv/s6jHByyRuBa7h0t+qVrnMrmCQd8BPw+x9KfyBX4xM+eomfvyUnb/fc79vHjnYcX4AgMSx0j+ovM4eEJfrH7GA/3DTrTfvr4Ub7zwAEOD5eG5tcj+j94+CCHhlI8vH+IPYdHefroGLc9dZzHDg7PS9sN02f3kVF+/fQJfv30CUC7dyxRd1v6CTuGHyAasm7TZDbPtx/Yz092HZm03W/e18eNjxya59YblgNG9BeZjP3K7o7M0BkXdx0ccabVEv10rsCRkQmuOXs1YLmGnMgfE/XRMGhh33NkFLAs/Xhosk9f59IHCAesh0L/WAalJufeH05mGUnlqpZdNBjcGNFfZLL2gBsdg62Uoq/fyri4y2WhT2Sn9uceHEqhFFyysRWwRvI6o3nNaM6GQXfQPnNsDMAekevh088WiIUssY/a/4+OTgCWUeAevavFfiCRYSxdnsPHYKjEiP4ik7EteG2NDyWzjKWtm//pY2P4fULQLzWFu9cuwnHB+lZ8YsX3O3l7jKXfMOhQTD36NhbyEwr4CPl9Tmw+lFI0AI5PX/fNVJZXdFv++421b6iBEf1FpmTpWze8+wbOFRSt0aCVWreGcOv1ujvjtMZCZe6djBnJ2TAkK66jFvZY2E+qintHR+8cdXXIu38n7lz8XmUXDQY3RvQXGe3T11kU9av6hvYoAK2xINGgn0wNS79vIEFnc5jmSJDWWNBy7ySNpd9oVGbT1OGa8VCgIg1DwWXpW7fpcdu9A+XlFfsGkqxrjSJSeuMzGKpRl+iLyIdE5CkReVJEfiAiERG5XkT22tO+KSJB1/LbRORxe527XdP3i8gee96j83FASw1thesY7L6BJEG/8LIzOgFoi4XqtvS7OuLOOsPJnEnW1oCkXMIe8vsIBaxbsCkcqIjTzzs+/YjdkXtsxLL0RSjrtO3tT3LWmmbWtUaNpW+oSU3RF5F1wAeAS5VS5wF+4M3A9cBZwPlAFLjOXr4V+DLwWqXUucAbKza5XSl1kVLq0jk7iiVMtmD79G1h7utPsrE9xhmrmgBojYXscnlTu2j6BpJ0O6IfZDiVdZK1mYFdjUMiU8Dvs0pf6lBN0O4dd+6dknvH5xNCAR/jmTx+n9DVEXdcOsWiYv+g9cC3phvRN0xNve6dABAVkQAQA44qpW5VNsDDwHp72bcCNyulDgIopSaPJDE4OO6dnHbvJJwbGCwBjwR9Uwr36ESOgUTWWac1FmIk5bb0jU+/UUhm8mxaGSPoF0fUwbL0tXtHKUXK5d6Bkl+/NRqku6PJEffjY2nSuSJdnXG6bdF3R/YYDJXUFH2l1BHgs8BB4BgwqpS6Xc+33TpvB26zJ50JtIlIj4jsFJF3uDcH3G5Pf89cHcRSRnfkTuQKFIqK/YMpujridHdYln5bPGSXyyswnMzy08fLB+bc/Vw//3rbswBlDwp39I5x73iTzhX44SMHKc6i/vBIpsgv9xyrOv+h3kH2Hh93viezeVoiQTatjDvx+WD59JOZPPtOjPNvtz9HoaiIud4EtF+/NRakuzPO/sEUhaJyxF8bColMnv5EhkJRccPDB+c9B38mX+Crd7/A/719L/e/MDCv+zLMDYFaC4hIG/A6oAsYAX4kIm9TSn3PXuTLwA6l1D2ubW4FrsFy+zwgIg8qpZ4DrlRKHRGRVcCvReRZpdQOj32+B3gPwOrVq+np6ZnRwSUSiRmvO5+42/VUnyXMJwaG+MWve8jmi0wMHOH53SfY3OIjOHaE1HieRE7x6R/dzQ+ezZI/tpeVUUsEPrYjxfGUIhaA5KGn6el/luHjWTL5ItlEBoDDx0/WdR4a9XzB/LRt54k8X9yVYeTQPl7U7q+9gge/2JfiN3c9xtd+J0bQdtu4+diOFOuaffzlxREAjvVPEPLDGS1+0nnlHNP4cIbB0QJ/d8N9PHisQEAg17+fnh47tULeemvz5SbIDh4mmy9y8213sWfAeqCf2LebkTFL4H92x32MJib44pN7GDi4j/M6ZnZs9fD0YIFPP2L1Nfz44Rf45ytjUy7fqL+xRm0XzH3baoo+8AqgTynVDyAiNwMvAb4nIh8HOoE/cy1/GBhUSiWBpIjsAC4EnrPfGlBKnRSRW4DLgEmir5T6GvA1gEsvvVRt27ZtRgfX09PDTNedT9zterK4D/Y+RzjWzIWXXgx39XDxeWdz9db1XL3dWv6F7+6kbyBJ25pV8OwLrDr9fKejt3Dfb3jLZav51OvPd7Z/NHqQH+/bg7Zf4y2tbNt2+bTa1WjMR9sGdh6GXU+wYv0ZbLts44y28cVdt6EocNnlV7IiFpw0P9XzK3zRZrZtewkAn9q1g/UrY3zlHeVdWneOPslTI0dJ+mK87Iwg3/3TF5fNb3t8BydS42w6rZNXvbSLbz31IKtPP59nCv1Egwe59pXb2XVohM8/dj/dZ53Hw7v2ABk2nH4W2y5aN6Njq4exJ47CI7u45qxV3LNvgJdd9XKnz8KLRv2NNWq7YO7bVo9P/yBwuYjERESwLPhnROQ64JXAW5RS7nfInwJXikhARGLAi+3l4yLSDCAiceB3gCfn7EiWKNl8yafvTqfrRvv0tbvG3VmXzOSJh8otubYK8THuHW90tMxsOj+PJ8vHWbjJF4qMpfNl2VJ14ZRK4nb0jrtD3o0eoNUWCzrz+waS9Nl9QCJCWywEwHAyRyKr7M/ZSduaS/So70s2tZEtFDk6MlFjDcNiU9PSV0o9JCI/Bh4D8sAuLCs8CRzAct+A1Xn7SaXUMyJyG7AbKALfUEo9KSLdwC32sgHg+0qp2ybv8dRCh2xO5ApOlsVKUYiGLJ++vsF0LHaxaHX4VT4kWu2bH8DvEyZMR64n+iE705w1haLiRMoSV6+QWl0AR183KB905aYpHCBXUOQKeadvxo326bfFQ3Q2h4mH/LboJzl33Qprnv2wH05lSeZs0U/Nb1qGYXssyMUbrPQfvQNJNrRP7eIxLC71uHdQSn0c+Hi96yqlPgN8pmJaL5abx+DCHaevLU93Bx5YCbcsS98SD22ZpnLeD4m2eMnSX90cdlI9GMrRI2Bnmrrg6MgEerCzl6WvBXcklUMphYiQzBQmXV/AickH6OpsmjRfR++siAYREbo64+w9Ps6h4Qlec+FaAFoiQXxi7S9hi777gTMfDKeyNIcDnLG6GYC+/gQvP7NzXvdpmB1mRO4ik3G5d3SWRS9LP5MrOlaVFv1qD4k2l6W/ZkXExOlXQb9ZHbAjYaaL2y3kZelrwc0XFeOZPNl8kWyhSFPI272jmdq9E7KXaWLngWEKReW8Gfh8woqoFbnluHfm2dIfSWVpjQfpaArRHA6YcQJLACP6i4xOr5DOFUrVlCpFP+gnWygymLSjcYZTZPKlPoDKh0Sry6d/WmvU+PSroM/fTH3RboHzGgvhFtyRZM55s6jm3gFrlO7a1uik+VGXTx+sEE0d7ut2B7XZYzQSOd2G+bb0c7TFQs7bh0nv3PgY0V9kdEdu3lUEvbJjVvtzBxJZVreEKSo4NJQi5ZTUKxeRcMDvuAtWNxtLvxruBGczEaty0fdy72TLPrvr4lair9emlTHP6JewHpylLf3OktC7Rb/VHqNRcu8sgKVvt8mMCF4aGNFfZNwZMAcTlkh4WfqaSza2AVZnbqKKewcsi68lEqApEiCdK5pRmh4kMgXW2Va1rmEwHXoHktjPY0/RH6kQfZ1mYSpL3y3mbhxLP16y9AHa46Gyjvs2O8Nqyb2zEJZ+qU1HRibMm2WDY0R/kcm6RH8gkSUU8BH0l1+WsIfo9w0kHXeQVwhgayxoj+a1ttXI6ZVHU7kpheLE2PzU+NUpEZojgbJ+kvE6C5H0DSRY32ydX6+3qTL3TipXNSTXPa2rY3InLriid2yB32yLfmWkj5WCY24t/ZFUtur1GU5lnTZ1dcRRCqf+r6ExMaK/yLhTJg8kMpNcO1Bu6W9oj9EeD7F/MOV0/HqJyPq2KOtao866jWx9vfE/7+fzv9nnOe+ZY2O8+J/voHd07tuvwyc3r7TSGgB87OY9/MX3d9VcN1cocnh4gg226Hv59EdSWUL2A3w4lS312Xh05K5sCuETOPu0Zs/9dTSFiYX8Tn9NSyTI+rYoZ60pX749HmQwmWUij12YJV9mWMyE13/lfifVh5t8och4Ou+0afNK6wFkCrk0NnWFbBrmjzL3TjLjKeARl+i3xaxIiaFkxok+8RKRT73+AgpFxW+esQpwT+QKtM514+cApax8QyerWPPH7ekHRuf+TSWZtQa2dTSFGLBdawcGk4xN1LaOrTBM6Iha/ndPSz+ZY0N7lN6BJMOpXOl6ebjjVjVHuP1DVzk5lyp564s3cs3Zq5x6uQA//LMraI5MHqOhRX5zR4znTiQYmciyqjlS85i8SGby9PYn6YiHJ80bsc+TtvQ7mq1lhuZ5QJhhdhhLf5HJ5ovofrvBRNbTVeO29NtsH64lItU7BtvtQTwlS78x3TsTuQLZfLGq+0kXlzmemvv2pzLWwDbLD24JlTtR3VRof317xLp41TpyV8bDrIhaRW2mcscBnL6qGV+VFAaRoJ9NK8tdOetao7REykdfu8N1tetnNi4e7fby6ujW50Bb+qXBYaZObyNjRH+RyeSLtEStm2UwkS0bpKPR/lywffV2ZSzt3ol5WPqV6zZq9SwtEFVF33Z/HU/OfUd0wq5Dq1NRgxVaOZbO1Yzb1+1uDfsQqdaRm7Ovl/WQnipkc65wp+DQ/QOzScWgRd+r6Lo+B/pBEw1a9X7ne0CYYXYY0V9ksvkiK2zRzxaKNd07rdGQIyLJTJ5o0D9lgiu9bq3C6ouFFqRq5SAnstbD4ERybi39fMF6u4iFArTFgiQyeSayBcYzeZQqpVCo2m5b2JqC2EVuvC39tljIKV+ZmMIdN1e4I3n0IK/ZWN5TFV3X106LvpX/JzjvEUOG2WFEf5HJ5AuO6IP3q78W7qZwgFDAV4rQyEzOu1Nt3XSDWvojtSx9W0z7JxS5OcwNn8yW/OutcUu09g+WRK2WcGlrtikkVjnLCtFXSlmWfjzouI+SmTw+KX9zm2vcKTi6OrV7Z3aWvjYqKmPw9bVzDwbUBomhcTGiv8hkXJY+eLtqoqHyGO22WJBcQdE/nqbJw59ftm6jW/q2IFWLMNFiWlBweHjuMji6/evaJeIWtVpCqYWtKSie5SxT2QLZQtGx9IeTOavjOBzATjo4L3j59Gcjwr0DSbZubPMsuq6vXVu8tE/9VmNoXIzoLzJZl08f8BTxSKA8Rlv/Pzw8MaU/H0qWvnaTNBpaOKpZ+u5kcbou7FzgTnmhz6db9HWeo2oM2+GYYT+EPcpZOoJo+/R1R+58unagZHUHBFbGQ7PysSul6OtP8KI1zaxvm1x0fTiVs8o+htzRZcbSb3SM6C8ylZa+l7tGW/raX6tv7CPDE1UjQZx1GzxOX4trtppPP1dAG8aVluZsKHPv2OfTvf2a7p2k1UkrYln6lZlMS66PEG2xIMmsVQ/BK9JqLtEpOJpCMmsf+1Ayy1g6b5dibPJw71gpGNxvLtr1aGhcjOgvIvlCkUJR1RT9SKA82ZZ+nR7P5D1TMJStG6w+YrQRqGXpp3NFWqNB4sHZFTupxD1QqmTpl94kagmleySql0+/ZOmX0iTU85CeC9piIZqCpc8ztbz1+e6uUnTdOgeVIaNBJ5W0oTExg7MWEZ0l0R1r7TUi1+cTwgGfy70z9UPCTSQ02dLP5Au8/b8e5q9++0x+a3M7b/zPB6xyjIE8V12lqsaKzwcjNUR/IlcgGvTTHiw4IvSt+/roG0jyD687r2zZX+w+xs93H+Urb9vqTPvz63fy6vNP4/cvsHLOP3lklL/76ZO8/fJNgLd7R2SyH/zGRw5x/wsDfP7NF9vtztlvCBminqKvwxmDtNsP6WeOj3FF98ppnJ2Z0RYPkktZGVmt/oTJD7AbHj7IA72DfOHNF/OTXUf4p188TWWUqu5n6eqIc3AoRSKT55J//LVj2Y9N5LhkU1v5vmMhJ5V05RgCQ2NgRH8Rydidf5Ggj0jQRzrnHbIJ8Ok3XMB5doUkd1ieV252N/otwS36vf1JHu4b4pH9Q7xoTTM7DwzT0RTmhdEiR0cnWN+2cJWPtDhW68hN5wpEgn5afOKM9Lz3+QGeO5HgHyqWvff5AX799Imyabc/dYIV0aAj+jsPDLPr4IiTviAeDhAN+QkHfAyncoT8Plqikzsjd+zr59Y9x/j0Gy4kFPAxnMqypbMJyBAJ+iaNQj1k559Z2xplVUuE91zVzUS2wCvPXTPtczRdPvqqs3hqz24A1rXGuO/5gUnL/Oqp49z9XD+fecOF3PnsSXIFxWvtYixu1qyIsLE9xqvPP40Dg6lJ1+l3zl1d9l27ykaSOSP6DYoR/UVEW/qhgI+IHQFSTfRf5ypu3eqO9qnh3gn6Bb9PyqJLtEU7nMo5onv1WZ3c+Ohh+gaSCyr6JUvf2/2kRT8CDNgumUQm7+muGkllyRet0M6g30euUCRfVGWdstrtsufIKFAazdwWC3F8LE1rLGgVIqnoyB1J5SjaycROX9VkZZe0o6kiQf+k6Ki+gSSrW8LO9fzbV589rfMyG152RieFI9ZxdXfGuemxw5PKNPYNJJ3j6RtIcsH6FfzjtedV2yQdTWH+7vfPqblvp05vKsvGlaZsYiNifPqLiLb0wwG/0+Faz2jNgN/n5Fyp5SMWESKB8uiSkuhnHRF0Z+9cSNyWvpcfOJ0rWm9CAXHqB6SyhSnz1+t5+r/bP687WPceHwdK56+USiCEOy1D5ba1X3t0opRHPhL0TxoH0TeQ9Kx1u9DoNrjHIGTzRQ7Z4a+9/Ymqxdhngn4QmgFajYsR/UUkW7CEQlv64B2y6YW2qOp5SOjC6hodpTKSyjmW9ozMhR0AACAASURBVJlrmon45zZCph60OBSVVUimkolcgWjITyQgTmriRCZfNe2BXsf93517Ru8vV1CIuCtSlSKjWu3OSK9t9w0kSGYL5ArK6VuJBv2kK9weluh7J09bSLToux/mh4ZL5SEf2T9EIuNdjH0m6AfhfBdvMcwcI/qLSNqx9EuiXyvuXqMFx6vjtxJdWF2jo1SGU1nHjdEWC7E67ltQS1+n5tXWtldnbjpXIBLwE/Fb8/MFq4B8rqDIV4zQdSKB7POq/5dXsCqJUTxUGihVGvhW29LXHaMlS99XlttoJJVlKJmdM+t5Nuh0x32uh7n78x3PnAS8i7HPBLd7x9CYGNFfRLTIWaJvXYp6Q/pap2npZ1w+fZ0xcSSVKxtEtCYm9M7hAKha6NS8q1qslLxenbkTuQIR29IHK75eu3nc1rVSyhF0L0tfu47cHbTumHl9PtviIVrj5WGHmXzBqXrV2590rFh3orF0vuAsrx+cjeDeiYb8rF0RKXuY68+nr2pyfgtz9YBaEQ16Rj8ZGgcj+ouIFrlQwDctnz64LP06lo+4RowOJ7OMpHL4fcJwyvrsEytsdE3cx+HhiaqdqnONFuA1LVaud6/9ZnJFx9IHy7Wjs4u6XTw6RbN7uv6fLRQd0R5OZZ1cMu5z54yBsEfQutfRIu/3iWXpux6UYFU2U6r0EHdEv0rpw4WmsmB570CS9niIizdYFRaqFWOfCX6f0BIxqRgaGSP6i4gWuXDA73Lv1OfTdyz9OtxB7iyQ+uY/a00zoxM5BpNZVkSD+HzC6rjPKnc3uDDl7rQ1uNoW/WqWfjTkI2xb+oOJjBNP7napuC1LPb18viVCI8lcKVwz5Bb9UoqLUl74bNn/s9Y0c3I84+QAanVZ+lByJ+kkZRsWMApqKro64vT2J1xvIglrlK39UKpWjH2mWKOAjaXfqBjRX0SyLvdONOgnYA/CqodSR27th4R7xKi2QrduarPrmSadba2JWze+V8GM+UD7xlc7lv7UPn2Ak2OZsnmV24KS28ft/hlJ5cgVioxn8k6kkpd7x+rILe+M1P0eer1dB4eBkqXv5DdyPVg3tEUJ1Xkt55uujibG0nlHiHVkUXeVOruzxaRiaGwa41e5DPnF7mM81Ds45TJun3446JtWBkbd8VhPH0DElQWybyBBwCecbw/06utPOuGKq2M+e5mFEX0tqmtsn36mIlOlUsqJ3onalv7Jcbfol4u6ppqlr5c5fVUTzZFA2blrqwjZ1OtY27bDWjdZ7pA7n7U6P3X6jGjIOm/PnRjnYzft5qHewYbw52u6nQieBMlMnhNjGSefDsy9G6otFuSpo2N8+EdP8OEfPcF3HzxQ97onxtL87S17+PCPnuDefZMHlS1HcoUin/v1czVrOMwVZnDWPPEvtz3D5pVxXjzFsPuS6Pu56ozOul07AC/uWslLT19Z10CqtliQxw+NAJagb2yPOfVMj46mOfu0FgDiQaElEuDoyNylMJ6KcTsEc2WT3ZFbKPfpZwtFlLIeWtooP+GqpeuOSHJHi+g3AHcfwbArPLU1FuTNv7WhrPzg+etXcNnmdi5Yv8Lx5R8bTTvrAmzd2M7WTW0cH03z2+esJmAXPdfunRseOcite46zrjXKq86b/5G39aIfQL39SafGbndHnK6OOC8/s5PfPnv1VKtPm+1nreK5EwkeeGGQsYkcP999lD+6bGNd6T1+uecY33/oICG/j2OjE1x5Rsectq0R2XlgmC/csY91rVHe9Fsb5n1/RvTnieFkDqWmtpjdHbnXXryOay9eN+Xybl60ppnrr7u8rmW7Opq48dHDJOwi110d8bK86+60Du3xhUuNq49fp5autPTTWZ2mohS9U27pl4dJVk6vDKN0l/f7379XPrp0VXOEG997BWCFkgZ84lSK0g+UVS1hbnrfSyYdR9gW/WeOjbN2RYT7PnZ1HUe/cKxvixKwO6G1K6qrM04o4OPb77pszvf3jis2844rNgNw/UMH+N+3PMnxsXRdncV9A0mawwGuOXsVj+wfnvO2NSL6zbpvcGHesI17Zx7I5oskMnmOjExMmdK41JE7v5ehZOkl2D+oRd9d7aj0eSH9sfr4nTj9irh7bclHg37sAcj0j1ez9HOTppf7/N3hqaWHnBcBv4+NK2POzTiSytr5kbzfxLSlv38w2TARO27cx6OPafPKhWmn1+CwqegdsM5hd2dTzftnueCI/gINjDSiPw+MTFjiouzcJtVwW/rzSbctRA+8MEg6V6SrM15m3bsrHy1kjdNMvkjI75sU/aLRN3wk6CPityz9E9U6clNZgvYy2tc/Yf8P+nV4asm9UwudStjadm7KB4V+GCjVGLH5Xujj6RtIsq41WvUBNvf7tfoN6g0O0J3MXukjlit6FPxC9aUZ0Z8H3J2KU6U1cHfkzicb22OIwB12B2RXR5yWSMAJ05tU47RG1ai5IpsvEgr4nIdeZZy+29IP+62Uxyddln65eyfHquZI2Xp6/qrmSLl7Jz61pQ/WOeobSFIsKqdYSDWiLgFthNQLXujjeaE/saAPptUtYaJBf11WbDpX4MjIRJnoL5T1u5joEfJ9g9bvbb4xoj8PuMMHpxrhms0X8fvE6RCcLyJBP+tao+w8YPlIuzuaEBEnW2elf38h3TvhgM956FXG6ZcsfT8iQjwUYCDh9t2Xlh9OZWmLB4kEfU4Vq3TO2r7up9BvA/WkrujqaCKTL3JsLG1b+tXfDtyFzrsb0L0DpeN56ujYgoq+iNgPnNojvQ8OpZy3JcclucAJABeafKHIwaEUK+Mhsnkrtfl8Y0R/HnD7l6eyVDL5AqF5FnxNV0ecQlERDfpZbYdIagu/tcK/n8wWqua3n0syOcvSDwe1pe/t03cPXCu4LKFKn35bLFQ2JkGnZdbFuq0Sh6G6wmLdlqa7SpYXbku/EfLteKGPp1BUC+6C6uqM1+W60G/F3R1NxMMBVreEFzzr60JzeHiCXEGx/axVwMK4eIzozwPaUvYqJu0mky86gjffuAfiOEnGKgqtA7TG9cCk+bf2s4WiZen7LdGsfNC4i8xAqcNXPygro3daY6Gy0ce66pYuGehV3q8a2mLvG0i4qmR5o6N3gn5h3RylM5hr3G8gC93Z3N0R59DwRE1Dwulk7rDCkLs66ntYLGX08V1jRH9poy39Sza2TXkRs3ZH5kKgrTv3Dd/qIfqlFATz79fP5IqEA/6alr4uDK9z5TRFAoQqagQMJy1Btyx9nYPHysWvO6ct8a7tzwdY1RwmFvLzQn+SkTot/Y3tsXl31c0UfTyw8G8j+i3z0PDU6T36BhJ0NodptitueRVjX25o99VvdbXTFA4sSGrzxvyFLjHSuQLPHBtzvo+ksoQCPs5Z28JgMstPdh3xHPC0kJa+Tp3rvuHbPN07C5caN1uwO3L93h25jk8/oEW/9D8atDKHDiWz3LrnGGPpPK22e8dt6VvunRDj6TyHh1N1W/raF/3AC4MU1dQRP0G/4JPG7cSF0vEsxttIvZ2ylYVnujviDCWz3LLr8LwOGCwWFb2j5b+9k2NpjlTsc8/hUSed99NHx2qGk+53peGuRClFz96T3Luvn5ZIgJXx0IK92RjRnwN+9vhRXvPFexnVuVpsN8J5a61UB//rh4/z0Zt2T1pvIX36Z69pJhTwccH6Vmdad2fTpPA9p8bpAoi+7sj1+YSgXya9/k+y9O0EafFQwMlh/7lfP8efX/8YABvaonat4XKf/sZ2y11wdDQ9rSRo565tYe8Jq8LWVCOfRYR1bVEu2rCi7m0vBhduaOXctSsW/G1Eh23WErTKCl7nrrVGin/oh0/wt7fsmbf23fv8AJ98IM2TdglNgI/etJu/sH9XYNU8fs2X7uWnjx9lKJnltV+6lxsePjjldt/69Qf519ue9Zz3cN8Q7/zvR7hrbz/nrl3h6vCef9E3I3LngJEJqzZrfyLDCjvDYFssxEtPX8ndf72N/+8Xzzg1Wd2kc0VH0OabVS0RHv7ba5x8MQDvflkXb79iU9lyJUt/Ydw7OlwzHPBPcu/o6JySpR9w/qdzBdL5AkOpLF0dcb7xx5fS3RHnll1HykQ/GvTz+kvWcdHG1ml3Yv7jtefxp1d2Ewr42Fyj3usvP3gVkQZJsFaNj7/mnLKO8IViRSzIyniI3oEkZ7R7LzM6kWMgkS27Pldsse6fj960u2x8xlxzzI6Y2Xt8nPPsnFR7j48zbNdUEBGesx/+e0+Ms3FljHxROQZBteM5Oprm2ePeyzx30opm+sG7L+eC9dY+uzri/Hz3UdsYmj9daOxf6RIhVygv0GF1KgYRETatjHPB+hUcG02TsvPAayayhbLIj/mmMnIl4PdNSti20O4dHa4ZCviqh2zaCc3coh8J+pnIFhhJZVnVHGZLpxWGWh69Y/n0RYQtnU2cubqZ4DSs3HDAz4vWNJd1flejKRxoWH++Jhzw112Zba6pFbapU164RV/fP+taY/P65unOPgrWfXl0NM1EruA8bPS83v6k46aayv++31k+4Vn7ua8/STTo5/Ludud33d0Zp6ist4r5pLF/pUuEXEGX5cs5/90df9rXu3+g/GKm84UFGxlZL9GQn3DAtyA1TnVHLlgD1Lx8+iKlaJ244+ax6g+k88VJ5zrqyiiqM3QaFp9args9z2ucw3yPEneXwoTyUcB6nI3ucO0bSLg+1z4ed0rr8vmJScaEOzHefGJEfw7I25b+cJml7xZ97/wjE9nGE33Qo3IXriMXLEt/UvSO/Sakb4xyS99HOltgJJVz0kwDhF31anUufsPi09UZ58RYhnTe273UO5DEJ7ChfbIbrS0eIp0rzlsenhF7BLqXmFfmxTk4lOL5k5bL5uR4hkSm/O1d01u2jclvOH0Dk/M0bZ5mnqKZYkR/DsgVLbEaSWVRSllC5Ir20HHHlRc/ky82pOi3LlDlo4w9YhYsS3+Se6fiTUi7oprCAaK2G6fyARsN+p03hrRdX9ew+OgO2uNJ71j93v4E69tinr7s1lh5JbO5Rm93v512Q4tuyO9zxL5vIEko4CNXUDzYO+QYK/urCLReHiZb7tl8kUPDE5NCZ1siQTqawsbSXwrk8trSzzGeyZMvqjKXQywU4LQVkUlDyi1LtvEuQdsCpWLI5KfuyJ3IFsv6PGLh0sjcSNDPYCJDvqhod51r7esH26dvLP2GQLs4j6e8Lf3KcE03Tj/TPOWE0q7MiVyBE+NpevuTrGmJ0G2PJE5m8hwfS/OSLVZtjEQm73yuliaibyDBpZvanJTWbg4Np6oGFXQvQARPXYojIh8SkadE5EkR+YGIRETkehHZa0/7pogEXctvE5HH7XXudk1/lb3O8yLysfk4oMUg77L09atiZVy3l0+zEX36YFXlWpCO3HzJp+/ZkZsvlI1jaHK5d6JBPyfs3Prucx21ff2lqluN91A9Fdm00kr6d8LD0ldKTSn68x1GPJzKoj2Eff3JUg1h+57VPn49ahZg25mdiHiPPVBK0def5MzVzWxsj0267/U6Xsfb1RGf93xDNe8IEVkHfAC4VCl1HuAH3gxcD5wFnA9Egevs5VuBLwOvVUqdC7zRnu4H/gP4XeAc4C0iUl7JYonidOROkbPdS/QXOnqnXqykawvg3nENTvPsyK04PzpOvykcIBws5eFpK7P0fRSKluAXispY+g1CJOhn7Yqop3vn5HiGVLZQNVndfIcRD6dybFlh/U567fTTXZ2W6B8cSrHvhOWWvWRTGy12YYezT2th7Yqop7++fzxD0j6ebo+8Q/q7p+h3xhlIZBhLz9/9V68ZFACiIhIAYsBRpdStygZ4GFhvL/tW4Gal1EEApdRJe/plwPNKqV6lVBa4AXjdXB3IYpJzdeQ6oh+fbOmPpHJOB2mxqGzRazxRaosFGZnIWbHwuYIzClGjlCKdKzgPu5lQLCqrI9fvFv0ixaJy9puq6Oh2u3fcDwP3udbLa4Ew0TuNQ3dnnGPJ0vXVD+3eKSxfmJswYv27qhynYPXBZVnfbNV1eOzAMMOpnFNOMl9U3P1cv9M+PbK9q4qgQ8nl435bcKdM7h1I0h4PeaYE0edg34nxeeu4rhm0q5Q6IiKfBQ4CE8DtSqnb9XzbrfN24IP2pDOBoIj0AM3AF5RS3wHWAYdcmz4MvHguDmKx0eI3kipZ+pUXVFsx+geh/deNaOmvjIcpFBVn/d1tADRHAuz46+1OHvq//MEufr77GOGAj1s/+DK2dE4//UDWPmfa0tfunT/51iPOTQZw1ZmdzufWqLX/FdFgWTrj1gqfPpTSWzfiQ/VUpbsjzj37Bpzf1WkrIuz4yHZXuKb378jt3vnEz55iJJXl82++eMp9FYuKV31hB3+x/XRed9E6Xv3v9/Ds8XFWxkPs+Mh2JxIsYffBNYesEbE37zpityXOCvv3dsuuI6xdESEWCrClM05vf4LOpjBdHXFueewISil+9sRRvvCbffzqQ1eVWfKHhiacFN06/YV2H3mxxdaJP/jKA3Q0hXn0/7yivpM7DWqKvoi0YVnkXcAI8CMReZtS6nv2Il8Gdiil7nFtcytwDZbb5wEReXA6jRKR9wDvAVi9ejU9PT3TWd0hkUjMeN3pcPS4VdjjxEiCHTufRoAXdj/CIX8pBvek/Vp76z2PcnFrhjt6dgBweH8vPT2HJm1zMdDna1VW8YcvClFQisEJxV2H8vzgtns4Z6UloPfsTdEeEYbSRX7e8yAXdk5/wE8yZ1k+h/b30dNzmNGhNMNjRXpPjnNWu4/zOqx9nd8xTk9PD4lEArV3J++7MIzvxDMcP1J6/X161yMcClnn+oA9/e4HHrW+v/AcPem+mZ+UGizUb2y6NGK7LggVed1mRTAU4vB4kQePpbnpth52HMoR9MHeXQ+yr8oguJAf9uztZc9AgUQOenomj3B3k8gqnjuR4tYHnqRp6DmePZ6iIyoMJLPceNvddNnunP6UdV8Gihlev8nHOc1Bwn6hePRpRgXecU6IVF6xZQX09PRwRVORM84PcPfdd1MYtgI3fnZ7Dzfvy9I7kOem23q4+1CegA+ee/whRoes7f/kN/dzrv2bfvZIivM6/J7XRynFu84LMZZVhP3K+e3P5bWs5259BdCnlOoHEJGbgZcA3xORjwOdwJ+5lj8MDCqrKnhSRHYAF9rT3aXe1wNHvHaolPoa8DWASy+9VG3btm06x+TQ09PDTNedDtcffBSOnyCVF4pNnaxtHeZ3rtletkyuUOT/3Hcb4Y4NNEWOc+ZFL4Y77+T8c17Etss2znsb68F9vn7fnnZsdIIrPnUnLetOZ9vlmxhP5xi97XZef8k6bn7sCN1nnsO2C9dOe1/94xm44zecc9aZbLt8Ez/vf4KdJ4+QK8LbrjqHt11enh5Ct02f1b4dvbDvGUTg1a/Y5lQBS+4+BnseY+MZZ8Oju7jo/HPZdsH021cvC/Ubmy6N2q6Vdrt2HhjmD75yP6u2nEt+4CBbVk1w9farqq7X8cAdRNva6T98jFxBccnlL6UlUj0JXt9AEu7sIb5yNRdedjbc/htedeEGvvfgQdo3ncW2i9YBsPvwCOy4j47mCNdde82k7UxZ4n7vSa5/9hFOO/NC0vv3AkN0dp9LfuAQWzpTXL39Ks4dS/Ovj9xBy7otbLtiM8lMnpHbfsUV521h27bTPTe7veL7XF/Lenz6B4HLRSQm1iiZa4BnROQ64JXAW5RSbufuT4ErRSQgIjEsF84zwCPAGSLSJSIhrM7gn83ZkSwi2uedLViVibw6pIJ+X1lPfmWBkEZldXPEKnenRyvao4rPt3OUJKsMTqlFZVH4sB0DDfWl/tXx9y2RoCP4gBOtoyM9GtF9Zihd497+pFUMvcY1b42F2HNk1PmN1MrYWRoomXN+Cxeub0WkPG5e9/00BWsX1pl8DDqRXKI0iGsgWea+0SmtKwd+LWaxnZqir5R6CPgx8Biwx17na8BXgdVY7pvHReTv7eWfAW4DdmN18H5DKfWkUioPvB/4FdZD4Eal1FNzf0gLT97VSfP8yer+OncET3qJiL7PJ2x2tVsPS3dEPzuzzqbK+sDuQTn1FPnQyc0qUyXraB19Mzf6+T1VaYuHaI0Fef5kgoODqZqi3xYP8oJLrGvFsmuhH3bVRl7dEplU2EgvF5+B6K9rixL0C3uOjHLSDh/edyLBwaGU8xuuzJ7p+PsXsaxmXc5YpdTHgY/Xu65S6jPAZzym3wrcOp0GLgUq48ur/YC7O+Pc+/wARRVxRH8pRJd0d8R52q4X0DeQRMQKWYOZW/rZCtHXg7SiQT+r7QLnU6HPW2WHuX4D0JaeEf3Gpcvu2M3Xkf208jrXimXXA7ncEXNtsdCkwix6XlNo+qLv91kJ4e56thR4cO/zA+QK5cfT1RF3suzq+2fzyga29A21yReVI1pQXfR1cerhtHKlDW78S6DjlXOFIn0DSda1RomHA4QDvlm4d6zjD7ncO2DlH/H5at+A2qKvZumPOJZ+45/fU5WujrhTqKRWQXl9nZsjATa0T12GFEoPfV0xDawoID3iVWe+1G8B8RkmH3Ufw3nrWkrHU1EM5tBQimzeun/WroguqjFi7og5IFcosqo57HzvrlJBST8MjieVkypgKVj6Trm7oRS9/SX/a1M4QDI7Q9HPaZ9+aUQu1O/r1OetchBctMLSNz79xqW7zBqeOuxXX+fujjjdHU1TpmmG0kN/dCLHoLb07epUiUzeCiTAcu+0RAJl/UIzOQYR2P6i0ojdMkvfTpl8cChF70Cy5gNuvjGiPwfkCsoR/ZDfx7o273J0+mIfTxVJ55eGTx9K/sfe/mRZdaNY2E8yMzOfvhOnX2Hp11vkRFvwk9w79nTj0298tNCviAZrlrHU17m7s8nykfcnPfPUa/RDXyk4OJQk6BfiIX8pfbH9pjCcyjnjT2Z2DNb21rVGHZdnSyRAu2ub+jh7+xP09Vfv81sojOjPAblCkY4mS/Q3rYxVtRp0T/7xZLFk6S8BUdIi//D+IRKZvDOIJh4KVE0tW4tMrsK9Y5+H+kXf272jz6eJ3ml89LWup0iNvs5dHdZI2GS24FjrXrjTiPT2J50CQtrw6nNEP+s5MrZe9L2gR9+CVY+6LE++7b/feWCYsXTeiP5yIF+wyh42hwNTXlDdk388qUjny0ekNjKtsRBtsSA/fMQaRFbm3qki+vfs6+crPS8A8MALg7ztGw/x9v96iJ0HhgG3pW8XUbHTMdQb1aBFvzVeaelb03UhbWPpNy465Xg9Lj3t3nGL61Sdue6UDX0DSeehsXZFlFDAx5d7nuetX3+Qxw+O1HzLmArdlu6OuNM5W3k8ulzkDRX3z2LR+IqzBMgVFAGfj3df1c2bL9sw5bJrW6MMp4ukl5ClD/AnL+3izNVNXH3WKi7eaBVXj4UDVUM2b9l1hC/3PA/ArXuO8VDfIA/2DvI/TxwFSnH62tK/YstK3rB1PefYr8i12Nge4w1b1/PyMzrLpocDPv7w0g1ctKGVN25dbzpyG5hYKMB7X76FP9i6vuayl2xs43UXreXK0zuqFiVyM5zKOW/fJ8czjjXv8wl/emUXa1oi5ApFzjqtmWvtgVozoaMpxDtfspnXXrSOaMjPe1++hTd4HM+fvHQzZ65uYvuLOrlkU9uM9zcXmMLoc0CuUCQUED5wzRk1l22LBUnklk6cvuYD15wx6fiawn7Hoq4klSkwns6TLxQZTmVZ3xYjHi4N8tLuHe3L39Ae47NvvLDu9gT9Ps/lRYR/fcMFdW/HsLh87HfPqmu5FbEgX7Dz7ayIBgkFfFOK/kgqS3eHlbESyt2AH33V5H329OybTrMdRIRPvPZc53u143n/1Wfw/qtr68NCYMygOSBftCz9emiLhUjkrNS/AZ9Mq1B3oxELVXfv6KiekYkcI6kcrbFgWYx0ZUeuwVAvPp/QtTI+ZYWpoWS2LEqmMsrrVMbccXNALl+sW7xbYyHyRcvnuFSs/GpM5dPXHbwjdrppa2BMnMPDKTL5wqSOXINhOlhpjb3DNieyBTL5IhvaS0EVs+msXW6YO24OyBWLBP31xfnq18xjo+klL/rxsJ9ktuAZOpeyQzmHUyVLv7vDilc+NJSa1JFrMEwHPWCwstYDlDpx2+MhWqPW/TabztrlhhH9OSBXUATqFH1tcRwfTS/5TsZYKEDBLgZTibb0h5Pllj5YIXR6cFa9D0uDwU1XR5xcQXF4eHKfUql6XdDJxW/cOyWWtuo0AMWiolBUdbt33Jb+UoncqYauWevl4tE+/RN2Kby2WJDNrqiLTL5IOOCrGZ9tMHhRGW/vppR2IeSIfWXN6lMZI/qzJGcXRa9b9O248tGJ3DJw72jRnxy2qR8E++2bsjUWYkU0SEdTqEz0DYaZ4Ixy9RB9d51q/WY9m1G3yw1z182SvJ3fu143hdviWOqWftzOc1M5KjebL5bynts3pXtwTa8t+iHjzzfMkLZYkBXRoGdnrk7B0RYrpXcw7p0SJk5/lmjRrzdkU9d5haUxGncq3HVGb3zkEGPpHBdtaC2rmVsS/dIw+rv29rO+LWosfcOMqcxTf2goxe1Pn0Apxf0vDAK2e8e28E1Hbgkj+rNER6EE6xSwUMBHxA/pwjKw9G3Rv+/5Ab5whzW4pbsjzrffdZmzzMEhq9KWvvm6Opq48dHDDCWzRvQNs2Jje4zHD40A8OWe5/nBw6Va090dcUIBH+eubWFje8yEbLowoj9L8tqnP43UrE0hIT2hloFP32r/7sPWjfeyMzp44tBIWbrlgl1VzO3eAdh7fJwVUWN9GWZOezzk+O/7x7OcubqJH7/vJUDJoHrdRet43SzSLCxHjKk1S3J5270zjZG1uh7nkrf0Q5bNsOfIGAGfcMH6FYyl84ynLdF3H5/uy9BRF8dG005mTYNhJrTGgk6qj5FUlvZ4iJZIkJZIcEmPvSZPvgAAGv9JREFUdJ9vzJmZJaXonWlY+rboL/U4fR2yOZDIsHFlzElwpfPx6LoC0aDfeavZ2B5DR2mGzY1pmAX67XFkIueMBTHUxtx1syRXmF7IJkCT/duMLIGqWVMRC5fa390Rd246PWBmvS367k60SNDPulZr+lLvyDYsLvrtccQuiWj89vVh7rpZUgrZrP9UxrWlv8RDFsMBv/OG09URd25CLfpa3CtvRu3XDxlL3zALtJExlMwxMpEzETp1Yu66WaKjd+pNwwAun/4St/ShFMHT1dHk3IS6OPT6NqtIRlu8/GbURSaMpW+YDfr3dmgoRaGojHunTsxdN0u0pT8dq7UppC39pX/6dWduV5l7xwrT1O6dapa+SbZmmA36zbLPGfVtLP16MCGbs0Rn+QtMJ2RzWVn61jF0d8ad4zkyPIHfJ6xZEQEmD4zpsgdvGfeOYTbosR+Vo74NU2NEf5aU3DvTCdm0/i/1OH2w3DuxkJ9VzVbkTsAnZPJFWiKBqkPgjXvHMBfEQ1afks6/U+lGNHhj7rpZMhP3TnvEWnY5RBusjIc5c3UzIoKIOK/Y8XCAjqYwfp+wuiVSts7a1ihN4YAZnGWYFdbvLVSW1M9QG2Ppz5LcDDpy1zX7uOl9V3DxhsUtkDwX/OO15zoPPrBuvIFElng4QGssxC1//hLOXN1cto7fJ9z0vpewpuJhYDBMl7ZYkP5xXQfXiH49GNGfJbni9EM2AbZuap+P5iw4p62Iln1vc1n6ABesb/Vc70Vrmj2nGwzTQVv3Ipg3xzox7p1ZkstPf0TuckbfhE3hpd9fYWh8tJHREgk69XANU2NEf5bkp1lEZbmjb8JYyLxEGuYf7dIxA7PqxyjVLNHFQqbj01/OtDmWvhF9w/zT6pRDNP78ejGiP0uc3Dt1FlFZ7uibL27cO4YFoBQWbCz9ejFKNUuc3DvLYHTtXOB05Br3jmEBKLl3jKVfL0apZkl2BiNylzMlS9+IvmH+0eNCjHunfozoz5KZZNlczlSGbBoM84mpgTt9jFLNklyhiE8w4WI27XETsmlYOLRbpzVuLP16MaI/S3LForHyXWzpbOID15zB1WetXuymGE4BujvifOCaM3jlueb3Vi/mHXyW5PLKiL4Ln0/4f3/7zMVuhuEUwfzepo9Rq1mSLxZNjL7BYFgyGNGfJbmCsfQNBsPSwajVLMkVigRNJ67BYFgiGNGfJflC0QzMMhgMS4a61EpEPiQiT4nIkyLyAxGJiMj1IrLXnvZNEQnay24TkVERedz++3vXdvaLyB57+qPzdVALSa6gzMAsg8GwZKgZvSMi64APAOcopSZE5EbgzcD1wNvsxb4PXAd8xf5+j1Lq96tscrtSamB2zW4ccgUTsmkwGJYO9YZsBoCoiOSAGHBUKXW7nikiDwPr56F9DY8RfYPBsJSoqVZKqSPAZ4GDwDFgtELwg8Dbgdtcq10hIk+IyC9F5Fz35oDbRWSniLxnTo5gEbhp52GOjkwAkC8qE7JpMBiWDKKUmnoBkTbgJuAPgRHgR8CPlVLfs+d/HUgqpf6X/b0FKCqlEiLyauALSqkz7HnrlFJHRGQV8GvgL5VSOzz2+R7gPQCrV6/eesMNN8zo4BKJBE1NTTNatxq5ouLdt6d4dVeQN70oxKcessT/b14crbHm/LZrLmjUdkHjts20a3qYdk2fmbRt+/btO5VSl3rOVEpN+Qe8Efgv1/d3AF+2P38c+Angm2L9/UCHx/RPAB+utf+tW7eqmXLXXXfNeN1qjCSzatNHf67e/e1HlFJKvf7L96m3fv2BRW/XXNCo7VKqcdtm2jU9TLumz0zaBjyqqmhqPc7og8DlIhITEQGuAZ4RkeuAVwJvUUoV9cIissZeDhG5DMuFNCgicRFptqfHgd8Bnqz3ydUoTOQKAPQNJAE7ZNP49A0GwxKhZkeuUuohEfkx8BiQB3YBXwOSwAHgAVvjb1ZKfRJ4A/A+EckDE8CblVJKRFYDt9jLBoDvK6Vum7TDBidti/6BwRSFoiJbUARM1SyDwbBEqCt6Ryn1cSxXTs11lVJfAr7kMb0XuHC6DWw0tKWfLRQ5OjJBvlAkFDAduQaDYWlgTNRpoi19gN6BJLlC0Vj6BoNhyWDUappMuES/rz9hjcg1IZsGg2GJYER/mmRyTp81fbalHzIduQaDYYlg1GqaaEs/EvTRO5A0g7MMBsOSwoi+B+PpHAOJjOc87dM/a00Lz50YJ50rmJBNg8GwZDBq5cE/3/oM7/6OdxJQbelfvLGVE2MZUtkCK6LBhWyewWAwzBhTI9eDgUSWk2PVLH3Lp//n207n984/DQWcv27FArbOYDAYZo4RfQ+y+SKZfMFznnbvtEQDXLq5fSGbZTAYDLPGuHc8yBWKTGSri74IJmLHYDAsSYxyeZArFEnnizoxXBkT2QLRoB87nYTBYDAsKYzoe5DNFykUFbnCZNFP5wtEgv5FaJXBYDDMHiP6HmRtsU97+PUnskWiRvQNBsMSxYi+B7mCFaGT9vDrp/MFwkFz2gwGw9LEqJcH2bwt+q6UC5q07dM3GAyGpYgRfQ+0pe9OrqYxPn2DwbCUMaLvgePe8RD9CWPpGwyGJYwRfQ8y+Sks/VyRiPHpGwyGJYpRLw+msvTTOePeMRgMSxcj+h7o+Hwj+gaDYblhRL+CQlFRKGrRnxy9M5EzPn2DwbB0MaJfgXbtgPHpGwyG5YdRrwqyLtGvdO8opYylbzAYljRG9CvQA7NgsqWvo3rCRvQNBsMSxYh+BbkyS7/cp68tf2PpGwyGpYoR/Qpy+VJmzUr3jn4ImOgdg8GwVFmWop8vFPnQDx/n/qP5aa+bLZSEXhdSKRYV/3b7XvaeGAcgGlqWp81gMJwCLMtyiQG/j3v2DXD2Cu/qV1OR9bD09w8m+eKdz/Oqc9cAEAkYS99gMCxNlq3J2t0R50Rqcpx9LbIeIZt9A0kAnj42BkAkZETfYDAsTZat6Hd1xDmenL7oe3XkatE/OJQCjKVvMBiWLstX9DvjjGVhdCI3rfVy+clx+r226GuixtI3GAxLlOUr+h1xAPZXCHYtMralHwv5HdHv6y/fhhmRazAYlirLVr22dFqi3zdN0deWfkskWObT72gKO8uYOH2DwbBUWbaiv6E9hjDZNVMLnWGzORIgnSuQzOQ5Ppbm6rM6nWVMnL7BYFiqLFvRDwf8dERl2pa+jtNviQZJ54rsH7TWv/KMTkJ+63QZ0TcYDEuVZSv6AGviPvoGEtNaR4/IbbEtff3QOL2ziU0rY4Dx6RsMhqXLslavNXGhrz+JUqr2wjY6Tr8lavn0dSfu5o4YXR1xfIJj8RsMBsNSY1mrV0fURzJbYCRVf9hm1tWRm84V6B1IsnZFhFgowNZNbWxsjyEi89Vkg8FgmFeWZRoGTcR2vU/kCrTVuU7OsfQDFBXsPT5Olx0J9O6XdfOuK7vmoaUGg8GwMCxrSz/ktyxyr1q31dCi3xwJAvD8yYQT8+/zCUHj2jEYDEuYZa1gIZelXy/avdMUtl6CsoUiXR1Nc942g8FgWAyWt+jbR+dV4Lwa2YIiFPCVDcDqti19g8FgWOosb9GfoXsn5PeV5dfpMqJvMBiWCctc9K3/0xH9bL5I0C9OLH7AJ6xvi85H8wwGg2HBqUv0ReRDIvKUiDwpIj8QkYiIXC8ie+1p3xSRoL3sNhEZFZHH7b+/d23nVfY6z4vIx+broDQhn2XpT8ennysUCQV8zqjbjStjBEznrcFgWCbUVDMRWQd8ALhUKXUe4AfeDFwPnAWcD0SB61yr3aOUusj++6S9HT/wH8DvAucAbxGRc+byYCopWfrT8ekXCfpLom/8+QaDYTlRrwkbAKIiEgBiwFGl1K3KBngYWF9jG5cBzyulepVSWeAG4HUzbXg9BGcYvRPylzpyjT/fYDAsJ2qKvlLqCPBZ4CBwDBhVSt2u59tunbcDt7lWu0JEnhCRX4rIufa0dcAh1zKH7WnzRtjuyM1UEf33f/8x/ubmPWXTtHunOWKFbJ6+yoRrGgyG5YPUyksjIm3ATcAfAiPAj4AfK6W+Z8//OpBUSv0v+3sLUFRKJUTk1cAXlFJniMgbgFcppa6zl3s78GKl1Ps99vke4D0Aq1ev3nrDDTfM6OBGxxJ88H7h9WcEee2W0KT5H9uRIhYU/v6KUkft53amGc0oPvGSKLv785y90k/QN7dpFxKJBE1NjfcwadR2QeO2zbRreph2TZ+ZtG379u07lVKXes5USk35B7wR+C/X93cAX7Y/fxz4CeCbYv39QAdwBfAr1/S/Af6m1v63bt2qZspdd92ltvzNL9Snb3vGc/5F//ArddWn7yyb9kdff1D9P/9x74z3WW+7GpFGbZdSjds2067pYdo1fWbSNuBRVUVT68m9cxC4XERiwARwDfCoiFwHvBK4Rinl9JTK/9/e3cfIVZ13HP8+Ozszu7NL2MV2XbCJX1I3bdqkBNwINS8SISoJSoG0tDiNmqRNioqaNLRqGihSiKr+UWjTRpQ0KSgE2tI4ggC1IpqSEkQrxZgaY8BOIIDtkDjGL8RvO/syO7tP/7hn1nfXO/vm3bmje34fabR3zszOPnPm7jNnnjn3XLOfBQ66u5vZ20hKSK+RfErYYGbrgP0kXwb/7tzetxauq1hgqHb6F7nj487xoVHGxid/0qmF8o6ISB7NmvTdfZuZ3Q/sAOrA08AdQBX4IbA1rDr5gCczda4GrjOzOsmbxKbwzlM3s08A/0UyA+gud9+9BM9pkq5igeH66TX9E8OjjDucGK5THxufmJZZq49P1PNFRPJmTtnN3W8mKeXM+rvufjtwe5PbHgYenk+AZ6qr2MFw7fSkfzS13PLxoVGWhXPgNo7IFRHJo9xnt+4mI/2jg7XU9qk3gFGVd0Qkx3Kf3ZKa/ulJ/1gq6ae3k2UYct8tIhKp3Ge37mJh2oOzjlZPje4nj/RdSV9Eciv32a1c7Jh2GYbJ5Z3USF/lHRHJsdxPU+kuFjh8cuS09vR5c48N1tj85CsMjY6FZRh0DlwRyafcJ/2uYmHapZWPDtborxQ5OVzn6OAoDz79E6ojdUbHVNMXkfzKfdJvVtM/NjhKf0+JQodxtFpj35EqI2GWj8o7IpJXuU/6XTPU9PsrJTrMeP7Vk5PeGDTSF5G8yn/SLzWZvTM4yqq+LjoMdv7o2KTbNNIXkbzKfXbr6ixQq48zPmWNnWODNfoqJfoqJUbHJt+mI3JFJK9yn90aJzifelRu44vc/koRSMpAy3qS5ZeLmr0jIjmV//JOKNUMj45TCUvqD4+OMTw6Tl+o6QOsW95Lb7nAa9UaRZV3RCSncp/dGiP9dF2/cTBWfyjvQHIu3MapEVXeEZG8yv9IP5zrNj1Xv7EEQ3+lSOOkWGuXV+gtJ6UefZErInkVTdJPL7rWWGCtr1LCJpV3ku7QlE0Ryatokv5I6ovcQ2FZhhVnlTivr4uVryuzcU0/XcUC557dxRtWtOe5MkVEzlTuk373xEj/1AFae45UMYPzz6lQ7iyw7S/fM3Hb1hsvbXmMIiKtkvs6RlexMXvn1Eh/75Eqq/u7KXcWsgpLRCQTuU/6EyP9SUl/gHXLVcIRkfjkPulPnb3j7uw9XGV9mJ4pIhKT6JL+4ZMjVGtjE3PyRURiEkHSP3VELiRf4gJK+iISpQiS/uSa/l4lfRGJWO6TfrHQQWeHTZR39h6pUurs4Ly+7owjExFpvdwnfZh89qw9h6usXVah0KGVNEUkPlEk/a7SqfPkvvLTKmuWqbQjInGKIun3lApUR5Kkf2KoTl93MeOIRESyEUXSr5Q6qY7UAaiO1Okp5371CRGRaUWR9HvLnQyM1HF3qrX6xGqaIiKxiSLp95QLDNaSs2WNO1TKWnNHROIURdKvlJPyzkAo8WikLyKxiiLp95aS8s5gLUn6PSUlfRGJUxRJv6fcyWBtbGKk36PyjohEKoqk31suUK3VGRhuJH2N9EUkTlEk/Uq5E3c4MpCcG1dJX0RiFUXSbyT5QyeHAX2RKyLxiiLp94YafuOE6JWSavoiEqcokn4lzNY5dCJJ+hrpi0isokj6vVPKOxVN2RSRSEWR9Cdq+idGKBU6KHVG8bRFRE4TRfbrKTVq+sOaoy8iUYsj6YeR/tHBUU3XFJGozSnpm9mfmtluM9tlZl8zsy4zu9fMXghtd5lZccrv/KqZ1c3s6lTbmJntDJcti/1kmkknei3BICIxmzXpm9kq4E+Aje7+y0AB2ATcC/wC8GagG/h46ncKwC3AI1MebsjdLwiXKxbnKcyuJzVFU+UdEYnZXMs7nUC3mXUCFeAn7v6wB8CTwOrU/T8JfAM4tKjRLlBnoYNy+PJW5R0RidmsSd/d9wN/B7wCHACOu/vECD6UdX4P+Fa4vgr4APClaR6uy8y2m9kTZnbVIsQ/Z41pm5qjLyIxs2SgPsMdzPpJRu3XAMeA+4D73f3fwu13AlV3vz5cvw/4vLs/YWZ3A9909/vDbavcfb+ZrQe+A1zq7i9P8zevBa4FWLly5UWbN29e0JMbGBigt7cXgE8/PsjhIeft53Xyh28pL+jxFks6rnbSrnFB+8amuOZHcc3fQmK75JJLnnL3jdPe6O4zXoDfBr6Suv5h4J/C9s3AQ0BH6va9wL5wGSAp8Vw1zePeDVw929+/6KKLfKEee+yxie3L/uFxX/OZb/pnH3puwY+3WNJxtZN2jcu9fWNTXPOjuOZvIbEB271JTp1LreMV4GIzqwBDwKXAdjP7OHAZyWh9PPUmsq6xnRrpPxQ+MQy6+4iZLQfeDtw6h7+/KBplHdX0RSRms2ZAd99mZvcDO4A68DRwB1AFfghsNTOAB9z9r2Z4qF8E/tnMxkm+S/gbd//eGcY/Zz1K+iIicxrp4+43k5Ry5vW77v7R1PZ3SaZ3ZqIxVbNHK2yKSMSiOCIXTh2UpZG+iMQsnqSvKZsiIjEl/aSsU1HSF5GIRZT0GyN91fRFJF7RJH1N2RQRiSjpr+7vplToYEVvtkfjiohkKZqkf8kbf4atN76bZUr6IhKxaJK+mSnhi0j0okn6IiKipC8iEhUlfRGRiCjpi4hERElfRCQiSvoiIhFR0hcRiYiSvohIRJT0RUQioqQvIhIRS06c3r7M7DDJuXgXYjlwZBHDWSyKa/7aNTbFNT+Ka/4WEtsad18x3Q1tn/TPhJltd/eNWccxleKav3aNTXHNj+Kav8WOTeUdEZGIKOmLiEQk70n/jqwDaEJxzV+7xqa45kdxzd+ixpbrmr6IiEyW95G+iIik5DLpm9l7zewFM3vJzG7IMI7zzewxM/ueme02s0+F9s+Z2X4z2xkul2cU3z4zey7EsD20nWNm3zazF8PP/hbH9MZUv+w0sxNmdn0WfWZmd5nZITPblWqbtn8scVvY5541swsziO1vzez58PcfNLO+0L7WzIZSffflFsfV9LUzsxtDn71gZpe1OK6vp2LaZ2Y7Q3sr+6tZjli6/czdc3UBCsDLwHqgBDwDvCmjWM4FLgzbZwE/AN4EfA748zboq33A8ilttwI3hO0bgFsyfi1fBdZk0WfAu4ALgV2z9Q9wOfCfgAEXA9syiO3Xgc6wfUsqtrXp+2UQ17SvXfhfeAYoA+vC/22hVXFNuf3zwGcz6K9mOWLJ9rM8jvTfBrzk7nvcvQZsBq7MIhB3P+DuO8L2SeD7wKosYpmHK4F7wvY9wFUZxnIp8LK7L/TgvDPi7v8D/HRKc7P+uRL4F088AfSZ2bmtjM3dH3H3erj6BLB6qf7+fOKawZXAZncfcfe9wEsk/78tjcvMDPgd4GtL8bdnMkOOWLL9LI9JfxXwo9T1H9MGidbM1gJvBbaFpk+Ej2d3tbqEkuLAI2b2lJldG9pWuvuBsP0qsDKb0ADYxOR/xHbos2b902773R+QjAgb1pnZ02b2uJm9M4N4pnvt2qXP3gkcdPcXU20t768pOWLJ9rM8Jv22Y2a9wDeA6939BPAl4A3ABcABko+WWXiHu18IvA/4YzN7V/pGTz5PZjK9y8xKwBXAfaGpXfpsQpb9MxMzuwmoA/eGpgPA6939rcCfAf9uZq9rYUht99pN8UEmDy5a3l/T5IgJi72f5THp7wfOT11fHdoyYWZFkhfzXnd/AMDdD7r7mLuPA3eyRB9pZ+Pu+8PPQ8CDIY6DjY+L4eehLGIjeSPa4e4HQ4xt0Wc075+22O/M7KPA+4EPhWRBKJ+8FrafIqmd/3yrYprhtcu8z8ysE/hN4OuNtlb313Q5giXcz/KY9P8P2GBm68JocROwJYtAQq3wK8D33f3vU+3pGtwHgF1Tf7cFsfWY2VmNbZIvAXeR9NVHwt0+AvxHq2MLJo2+2qHPgmb9swX4cJhdcTFwPPXxvCXM7L3AXwBXuPtgqn2FmRXC9npgA7CnhXE1e+22AJvMrGxm60JcT7YqruA9wPPu/uNGQyv7q1mOYCn3s1Z8Q93qC8k33D8geYe+KcM43kHysexZYGe4XA78K/BcaN8CnJtBbOtJZk48A+xu9BOwDHgUeBH4b+CcDGLrAV4Dzk61tbzPSN50DgCjJLXTjzXrH5LZFF8M+9xzwMYMYnuJpN7b2Ne+HO77W+E13gnsAH6jxXE1fe2Am0KfvQC8r5Vxhfa7gT+act9W9lezHLFk+5mOyBURiUgeyzsiItKEkr6ISESU9EVEIqKkLyISESV9EZGIKOlL9MxszCav7LloK7OGFRuzOqZA5DSdWQcg0gaG3P2CrIMQaQWN9EWaCGus32rJOQeeNLOfC+1rzew7YQGxR83s9aF9pSXr2D8TLr8WHqpgZneG9dIfMbPuzJ6URE9JXwS6p5R3rknddtzd3wzcDnwhtP0jcI+7v4VkUbPbQvttwOPu/iska7fvDu0bgC+6+y8Bx0iO+BTJhI7IleiZ2YC7907Tvg94t7vvCYtiveruy8zsCMlSAqOh/YC7Lzezw8Bqdx9JPcZa4NvuviFc/wxQdPe/XvpnJnI6jfRFZuZNtudjJLU9hr5Lkwwp6YvM7JrUz61h+7skq7cCfAj437D9KHAdgJkVzOzsVgUpMlcacYiEmn7q+rfcvTFts9/MniUZrX8wtH0S+KqZfRo4DPx+aP8UcIeZfYxkRH8dycqOIm1DNX2RJkJNf6O7H8k6FpHFovKOiEhENNIXEYmIRvoiIhFR0hcRiYiSvohIRJT0RUQioqQvIhIRJX0RkYj8P6PRj7X5GMkSAAAAAElFTkSuQmCC\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(evaluation_accuracy[200:])\n", + "plt.grid()\n", + "plt.title('Accuracy (%) on the test data')\n", + "plt.xlabel('Epoch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 368 + }, + "id": "dI40Dk1xLSk1", + "outputId": "17881deb-9296-48bf-988e-337723bf03eb" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 0, 'Epoch')" + ] + }, + "metadata": {}, + "execution_count": 14 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAFNCAYAAAAQFpQ2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxU5Z3v8c+vV5ruhga6aZCtQRaDKCjgEo1CjImaRc2mJjHRxPFOJjOTe52sk4xmYpJxkmuuk8U4JjFklZkkZkxITNQIbqAICMoi+7413dD0Rq/1u3/U6aZpqpdqqrvqVH/fr1e9uqrOqVO/OjTffuo5z3mOuTsiIhJ+GckuQEREEkOBLiKSJhToIiJpQoEuIpImFOgiImlCgS4ikiYU6DLomNkyM7sj2XWcKTP7ipn9Itl1SOpQoEu3zOxDZrbKzGrN7KCZPWFml5/hNneZ2dsSVWMP79WvoZeoz2Jmt5nZC4moqYvtLzKzr/XX9iU1KNClS2Z2F/AA8A2gFJgIPAhcn8y6RKQL7q6bbqfdgOFALfCBbtbJJRr4B4LbA0BusKwYWAJUAUeB54k2IH4ORIATwfY/18W2/wbYFrz298BZHZY58LfA1mD73wcsxjauAZqA5uC91gXPLwPuBV4EaoAngeIOr7sEWB5sex2woIsaY36W7l4P3AbsCN53J/Bh4E1AA9AabKeqi/ebDDwbvPYp4HvALzos/zVwCDgOPAecGzx/Z7APmoLt/yF4/gvA9mB7G4Ebk/17p9uZ3ZJegG6peQvCsAXI6madrwIvAaOBkiDE7g2W/RvwEJAd3N7SFrrALuBt3Wz3rUAFcGHwR+O7wHMdlnvwx6KI6LeGI8A1XWzrKx1DL3huWRBk04G84PF9wbJxQCVwHdE/QFcHj0u62P4pn6W71wP5QDUwI1h3bIfQvQ14oYd/kxXAt4N9ckUQxB0D/eNAISf/0K7tsGwR8LVO2/sAcFZQ501AHTA22b97uvX9pi4X6coooMLdW7pZ58PAV9293N2PAP8K3BosayYaWJPcvdndn/cgRXrhw8Aj7r7G3RuBLwKXmllZh3Xuc/cqd98DLAXm9PqTRf3E3be4+wngvzu8/iPAn9z9T+4ecfengFVEA7o3enp9BJhlZnnuftDdN/Rmo2Y2EZgP/Iu7N7r7c8AfOq7j7o+4e02wz74CzDaz4V1t091/7e4Hgjr/i+g3not6+TklBSnQpSuVQLGZZXWzzlnA7g6PdwfPAXyLaJfJk2a2w8y+EMd7n7Jdd68N6hnXYZ1DHe7XAwVxbL+7108CPmBmVW034HKif5x6o8vXu3sd0Zbw3wIHzeyPZnZOL7d7FnAs2Eab9n1kZplmdp+ZbTezaqLfHCDa9RWTmX3UzNZ2qHNWd+tL6lOgS1dWAI3ADd2sc4BogLWZGDxH0FL8J3efArwHuMvMrgrW66mlfsp2zSyf6DeG/XF9gt69V2d7gZ+7e1GHW76739fL7Xf7enf/i7tfTfQPxBvAD3tZ50FgRLAv2kzscP9DRA9Wv43o8Y+y4HmLtX0zmxS8998Do9y9CFjfYX0JIQW6xOTux4G7ge+b2Q1mNtTMss3sWjP7ZrDao8CXzazEzIqD9X8BYGbvMrOpZmZED9K1Eu1uADgMTOnm7R8FbjezOWaWS3SUzcvuvqsPH+UwUGZmvf1d/wXwbjN7R9DqHWJmC8xsfDfbn9Kb15tZqZldH4RyI9EDlB33yXgzy4n1Ju6+m2jXzb+aWU4wdPTdHVYpDLZZCQwlus+6qzOfaMgfATCz24m20CXEFOjSJXe/H7gL+DLR//h7ibbo/idY5WtEQ+Y14HVgTfAcwDTgaaKhtQJ40N2XBsv+jegfgioz+0yM930a+Bfgt0RbpmcDN/fxY/w6+FlpZmt6Wtnd9xJt6f4zJz/zZ+n6/8opn6WH12cQ3Z8HiI7euRL4ZLCdZ4ANwCEzq+jivT4EXBy89h7gZx2W/YxoF8x+oiNWXur02h8DM4M6/8fdNwL3E/23OQycR3TUj4RY26gDEREJObXQRUTShAJdRCRNKNBFRNKEAl1EJE0o0EVE0kR3ZwH2q+LiYi8rK+vTa+vq6sjPz+95xSRI1dpUV3xUV3xUV/z6Wtvq1asr3L0k5sJkTSIzd+5c76ulS5f2+bX9LVVrU13xUV3xUV3x62ttwCrX5FwiIulNgS4ikiYU6CIiaUKBLiKSJhToIiJpQoEuIpImFOgiImlCgS4ikiZ6DHQzm2BmS81so5ltMLNPx1jnw2b2mpm9bmbLzWx2/5QrIiJd6U0LvQX4J3efCVwCfMrMZnZaZydwpbufB9wLPJzYMk/aU1nPsr3NHK9v7q+3EBEJpR4D3d0Puvua4H4NsIlTr76Ouy9392PBw5eArq6/eMZe33+cRRuaOFzT0F9vISISSnH1oZtZGXAB8HI3q30CeKLvJXUvI7gmeUSXzhMROUWvrylqZgXAs8DX3f2xLtZZCDwIXO7ulTGW3wncCVBaWjp38eLFcRe86lAL31vbyFffPISJwzLjfn1/q62tpaCgINllnEZ1xUd1xUd1xa+vtS1cuHC1u8+LubCrWbs63oBs4C/AXd2scz6wHZjem232dbbFP68/6JM+v8TX76/q0+v7W6rO7qa64qO64qO64peU2RbNzIAfA5vc/dtdrDMReAy41d23xP0nJw4ZFu1zUY+LiMipenOBi8uAW4HXzWxt8Nw/AxMB3P0h4G5gFPBgNP9p8a6+Epwh9aGLiMTWY6C7+wuA9bDOHcAdiSqqO20t9IjyXETkFOE7U1QtdBGRmEIX6OpDFxGJLYSBHv3pSnQRkVOEMNDVhy4iEkvoAr3t6Kz60EVEThW+QFcfuohITKELdPWhi4jEFr5Az1AfuohILKELdPWhi4jEFr5Ab+tDT3IdIiKpJnSBrrlcRERiC2Ggt41yUaCLiHQUukC3thZ6JLl1iIikmtAFeob60EVEYgpdoJv60EVEYgpdoKsPXUQkttAGuk4sEhE5VegCXV0uIiKxhS7QT87lktw6RERSTegC3dq7XJToIiIdhS7QdQk6EZHYQhfompxLRCS20AW6WugiIrGFLtA1ykVEJLbQBXrbBS6U5yIipwpdoKsPXUQkttAFuibnEhGJLYSBHv2pFrqIyKlCF+imuVxERGIKYaBHf2q2RRGRU4Uu0DUOXUQkthAGevSn+tBFRE4VukBXH7qISGwhDPToT/Whi4icKnSBnqHpc0VEYgphoEd/Ks9FRE4VwkBXH7qISCyhC3TNtigiElv4Ap22cegKdBGRjnoMdDObYGZLzWyjmW0ws0/HWMfM7Dtmts3MXjOzC/unXPWhi4h0JasX67QA/+Tua8ysEFhtZk+5+8YO61wLTAtuFwM/CH4mnPrQRURi67GF7u4H3X1NcL8G2ASM67Ta9cDPPOoloMjMxia8WtSHLiLSlbj60M2sDLgAeLnTonHA3g6P93F66CeEmfrQRURisd4Go5kVAM8CX3f3xzotWwLc5+4vBI//Cnze3Vd1Wu9O4E6A0tLSuYsXL+5T0bf/uZZ3nZ3D+6bl9On1/am2tpaCgoJkl3Ea1RUf1RUf1RW/vta2cOHC1e4+L+ZCd+/xBmQDfwHu6mL5fwK3dHi8GRjb3Tbnzp3rfTXlC0v8m3/e1OfX96elS5cmu4SYVFd8VFd8VFf8+lobsMq7yNXejHIx4MfAJnf/dher/R74aDDa5RLguLsfjOvPThwMHRQVEemsN6NcLgNuBV43s7XBc/8MTARw94eAPwHXAduAeuD2xJfagemgqIhIZz0Gukf7xa2HdRz4VKKK6kkG6CrRIiKdhO5MUYgOXVQLXUTkVOEMdNSHLiLSWTgDXS10EZHThDPQ0VwuIiKdhTPQTWeKioh0FspAz0B96CIinYUy0NWHLiJyulAGOpha6CIinYQy0KMXuVCii4h0FMpANyASSXYVIiKpJZyBrj50EZHThDPQ0SgXEZHOwhnoBq4+dBGRU4Qz0NGZoiIinYUy0DPUhy4icppQBjqoD11EpLNQBrrmchEROV0oAz0D9aGLiHQWykDXOHQRkdOFM9BRoIuIdBbOQDdTl4uISCfhDHQ0ykVEpLNwBrpGuYiInCacgY760EVEOgtnoJu6XEREOgtnoKPLW4iIdBbKQM9QH7qIyGlCGejqQxcROV04A910CToRkc7CGejoAhciIp2FM9A1ykVE5DThDHR0UFREpLNwBrpa6CIipwlnoKMWuohIZ+EMdDO10EVEOglloEevWKREFxHpKJSBjvrQRUROE8pA1zh0EZHThTLQM3SmqIjIaUIZ6JrLRUTkdD0Gupk9YmblZra+i+XDzewPZrbOzDaY2e2JL7Pze6JrioqIdNKbFvoi4Jpuln8K2Ojus4EFwP1mlnPmpXVNfegiIqfrMdDd/TngaHerAIVmZkBBsG5LYsqLTWeKioicLisB2/ge8HvgAFAI3OTu/XrIUn3oIiKns96coGNmZcASd58VY9n7gcuAu4CzgaeA2e5eHWPdO4E7AUpLS+cuXry4T0V/d3Ut++oy+Pcrhvbp9f2ptraWgoKCZJdxGtUVH9UVH9UVv77WtnDhwtXuPi/mQnfv8QaUAeu7WPZH4C0dHj8DXNTTNufOnet9ddMDT/gV33ymz6/vT0uXLk12CTGprviorviorvj1tTZglXeRq4kYtrgHuArAzEqBGcCOBGy3SxrlIiJyuh770M3sUaKjV4rNbB9wD5AN4O4PAfcCi8zsdaLd259394p+qxjIwNSHLiLSSY+B7u639LD8APD2hFXUC2qhi4icTmeKioikiXAGulroIiKnCWegoxa6iEhn4Qx0nSkqInKacAY6umKRiEhn4Qx0Q1NziYh0Es5AB1rV5yIicopQBnqmQUSBLiJyilAGeoYZzboGnYjIKUIZ6FkZ6nIREekslIGeYdDc6hrpIiLSQSgDPdOiP9VKFxE5KdSB3qJAFxFpF85Az4gmugJdROSkcAZ6W5dLqwJdRKRNIi4SPeCCBrqGLopIymtpjbD32Am2ldey/UgtbxysZsGM0RT1w3uFMtCzgu8VLWqhi0iKqGtsYceROrYdqWF7eR3bj9SyrbyWXZV1NHfIqjHDhjBr3HAFepuM9oOiaqGLyMCqbmhmy6EathyuZWt5DdvKo8F98HhD+zqZGcakUUM5u6SAq95UytTRBZxdks+UkgKG52UDsGzZnoTXFspAbx/loha6iPSTE02t7KyoY0dFLTuO1LHjSC2bDtaw+XBN+zpDczI5u6SAS6eM4uzRBZxdUsDU0flMHJlPTtbAH6IMZ6BrlIuIJEh9Uwt7j57g1T3H2HiwOhriR+rYX3XilPXGFeUxrbSAd50/llnjhjOttICzhueR0dZlkALCGejqchGRODU0t7LhQDUv7ajkQNUJDh1v4I1DNacEd2FuFlNK8rlo8kgmF+czpSSfKcUFTC7OJy8nM4nV9064A11dLiLSibtzpLaRbYdr+c2WJn607WV2VtRx4PiJ9msRj8zPYXRhLhdOGsEtF01g0qh8zhlTyNTRBZilTos7XqEM9AydKSoy6EUizv6qE7xxqIbNh6rZVl7Ljoo6dh6po6axBYhmxaxxzcwvG0FZ8XhmlBZyyZRRjMjPSXL1/SOUgX5y2KK6XEQGg6aWCFvLa9iwv5rX9x9nw4HjbD5UQ11Ta/s644rymFKSz3svHMeUkmg3Se3u9Vx39eVJrHxghTLQM00HRUXSVX1TC9vL69hyuIat5bWs21vFmj3HaGyJNuAKcrOYOXYY7587nhljhjFjTCEzxhRSkHt6nC07EN7uk74IZaBnqA9dJC00trSyfn81L2ytYN2+KrYcrmHfsZMHKbMzjemlhXzkkknMmVDEuWcNo2xUfkqNLEkloQz0zLYuF41yEQmN9m6TA9W8tq+K1/YdZ9PBappbHTOYUVrIBRNH8MF5E5heWsDU0YVMGjWU7MxQTjmVFOEMdLXQRVJaa8TZc7Se1/cfZ8X2StbtrWJreU37KfAFuVmcN244H798MnPGF6X1gcqBFO5AVwtdJOkaW1rZXtXKzhd38tq+6MHK7Udq2/u8C3OzmDOxiCumT2HmWcM496xhTFa3Sb8IaaDroKhIMrg7uyvrWbu3irV7q3h1bxWbDlTT1BoBNlI6LJdzxgzjsqmjmFZayDljCpk5dhhZ6jYZEOEMdM22KDIgquqb2sN77d4q1u2t4lh9MwB52ZmcN344t19WRlb1fj5y7WWMHZ6X5IoHt3AGuk4sEkm4ppYIGw9Ws3bPsfYA31VZD4AZTBtdwNUzS5kzYQRzJhQxvbSgveW9bNlhhXkKCGWgnxy2qD50kb5qaY2wcudRnt5Uzpo9x9jY3nUCJYW5zJlQxAfmTeCCCUWcN344hUOyk1yx9CSUgd7W5dKsFrpIrzQ0t7KtvJbdlfX8ZcMhlm+voLKuCXfIzcrg/PHD+dibJ0Vb3xOLOGv4kFDPaTJYhTPQg1+0VrXQRWJqaY2wYkclO47UsXLXUZa9Ud5+mvyo/ByunFHC+KI8zhk7jAUzShiaE8ookE5C+a+oPnSRk9ydQ9UN7Dt2gqc3HWbJuoOnTAlbXJDDe+aM4/KpxZQOy+WCiSParykg6SWcgd5+pqgCXQanitpGVmyvZM/Ren67Zh87jtQBkJVhXD6tmPfMOYvZ44dz4cQRlBTmqvtkkAhnoOugqAwy7s6mgzU8uauZX/1sFS9sq6A+6EKZM6GIr7x7JhNGDuWCiSMYqTMuB61QBrrmQ5fBoKG5lTV7jvHMpnKeWH+ovRtl4sgarp9zFjfNn0jZqKEUDVWAS1RIA93IMJ1YJOmlvLqBjQer+dHzO3lpR2V7gyUnM4O3TCvm02+bRuaRrbzv2oVJrlRSVY+BbmaPAO8Cyt19VhfrLAAeALKBCne/MpFFxpKVmUGz5nKREGtujfDCtgpe3FrBM5vL2/vB87Izue3NZeTnZnH++OFcNHlk+xjwZcu2J7NkSXG9aaEvAr4H/CzWQjMrAh4ErnH3PWY2OnHldS0rw2hVC11C5kDVCV7YWsGaPcd4elM5FbWNZBhcOb2E9104nnmTRnDOmGEMH6qTeCR+PQa6uz9nZmXdrPIh4DF33xOsX56Y0rqXlWHqQ5eU5u6s2VPFUxsPc/xEM6t3H2XL4VoAioZmc/HkkXxw3gTmlY1keJ4CXM5cIvrQpwPZZrYMKAT+w91jtuYTKTszQ9PnSsppbGll9e5j/PG1g6zYXsmOijqyM4287EzOH1/EB+dN4IrpJUwL+dXlJTWZe8+t3KCFviRWH7qZfQ+YB1wF5AErgHe6+5YY694J3AlQWlo6d/HixX0qura2li+/ksH5JZl8fFZun7bRX2praykoKEh2GadRXfHpbV2tEWd9ZSs7qiI0tjqvHGqlssHJyYDpIzOZPyaTi8ZkkZeVmPAO+/4aaKlaF/S9toULF65293mxliWihb4PqHT3OqDOzJ4DZgOnBbq7Pww8DDBv3jxfsGBBn95w2bJl5OdFKBk9igULZve58P6wbNky+vq5+pPqik93dbW0Rnhi/SEeeXEnWw/XUtvYAkSvf3nhxBF8/bIyLp1S3C/94GHcX8mUqnVB/9SWiEB/HPiemWUBOcDFwP9LwHa7lZlhtKrLRQbIiaZWntt6hEdX7mH9/uNU1DYxpTifGy8Yx+XTilk4YzQ5WbqIgyRXb4YtPgosAIrNbB9wD9Hhibj7Q+6+ycz+DLwGRIAfufv6/is5KivTNNui9JuqxghPbTzMgaoTvLSjkmfeKKexJUJJYS6XTS3muvPGcvWbSnUZNUkpvRnlcksv1vkW8K2EVNRL2RkZGrYoCeXurN9fze9e3c9Pl5+g1VcBMDwvm1sumsjbZ5Yyf/JIXYVeUlYozxSFoIWuuVzkDDW3RthxpI7H1+7n6U2H2XK4luxMY/6YTD57w8WMK8qjaGg2Q7Izk12qSI9CG+h52ZmcaG5NdhkSQi/tqGTpG+XUNrbw2zX7aGiOkJlhzC8bwb03zOI9s8/i1ZdfZO6kEckuVSQu4Q30nExqGlqSXYaEyK6KOn66Yhc/eXEXAEOyM7h21lgunjySq95USklhag2BFYlXeAM9O5Py6sZklyEpbsOB4zy2Zj8vbqtga3ktrRHnpnkTuPvdM8nOzNDIFEkroQ30oTmZ1DerhS6nikScjQereXbLEZ7dfISVu46Sk5XBJVNG8eazi7nziimMGT4k2WWK9IvQBnpeThYnmnRQVOBITSNPbjzEql3HeH5rBRW10W9us8YN4/+8bTq3XVamuVJkUAhtoA/NyeREk1rog9nKnUdZsb2SHz2/g5rGFkbl53DZ1GKunF7CW6YXM7pQLXEZXEId6PXNrbi7JjkaRBpbWnlq42EWr9zLC9sqAHjLtGK+9M43MaO0UL8LMqiFNtDzcjJxh8aWiMYIDwJ7j9az+JU9/OrlPRyrb+as4UP43DUzeM/ssxhXlKcgFyHMgR6EeH1TqwI9DbVGnOoTzfxm9T5e3lnJ05ui0+xfO2sMt1w0kcumFpOp0+5FThHaQB+a0xboLbrKeRrZXVnHH9Yd4Mcv7ORYfTMAowtzuePyyXzo4olMKUnNqVBFUkFoAz0vJ1p6g84WTQt7j9bzL4+vZ/m2SppaI0wpzufWS8u47rwxnDNmWLLLEwmF0Ab60A5dLhJO28prKK+P8PzWI3x68VqaWyJ8cP54br2kjKmjC9SlIhKn8AZ6jgI9zJ7eeJhP/nI1za0Oz63k7JJ8fvSx+Uwuzk92aSKhFdpAzwsC/YQCPVSqG5r59pNb+OXLu5k0Kp85wxuZPXM67587of3fVET6JvSBrhZ66otEnKc3HWb59uiFIg5UneAd547hG+89j1dffpEFl5Ylu0SRtBDaQB+aHS29XmeLpqwDVSf4+p828dzmI9Q0tjAkO4MLJozg6zfO4i3TSpJdnkjaCW2gt7XQNcol9by8o5JXdh3lpyt2c6yuieF52Xz68mn8/Vun6mo/Iv0otIGug6Kp53h9Mw8+u40fPreDiMPMscP41R0XM620MNmliQwKoQ30tnmsm1o042KyHatrYs2eY3z3mW2s3VvFldNLuO995zFm2BCdki8ygEIb6FnBGGVdVzR53J0/vX6Iry7ZwOHqRoZkZ/DQRy7kmlljk12ayKAU2kA3M3KyMmhUoCdFY0srX/jt6/zu1f1MHDmUB26aw2VTi3UZN5EkCm2gA+RmZqjLZYA9vfEwq3Yf42crdlHf1MpdV0/n7xdOJUNndYokXagDPTtLgT5Q6hpb+M3qfdzz+w0AnD9+OHddPZ0FM0YnuTIRaRPqQM/JzFAfej9zd5ZtOcKXHnudA8cbyMww7rp6OrdeOolhQ3RZN5FUEu5AVwu931TVN/HYmv08unIPW8trObskn3tvmMX8shGa/VAkRYU60LMzjSa10BNu9e6jfOyRV6htbGHOhCL+/X3ncf2ccbqQiEiKC3Wg52Rl0tTiyS4jbTQ0t/LYmv382xObKCnM5b//16XMPEutcZGwCHmgZ6iFngDHTzRz6HgD9y7ZyAvbKpg0aii/vONizirKS3ZpIhKHUAd6dNiiTv3vq5qGZh5+bgePrtxLRW0jZvC1G2Zx0/wJmnNFJIRCHejZWUZDs1ro8aqqb+KlHZUsWr6Ll3YcpWhoNh+7dBLvmTOOuZNGJLs8EemjUAd6TmYG1Sc0fW483J2/++Ualm+vBODbH5zNjReM05wrImkg3IGuYYtxiUScH7+wk+XbK5lfNoKPXDKJ6+eMS3ZZIpIgoQ70bJ1Y1Ct7j9bzg7UN/OOyJ6luaOHNZ4/iF5+4WKfri6SZUAd6TlYGjWqhd+v5rUe45/EN7KiIHjz+h7dO5Y63TFGYi6ShUAd6roYtdmv17mN8fNEr5GZlctu5OVx/5TwumKiDniLpKtSBri6X0x2vb2ZreQ17jtbzud+8RumwIfzxHy9n7crlCnORNBfqQM/R9Lmn2F1Zx80Pv8TB4w0AXDCxiEc+Np+ioTlJrkxEBkK4A12jXNq5O3c/voHahhb+4+Y5NLZEeMfMMQwfqhkRRQaLHgPdzB4B3gWUu/usbtabD6wAbnb33ySuxK5lZ2bQEnEiER/UB/lW7TrKZ369jl2V9Xzm7dM1FFFkkOrN+d2LgGu6W8HMMoF/B55MQE291n6h6EHaj94acf666TB/+4s1tESc+z8wm08umJrsskQkSXpsobv7c2ZW1sNq/wD8FpifgJp6LbdDoA/GqV3/75Ob+cGy7QD88KPzuHpmaZIrEpFkOuM+dDMbB9wILGSAA71tAqnB2I/++Nr9/GDZdqaOLuB9F47nqnN0KTiRwc7ce55PPGihL4nVh25mvwbud/eXzGxRsF7MPnQzuxO4E6C0tHTu4sWL+1R0bW0tBQUFLNvbzKINTXx7QR4jh6TG7IBttfWXzUdb+e6rDdQ2w/QRGXxu/hCyenH8oL/r6ivVFR/VFZ9UrQv6XtvChQtXu/u8mAvdvccbUAas72LZTmBXcKsFyoEbetrm3Llzva+WLl3q7u6/WbXXJ31+ie+qqO3zthKtrbZEi0Qi/uDSbT7rnj/7pM8v8Xd953k/UtOQ9LrOlOqKj+qKT6rW5d732oBV3kWunnGXi7tPbrvfoYX+P2e63d7IzhocXS4rdx7llh++RGvEWTijhHvefS5lxfnJLktEUkxvhi0+CiwAis1sH3APkA3g7g/1a3U9yMlM71Eumw/VsLOijl+t3ENrxLnvvedx0/wJmupWRGLqzSiXW3q7MXe/7YyqiVNudjTQ0/EiF2v2HOO9Dy5vf/zZd8zg5osmJrEiEUl1qXEksY9GBqe0H61rSnIlibV+/3He94OTYX7OmELueMvkbl4hIhLyU/9HD8sFoLymIcmVJE5tYwsPPL2VIVmZPPvZBQzJySQ3K4PcrME3zl5E4hPqQC8uyMUMyqsbk13KGWuNOK/uOcbtP3mFmsYW/vGqaYweNiTZZYlIiIQ60LMzMxg5NIfymnAHelNLhOu+8zzbymsZXZjLz++4mDkTipJdloiETKj70AFKCnM5EuIul+bWCA88vYVt5bUA3DIn4sMAAA4ASURBVHjhOIW5iPRJqFvo0Bbo4W2hf3/pNh5ctp1xRXn83cKzNVOiiPRZ6AN9dOGQ9tZt2DS2tPLoyj1MHV3AL++4mFL1mYvIGQh9l8voYdEWeiTS85w0qaSlNcL133uRw9WNfOm6NynMReSMpUELPZeWiHOsvolRBbnJLqdH7s6GA9Ws3HmUNw7V8NXrz2WhZkoUkQRIg0CPtmzLaxpDEejf+stmHgzmMAf44LwJSaxGRNJJ+AO9/eSiRt40NsnF9GD9/uM8uGw77zx/LMPzspkwYuigvDCHiPSP8Ad6YRDo1ak9dLG2sYX7nniDgtwsvnHjeQzP08WbRSSxwn9QtEOXSyq7+/H1LN9eweeumaEwF5F+EfoWel5OJoW5WSk7Fr25NcL9T27hsTX7+fhlk/nopWXJLklE0lToAx2gZFhuyk3Q1dDcyss7j7L0jXIWLd/FuKI8/uYKzZgoIv0nPQK9IJeKmtSaQve7z2zl+0ujo1k+dPFEvnHjeUmuSETSXej70AFGDM3hWH3qBHpdYws/X7G7/fHHL1PLXET6X1q00EfkZ3Nsd3OyywDgyV3N3PnUUzS1RvjIJRPJycxk6ujUvOq4iKSXtAj0oqE5VNU34e5Jv97mr944+U3h3utnJb0eERk80qTLJZuWiDP5i39i3d6qAX//SMRZteso6/cfb3/uvveepzAXkQGVNi30Ni9sq2D2AM8n/k+/XsfvXt3f/vjpu65g6ujCAa1BRCQtAn1Eh0AfmZ/TzZqJt7Oijv9Zu5/ZE4qobWjmwhFNCnMRSYo0CfTknXn529X7yDDjhx+dy+jCISxbtixptYjI4JYWfegdu1zqm1oH9L1f3F7B7PHD26cgEBFJlrRooRd1aKGfaGrp1/eqbWwhLzuTR1fuIT83k9f2Hedvr5zSr+8pItIb6RHoHSa76o8W+o4jtUwalc+x+iYWfGsZ40fk8cahmvbll00tTvh7iojEKy26XLIyM3j0by4BEh/oeyrreev9z/LA01v4r1f2UtvYwhuHatpnTBxdmMulU0Yl9D1FRPoiLVroAJeePYoxw4ZQn+Auly2Hoy3x7z6zDQAzcId7b5hFS2uE+WUjNd5cRFJC2gQ6wNCczIS30Dcfrjnl8WOffDNvHKrhulljyMpMiy84IpIm0ivQczNZ8tpBxg7fyJfeOTMh29zcoa/8M2+fzgUTR3DBxBEJ2baISCKlVRNzaHb079MPn9+ZsG2+cai6/f7cSSMTtl0RkURLqxZ6Xk5iL7h8uLqBLYdruevq6cwvG8mlZ+vgp4ikrrQK9CHZJ79wJGLmxb9uKgfgHeeOYcYYnc4vIqktrbpcTjRH2u83dLgfj+XbKvjjawcBeGL9QSaMzGN6qeYzF5HUl1Yt9OoTJy9ycfxEc9xdMJGI86EfvQzAv/4hl/KaRv7xqmkaligioZBWLfTqhlMDPV7Pbj3Sfr+8ppHMDOMDc8cnpDYRkf6WVi30hg5j0OMN9LrGFr78u/WMH5HHLRdNZPb4Is6fMJxhQ5I3k6OISDzSqoX+n7fOY05wcYt4A33V7mPsrzrBV68/l08tnMrl04oV5iISKmkV6OeNH853br4AgNW7j7Gnsv6U5R/8zxXc/fh63B13P2XZroo6AGadNXxgihURSbC0CnSgfdKsh57dzhXfWkpVffSize7Oyp1H+dmK3dy7ZBMz7/7LKaG+s6KO/JxMSgpzk1K3iMiZ6jHQzewRMys3s/VdLP+wmb1mZq+b2XIzm534MnuvcMiphwX2HTsBwKHqhvbnHnlxJyeaWzlWf7JbZldlHWXF+RrRIiKh1ZsW+iLgmm6W7wSudPfzgHuBhxNQV59lZBgjhmYzbXR07PjB49Eg33mk7rR1P/mL1azbW8Vja/axbPMRyorzB7RWEZFE6nGUi7s/Z2Zl3Sxf3uHhS0DSx/mt/vLVVNQ2ctE3/srB49EW+vaK0wP95Z1Huf77L7Y/viA4oCoiEkaJ7kP/BPBEgrcZt4wMo7ggl6wMY+2eKuoaW3h9XxUAE0bm8cVrz2lft2hoNh+YO551d7+dT1w+OVkli4icMes82iPmStEW+hJ3n9XNOguBB4HL3b2yi3XuBO4EKC0tnbt48eI+lAy1tbUUFPR8Ov6nl9ZzvPHk51swPovbZkUPet7252iLfdE1ie1m6W1tA011xUd1xUd1xa+vtS1cuHC1u8+LubBtCF93N6AMWN/N8vOB7cD03mzP3Zk7d6731dKlS3u13qTPL2m/XXX/Mq9taG5ftmxzua/YXtHnGs60toGmuuKjuuKjuuLX19qAVd5Frp7xmaJmNhF4DLjV3bec6fYS6YrpJTy35QiLbp/P3EkjyM89+XGvnF6SxMpERBKvx0A3s0eBBUCxme0D7gGyAdz9IeBuYBTwYDDkr8W7+jowwB6+dS5NrRGd8Skig0JvRrnc0sPyO4A7ElZRAg3JzmRIdmIveiEikqrS7kxREZHBSoEuIpImFOgiImlCgS4ikiYU6CIiaUKBLiKSJhToIiJpQoEuIpImFOgiImlCgS4ikiZ6NX1uv7yx2RFgdx9fXgxUJLCcRErV2lRXfFRXfFRX/Ppa2yR3jzm7YNIC/UyY2apUmQCss1StTXXFR3XFR3XFrz9qU5eLiEiaUKCLiKSJsAb6w8kuoBupWpvqio/qio/qil/CawtlH7qIiJwurC10ERHpJHSBbmbXmNlmM9tmZl9Ici27zOx1M1trZquC50aa2VNmtjX4OWIA6njEzMrNbH2H52LWYVHfCfbfa2Z24QDX9RUz2x/ss7Vmdl2HZV8M6tpsZu/ox7ommNlSM9toZhvM7NPB80ndZ93UlQr7bIiZrTSzdUFt/xo8P9nMXg5q+C8zywmezw0ebwuWlw1wXYvMbGeHfTYneH7Afv+D98s0s1fNbEnwuH/3V1dXj07FG5AJbAemADnAOmBmEuvZBRR3eu6bwBeC+18A/n0A6rgCuBBY31MdwHXAE4ABlwAvD3BdXwE+E2PdmcG/Zy4wOfh3zuynusYCFwb3C4EtwfsndZ91U1cq7DMDCoL72cDLwb74b+Dm4PmHgE8G9/8OeCi4fzPwXwNc1yLg/THWH7Df/+D97gJ+BSwJHvfr/gpbC/0iYJu773D3JmAxcH2Sa+rseuCnwf2fAjf09xu6+3PA0V7WcT3wM496CSgys7EDWFdXrgcWu3uju+8EthH99+6Pug66+5rgfg2wCRhHkvdZN3V1ZSD3mbt7bfAwO7g58FbgN8HznfdZ2778DXCVWfQq8gNUV1cG7PffzMYD7wR+FDw2+nl/hS3QxwF7OzzeR/e/8P3NgSfNbLWZ3Rk8V+ruB4P7h4DS5JTWZR2psA//Pvi6+0iHLqmk1BV8tb2AaMsuZfZZp7ogBfZZ0H2wFigHniL6jaDK3VtivH97bcHy48CogajL3dv22deDffb/zCy3c10xak60B4DPAZHg8Sj6eX+FLdBTzeXufiFwLfApM7ui40KPfn9K+jCiVKkj8APgbGAOcBC4P1mFmFkB8Fvgf7t7dcdlydxnMepKiX3m7q3uPgcYT/SbwDnJqKOzznWZ2Szgi0Trmw+MBD4/kDWZ2buAcndfPZDvG7ZA3w9M6PB4fPBcUrj7/uBnOfA7or/kh9u+wgU/y5NUXld1JHUfuvvh4D9gBPghJ7sIBrQuM8smGpq/dPfHgqeTvs9i1ZUq+6yNu1cBS4FLiXZZZMV4//baguXDgcoBquuaoPvK3b0R+AkDv88uA95jZruIdg2/FfgP+nl/hS3QXwGmBUeKc4gePPh9Mgoxs3wzK2y7D7wdWB/U87FgtY8Bjyejvm7q+D3w0eBo/yXA8Q7dDP2uU3/ljUT3WVtdNwdH+ycD04CV/VSDAT8GNrn7tzssSuo+66quFNlnJWZWFNzPA64m2se/FHh/sFrnfda2L98PPBN86xmIut7o8IfZiPZTd9xn/f5v6e5fdPfx7l5GNKeecfcP09/7K5FHdAfiRvQo9Rai/XdfSmIdU4iOMFgHbGirhWi/11+BrcDTwMgBqOVRol/Fm4n2y32iqzqIHt3/frD/XgfmDXBdPw/e97Xgl3hsh/W/FNS1Gbi2H+u6nGh3ymvA2uB2XbL3WTd1pcI+Ox94NahhPXB3h/8HK4kekP01kBs8PyR4vC1YPmWA63om2GfrgV9wciTMgP3+d6hxASdHufTr/tKZoiIiaSJsXS4iItIFBbqISJpQoIuIpAkFuohImlCgi4ikCQW6pC0za+0w295aS+DsnGZWZh1mkRRJBVk9ryISWic8ekq4yKCgFroMOhadx/6bFp3LfqWZTQ2eLzOzZ4IJnf5qZhOD50vN7HcWnXN7nZm9OdhUppn90KLzcD8ZnKkokjQKdElneZ26XG7qsOy4u58HfI/orHgA3wV+6u7nA78EvhM8/x3gWXefTXR+9w3B89OA77v7uUAV8L5+/jwi3dKZopK2zKzW3QtiPL8LeKu77wgmwzrk7qPMrILoafXNwfMH3b3YzI4A4z060VPbNsqITtU6LXj8eSDb3b/W/59MJDa10GWw8i7ux6Oxw/1WdExKkkyBLoPVTR1+rgjuLyc6Mx7Ah4Hng/t/BT4J7RdTGD5QRYrEQy0KSWd5wZVs2vzZ3duGLo4ws9eItrJvCZ77B+AnZvZZ4Ahwe/D8p4GHzewTRFvinyQ6i6RISlEfugw6QR/6PHevSHYtIomkLhcRkTShFrqISJpQC11EJE0o0EVE0oQCXUQkTSjQRUTShAJdRCRNKNBFRNLE/wcyiLOpTL1ZGAAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(evaluation_cost)\n", + "plt.grid()\n", + "plt.title('Cost on the test data')\n", + "plt.xlabel('Epoch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 368 + }, + "id": "Nc-KFxLnLSpd", + "outputId": "405a7bab-1d0d-4c99-9636-c5fe9bc4339d" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 0, 'Epoch')" + ] + }, + "metadata": {}, + "execution_count": 15 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAFNCAYAAADsA7/8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xcdX3/8dc72dwTyQVcQwgkSLwAYppNEVSEQBXB2mhFhVpBSo1WVGxtBdr+CtWi1AcVRaqigkBVIuItRS5C2KgUAQMGSIBIuCYhAQK5sLnv7uf3x/luMpmd3WRmdncmZ97Px2Mee+Z7bp852bz3O99z5owiAjMzawyDal2AmZkNHIe+mVkDceibmTUQh76ZWQNx6JuZNRCHvplZA3HoW92RNEzSw5ImVrDuEZLu6o+69nD/x0laUav9F5N0oKQ2SYP7ctk+qOsjku7s7/1Ydw79vYykBZLWShpW61r60RzgNxGxCkDSX0laJekpSbO6FpL0akl3FYZURDwIrJP07oEoVFJIOqSftl11MEbEMxExOiI6+nLZgSTpQknfr3UdeeHQ34tImgIcAwTwFwO876YB3N3Hgf8p2O/FwAzgk8DXC5a7DPj7EiH1A+BjA1BnzQ1Er9zyxaG/dzkduBu4GjijcIakyZJ+KukFSS9Kurxg3kclPSLp5TRsMiO179JLlXS1pP9I08dJWiHpXEmrge9JGifpxrSPtWn6gIL1x0v6nqRn0/yfp/bFhT1vSUMkrZH0J8UvUNKBwMHAPalpArAy9fpvT/OQdEpqv6d4G8AC4ISe3g1J2l/SPEkvSVom6aMF8y6UdL2ka9PxWiJpZg/b+U2afCANi3ywYN5nJT2f3qGcWdA+TNIlkp6R9Jykb0kaUWLbrwe+BRydtr0utV8t6ZuSbpK0EZgl6V2S/iBpg6Tlki4s2M6U9O/clJ4vkPQFSf+XXt+vJO1b7rJp/umSnk6/b/8vvRP7sx6O1YR0zDdIuhd4ddH8r6XaN0i6T9Ixqf2dwD8DH0zH4YHUfmbB7/QTkhrij3yfiAg/9pIHsAz4BNACbAeaU/tg4AHgUmAUMBx4a5r3fmAl8KeAgEOAg9K8AA4p2P7VwH+k6eOAduA/gWHACLIAfh8wEhgD/Bj4ecH6vwR+BIwDhgDHpvbPAT8qWG428FAPr/FdwJKC54OAPwIHAO8Gfp/2vQiY0Mux2gAc0cO83wDfSMdpOvACcHyadyGwBTg5HdcvAXf3sp/iY9h13D6fjsHJwCZgXJp/KTAPGJ9ex/8CX+ph2x8B7ixquxpYD7wlHZvhaZ9vSM+PAJ4D3pOWn5JqbErPFwCPA69J/6YLgIsrWPZQoA14KzAUuITsd/LPengtc4HryX4/Dyf7nbyzYP5fk/1+NQGfBVYDwwv+Tb5f4vfk1WS/08emYzyj1v9H94ZHzQvwYw//obL/XNuBfdPzR8mGNgCOTsHVVGK9W4Fzetjm7kJ/W9d/vB7Wnw6sTdMTgc6ucCtabn/gZeAV6fkNwOd62OaHKApZ4ASydzi/Tvv8CnAWMAtoTa/x8KJ1VgJvK7H9yUAHMKag7UvA1Wn6QuD2gnmHApt7OQalQn9z4b8F8DxwVAqojcCrC+YdDTzZw7Y/QunQv3Y3vytfBS5N01PoHuT/WrDsJ4BbKlj234DrCuaNTL8v3UKf7I/nduB1BW1fLH5tReusBd5Y8G/y/d285p/Tw++5H7s+PLyz9zgD+FVErEnPf8jOIZ7JwNMR0V5ivclkvbVKvBARW7qeSBop6Yr0ln4DWY95bBpXngy8FBFrizcSEc8C/we8T9JY4CSycfdS1pL1gAvXnx8RR0XEsWShNJMUfmTB+AXgu0XbGQOsK7H9/VOdLxe0PQ1MKni+umB6EzC8zHMaLxb9W2wCRgP7kYXjfZLWpSGbW1J7OZYXPpH0JkmtadhtPdk5kX1Lrwp0f32jK1h2/8I6ImIT8GIP29iPrAdfWPfTRa/hH9Nwzfp0XPbp7TVIOknS3WmIbh3ZO6reXrMlA3lyziqUxnw/AAxO4+uQDbmMlfRGsv9MB0pqKhH8yykaPy2wiSyEurwKKLzcsPgWrJ8FXgu8KSJWS5oO/IGsB7scGC9pbESUCttrgL8l+537XUSs7KGmB4GppV6LJAGXA58i+w8+OCKeTsfkiILlJpENOSwtsf1nU51jCoL/QLJ3Bv1tDdm7gMN6ef2FeroFbnH7D8mOy0kRsUXSV+n/AFxF9rsA7PgdndDDsi+QDXlNJnuHCtkx71r3GLIhwBPIhvY6Ja0l+72CotebztX8hOwc1y8iYruy80fCdss9/b3De8iGJA4lG96YDrwe+C3ZL/69ZP8JL5Y0StJwSW9J634X+EdJLcocIumgNG8R8FeSBqcTZsfupo4xZKG1TtJ44IKuGZGdaL0Z+IayE75DJL2tYN2fk12Bcw5ZD72kiFhBdu7iyBKz/xa4PyIWkfUqR0g6lGyY54mC5Y4F7oiIrSW2vxy4C/hSOk5HkA0VVXpJ4HOkk8u7ExGdwHeASyW9ErI/UJJO7GXbB0gauptNjyF797JF0pHAX+1Z6VW5AXi3pDen+i6kh9CN7OqqnwIXpneLh7LrhQhjyP4ovAA0Sfo34BUF858DpkjqyquhZJ2eF4B2SScB7+izV5ZzDv29wxnA9yK7jnp114Osd/chsv9s7yY7SfsMWW/9gwAR8WPgIrLe4Mtk4Ts+bfectN66tJ2f76aOr5Kd0FtDNsZ+S9H8D5ON3T5KNo79ma4ZEbGZrHc2lSwAenNF2tYO6aqRc4D/l7bXTnYJ5x1kV7l8qmDxD6W2npxGNn79LPAz4IKIuH03NfXkQuCaNFzzgT1Y/lyyP2p3pyGy2ynoMRe5A1gCrJa0podlIBtr/7ykl8nG2q/f0+IrFRFLyI75XLIORxvZv3m3P7TJJ8mGhlaTDc19r2DerWS/S38kG/bZwq5DQT9OP1+UdH96h/Zpste5luyP3LyqX1SDUDoJYtbvUg/uNRHx17tZbhjZsNEJ6R1EOfs4ArgiIo6uvFIrl6TRZJ2HaRHxZK3rsZ459G1ApOGgPwAfjojf7G55q3/KPnsxn+yd5n8BbyK7bNKhUsc8vGP9TtmHn5YDNzvwc2U22RDZs8A04FQHfv1zT9/MrIG4p29m1kAc+mZmDaSuP5y17777xpQpUypef+PGjYwaNarvCuojrqs8rqs8rqt89VpbpXXdd999ayKi9Ce9a30fiN4eLS0tUY3W1taq1u8vrqs8rqs8rqt89VpbpXUBC8P33jEzM4e+mVkDceibmTUQh76ZWQNx6JuZNRCHvplZA3Hom5k1kN2GvqSrJD0vaXFB23hJt0l6LP0cl9ol6TJJyyQ9KGlGwTpnpOUfk3RGqX2ZmVn/2pOe/tXAO4vazgPmR8Q0slurnpfaTyK72940YA7wTdhxW90LyG69eiRwQdcfCjMzGzi7vQ1DRPxG0pSi5tnAcWn6GmAB2TcCzQauTZ8Iu1vSWEkT07K3RcRLAJJuI/tDcl3VryBnnlyzkbuf6On7pfvX0uXbWXXvMzXZd29cV3lcV/nqsbYDx4/c/UIVqPTeO82x8xuNVgPNaXoSu37N2YrU1lN7N5LmkL1LoLm5mQULFlRYIrS1tVW1fn/pquv5TZ3c8uR2xg8XT6zvBODhFzvY0lHD4pY8VMOd98J1lcd1la/OajvyVYM5/ZD2Ps+wqm+4FhEhqc9uyh8R3wa+DTBz5sw47rjjKt7WggULqGb9cnV0Bs+u28zk8SN5aMV6vvXrx7ntkec48bBX0TRI/PKhVRDQ2SkGDdpMe2cnnenIHTRhJCOGDOZNrx7OP5/8evYZMWTA6u7yu9/dxdFHv3nA97s7rqs8rqt89Vjb8CGDWHTvXX2eYZWG/nOSJkbEqjR883xqXwlMLljugNS2kp3DQV3tCyrcd93679ZlfOW2PzLjwLHc/8w6AN40dTz/+8CzALxvxgG88hXDeObpZzjwoAMZLDHjoLE8t2ErH5w5mUGDVMvyGTd8EK/aZ3hNayjFdZXHdZWvnmvra5WG/jzgDODi9PMXBe2flDSX7KTt+vSH4VbgiwUnb98BnF952fVlTdtWzvvJg9z+SPa3b+nqlzntyMmc9dapHPLKMTz23Mt0Brz2VWMAWLBgNccd97palmxmDWq3oS/pOrJe+r6SVpBdhXMxcL2ks4CngQ+kxW8CTgaWAZuAMwEi4iVJXwB+n5b7fNdJ3b3dqvWb+dB37uGJNRsBuOT9b+SUlgN2WWZa85halGZm1s2eXL1zWg+zTiixbABn97Cdq4CryqpuL/CvP1vMcxu28KM5R9ERwVFTJ9S6JDOzHtX1N2fVq2//5nF+ev9KXtq4jedf3srn3vla3nSww97M6p9Dv0zPb9jCF296FIDmVwzjX9/1ej589EE1rsrMbM849Hdjy/YOhg4exGV3PMbS1S+zeXsHQwaLGz7+Zg7ebxRjhg/8pZVmZpVy6PeiozM46Wu/ZU3bVl7e0r6j/YyjD+KNk8fWsDIzs8o49Hvw+AttzLl2IU+mq3I+OesQTnrDq3hqzSbefmjzbtY2M6tPDv0efPe3T/D4C1ngL/n3Exk1LDtUh+2/Ty3LMjOriu+nX8L1v1/OdfcuZ9/Rw5g756gdgW9mtrdzmiVPv7iRT1/3BxY/u4GOdEOcjx97MEf5UkwzyxGHfnLxzY/ywIr1O55/9JipvL9lci9rmJntfRz6wPKXNnHz4tV86vhDOPXIA+nsDCb3072szcxqyaEPzEt3wfzAzMlMGjuixtWYmfUfn8gF7nj0ef7kwLHu3ZtZ7jV86K/btI3lL21i2itH17oUM7N+19DDO79/6iXe/63fAbC/h3XMrAE0dE//90/tvKW/x/LNrBE0dOg/uurlHdMOfTNrBI0d+qs37JieNM6hb2b517Chv629kyfSvXWAhvlSZDNrbA17IvfZdZtp7ww+duzBTJkwimFNg2tdkplZv2vo0Ac49jX78eZX71vjaszMBkbDDu+sSKHvE7hm1kgaNvSfXbcZyWP5ZtZYGjb0V67dzH6jh3ks38waSsOG/rPrN/syTTNrOA0b+ivXbvatF8ys4TRk6Hd2Bs+u3+KTuGbWcKoKfUnnSFosaYmkz6S2CyWtlLQoPU4uWP58ScskLZV0YrXFV+rFjdvY1t7p0DezhlPxdfqSDgc+ChwJbANukXRjmn1pRFxStPyhwKnAYcD+wO2SXhMRHZXWUKmV6XJND++YWaOppqf/euCeiNgUEe3Ar4G/7GX52cDciNgaEU8Cy8j+YAy4Z32Nvpk1qGpCfzFwjKQJkkYCJwNd3yT+SUkPSrpK0rjUNglYXrD+itQ24B5dnd1d06FvZo1GEVH5ytJZwCeAjcASYCvwJWANEMAXgIkR8TeSLgfujojvp3WvBG6OiBuKtjkHmAPQ3NzcMnfu3Irra2trY/ToXb8R6+EXO/jy77cwdBBc8faRSKp4+31ZVz1wXeVxXeWp17qgfmurtK5Zs2bdFxEzS86MiD55AF8EPlHUNgVYnKbPB84vmHcrcHRv22xpaYlqtLa2dmv7+vw/xkHn3hjzH1ld1barUaqueuC6yuO6ylOvdUXUb22V1gUsjB5ytdqrd16Zfh5INp7/Q0kTCxZ5L9kwEMA84FRJwyRNBaYB91az/0o88cJGJu4znONf1zzQuzYzq7lq77L5E0kTgO3A2RGxTtLXJU0nG955CvgYQEQskXQ98DDQnpYf8Ct3Hl+zkYP3GzXQuzUzqwtVhX5EHFOi7cO9LH8RcFE1+6xGRPDE8228d0ZNzh+bmdVcQ30i98WN23h5aztT93VP38waU2OFfts2APYbM6zGlZiZ1UZDhf66TVnojx0xtMaVmJnVRmOF/ubtAIwdOaTGlZiZ1UZjhX5XT9+hb2YNqsFCv6un7+EdM2tMjRX6m7fTNEiMGuqvSDSzxtRYob9pO2NHDq3J/XbMzOpBg4X+No/nm1lDa7DQ387YEQ59M2tcjRX6m7e7p29mDa1hQj8iWLV+M/uO9qdxzaxxNUzoP/PSJtZt2s4bDtin1qWYmdVMw4T+ouXrAJg+eWyNKzEzq52GCP2HVqznnLmLGNY0iNc2j6l1OWZmNdMQoX//M2sB+MRxh9A0uCFesplZSQ2RgBu3tQPwsWMPrnElZma11RChv3lbB4MEw5oa4uWamfWoIVJw07YORg5t8u0XzKzhNUjotzPCN1kzM2uU0O/wnTXNzGiQ0N+4tYMRQ5tqXYaZWc01ROhv3t7unr6ZGQ0S+llP36FvZtYQob95WwcjHfpmZo0R+hu3tTPKY/pmZo0R+pu3eXjHzAyqDH1J50haLGmJpM+ktvGSbpP0WPo5LrVL0mWSlkl6UNKMvngBe2LjtnZGDXNP38ys4tCXdDjwUeBI4I3An0s6BDgPmB8R04D56TnAScC09JgDfLOKuvdYR2ewZXsnI4a4p29mVk1P//XAPRGxKSLagV8DfwnMBq5Jy1wDvCdNzwaujczdwFhJE6vY/x7ZvL0DwCdyzcyoLvQXA8dImiBpJHAyMBlojohVaZnVQHOangQsL1h/RWrrV5vSHTZHenjHzAxFROUrS2cBnwA2AkuArcBHImJswTJrI2KcpBuBiyPiztQ+Hzg3IhYWbXMO2fAPzc3NLXPnzq24vra2NjZqJOf+djMffcNQ3jKpPr4Uva2tjdGjR9e6jG5cV3lcV3nqtS6o39oqrWvWrFn3RcTMkjMjok8ewBfJ/gAsBSamtonA0jR9BXBawfI7luvp0dLSEtVobW2Nh1asi4POvTFuWbyqqm31pdbW1lqXUJLrKo/rKk+91hVRv7VVWhewMHrI1Wqv3nll+nkg2Xj+D4F5wBlpkTOAX6TpecDp6Sqeo4D1sXMYqN+s27QdgH1G1Ecv38yslqod6P6JpAnAduDsiFgn6WLg+jT08zTwgbTsTWTj/suATcCZVe57j6zf7NA3M+tSVehHxDEl2l4ETijRHsDZ1eyvEl2hP3akQ9/MLPefyHVP38xsp4YI/SGD5Q9nmZnRIKG/z4gh/n5cMzMaIPQ3bN7OKzy0Y2YGNEDod/X0zczMoW9m1lByH/rrNm9z6JuZJbkP/Ze3tPOK4Q59MzNogND3t2aZme2U69DvjGBreyfDfY2+mRmQ89Df3pn9HD4k1y/TzGyP5ToN05dm+dO4ZmZJrkN/W2f2BTEe3jEzy+Q79N3TNzPbRc5Dv6unn+uXaWa2x3Kdhl09fQ/vmJll8h36O67eceibmUHeQz8N73hM38wsk+/Qd0/fzGwX+Q59n8g1M9tFrtPQl2yame0q16HfdRuGYQ59MzMg56G/1Sdyzcx2kevQ394BgwRDBvtL0c3MIOehv60jGDFkMJJD38wM8h76nb5c08ysUFWhL+nvJS2RtFjSdZKGS7pa0pOSFqXH9LSsJF0maZmkByXN6JuX0LNtHQ59M7NCTZWuKGkS8Gng0IjYLOl64NQ0+58i4oaiVU4CpqXHm4Bvpp/9Zltn+Bp9M7MC1SZiEzBCUhMwEni2l2VnA9dG5m5grKSJVe6/V+2dMLTJPX0zsy4Vh35ErAQuAZ4BVgHrI+JXafZFaQjnUknDUtskYHnBJlaktn7T3glDfeWOmdkOiojKVpTGAT8BPgisA34M3ADMB1YDQ4FvA49HxOcl3QhcHBF3pvXnA+dGxMKi7c4B5gA0Nze3zJ07t6L6AL70uzY6NJh/PWpExdvoD21tbYwePbrWZXTjusrjuspTr3VB/dZWaV2zZs26LyJmlpwZERU9gPcDVxY8Px34RtEyxwE3pukrgNMK5i0FJva2j5aWlqjGOy6+KT7wrbuq2kZ/aG1trXUJJbmu8riu8tRrXRH1W1uldQELo4dcrWZM/xngKEkjlV0IfwLwSNc4fWp7D7A4LT8POD1dxXMU2XDQqir2v1vZmL5P5JqZdan46p2IuEfSDcD9QDvwB7LhnJsl7QcIWAR8PK1yE3AysAzYBJxZRd17pCOgaZDH9M3MulQc+gARcQFwQVHz8T0sG8DZ1eyvXB0BTYPd0zcz65LrRGzvDIY69M3Mdsh1ImY9fQ/vmJl1yXfod0LToFy/RDOzsuQ6ETvCt1U2MyuU69Bv7wyGeEzfzGyHXCeix/TNzHaV79DvxD19M7MCuU7Edo/pm5ntIrehHxF0hq/eMTMrlNtE3N6R3T3UPX0zs51yG/rtnZ2Ax/TNzArlNhG3t2c9fd97x8xsp9wm4vYdPX0P75iZdclt6LenMX2fyDUz2ym3ibi9wz19M7NiDRD6uX2JZmZly20itnd2nch1T9/MrEtuQ989fTOz7nKbiP5wlplZd7kN/fbU0/fVO2ZmO+U2Ebt6+h7TNzPbKcehn/X0/cXoZmY75TYRu+6949swmJntlNtE3DG8M8jDO2ZmXXIc+ml4pym3L9HMrGy5TcR29/TNzLrJbej7w1lmZt1VlYiS/l7SEkmLJV0nabikqZLukbRM0o8kDU3LDkvPl6X5U/riBfSk6zYMDn0zs50qTkRJk4BPAzMj4nBgMHAq8J/ApRFxCLAWOCutchawNrVfmpbrN109fV+nb2a2U7Xd4CZghKQmYCSwCjgeuCHNvwZ4T5qenZ6T5p8gqd8SecdtGPyJXDOzHSpOxIhYCVwCPEMW9uuB+4B1EdGeFlsBTErTk4Dlad32tPyESve/O51peMeZb2a2U1OlK0oaR9Z7nwqsA34MvLPagiTNAeYANDc3s2DBgoq289iT2wC46847GdZUX0M8bW1tFb+u/uS6yuO6ylOvdUH91tYvdUVERQ/g/cCVBc9PB74JrAGaUtvRwK1p+lbg6DTdlJZTb/toaWmJSn2jdVkcdO6NsXlbe8Xb6C+tra21LqEk11Ue11Weeq0ron5rq7QuYGH0kKvVDH48AxwlaWQamz8BeBhoBU5Jy5wB/CJNz0vPSfPvSMX1i8606f47a2BmtvepZkz/HrITsvcDD6VtfRs4F/gHScvIxuyvTKtcCUxI7f8AnFdF3XtSHwCDnPpmZjtUPKYPEBEXABcUNT8BHFli2S1kQ0IDIp3HxZFvZrZTbq9t6Ro4ck/fzGyn3Ia+x/TNzLrLbehHBAL68fNfZmZ7ndyGfme4l29mVizHoR8+iWtmViTHoe+evplZsdyGfrinb2bWTX5DH/f0zcyK5Tb0Ozuru8eEmVke5TYXPaZvZtZdjkPfY/pmZsVyG/oR4Z6+mVmR3IZ+Z/hma2ZmxXIb+oF7+mZmxXIb+llP36lvZlYot6EfEQxy5puZ7SK3od/Z6TF9M7Ni+Q19X71jZtZNjkPfPX0zs2K5DX1fp29m1l1+Qx/39M3MiuU29Dt99Y6ZWTc5Dn339M3MiuU49D2mb2ZWLLeh7xO5Zmbd5Tb0/eEsM7Pu8hv6vp++mVk3FYe+pNdKWlTw2CDpM5IulLSyoP3kgnXOl7RM0lJJJ/bNSygt+45cx76ZWaGmSleMiKXAdABJg4GVwM+AM4FLI+KSwuUlHQqcChwG7A/cLuk1EdFRaQ27qc+XbJqZFemr4Z0TgMcj4ulelpkNzI2IrRHxJLAMOLKP9t+NL9k0M+tOEVH9RqSrgPsj4nJJFwIfATYAC4HPRsRaSZcDd0fE99M6VwI3R8QNRduaA8wBaG5ubpk7d25FNX1l4RbWb2nn3986usJX1X/a2toYPdp17SnXVR7XVb56ra3SumbNmnVfRMwsOTMiqnoAQ4E1QHN63gwMJnsXcRFwVWq/HPjrgvWuBE7pbdstLS1RqQ9feU/M+uJNFa/fn1pbW2tdQkmuqzyuqzz1WldE/dZWaV3AwughV/tieOcksl7+c+mPyHMR0RERncB32DmEsxKYXLDeAamtX4Sv3jEz66YvQv804LquJ5ImFsx7L7A4Tc8DTpU0TNJUYBpwbx/sv6QI/OEsM7MiFV+9AyBpFPB24GMFzV+WNJ3sqsmnuuZFxBJJ1wMPA+3A2dFPV+6Ab7hmZlZKVaEfERuBCUVtH+5l+YvIxvn7nT+cZWbWXY4/kevhHTOzYrkNfZ/INTPrLreh756+mVl3OQ599/TNzIrlNvSzSzYd+2ZmhXIc+n3zyTMzszzJbS56TN/MrLsch77H9M3MiuU49N3TNzMrltvQ93X6Zmbd5Tj03dM3MyuW29D3mL6ZWXe5Dn3fZdPMbFe5Df3wd+SamXWT29DvjPCYvplZkRyHvnv6ZmbFchz64XvvmJkVyW3oe0zfzKy7HIe+r94xMyuW29D3bRjMzLrLcej7w1lmZsVyHPru6ZuZFctt6PuGa2Zm3eU39HFP38ysWG5D32P6Zmbd5Tf0O33JpplZsYpDX9JrJS0qeGyQ9BlJ4yXdJumx9HNcWl6SLpO0TNKDkmb03cvozh/OMjPrruLQj4ilETE9IqYDLcAm4GfAecD8iJgGzE/PAU4CpqXHHOCb1RS+O77hmplZd301vHMC8HhEPA3MBq5J7dcA70nTs4FrI3M3MFbSxD7afze+4ZqZWXd9FfqnAtel6eaIWJWmVwPNaXoSsLxgnRWprV/4hmtmZt0pIqrbgDQUeBY4LCKek7QuIsYWzF8bEeMk3QhcHBF3pvb5wLkRsbBoe3PIhn9obm5umTt3bkV1/e2vNjJr/+BDh4+u7IX1o7a2NkaPdl17ynWVx3WVr15rq7SuWbNm3RcRM0vOjIiqHmTDNr8qeL4UmJimJwJL0/QVwGmlluvp0dLSEpU65J9/GWdfcWvF6/en1tbWWpdQkusqj+sqT73WFVG/tVVaF7AwesjVvhjeOY2dQzsA84Az0vQZwC8K2k9PV/EcBayPncNAfc5j+mZm3TVVs7KkUcDbgY8VNF8MXC/pLOBp4AOp/SbgZGAZ2ZU+Z1az793x1TtmZt1VFfoRsRGYUNT2ItnVPMXLBnB2Nfsroy5fp29mVkIuP5HbdW7aPX0zs13lM/TTT2e+mdmuchn6namr756+mdmuch36uXxxZmZVyGUuekzfzKy0XIa+h3fMzErLaehnP+VTuWZmu8hp6Keefo3rMDOrN7kMfY/pm5mVltPQ99U7Zmal5DIXO93TNzMrKaeh76t3zMxKyXn0hJsAAAj6SURBVHfo17gOM7N6k8vQ33Eit7ZlmJnVnXyHvlPfzGwXuQx9j+mbmZWW69DP5YszM6tCLnPRwztmZqXlMvR99Y6ZWWk5Df3sp9zVNzPbRU5D3z19M7NSchn6HtM3Mystp6Hvq3fMzErJZS76hmtmZqXlNPQ9pm9mVkq+Q9+pb2a2i1yGvm+4ZmZWWlWhL2mspBskPSrpEUlHS7pQ0kpJi9Lj5ILlz5e0TNJSSSdWX35pvnrHzKy0pirX/xpwS0ScImkoMBI4Ebg0Ii4pXFDSocCpwGHA/sDtkl4TER1V1tCNx/TNzEqruKcvaR/gbcCVABGxLSLW9bLKbGBuRGyNiCeBZcCRle6/N9OaR3Pjp97Ka8YN7o/Nm5nttaoZ3pkKvAB8T9IfJH1X0qg075OSHpR0laRxqW0SsLxg/RWprc+NHNrE4ZP2YeQQ9/XNzAqp64NMZa8ozQTuBt4SEfdI+hqwAbgcWAME8AVgYkT8jaTLgbsj4vtp/SuBmyPihqLtzgHmADQ3N7fMnTu3slcGtLW1MXr06IrX7y+uqzyuqzyuq3z1Wluldc2aNeu+iJhZcmZEVPQAXgU8VfD8GOCXRctMARan6fOB8wvm3Qoc3ds+Wlpaohqtra1Vrd9fXFd5XFd5XFf56rW2SusCFkYPuVrx8E5ErAaWS3ptajoBeFjSxILF3gssTtPzgFMlDZM0FZgG3Fvp/s3MrHzVXr3zKeAH6cqdJ4AzgcskTScb3nkK+BhARCyRdD3wMNAOnB39cOWOmZn1rKrQj4hFQPG40Yd7Wf4i4KJq9mlmZpXL5SdyzcysNIe+mVkDceibmTUQh76ZWQNx6JuZNZCKP5E7ECS9ADxdxSb2Jft0cL1xXeVxXeVxXeWr19oqreugiNiv1Iy6Dv1qSVoYPX0UuYZcV3lcV3lcV/nqtbb+qMvDO2ZmDcShb2bWQPIe+t+udQE9cF3lcV3lcV3lq9fa+ryuXI/pm5nZrvLe0zczswK5DH1J70xfvr5M0nk1ruUpSQ+lL4lfmNrGS7pN0mPp57jdbaeParlK0vOSFhe0laxFmcvSMXxQ0owBrutCSSvTcVsk6eSCeeenupZKOrGfaposqVXSw5KWSDontdf0ePVSV02PV9rPcEn3Snog1fbvqX2qpHtSDT9Kd+Ul3Wb9R6n9HklTBriuqyU9WXDMpqf2AfvdT/sbrOzbB29Mz/v3ePV0o/299QEMBh4HDgaGAg8Ah9awnqeAfYvavgycl6bPA/5zgGp5GzCD9MU2vdUCnAzcTPb98kcB9wxwXRcC/1hi2UPTv+kwsq/sfBwY3A81TQRmpOkxwB/Tvmt6vHqpq6bHK+1LwOg0PQS4Jx2L64FTU/u3gL9L058AvpWmTwV+NMB1XQ2cUmL5AfvdT/v7B+CHwI3peb8erzz29I8ElkXEExGxDZhL9qXs9WQ2cE2avgZ4z0DsNCJ+A7y0h7XMBq6NzN3AWO36BTn9XVdPZgNzI2JrRDwJLCP7N+/rmlZFxP1p+mXgEbLvdK7p8eqlrp4MyPFK9UREtKWnQ9IjgOOBrq9FLT5mXcfyBuAESX3+xda91NWTAfvdl3QA8C7gu+m56OfjlcfQH7AvYN9DAfxK0n3Kvv8XoDkiVqXp1UBzbUrrtZZ6OI6fTG+vryoYAhvwutLb6D8h6yHWzfEqqgvq4HiloYpFwPPAbWTvLNZFRHuJ/e+oLc1fD0wYiLoiouuYXZSO2aWShhXXVaLmvvZV4HNAZ3o+gX4+XnkM/Xrz1oiYAZwEnC3pbYUzI3uvVheXUNVTLcA3gVcD04FVwH/VoghJo4GfAJ+JiA2F82p5vErUVRfHKyI6ImI6cADZO4rX1aKOYsV1STqc7Hu7Xwf8KTAeOHcga5L058DzEXHfQO43j6G/Ephc8PyA1FYTEbEy/Xwe+BnZf4Tnut4upp/P16q+Xmqp6XGMiOfSf9RO4DvsHJIYsLokDSEL1h9ExE9Tc82PV6m66uF4FYqIdUArcDTZ8EjXt/QV7n9HbWn+PsCLA1TXO9NQWUTEVuB7DPwxewvwF5KeIhuGPh74Gv18vPIY+r8HpqUz4EPJTnjMq0UhkkZJGtM1DbyD7Ivi5wFnpMXOAH5Ri/qSnmqZB5yermQ4ClhfMKzR74rGUN9Ldty66jo1XckwFZgG3NsP+xdwJfBIRHylYFZNj1dPddX6eKUa9pM0Nk2PAN5Ods6hFTglLVZ8zLqO5SnAHend00DU9WjBH2+RjZsXHrN+/7eMiPMj4oCImEKWU3dExIfo7+PVl2eh6+VBdvb9j2Tjif9SwzoOJrty4gFgSVctZONw84HHgNuB8QNUz3Vkb/23k40VntVTLWRXLvx3OoYPATMHuK7/Sft9MP2yTyxY/l9SXUuBk/qppreSDd08CCxKj5Nrfbx6qaumxyvt5wjgD6mGxcC/Ffw/uJfsJPKPgWGpfXh6vizNP3iA67ojHbPFwPfZeYXPgP3uF9R4HDuv3unX4+VP5JqZNZA8Du+YmVkPHPpmZg3EoW9m1kAc+mZmDcShb2bWQBz61vAkdRTcaXGR+vDOrJKmqODuoWa11rT7Rcxyb3NkH9E3yz339M16oOy7EL6s7PsQ7pV0SGqfIumOdKOu+ZIOTO3Nkn6m7L7tD0h6c9rUYEnfUXYv91+lT4Wa1YRD3wxGFA3vfLBg3vqIeANwOdkdEQG+DlwTEUcAPwAuS+2XAb+OiDeSfT/AktQ+DfjviDgMWAe8r59fj1mP/Ilca3iS2iJidIn2p4DjI+KJdJOz1RExQdIastscbE/tqyJiX0kvAAdEdgOvrm1MIbuV77T0/FxgSET8R/+/MrPu3NM36130MF2OrQXTHfhcmtWQQ9+sdx8s+Pm7NH0X2V0RAT4E/DZNzwf+DnZ8acc+A1Wk2Z5yj8MsjekXPL8lIrou2xwn6UGy3vppqe1TwPck/RPwAnBmaj8H+Laks8h69H9HdvdQs7rhMX2zHqQx/ZkRsabWtZj1FQ/vmJk1EPf0zcwaiHv6ZmYNxKFvZtZAHPpmZg3EoW9m1kAc+mZmDcShb2bWQP4/ANH9nhD2HlIAAAAASUVORK5CYII=\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(training_accuracy)\n", + "plt.grid()\n", + "plt.title('Accuracy (%) on the training data')\n", + "plt.xlabel('Epoch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 352 + }, + "id": "CaBm9Kt_Manj", + "outputId": "6ddf47c6-70d5-40cf-a276-4dcbcc66bdd8" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 16 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAE9CAYAAAD09vLJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1f3/8dfJRshGCEvYAmHfEyDsuETFrUWUol+wqMVqte1Xa+v3a9V+baUu/VmltdW61brVBWgREa0VtSaKOzsICCQIJIEAWcmezMzn98edDAGyTIYkk7nzeT4e85gld+58DgPvHM4991wjIiillLKHEH8XoJRSqu1oqCullI1oqCullI1oqCullI1oqCullI1oqCullI2E+euDe/bsKcnJyT69t6Kigujo6LYtyM/s1ia7tQfs1ya7tQfs16bG2rNx48YCEenV1Hv8FurJycls2LDBp/dmZmaSnp7etgX5md3aZLf2gP3aZLf2gP3a1Fh7jDEHmntPi8MvxpjnjTFHjTFfN/FzY4x5zBiTZYzZZoyZ1JqilVJKtR1vxtRfBC5p5ueXAsPdt5uAp868LKWUUr5oMdRF5GOgqJlNLgf+LpYvgHhjTN+2KlAppZT32mL2S38gp8HzXPdrSimlOpjxZkEvY0wy8LaIjGvkZ28DD4nIJ+7n/wHuFJHTjoIaY27CGqIhMTExbfny5T4VXV5eTkxMjE/v7azs1ia7tQfs1ya7tQfs16bG2nPeeedtFJHJTb5JRFq8AcnA10387Bng6gbPdwN9W9pnWlqa+CojI8Pn93ZWdmuT3dojYr822a09IvZrU2PtATZIM9naFsMva4Dr3LNgpgOlInK4DfarlFKqlVqcp26MWQakAz2NMbnAvUA4gIg8DbwDfAfIAiqB69urWKWUUs1rMdRF5OoWfi7Af7dZRUoppXzmtzNKlVKqM6h1uKiocVDuvlXUOCircVBe7aCy1kGXsFBiuoQR3SWM2EjrPsZ9iwwPwRjj7yacRENdKdUpiAiHS6vJPlZO1tFyz31xRR3RXUJPCtPGAjYqIpQthx0cWX+QsmoHFTVOymvqKK9xesK6vPr08K51uHyuOTTEeD6/S3gI3sb7bbNHMDe1n8+f2xwNdaWUzxxOF3uOlLM1t4StOSUcLq0mukuoO+jCiekSSswp4RvTJYyuEaEcKqki+1iFJ8Czj5ZTUev07DsuMoxhvWNI7hlFZa2TsmoH+aXVnlAur3HQ6Izsrds9D7uGW59/4pdBKP3iu55UV2yXU+prUG9URCi1DteJz6x2UFFb/8vB+qVRUWPVVu1wNlJM47pHhZ/JH3uzNNSVCkIOp4sqh+ByCSEh3vUvRYTc4ipPgG/NKWV7XilVdVaYdesazsCEKHKL63vJVgC2dCpMv26RDO0dw1WTkxjWO4ahvWIY1juGnjERzQ5tiAhVdU5P77uixsm2zRtJP3uGFeARoYSFBt/q4hrqSnUAEaG4so780mqOHK8m/3g1R4/XUOf07r/+ISHmRI8yMszqaXYJJ7pLKLFdwt29y1C6hIVSXuM48Tml1mfV39e/VlBeg0uAD94hOiK0wX5PDG80HDfOPlbB1pwSCitqAYgIC2FcvzgWTk1iQlI8qQPiGdQj6rQQdrncwduwp1vjoKLWSZ+4SIb0iia6i28xZIwhKiKMqIgwertfK8wKoX98V5/2Zxca6kq1AYfTRU5xFVlHyzlQWOEO7hryS6vcYVrT6NhtqJe9ZKer5TO/AcJCDI5Gtu3WNZw+cZEkdotkVJ9Y+sRFciTvIH0GDDppfLl+3LmootLzekWtk+QeUZw/qjepSfFMSIpnRGIsEWEt94JDQgzR7l8SiV61QJ0pDXWlWqGy1sG+BuPA9ff7CyqpbdDrjgwPsUI0LpJJA7t7HvfpduK+d2wXwr0cHnC5hIraUw7+nXLQr/6xJ8Ddn9MnLpKuEaGn7TMz8zDp6SPa7M9GdQ4a6ko1Q0T48tsiXvj0WzZkV1L47lrPz0JDDIMSohjSK4bzRyUytFc0w3rHMLhnNN26hrfpVLeQEENsZDixkeFAZJvtV9mPhrqynRqHk4jQM5s/LCJ8+M1RnszMZuOBYnrGRDCiewgzxw1lWG/rQN7AHlF0CTu9B6yUP2moK1s4crya93YeYe3X+Xyxr5A+3SK5aEwfLh6byOTkhFaNXf9r+2GezMjim/wy+sd35f7Lx3LV5CS++HQd6enD27klSp0ZDXUVsL4tqGDtjnzW7shn88ESAIb0jOYHM5PZX1DBK18c4PlPv6VHdASzRydyybg+zBzWo9HedY3DyapNeTz9UTYHCisZ2iuaP1yVytwJ/bwe91aqM9BQVwFDRNhx6LgnyPccKQdgfP9u/O9FI7h4bB+G9Y7xDLuU1zjI3H2UtTuO8K/th1mxIYeYLmGkj+zFxWP7kD6yFyHGsOyrgzy7bh9Hjtcwvn83nr4mjYvGJHo9f1upzkRDXXV6pZV1vPT5flaszyGvpIoQA1MHJ3DvZWO4aGyfJuclx3QJY05KP+ak9KPG4eSz7ELe25HP+zuP8Pa2w0SEhhAZHsLxagfThySw9KpUzhrWs9Ot5aFUa2ioq07raFk1z33yLa98foCKWifnjOjFbRcM54LRvekR06VV++oSFsp5I3tz3sjePHCFsOlgMWu/zqegvIZrZwwibVBCO7VCqY6loa46nZyiSp75OJt/bMjF4XQxJ6UfP0kfyui+cW2y/9AQw5TkBKYka5Ar+9FQV53GniNlPJWZzZqthwgxcGXaAG4+ZyjJPaP9XZpSAUNDXfndlpwSnszI4r2dR+gaHsr1M5O58ewh9OmmJ9ko1Voa6sovRITPswt5IjOLT7MK6dY1nJ9dMJzrZybTPTrC3+UpFbA01FWHcrmED3Yd4cnMbLbklNArtgt3XzqKRdMHEePjan1KqRP0X5HqEA6ni7e3HebJzCz2HCknKaErD1wxjivTBhAZrqfaK9VWNNRVu6quc/L6plye/iibnKIqRiTG8KcFE5iT0jcoL2CgVHvTUFftosoh/PXjbJ5d9y3HympITYrnN3PGcsGo3nqmplLtSENdtaiy1sG7X+fz9rbDlFTWevWe3Ycrqaj7hrOG9eTPCyYwY2gPPVNTqQ6goa4a5XIJX3xbyOsb8/j314eprHUyoHtXBns5ZzylZyh3fm86E5Li27lSpVRDGurqJPuOlbNqUx5vbM4jr6SKmC5hXJbSj/lpA5iS3N3r3nZmZqYGulJ+oKGuKK2s461th3h9Uy6bD5YQYuCs4b345SUjuWhMn0YvhaaU6pw01IOYyyX8+T97eSozm1qnixGJMdx96SiumNifxDg9m1OpQKShHqRKK+v4+YrNZOw+xmWp/bj5nCGM7RenBzOVCnAa6kHom/zj3PzyRg6VVHH/FeO4ZtpADXOlbEJDPci8tfUQv1y5jZjIMJbfNF3XEVfKZjTUg4TD6eL3737Ds+u+JW1Qd55aNIneOm6ulO1oqAeBwvIabl22mc+yC7l2+iB+PWcMEWF6ir5SdqShbnPbc0v58SsbOVZewyNXpnDV5CR/l6SUakca6jb2zw05/N/qr+kZHcHKH88gZYCeDKSU3Wmo20xVrZP1+4t4c4t1MtHMoT14/OqJrb5Qs1IqMGmoBziH08X2vFI+zSrgk6wCNh0oodbpIjzUcPM5Q7jj4pG6xK1SQURDPcCICNnHyvk0q5BPsgr4IruQshoHAGP6xrF4VjIzh/Zg6uAEoiL061Uq2Oi/ej+rrnNyx8ptbPu2ipht61rc/lhZDUfLagAYmBDFnNS+zBrWkxlDeugQi1JKQ92fRIRfvbGdt7YeIqVXKL27tTxvfHjvGKYP6cGsYT1JSojqgCqVUoFEQ92PXvniAKs25fHz2cOZEHaI9PQp/i5JKRXg9Aian2w8UMRv39rJBaN687Pzh/u7HKWUTWio+8HR49X85JVN9O/elT8umKDX7FRKtRkN9Q5W63Dx01c3UVbt4Jlr0+jWNdzfJSmlbETH1DvY797ZxYYDxTx29URG9YnzdzlKKZvRnnoHWrUplxc/28+NZw1mbmo/f5ejlLIhDfUO8nVeKXev2s70IQncdekof5ejlLIpr0LdGHOJMWa3MSbLGHNXIz8fZIz5jzFmmzEm0xgzoO1LDVwllbX8+JWNdI+K4C/fn6Sn7Sul2k2L6WKMCQWeAC4FxgBXG2PGnLLZUuDvIpIC3Af8v7YuNFA5XcLPlm/h6PEanrpmEj31rE+lVDvypss4FcgSkX0iUgssBy4/ZZsxwIfuxxmN/DxoPfr+Hj7ec4wlc8cycWB3f5ejlLI5b0K9P5DT4Hmu+7WGtgLfcz+eB8QaY3qceXmBbe2OfP6SkcWCyUlcPVUvTqGUan9GRJrfwJgrgUtE5Eb382uBaSJyS4Nt+gF/AQYDHwPzgXEiUnLKvm4CbgJITExMW758uU9Fl5eXExMT49N7O0pemYv7v6iib3QId0+LJCK0+ROMAqFNrWG39oD92mS39oD92tRYe84777yNIjK5yTeJSLM3YAawtsHzu4G7m9k+Bshtab9paWniq4yMDJ/f2xE+3XtMxt/7rqTd/77kFld69Z7O3qbWslt7ROzXJru1R8R+bWqsPcAGaSZbvRl+WQ8MN8YMNsZEAAuBNQ03MMb0NMbU7+tu4Hkv9mtLy786yHXPf0ViXCRv/HQm/eO7+rskpVQQaTHURcQB3AKsBXYB/xCRHcaY+4wxc92bpQO7jTF7gETgwXaqt9NyuoQH/7WTu1ZtZ9awnrz+05m6NK5SqsN5tUyAiLwDvHPKa79p8HglsLJtSwscFTUOblu+hQ92HeEHMwbx6zljdC66UsovdO2XM3SopIobXtrA7vzj/HbuWH4wM9nfJSmlgpiG+hnYllvCjS9toLLWyfOLp5A+sre/S1JKBTkNdR+9s/0wt/9jCz1juvDyDdMY2SfW3yUppZSGemuJCE9mZvPI2t1MGhjPX6+brKf+K6U6DQ31VhAR7np9Oys25HD5hH78fn4KkeGh/i5LKaU8NNRbYdWmPFZsyOEn6UP55cUjMUYvQ6eU6lx03p2XjpZVc9/bO5k8qDt3XKSBrpTqnDTUvXTvmzuoqnPy+ytT9ELRSqlOS0PdC//efph/f53Pz2cPZ2gv+ywWpJSyHw31FhRX1PLrN3cwrn8cN509xN/lKKVUs/RAaQvuf3snJZW1/P2HU/XUf6VUp6cp1YyMb46yanMeP0kfyph+cf4uRymlWqSh3oSy6jp+9cZ2hveO4Zbzh/m7HKWU8ooOvzThoX9/Q/7xal7/yUy6hOkJRkqpwKA99UZ8nl3Iq18e5IZZg5mkF4tWSgUQDfVTVNU6uWvVNgb1iOJ/Lhrp73KUUqpVdPjlFH94bzcHCitZ9qPpdI3QYRelVGDRnnoDmw8W8/yn3/L9aQOZMbSHv8tRSqlW01B3q3E4+eXKbSTGRXL3paP8XY5SSvlEh1/cnvgwi71Hy3lh8RRiI8P9XY5SSvlEe+rArsPHeTIzm3kT+3PeKL0knVIqcGmoA8+u20fX8FB+M2eMv0tRSqkzEvShXlnr4N2v8/nO+L50j47wdzlKKXVGgj7U39txhMpaJ/Mm9fd3KUopdcaCPtRf35RL//iuTE1O8HcpSil1xoI61I8er+bTrALmTeyvVzNSStlCUIf6m1sO4RJ06EUpZRtBHeqrNueROqCbXqJOKWUbQRvquw4fZ9fh48ybqL10pZR9BG2ov7E5j7AQw2Wp/fxdilJKtZmgDHWnS3hzSx7pI3vRI6aLv8tRSqk2E5Sh/ll2AUeO1zBv4gB/l6KUUm0qKEP9jU15xEaGccFoXedFKWUvQRfqFTUO3t2Rz3fH9yUyXC+CoZSyl6AL9bU78q1lAXTWi1LKhoIu1N/YnMeA7l2ZossCKKVsKKhC/YguC6CUsrmgCvU3t+RZywLo0ItSyqaC6nJ2qzblkZoUzxBdFkCp4CIClYVQvN+6leVDXaV1q62Eugr3fVWDx+7nXbtDbB/3ra91H9PgeVQPCOk8/eOgCfWdh47zTX4Z910+1t+lKBW8XE449g3kbQIEImKgSxx0iYUuMdZ9hPs+1MtrBbtc4HKAs5aoilzY896J8C45cOJxbfnp7w0Jh4goCHffIqIgPBoi46zQDouEqmIoOQg5X1q/GE7bRxjEJEJ0T6st9fV72hN7evt6j4a49jmbPWhC/Y3NuYSFGOak6LIASnWY6uOQtwFyvrJCMXcD1Bz37r1hkVYIhkVaoe25Od0hXmfdI563TAVYX//+rtA92boln3XicfdkK7AjYrz/xVHPUQPlR6DsCJQdtnr85fnWfcUxqCmH47lQU2Y9rikDZ83p+/nuH2HKDa37bC8FRahbywIcIn1kbxL0knUqUNRVQ8FuOLIDqkut3l3ieIju0faf5ayzArim1Aqi6uNW+FYft3q4IaFWSIZHWj3aMPf9Sc+7QnXJiQDP+cqqHQEMJI6D8VdB0jQYMBlCI6x915SdfDv1NUcNhIZZPWLPLbTB43DP8515JYyZ+R2IHwQxvcG08YSIsC4QP9C6ectRe3Kbasut+tpJUIT6p1kFHC2r4Xu6brrqjETgeJ4VgEe+dt/vgIK9IM7Tt4/tC33GWyHZZxz0SYGEIVawncpRA6W5UJoDJTkn3U89sg82OKzgdlS1bZu6xFnBPfoySJoK/SdbQxrt7GhmJmOSprb757RKWASEJUBUx0yjDopQX7Upl9jIMM4fpcsCqHbiclm91IoCawzWUeU+6Oa+eZ5XWwfgHNVWj60w2wry6tIT+4ofaAX26Msgcaz1OLLbidDP3w75X0P2h+7hB6zecu/R0HuMtf/6AC/Lp+HwBBjrl0J8EuUxg4hKGmYFcGQ3931cg/tY93h3nPXLpa7SXX9V0+0L7woDpkCvUY3/klHtzvahXlHjYO2OI1wxsb8uC6BOVvQt5G+zesotERd9D62Hj9ZDZYE1flpR4L4dsw6gNdarbkxImHsooyskDIZx80+Ed+/RVsA2JqY3DD3vxHNHjXXQMf/rE2G/+9/WAbluSTD0fOs+fiDEJ1mP4/pbPUdgZ2YmvdPTvatZBQzbh/q7X+dTVefUoRdlKciCXW/Czjfh8NZWvXUkwB6gSzdrXDu6l3XQbcBk63F0T+s+Mt6aRdHUuHNrD841JawL9E21bkq52T7U39icR1JCVyYP6u7vUpS/HP3GCvGdb8LRHdZrA6bCRQ9A8tnWATsvfL55JzNmX2aFqVKdlFehboy5BPgzEAr8TUQeOuXnA4GXgHj3NneJyDttXGur5ZdW82l2AbeeNwzT1kfBVeclYo0/1wd5wW7AwMAZcMnvrbHqbq3/n1tN5FENdNXptRjqxphQ4AngQiAXWG+MWSMiOxtsdg/wDxF5yhgzBngHSG6Heltl9ZY8RGDeJL0YRqcjAluXwYcPWmfsjbgIhl9kzZII9eE/kLUVsP8TyPoA9r4Pxd+CCbHmJ0/9kRXksX3avh1KdTLe/OuZCmSJyD4AY8xy4HKgYagLUD9fqRtwqC2L9NXqzXlMHBjP4J7R/i5FNXR0F7x9Oxz8DPqnWQcNP/kTrPuDNR49bLYV8MMusMapGyMCR3daIZ71Hzj4OThrrbHr5LPhrJ/DqDlNv18pm/Im1PsDOQ2e5wLTTtlmCfCeMeZWIBqY3SbVnYGqWiff5Jfxi9kj/F2KfdTPEvF1KKu2Aj76PXz+hDVd7rLHYOK11roZVSWwL8PqZe99D75eCRjrIOTwi2D4hdYJG/syrRDP/o91Rh9A77Ew7Wbrl8HAGTpEooKakRamcxljrgQuEZEb3c+vBaaJyC0Ntrndva8/GGNmAM8B40TEdcq+bgJuAkhMTExbvny5T0WXl5cTE9P8olyHyl386pMqbkrpwsx+nf94sDdt8qe40l2M3fF7nKGRHOs1i2O9ZlEeM7jJgD+1PT0KvmT43meJrDnG4T6z2TfkB9RFNHEyiriILcsmoWgjPQo3EFuWhWkw17ouLIbi7qkUJUyiKGEitV3a4QzLRnT276i17NYesF+bGmvPeeedt1FEJjf5JhFp9gbMANY2eH43cPcp2+wAkho83wf0bm6/aWlp4quMjIwWt/lw1xEZdOfbsv7bQp8/pyN50ya/2b1W5P5EkT9PEHnpcpEl3UXujbOev79E5NAWEZfrpLd42lO0X+TVBdb2T0wX2f9Z6z+/7KjIlmUiHz0icvBLEUfdmbfJB536O/KB3dojYr82NdYeYIM0k63edGHXA8ONMYOBPGAh8P1TtjkIXAC8aIwZDUQCx7zYd7vJKa4EICkhyp9lBL5t/4DVP7HOVLxmFcT0gopC+OYt2LEaPv0zfPJH6D4Yxl4BY+dBnxSMq84aI//oEeuA5YX3w/Sf+DZHO6YXpC5s+7YpZUMthrqIOIwxtwBrsaYrPi8iO4wx92H9xlgD/A/wrDHmF1gHTRe7f6P4TU5RJRFhIfSK0fFVn33xNLx7p3XgceFrJ9buiO4BaYut20kB/xh88ih0H8yU6hqoOmTNOrnkIeimM5CU6gheDTaLNef8nVNe+02DxzuBWW1b2pnJLa5iQPeuetk6X4hAxu/g44etGSTzn7POimzMaQH/Nux4A+fRA/D9f1pTFZVSHabzH0H0UU5xJQO669BLq7mc8M7/wobnrZkpc/7k/bzx6B6Q9gNI+wEbMzNJH5HerqUqpU7Xea7B1MZyiqpI6t7V32UEFkcNvH6DFeizfg5zH/ftRCCllN/Y8l/s8eo6SqvqgvcgaXWpFcwh4dB/krXgU0QLJ2DVlMOKa6y54hc9ADNv7ZhalVJtypahnltkLfifFGzDLy4XbHkF/nOftRxsPRNirW/db6J16z/JWua1/iSdikJ47So4tAUufxImLvJP/UqpM2bLUK+fzjggmIZfDn4B/74TDm+xLhe26J8Q2w8ObYZDm6z7PWthy6vW9iHh1hre/SfB/k+tC/MueAVGfcevzVBKnRl7hnpREM1RL82DD+6F7e4Q/97fYPyVJ870HHmJdQNrVktpjnUl9/qg377SukLNtW9AcqeawKSU8oEtQz23uIroiFC6R7XRxQg6o7pq+PxxWPdHa8bKOXfAWb9ofuzcmBMXzR17hfWaywXi0gOiStmELf8l5xZXkpQQZc811EVg11vw3v9ByUHr5J6LHrCuwOOLkBBsPAlKqaBjy1DPKaoiKcFm4+kul7VU7Ue/h28/tk7bv24NDDnX35UppToR24W6iJBTXMmMoR2zcl+7K94PW5fDlteg5IB1QYnvLIW063XIRCl1GtulQnFlHZW1zsA+SFpTDrvWWEG+fx1grB75+fdYp+1HBHDblFLtynah7pn5EmjTGcVlTS3c8qq1OFZdBSQMsYI8ZSHEJ/m7QqVUALBfqHvmqAdIb7YkB7a8xrQvn4eP8iEiFsbPhwmLrPnmdjzYq5RqN/YL9fqzSTvzgVJHjbWa4eZXIDsDgOr48XT9zv06vKKUOiO2C/Xc4krio8KJjeyEc9Tzt1tBvm0FVBVDt4GQfhdM+D5bt+wjPSXd3xUqpQKc7UI9p7iqc635UlViXUR508vWKfyhEVZvfNK1MDjdPU8crCsAKqXUmbFdqOcWVTKyT6y/y4DiA/DhA9YsFkc1JI6HSx+G8VdBVIK/q1NK2ZStQt3lEnKLq5g9JtG/hVSVwCvzoewwTLzGuvWdoAc9lVLtzlahfqy8hlqny7/TGV1O60ITxd9aZ3zqIllKqQ5kq1Cvn6M+wJ8nHr3/G8j6AC77swa6UqrD2Wolp/o56n7rqW95DT7/C0y92boQs1JKdTB7hbp7jrpfTjw6+CW8dRsMPhcu/l3Hf75SSmGzUM8trqRXbBciw0M79oNLc63re3YbAFe9qAttKaX8xlbpk1NU1fFDL7WVsPz7UFcFi9/W6YpKKb+yVU89p7iyY4deRODNn8LhbXDl89BrZMd9tlJKNcI2PXWH08Xh0uqOXfPl46Ww4w248D4YcVHHfa5SSjXBNj31w6XVOF3ScUsE7HoLMh6wlsWd+bOO+UyllGqBbULdM52xI+ao538Nq26G/pOt+eh6pqhSqpOwTajneqYztvPwS0UBLLsaIrvBwlchPLJ9P08ppVrBNmPqOcWVhBjoF9+Ooe6ohRXXQsVRuP7fENun/T5LKaV8YJtQzy2uom+3roSHttN/PkTgX7+Ag5/B/Oeg/6T2+RyllDoDthl+ySmqbN+hl8+fsC5wcc4vYfyV7fc5Sil1BuwT6u05R33PWnjvHhg9F9Lvbp/PUEqpNmCLUK+uc3LkeE37zFE/shNW3gB9U2De0w2uVKSUUp2PLRLqUIn7YtNt3VOvKIBlCyAiGhYus+6VUqoTs8WB0pxid6i35Rx1R421SFf5UVj8DnTr33b7VkqpdmKPUK+/OEZbHSgVgbdvh4OfWzNdBqS1zX6VUqqd2WL4Jae4kvBQQ2JcG50I9PlfYMsrcO6dOtNFKRVQbBHqucVV9I/vSmhIG5yuv/tdeO/XMOZyOPeuM9+fUkp1IHuEelFl24ynH9lpXTS6bwpcoTNdlFKBxxaplVNcdebj6eXH3DNdYtwzXfx48WqllPJRwB8orahxUFRRe2YnHjWc6XK9znRRSgWugA/13LaYzvjZY5DzhXtNF53popQKXAE//FI/ndHna5O6nLDhRRhyns50UUoFvMAP9eL6Oeo+9tT3vg/HcyFtcdsVpZRSfhL4oV5URdfwUHrGRPi2g40vQnRvGPXdNq1LKaX8watQN8ZcYozZbYzJMsacNnnbGPOoMWaL+7bHGFPS9qU2LrfYWnLX+HJJudJc2LsWJl4DoeFtX5xSSnWwFg+UGmNCgSeAC4FcYL0xZo2I7KzfRkR+0WD7W4GJ7VBro3KKq3w/SLrpZRAXTLqubYtSSik/8aanPhXIEpF9IlILLAcub2b7q4FlbVFcS0SEXF8vjuF0wOaXYej5kDC47YtTSik/8CbU+wM5DZ7nul87jTFmEDAY+PDMS2vZ8SoHZTUO35bczXofjudB2vVtX5hSSvlJW89TXwisFBFnYz80xtwE3ASQmJhIZmamTx9SXl5OZkqP8UwAABpqSURBVGYm+0utjyk9tI/MzIOt2sf4bUuJiejOF/lRyFHf6mhL9W2yC7u1B+zXJru1B+zXJp/aIyLN3oAZwNoGz+8G7m5i283AzJb2KSKkpaWJrzIyMkRE5J1th2TQnW/L9tyS1u2g+KDIkniRD37rcw1trb5NdmG39ojYr012a4+I/drUWHuADdJMtnoz/LIeGG6MGWyMicDqja85dSNjzCigO/B5636t+K5+jnqrh182v2ytmT7pB+1QlVJK+U+LoS4iDuAWYC2wC/iHiOwwxtxnjJnbYNOFwHL3b5IOkVtcRWxkGN2iWjEd0emATX+HYRdA90HtV5xSSvmBV2PqIvIO8M4pr/3mlOdL2q4s7+QUVba+l773PSg7DN95pH2KUkopPwroM0qtOeqtnM648QWI6QMjLmmfopRSyo8CNtRFhNziVvbUS3KstV4mXatnkCqlbClgQ72gvJbqOlfrTjza9HfrXs8gVUrZVMCGumfmi7dLBNSfQTpsNsQPbMfKlFLKfwI31ItaGep711oHSCfrGaRKKfsK2FCvv+KR18MvG16A2L4w/OJ2rEoppfwrgEO9kh7REURFeDErs/gAZH0AE6+F0IC/gp9SSjUpYEM9p6iKAd4OvWx+GYzRA6RKKdsL3FAvrvTuuqTOOmvd9GEXQnxS+xemlFJ+FJCh7hLhUImXF8fY8y6U5+s1SJVSQSEgQ724WqhzincHSTe+CLH9YPhF7V6XUkr5W0CGekGVtWZYi2eTFh+ArP9YY+l6gFQpFQQCMtSPVbkAL+aob3rJfYD02g6oSiml/C8wQ71SMAb6xUc2vZEIbHnNGnbpNqDjilNKKT8KyFAvqBISYyPpEhba9EYVBdYZpEPSO6ospZTyuwANdVfLS+4W7LHuewxv/4KUUqqTCMhQP1YlLR8kLdxr3ffUUFdKBY+AC/Vah4viamn5bNKCvRAWCd30hCOlVPAIuFA/XFqF4MVCXoVZkDAUQgKuiUop5bOAS7ycImt1xhaHXwr2Qs9hHVCRUkp1HoEX6p6LYzTTU3fUQvF+PUiqlAo6ARfqocbQJ9rQJ66ZOerF+0GcepBUKRV0Au7c+f+akkTvimzCQpv5fVQ/80V76kqpIBNwPXWvFNRPZ9QxdaVUcLFvqEf3hshu/q5EKaU6lD1DvXCvjqcrpYKSPUO9QENdKRWc7BfqlUVQVaQHSZVSQcl+oV6ga74opYKX/ULdM51RZ74opYKP/UK9YC+EhEP8IH9XopRSHc5+oV6YBQlD9JqkSqmgZL9QL9ij4+lKqaBlr1B3OqDoWx1PV0oFLXuFeskBcNVpT10pFbTsFeqe6Ywj/FuHUkr5ib1CXaczKqWCnL1CvWAvRPWAqAR/V6KUUn5hr1AvzNLlAZRSQc1eoa7XJVVKBTn7hHpVCVQc1Z66Uiqo2SfUC7Ose53OqJQKYvYJ9QK9LqlSStkn1Av3ggmF7sn+rkQppfzGPqFesBcSBkNYhL8rUUopv7FPqOt0RqWUskmou5xQmK3TGZVSQc+rUDfGXGKM2W2MyTLG3NXENv9ljNlpjNlhjHmtbctsQWkOOGu0p66UCnotXknCGBMKPAFcCOQC640xa0RkZ4NthgN3A7NEpNgY07u9Cm5UgU5nVEop8K6nPhXIEpF9IlILLAcuP2WbHwFPiEgxgIgcbdsyW1Cwx7rXnrpSKsgZEWl+A2OuBC4RkRvdz68FponILQ22WQ3sAWYBocASEXm3kX3dBNwEkJiYmLZ8+XKfii4vLycmJsbzfPiep+h9dB2fznoVjPFpn/52apsCnd3aA/Zrk93aA/ZrU2PtOe+88zaKyOSm3tNWF/IMA4YD6cAA4GNjzHgRKWm4kYj8FfgrwOTJkyU9Pd2nD8vMzOSk9+5fComjST/vPJ/21xmc1qYAZ7f2gP3aZLf2gP3a5Et7vBl+yQOSGjwf4H6toVxgjYjUici3WL32jhsLKczS8XSllMK7UF8PDDfGDDbGRAALgTWnbLMaq5eOMaYnMALY14Z1Nq2mDMoOa6grpRRehLqIOIBbgLXALuAfIrLDGHOfMWaue7O1QKExZieQAdwhIoXtVfRJ6hfy0oOkSinl3Zi6iLwDvHPKa79p8FiA2923jqXTGZVSyiPwzygt3AsmBBKG+LsSpZTyu7aa/eI/BXsgfiCEdfF3JaqD1dXVkZubS3V1dYd8Xrdu3di1a1eHfFZHsFt7wF5tioyMxPgwRdsGoa4LeQWr3NxcYmNjSU5O9ukvf2uVlZURGxvb7p/TUezWHrBPm0SEwsJCoqOjW/3ewB5+cbl0OmMQq66upkePHh0S6Ep1JGMMPXr0IDQ0tNXvDexQP54HjirooaszBisNdGVXvv7dDuxQL3Rfwk576sqPVq9ejTGGb775xt+ldLgXX3yRQ4cOeZ4nJydTUFDg077279/Pa6/5tsDrzJkzW9zmxhtvZOfOnS1u11pLlixh6dKlzW6zevXqdvnsxgR2qHumM47wbx0qqC1btoyzzjqLZcuWtevnOJ3Odt2/L04N9TPRXKg7HI5m3/vZZ5+1uP+//e1vjBkzxqfazpSGurcK90JELMQk+rsSFaTKy8v55JNPeO6552i4QJ3T6eR///d/GTduHCkpKTz++OMArF+/npkzZ5KamsrUqVMpKyvjxRdf5JZbPOvjMWfOHDIzMwGIiYnhf/7nf0hNTeXzzz/nvvvuY8qUKYwbN46bbrqJ+gX5srKymD17NqmpqUyaNIns7Gyuu+46Vq9e7dnvokWLePPNN0+qX0S44447GDduHOPHj2fFihXAiTVHrrzySkaNGsWiRYs4dfG/lStXsmHDBhYtWsSECROoqqoC4PHHH2fSpEmMHz/e87+XiooKfvjDHzJ16lQmTpx4Wh0Ad911F+vWrWPChAk8+uijvPjii8ydO5fzzz+fCy64gPLyci644ALPvhvuo37Rq3Xr1jVZd3p6Ohs2bPBs/3//93+kpqYyffp0jhw5AkB2djbTp09n/Pjx3HPPPU0uDvbggw8yYsQIzjrrLHbv3u15/dlnn2XKlCmkpqYyf/58Kisr+eyzz1izZg133HEHEyZMIDs7u9Ht2kpgz34p2Gtd7UjHVYPeb9/awc5Dx9t0n2P6xXHvZWOb3ebNN9/kkksuYcSIEfTo0YONGzeSlpbGX//6V/bv38+WLVsICwujqKiI2tpaFixYwIoVK5gyZQrHjx+na9euze6/oqKCadOm8Yc//MGqacwYfvMb67y/a6+9lrfffpvLLruMRYsWcddddzFv3jyqq6txuVzccMMNPProo1xxxRWUlpby2Wef8dJLL520/1WrVrFlyxa2bt1KQUEBU6ZM4ZxzzgFg8+bN7Nixg379+jFr1iw+/fRTzjrrLM97r7zySv7yl7+wdOlSJk8+sWhgz5492bRpE08++SRLly7lb3/7Gw8++CDnn38+zz//PCUlJUydOpXZs2efNLvjoYceYunSpbz99tuA9b+ATZs2sW3bNhISEnA4HLzxxhvExcVRUFDA9OnTmTt37mljzy3VXf/nOn36dB588EF++ctf8uyzz3LPPfdw2223cdttt3H11Vfz9NNPN/qdbNy4keXLl7NlyxYcDgeTJk0iLS0NgO9973v86Ec/AuCee+7hueee49Zbb2Xu3LnMmTOHK6+8EoD4+PhGt2sLAd5T1+mMyr+WLVvGwoULAVi4cKFnCOaDDz7g5ptvJizM6jclJCSwe/du+vbty5QpUwCIi4vz/LwpoaGhzJ8/3/M8IyODadOmMX78eD788EN27NhBWVkZeXl5zJs3D7DmN0dFRXHuueeyd+9ejh07xrJly5g/f/5pn/fJJ59w9dVXExoaSmJiIueeey7r168HYOrUqQwYMICQkBAmTJjA/v37vfoz+d73vgdAWlqa5z3vvfceDz30EBMmTCA9PZ3q6moOHjzY4r4uvPBCEhISAOt/Fb/61a9ISUlh9uzZ5OXleXrYDXlTd0REBHPmzDmtzs8//5yrrroKgO9///uN1rRu3TrmzZtHVFQUcXFxzJ071/Ozr7/+mrPPPpvx48fz6quvsmPHjkb34e12vgjcnnpthXUZu57X+bsS1Qm01KNuD0VFRXz44Yds374dYwxOpxNjDI888kir9hMWFobL5fI8b3gyVWRkpGdaW3V1NT/96U/ZsGEDSUlJLFmypMUTr6677jpeeeUVli9fzgsvvNCqurp0OXFCX2hoaIvj2qe+r+F7RITXX3+dkSNHtqqGhj35V199lWPHjrFx40bCw8NJTk5utP3e1B0eHu7p4bembS1ZvHgxq1evJjU1lRdffNEzjObrdr4I3J56YbZ1r9MZlZ+sXLmSa6+9lgMHDrB//35ycnIYPHgw69at48ILL+SZZ57xhEVRUREjR47k8OHDnp5wWVkZDoeD5ORktmzZgsvlIicnh6+++qrRz6sPsJ49e1JeXs7KlSsBiI2NZcCAAZ7x85qaGs8Y7eLFi/nTn/4E0OhBwrPPPpsVK1bgdDo5duwYH3/8MVOnTvX6zyA2NpaysrIWt7v44ot5/PHHPePbmzdvbvW+SktL6d27N+Hh4WRkZHDgwAGv6/TW9OnTef311wFo6iI+55xzDqtXr6aqqoqysjLeeustz8/Kysro27cvdXV1vPrqq57XT21bU9u1hQAOdZ3OqPxr2bJlniGPevPnz2fZsmXceOONDBw4kJSUFFJTU3nttdeIiIhgxYoV3HrrraSmpnLhhRdSXV3NrFmzGDx4MGPGjOFnP/sZkyZNavTz6sdhx40bx8UXX+wZxgF4+eWXeeyxx0hJSWHmzJnk5+cDkJiYyOjRo7n++usb3ee8efM8NZ5//vk8/PDD9OnTx+s/g8WLF/PjH//4pAOljfn1r39NXV0dKSkpjB07ll//+tenbZOSkkJoaCipqak8+uijp/180aJFbNiwgfHjx/P3v/+dUaNGeV2nt/70pz/xxz/+kZSUFLKysujWrdtp20yaNIkFCxaQmprKpZdeetL3cP/99zNt2jRmzZp1Un0LFy7kkUceYeLEiWRnZze5XZsQEb/c0tLSxFcZGRkiGQ+J3BsnUlPh8346k4yMDH+X0KY6oj07d+5s989o6Pjx4x36eW2hoqJChgwZIiUlJaf9LBDb05IzbVNFRYW4XC4REVm2bJnMnTu3Lcry2aZNm057DdggzWRr4I6pF+6FbkkQEeXvSpTqlD744ANuuOEGfvGLXzTa41Sn27hxI7fccgsiQnx8PM8//7y/S2q1wA31gr069KJUM2bPnt0u4852dvbZZ7N161Z/l3FGAnNMXUSnMyqlVCMCMtQjaougtlx76kopdYqADPWoyjzrgU5nVEqpkwR2qGtPXSmlThK4oR4eBbH9/F2KUrr0bidYehfgd7/7nVfbnbqAWmMyMzO9WvmxMwrIUO9alQc9hkJIQJavbEaX3m3/pXe94W2oe0NDvYNFVebpzBfVKejSu+239K7T6eSOO+5gypQppKSk8MwzzwBw+PBhzjnnHCZMmMC4ceNYt24dd911F1VVVcyaNYtFixadtu8XXniBESNGMHXqVD799FPP62+99RbTpk1j4sSJzJ49myNHjrB//36efvppHn30USZMmMC6desa3a7Tau7MpPa8+XxGaW2VuO7tJvLhg769v5PSM0pb76QzSt+5U+T577Tt7Z07T/q8xs5WfOWVV+SHP/yhiIjMmDFDNmzYICIiTz75pMyfP1/q6upERKSwsFBqampk8ODB8tVXX4mISGlpqdTV1ckLL7wg//3f/+3Z53e/+13Pnx8gK1as8PyssLDQ8/iaa66RNWvWiIjI1KlTZdWqVSIiUlVVJRUVFZKZmSmXX365iIiUlJRIcnKyp5769qxcuVJmz54tDodD8vPzJSkpSQ4dOiQZGRkSFxcnOTk54nQ6Zfr06bJu3brT2n/uuefK+vXrPc8HDRokjz32mIiIPPHEE3LDDTeIiMjdd98tL7/8soiIFBcXy/Dhw6W8vPykfWVkZMh3v/tdz/NnnnlG7r//fhERqa6ulrS0NNm3b58sXbpUHnjgARERcTgcnu8lOjq60e/o0KFDkpSUJEePHpWamhqZOXOm58+7qKjIcwbps88+K7fffruIiNx7773yyCOPePbR1HbtLTjOKC3ah0H0akeqU1i2bBm33XYbcGLp3bS0ND744AN+/OMfn7T07vbt209bercljS29+/DDD1NZWUlRURFjx44lPT39tKV3Ac4991x++tOfcuzYMV5//fVWLb0bFxfnWcIW8Cxhe+q65I1puPTuqlWrAGvp3TVr1ngu+1a/9O7o0aOb3M97773Htm3bPAuXlZaWsnfvXqZMmcIPf/hD6urquOKKK5gwYUKz9Xz55Zekp6fTq1cvABYsWMCePXsAyM3NZcGCBRw+fJja2loGDx7c6D683a4zCLxQr1/IS6czqoYufajDP1KX3m3+fWe69K6I8Pjjj3PxxRef9rOPP/6Yf/3rXyxevJjbb7+d667zbQnuW2+9ldtvv525c+eSmZnJkiVLzmi7ziDwxtQLNNRV56BL77bv0rsXX3wxTz31FHV1dQDs2bOHiooKDhw4QGJiIj/60Y+48cYb2bRpE2CtkV6/bUPTpk3jo48+orCwkLq6Ov75z396flZaWkr//v0BTroq1Km1NLVdZxR4oT7lBjZNfBi6NH7tQKU6ii69275L7954442MGTOGSZMmMW7cOG6++WYcDgeZmZmkpqYyceJEVqxY4Rn+uummm5gxY8ZpB0r79u3LkiVLmDFjBrNmzTppyGfJkiVcddVVpKWl0bNnT8/rl112GW+88YbnQGlT23VGpv43Z0ebPHmy1F8EtrXqj8zbid3a1BHt2bVrV7Njsm2trKyM2NjYDvu8tlBZWcn48ePZtGnTaSs1BmJ7WmK3Nm3evJmJEyee9JoxZqOITG7iLQHYU1dKeeWDDz5g9OjR3Hrrrbr0bhAJvAOlSimv6NK7wUl76kopZSMa6iqg+euYkFLtzde/2xrqKmBFRkZSWFiowa5sR0QoLCz0ab0fHVNXAWvAgAHk5uZy7NixDvm86upqz9madmC39oC92hQZGUlFRUWr36ehrgJWeHh4h56unZmZedr0skBmt/aA/drky4FuHX5RSikb0VBXSikb0VBXSikb8dsyAcaYY4CvZ0b0BHy7ZlbnZbc22a09YL822a09YL82NdaeQSLSq6k3+C3Uz4QxZkNzax8EIru1yW7tAfu1yW7tAfu1yZf26PCLUkrZiIa6UkrZSKCG+l/9XUA7sFub7NYesF+b7NYesF+bWt2egBxTV0op1bhA7akrpZRqRMCFujHmEmPMbmNMljHmLn/Xc6aMMfuNMduNMVuMMb5dCsrPjDHPG2OOGmO+bvBagjHmfWPMXvd9d3/W2BpNtGeJMSbP/T1tMcZ8x581tpYxJskYk2GM2WmM2WGMuc39ekB+T820J2C/J2NMpDHmK2PMVnebfut+fbAx5kt35q0wxkQ0u59AGn4xxoQCe4ALgVxgPXC1iOz0a2FnwBizH5gsIgE7t9YYcw5QDvxdRMa5X3sYKBKRh9y/fLuLyJ3+rNNbTbRnCVAuIkv9WZuvjDF9gb4isskYEwtsBK4AFhOA31Mz7fkvAvR7MsYYIFpEyo0x4cAnwG3A7cAqEVlujHka2CoiTzW1n0DrqU8FskRkn4jUAsuBy/1cU9ATkY+BolNevhyov+z6S1j/4AJCE+0JaCJyWEQ2uR+XAbuA/gTo99RMewKWWMrdT8PdNwHOB1a6X2/xOwq0UO8P5DR4nkuAf5FYX9p7xpiNxpib/F1MG0oUkcPux/lAoj+LaSO3GGO2uYdnAmKYojHGmGRgIvAlNvieTmkPBPD3ZIwJNcZsAY4C7wPZQImIONybtJh5gRbqdnSWiEwCLgX+2/1ff1sRa4wvcMb5GvcUMBSYABwG/uDfcnxjjIkBXgd+LiLHG/4sEL+nRtoT0N+TiDhFZAIwAGtkYlRr9xFooZ4HJDV4PsD9WsASkTz3/VHgDawv0g6OuMc968c/j/q5njMiIkfc/+BcwLME4PfkHqd9HXhVRFa5Xw7Y76mx9tjhewIQkRIgA5gBxBtj6q990WLmBVqorweGu48GRwALgTV+rslnxpho90EejDHRwEXA182/K2CsAX7gfvwD4E0/1nLG6oPPbR4B9j25D8I9B+wSkT82+FFAfk9NtSeQvydjTC9jTLz7cVesCSG7sML9SvdmLX5HATX7BcA9RelPQCjwvIg86OeSfGaMGYLVOwfrKlSvBWJ7jDHLgHSsFeWOAPcCq4F/AAOxVuP8LxEJiIOPTbQnHeu/9ALsB25uMBbd6RljzgLWAdsBl/vlX2GNQwfc99RMe64mQL8nY0wK1oHQUKwO9z9E5D53TiwHEoDNwDUiUtPkfgIt1JVSSjUt0IZflFJKNUNDXSmlbERDXSmlbERDXSmlbERDXSmlbERDXdmWMcbZYLW+LW25qqcxJrnhKo5KdRZhLW+iVMCqcp9yrVTQ0J66CjruNewfdq9j/5UxZpj79WRjzIfuxaD+Y4wZ6H490Rjzhnud663GmJnuXYUaY551r339nvssQKX8SkNd2VnXU4ZfFjT4WamIjAf+gnWGMsDjwEsikgK8Cjzmfv0x4CMRSQUmATvcrw8HnhCRsUAJML+d26NUi/SMUmVbxphyEYlp5PX9wPkiss+9KFS+iPQwxhRgXXihzv36YRHpaYw5BgxoeGq2e7nX90VkuPv5nUC4iDzQ/i1TqmnaU1fBSpp43BoN199woseoVCegoa6C1YIG95+7H3+GtfInwCKsBaMA/gP8BDwXMejWUUUq1Vras1B21tV9FZl674pI/bTG7saYbVi97avdr90KvGCMuQM4Blzvfv024K/GmBuweuQ/wboAg1Kdjo6pq6Bjh4t9K9UUHX5RSikb0Z66UkrZiPbUlVLKRjTUlVLKRjTUlVLKRjTUlVLKRjTUlVLKRjTUlVLKRv4/9lRCalmXKGoAAAAASUVORK5CYII=\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(np.array(training_accuracy[:30])/1000)\n", + "plt.plot(np.array(evaluation_accuracy[:30])/10000)\n", + "\n", + "plt.grid()\n", + "plt.xlabel('Epoch')\n", + "plt.legend(['Accuracy on the training data','Accuracy on the test data'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h_q1Kot67m7E" + }, + "source": [ + "### chapter 3 - Regularization (weight decay) example 1 (only 1000 of training data and 30 hidden neurons)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-0LRIw8A7qIg", + "outputId": "ec44d2bd-8ba7-44d0-9962-97ec363802ad" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 0 training complete\n", + "Cost on training data: 1189.527095290911\n", + "Accuracy on training data: 698 / 1000\n", + "Cost on evaluation data: 1189.8550511013198\n", + "Accuracy on evaluation data: 6009 / 10000\n", + "Epoch 1 training complete\n", + "Cost on training data: 1179.5939837457465\n", + "Accuracy on training data: 799 / 1000\n", + "Cost on evaluation data: 1180.0144219542087\n", + "Accuracy on evaluation data: 6744 / 10000\n", + "Epoch 2 training complete\n", + "Cost on training data: 1170.4059690893648\n", + "Accuracy on training data: 840 / 1000\n", + "Cost on evaluation data: 1170.8789797423315\n", + "Accuracy on evaluation data: 7073 / 10000\n", + "Epoch 3 training complete\n", + "Cost on training data: 1161.0808386660801\n", + "Accuracy on training data: 890 / 1000\n", + "Cost on evaluation data: 1161.6027380549638\n", + "Accuracy on evaluation data: 7468 / 10000\n", + "Epoch 4 training complete\n", + "Cost on training data: 1152.0774786279164\n", + "Accuracy on training data: 896 / 1000\n", + "Cost on evaluation data: 1152.6463446673986\n", + "Accuracy on evaluation data: 7633 / 10000\n", + "Epoch 5 training complete\n", + "Cost on training data: 1143.0399798466524\n", + "Accuracy on training data: 914 / 1000\n", + "Cost on evaluation data: 1143.6532869721755\n", + "Accuracy on evaluation data: 7719 / 10000\n", + "Epoch 6 training complete\n", + "Cost on training data: 1134.1816383544956\n", + "Accuracy on training data: 926 / 1000\n", + "Cost on evaluation data: 1134.8491241852917\n", + "Accuracy on evaluation data: 7821 / 10000\n", + "Epoch 7 training complete\n", + "Cost on training data: 1125.4297207375437\n", + "Accuracy on training data: 939 / 1000\n", + "Cost on evaluation data: 1126.1285910287268\n", + "Accuracy on evaluation data: 7949 / 10000\n", + "Epoch 8 training complete\n", + "Cost on training data: 1116.6161456979278\n", + "Accuracy on training data: 946 / 1000\n", + "Cost on evaluation data: 1117.3613469061077\n", + "Accuracy on evaluation data: 7967 / 10000\n", + "Epoch 9 training complete\n", + "Cost on training data: 1107.9494622166937\n", + "Accuracy on training data: 959 / 1000\n", + "Cost on evaluation data: 1108.722844563538\n", + "Accuracy on evaluation data: 8028 / 10000\n", + "Epoch 10 training complete\n", + "Cost on training data: 1099.2588907973695\n", + "Accuracy on training data: 962 / 1000\n", + "Cost on evaluation data: 1100.0748757805063\n", + "Accuracy on evaluation data: 7992 / 10000\n", + "Epoch 11 training complete\n", + "Cost on training data: 1090.6613318197058\n", + "Accuracy on training data: 969 / 1000\n", + "Cost on evaluation data: 1091.4972027870822\n", + "Accuracy on evaluation data: 8091 / 10000\n", + "Epoch 12 training complete\n", + "Cost on training data: 1082.0515239269528\n", + "Accuracy on training data: 978 / 1000\n", + "Cost on evaluation data: 1082.907839514214\n", + "Accuracy on evaluation data: 8160 / 10000\n", + "Epoch 13 training complete\n", + "Cost on training data: 1073.477041098918\n", + "Accuracy on training data: 979 / 1000\n", + "Cost on evaluation data: 1074.3561588881428\n", + "Accuracy on evaluation data: 8178 / 10000\n", + "Epoch 14 training complete\n", + "Cost on training data: 1064.9624355672843\n", + "Accuracy on training data: 980 / 1000\n", + "Cost on evaluation data: 1065.8739948439502\n", + "Accuracy on evaluation data: 8123 / 10000\n", + "Epoch 15 training complete\n", + "Cost on training data: 1056.386980299776\n", + "Accuracy on training data: 980 / 1000\n", + "Cost on evaluation data: 1057.3165648046704\n", + "Accuracy on evaluation data: 8146 / 10000\n", + "Epoch 16 training complete\n", + "Cost on training data: 1047.915010391618\n", + "Accuracy on training data: 981 / 1000\n", + "Cost on evaluation data: 1048.855967807932\n", + "Accuracy on evaluation data: 8178 / 10000\n", + "Epoch 17 training complete\n", + "Cost on training data: 1039.4272968094797\n", + "Accuracy on training data: 981 / 1000\n", + "Cost on evaluation data: 1040.3822068386899\n", + "Accuracy on evaluation data: 8183 / 10000\n", + "Epoch 18 training complete\n", + "Cost on training data: 1030.99832823966\n", + "Accuracy on training data: 987 / 1000\n", + "Cost on evaluation data: 1031.9648489852393\n", + "Accuracy on evaluation data: 8207 / 10000\n", + "Epoch 19 training complete\n", + "Cost on training data: 1022.625748354909\n", + "Accuracy on training data: 986 / 1000\n", + "Cost on evaluation data: 1023.6001191010827\n", + "Accuracy on evaluation data: 8211 / 10000\n", + "Epoch 20 training complete\n", + "Cost on training data: 1014.2246353500093\n", + "Accuracy on training data: 989 / 1000\n", + "Cost on evaluation data: 1015.2148797935556\n", + "Accuracy on evaluation data: 8233 / 10000\n", + "Epoch 21 training complete\n", + "Cost on training data: 1005.9057686080109\n", + "Accuracy on training data: 991 / 1000\n", + "Cost on evaluation data: 1006.8991409619174\n", + "Accuracy on evaluation data: 8235 / 10000\n", + "Epoch 22 training complete\n", + "Cost on training data: 997.612695350241\n", + "Accuracy on training data: 991 / 1000\n", + "Cost on evaluation data: 998.6215080955571\n", + "Accuracy on evaluation data: 8255 / 10000\n", + "Epoch 23 training complete\n", + "Cost on training data: 989.3777006220398\n", + "Accuracy on training data: 991 / 1000\n", + "Cost on evaluation data: 990.3875791492827\n", + "Accuracy on evaluation data: 8233 / 10000\n", + "Epoch 24 training complete\n", + "Cost on training data: 981.1517764972461\n", + "Accuracy on training data: 991 / 1000\n", + "Cost on evaluation data: 982.1780094072955\n", + "Accuracy on evaluation data: 8267 / 10000\n", + "Epoch 25 training complete\n", + "Cost on training data: 972.9777093172275\n", + "Accuracy on training data: 994 / 1000\n", + "Cost on evaluation data: 974.0061707896338\n", + "Accuracy on evaluation data: 8276 / 10000\n", + "Epoch 26 training complete\n", + "Cost on training data: 964.8280069283617\n", + "Accuracy on training data: 994 / 1000\n", + "Cost on evaluation data: 965.8728798143871\n", + "Accuracy on evaluation data: 8249 / 10000\n", + "Epoch 27 training complete\n", + "Cost on training data: 956.7239902110755\n", + "Accuracy on training data: 994 / 1000\n", + "Cost on evaluation data: 957.7886865139669\n", + "Accuracy on evaluation data: 8243 / 10000\n", + "Epoch 28 training complete\n", + "Cost on training data: 948.704749265818\n", + "Accuracy on training data: 993 / 1000\n", + "Cost on evaluation data: 949.7626920727115\n", + "Accuracy on evaluation data: 8277 / 10000\n", + "Epoch 29 training complete\n", + "Cost on training data: 940.6985409374273\n", + "Accuracy on training data: 995 / 1000\n", + "Cost on evaluation data: 941.7710535036861\n", + "Accuracy on evaluation data: 8263 / 10000\n", + "Epoch 30 training complete\n", + "Cost on training data: 932.776410365119\n", + "Accuracy on training data: 995 / 1000\n", + "Cost on evaluation data: 933.8466344287627\n", + "Accuracy on evaluation data: 8287 / 10000\n", + "Epoch 31 training complete\n", + "Cost on training data: 924.8747684408047\n", + "Accuracy on training data: 996 / 1000\n", + "Cost on evaluation data: 925.9549043145403\n", + "Accuracy on evaluation data: 8285 / 10000\n", + "Epoch 32 training complete\n", + "Cost on training data: 917.0569232013402\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 918.1385649043116\n", + "Accuracy on evaluation data: 8300 / 10000\n", + "Epoch 33 training complete\n", + "Cost on training data: 909.2765266165737\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 910.3597078875425\n", + "Accuracy on evaluation data: 8284 / 10000\n", + "Epoch 34 training complete\n", + "Cost on training data: 901.5485450776542\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 902.6379441129747\n", + "Accuracy on evaluation data: 8277 / 10000\n", + "Epoch 35 training complete\n", + "Cost on training data: 893.8880260487902\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 894.9864783294919\n", + "Accuracy on evaluation data: 8300 / 10000\n", + "Epoch 36 training complete\n", + "Cost on training data: 886.2785717758215\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 887.3786303921989\n", + "Accuracy on evaluation data: 8296 / 10000\n", + "Epoch 37 training complete\n", + "Cost on training data: 878.7238634271453\n", + "Accuracy on training data: 998 / 1000\n", + "Cost on evaluation data: 879.8229689671816\n", + "Accuracy on evaluation data: 8293 / 10000\n", + "Epoch 38 training complete\n", + "Cost on training data: 871.2133771575683\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 872.3190025655961\n", + "Accuracy on evaluation data: 8284 / 10000\n", + "Epoch 39 training complete\n", + "Cost on training data: 863.7752728350879\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 864.8847464155103\n", + "Accuracy on evaluation data: 8305 / 10000\n", + "Epoch 40 training complete\n", + "Cost on training data: 856.3931207076041\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 857.4991441139953\n", + "Accuracy on evaluation data: 8304 / 10000\n", + "Epoch 41 training complete\n", + "Cost on training data: 849.0587521340411\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 850.179899163105\n", + "Accuracy on evaluation data: 8313 / 10000\n", + "Epoch 42 training complete\n", + "Cost on training data: 841.7793118347722\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 842.8976200997531\n", + "Accuracy on evaluation data: 8312 / 10000\n", + "Epoch 43 training complete\n", + "Cost on training data: 834.5693840507612\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 835.6915705460643\n", + "Accuracy on evaluation data: 8307 / 10000\n", + "Epoch 44 training complete\n", + "Cost on training data: 827.3953297991724\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 828.5192835647074\n", + "Accuracy on evaluation data: 8314 / 10000\n", + "Epoch 45 training complete\n", + "Cost on training data: 820.2889528339799\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 821.4193220773445\n", + "Accuracy on evaluation data: 8318 / 10000\n", + "Epoch 46 training complete\n", + "Cost on training data: 813.2342652084045\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 814.3653496263973\n", + "Accuracy on evaluation data: 8300 / 10000\n", + "Epoch 47 training complete\n", + "Cost on training data: 806.2602757176262\n", + "Accuracy on training data: 999 / 1000\n", + "Cost on evaluation data: 807.3955709156385\n", + "Accuracy on evaluation data: 8316 / 10000\n", + "Epoch 48 training complete\n", + "Cost on training data: 799.3364646916554\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 800.4674124503357\n", + "Accuracy on evaluation data: 8325 / 10000\n", + "Epoch 49 training complete\n", + "Cost on training data: 792.479127079805\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 793.6163844044522\n", + "Accuracy on evaluation data: 8315 / 10000\n", + "Epoch 50 training complete\n", + "Cost on training data: 785.6723730974705\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 786.8101275028713\n", + "Accuracy on evaluation data: 8329 / 10000\n", + "Epoch 51 training complete\n", + "Cost on training data: 778.9158501835495\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 780.0533767671817\n", + "Accuracy on evaluation data: 8322 / 10000\n", + "Epoch 52 training complete\n", + "Cost on training data: 772.2248675822383\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 773.3604203874336\n", + "Accuracy on evaluation data: 8339 / 10000\n", + "Epoch 53 training complete\n", + "Cost on training data: 765.5900642399048\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 766.7280889519577\n", + "Accuracy on evaluation data: 8342 / 10000\n", + "Epoch 54 training complete\n", + "Cost on training data: 758.9977005187674\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 760.1394675810003\n", + "Accuracy on evaluation data: 8343 / 10000\n", + "Epoch 55 training complete\n", + "Cost on training data: 752.4630873934227\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 753.6085438450476\n", + "Accuracy on evaluation data: 8344 / 10000\n", + "Epoch 56 training complete\n", + "Cost on training data: 745.987290914852\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 747.1331974990774\n", + "Accuracy on evaluation data: 8366 / 10000\n", + "Epoch 57 training complete\n", + "Cost on training data: 739.5526323231533\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 740.695445067358\n", + "Accuracy on evaluation data: 8371 / 10000\n", + "Epoch 58 training complete\n", + "Cost on training data: 733.1928039121759\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 734.3368420179984\n", + "Accuracy on evaluation data: 8365 / 10000\n", + "Epoch 59 training complete\n", + "Cost on training data: 726.8813269174035\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 728.0186363701339\n", + "Accuracy on evaluation data: 8362 / 10000\n", + "Epoch 60 training complete\n", + "Cost on training data: 720.6171286067117\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 721.758423612669\n", + "Accuracy on evaluation data: 8362 / 10000\n", + "Epoch 61 training complete\n", + "Cost on training data: 714.4173295937903\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 715.5592874167378\n", + "Accuracy on evaluation data: 8373 / 10000\n", + "Epoch 62 training complete\n", + "Cost on training data: 708.2793383719294\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 709.4171615402431\n", + "Accuracy on evaluation data: 8376 / 10000\n", + "Epoch 63 training complete\n", + "Cost on training data: 702.167108124921\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 703.3049960579104\n", + "Accuracy on evaluation data: 8372 / 10000\n", + "Epoch 64 training complete\n", + "Cost on training data: 696.1316471296383\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 697.2723190766797\n", + "Accuracy on evaluation data: 8365 / 10000\n", + "Epoch 65 training complete\n", + "Cost on training data: 690.145763297733\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 691.2838335104531\n", + "Accuracy on evaluation data: 8385 / 10000\n", + "Epoch 66 training complete\n", + "Cost on training data: 684.2109672207824\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 685.3517146678568\n", + "Accuracy on evaluation data: 8372 / 10000\n", + "Epoch 67 training complete\n", + "Cost on training data: 678.3335124203676\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 679.4730390083978\n", + "Accuracy on evaluation data: 8388 / 10000\n", + "Epoch 68 training complete\n", + "Cost on training data: 672.495143453233\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 673.6368683983908\n", + "Accuracy on evaluation data: 8385 / 10000\n", + "Epoch 69 training complete\n", + "Cost on training data: 666.7228456823153\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 667.8574013163882\n", + "Accuracy on evaluation data: 8397 / 10000\n", + "Epoch 70 training complete\n", + "Cost on training data: 660.9952054606828\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 662.1317537291314\n", + "Accuracy on evaluation data: 8385 / 10000\n", + "Epoch 71 training complete\n", + "Cost on training data: 655.3205144108235\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 656.4547624233277\n", + "Accuracy on evaluation data: 8397 / 10000\n", + "Epoch 72 training complete\n", + "Cost on training data: 649.6903837995117\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 650.8268055066211\n", + "Accuracy on evaluation data: 8394 / 10000\n", + "Epoch 73 training complete\n", + "Cost on training data: 644.1127366761502\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 645.2509609703478\n", + "Accuracy on evaluation data: 8386 / 10000\n", + "Epoch 74 training complete\n", + "Cost on training data: 638.5895040441515\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 639.7274235337444\n", + "Accuracy on evaluation data: 8395 / 10000\n", + "Epoch 75 training complete\n", + "Cost on training data: 633.1133568257491\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 634.2446828607001\n", + "Accuracy on evaluation data: 8409 / 10000\n", + "Epoch 76 training complete\n", + "Cost on training data: 627.6920480080773\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 628.8181630454156\n", + "Accuracy on evaluation data: 8399 / 10000\n", + "Epoch 77 training complete\n", + "Cost on training data: 622.3152430709075\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 623.4484822672671\n", + "Accuracy on evaluation data: 8393 / 10000\n", + "Epoch 78 training complete\n", + "Cost on training data: 616.9888932271684\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 618.1195098829282\n", + "Accuracy on evaluation data: 8406 / 10000\n", + "Epoch 79 training complete\n", + "Cost on training data: 611.7073272133292\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 612.8365473868815\n", + "Accuracy on evaluation data: 8412 / 10000\n", + "Epoch 80 training complete\n", + "Cost on training data: 606.4784208342605\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 607.605176734595\n", + "Accuracy on evaluation data: 8402 / 10000\n", + "Epoch 81 training complete\n", + "Cost on training data: 601.297661805465\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 602.4212901857181\n", + "Accuracy on evaluation data: 8414 / 10000\n", + "Epoch 82 training complete\n", + "Cost on training data: 596.1567913610725\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 597.2823683845928\n", + "Accuracy on evaluation data: 8409 / 10000\n", + "Epoch 83 training complete\n", + "Cost on training data: 591.0725741723302\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 592.1985339366925\n", + "Accuracy on evaluation data: 8419 / 10000\n", + "Epoch 84 training complete\n", + "Cost on training data: 586.0394228514707\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 587.1588712986452\n", + "Accuracy on evaluation data: 8419 / 10000\n", + "Epoch 85 training complete\n", + "Cost on training data: 581.047343319327\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 582.1652248269108\n", + "Accuracy on evaluation data: 8417 / 10000\n", + "Epoch 86 training complete\n", + "Cost on training data: 576.0966111213579\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 577.2195005851567\n", + "Accuracy on evaluation data: 8420 / 10000\n", + "Epoch 87 training complete\n", + "Cost on training data: 571.1959561785774\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 572.3122401716006\n", + "Accuracy on evaluation data: 8418 / 10000\n", + "Epoch 88 training complete\n", + "Cost on training data: 566.3371730091762\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 567.4552805401123\n", + "Accuracy on evaluation data: 8429 / 10000\n", + "Epoch 89 training complete\n", + "Cost on training data: 561.5313120391985\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 562.6478113122239\n", + "Accuracy on evaluation data: 8426 / 10000\n", + "Epoch 90 training complete\n", + "Cost on training data: 556.7657283679526\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 557.881081898345\n", + "Accuracy on evaluation data: 8437 / 10000\n", + "Epoch 91 training complete\n", + "Cost on training data: 552.0491214097489\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 553.159295298081\n", + "Accuracy on evaluation data: 8433 / 10000\n", + "Epoch 92 training complete\n", + "Cost on training data: 547.377435764872\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 548.487415876447\n", + "Accuracy on evaluation data: 8429 / 10000\n", + "Epoch 93 training complete\n", + "Cost on training data: 542.745477343354\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 543.8539812937332\n", + "Accuracy on evaluation data: 8441 / 10000\n", + "Epoch 94 training complete\n", + "Cost on training data: 538.1584042728306\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 539.270474323244\n", + "Accuracy on evaluation data: 8435 / 10000\n", + "Epoch 95 training complete\n", + "Cost on training data: 533.6091269037969\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 534.7175967875492\n", + "Accuracy on evaluation data: 8442 / 10000\n", + "Epoch 96 training complete\n", + "Cost on training data: 529.1117657433148\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 530.2209979939807\n", + "Accuracy on evaluation data: 8439 / 10000\n", + "Epoch 97 training complete\n", + "Cost on training data: 524.6565312441514\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 525.7647403712916\n", + "Accuracy on evaluation data: 8438 / 10000\n", + "Epoch 98 training complete\n", + "Cost on training data: 520.2379914659955\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 521.3409627228912\n", + "Accuracy on evaluation data: 8452 / 10000\n", + "Epoch 99 training complete\n", + "Cost on training data: 515.8622594400932\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 516.965605515279\n", + "Accuracy on evaluation data: 8454 / 10000\n", + "Epoch 100 training complete\n", + "Cost on training data: 511.52823823469976\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 512.6283740889382\n", + "Accuracy on evaluation data: 8449 / 10000\n", + "Epoch 101 training complete\n", + "Cost on training data: 507.23950075070366\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 508.3412967750972\n", + "Accuracy on evaluation data: 8445 / 10000\n", + "Epoch 102 training complete\n", + "Cost on training data: 502.9899666789272\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 504.0913930719927\n", + "Accuracy on evaluation data: 8446 / 10000\n", + "Epoch 103 training complete\n", + "Cost on training data: 498.78030949910624\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 499.87865230601597\n", + "Accuracy on evaluation data: 8455 / 10000\n", + "Epoch 104 training complete\n", + "Cost on training data: 494.6064780022861\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 495.6972185263593\n", + "Accuracy on evaluation data: 8470 / 10000\n", + "Epoch 105 training complete\n", + "Cost on training data: 490.4670491385514\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 491.5647407085375\n", + "Accuracy on evaluation data: 8450 / 10000\n", + "Epoch 106 training complete\n", + "Cost on training data: 486.3765640379435\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 487.4694403904721\n", + "Accuracy on evaluation data: 8471 / 10000\n", + "Epoch 107 training complete\n", + "Cost on training data: 482.3290355978948\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 483.42142974436894\n", + "Accuracy on evaluation data: 8465 / 10000\n", + "Epoch 108 training complete\n", + "Cost on training data: 478.3124987905168\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 479.40172515303806\n", + "Accuracy on evaluation data: 8469 / 10000\n", + "Epoch 109 training complete\n", + "Cost on training data: 474.33895532798545\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 475.42971552177454\n", + "Accuracy on evaluation data: 8461 / 10000\n", + "Epoch 110 training complete\n", + "Cost on training data: 470.3963670284065\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 471.4835576548555\n", + "Accuracy on evaluation data: 8481 / 10000\n", + "Epoch 111 training complete\n", + "Cost on training data: 466.49436513928185\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 467.57858866604306\n", + "Accuracy on evaluation data: 8465 / 10000\n", + "Epoch 112 training complete\n", + "Cost on training data: 462.63256209247135\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 463.720872163426\n", + "Accuracy on evaluation data: 8481 / 10000\n", + "Epoch 113 training complete\n", + "Cost on training data: 458.80757457230425\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 459.8861313611991\n", + "Accuracy on evaluation data: 8485 / 10000\n", + "Epoch 114 training complete\n", + "Cost on training data: 455.01325439799064\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 456.09510027149594\n", + "Accuracy on evaluation data: 8483 / 10000\n", + "Epoch 115 training complete\n", + "Cost on training data: 451.26567719445075\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 452.34128634643236\n", + "Accuracy on evaluation data: 8492 / 10000\n", + "Epoch 116 training complete\n", + "Cost on training data: 447.54999190149204\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 448.62303999178124\n", + "Accuracy on evaluation data: 8491 / 10000\n", + "Epoch 117 training complete\n", + "Cost on training data: 443.86609452976427\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 444.9426107256536\n", + "Accuracy on evaluation data: 8490 / 10000\n", + "Epoch 118 training complete\n", + "Cost on training data: 440.22120029093475\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 441.29374340303735\n", + "Accuracy on evaluation data: 8484 / 10000\n", + "Epoch 119 training complete\n", + "Cost on training data: 436.61010320820264\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 437.677870960006\n", + "Accuracy on evaluation data: 8499 / 10000\n", + "Epoch 120 training complete\n", + "Cost on training data: 433.0290242485104\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 434.098173198043\n", + "Accuracy on evaluation data: 8505 / 10000\n", + "Epoch 121 training complete\n", + "Cost on training data: 429.49218037519546\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 430.5611971622741\n", + "Accuracy on evaluation data: 8494 / 10000\n", + "Epoch 122 training complete\n", + "Cost on training data: 425.98765770598374\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 427.0550960462521\n", + "Accuracy on evaluation data: 8490 / 10000\n", + "Epoch 123 training complete\n", + "Cost on training data: 422.50922746002414\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 423.5753444978525\n", + "Accuracy on evaluation data: 8505 / 10000\n", + "Epoch 124 training complete\n", + "Cost on training data: 419.0727478765157\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 420.1362967730769\n", + "Accuracy on evaluation data: 8498 / 10000\n", + "Epoch 125 training complete\n", + "Cost on training data: 415.6650021236998\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 416.7279957864222\n", + "Accuracy on evaluation data: 8500 / 10000\n", + "Epoch 126 training complete\n", + "Cost on training data: 412.2907713338565\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 413.3508998890157\n", + "Accuracy on evaluation data: 8502 / 10000\n", + "Epoch 127 training complete\n", + "Cost on training data: 408.95294635765083\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 410.00955230208\n", + "Accuracy on evaluation data: 8508 / 10000\n", + "Epoch 128 training complete\n", + "Cost on training data: 405.6389050096784\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 406.69750632637056\n", + "Accuracy on evaluation data: 8504 / 10000\n", + "Epoch 129 training complete\n", + "Cost on training data: 402.35748795439025\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 403.41099306225215\n", + "Accuracy on evaluation data: 8517 / 10000\n", + "Epoch 130 training complete\n", + "Cost on training data: 399.11175123802417\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 400.16940693127646\n", + "Accuracy on evaluation data: 8508 / 10000\n", + "Epoch 131 training complete\n", + "Cost on training data: 395.898627792672\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 396.95573917116667\n", + "Accuracy on evaluation data: 8514 / 10000\n", + "Epoch 132 training complete\n", + "Cost on training data: 392.71897099912906\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 393.7682201033877\n", + "Accuracy on evaluation data: 8527 / 10000\n", + "Epoch 133 training complete\n", + "Cost on training data: 389.56604953344424\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 390.61613918938923\n", + "Accuracy on evaluation data: 8521 / 10000\n", + "Epoch 134 training complete\n", + "Cost on training data: 386.44451884085976\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 387.4936436492131\n", + "Accuracy on evaluation data: 8522 / 10000\n", + "Epoch 135 training complete\n", + "Cost on training data: 383.3593901301572\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 384.4031930923719\n", + "Accuracy on evaluation data: 8533 / 10000\n", + "Epoch 136 training complete\n", + "Cost on training data: 380.29870918179716\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 381.3398245398442\n", + "Accuracy on evaluation data: 8528 / 10000\n", + "Epoch 137 training complete\n", + "Cost on training data: 377.2655428780794\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 378.30810552053566\n", + "Accuracy on evaluation data: 8523 / 10000\n", + "Epoch 138 training complete\n", + "Cost on training data: 374.26468379851906\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 375.31020866263026\n", + "Accuracy on evaluation data: 8519 / 10000\n", + "Epoch 139 training complete\n", + "Cost on training data: 371.29509066340984\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 372.33553795338486\n", + "Accuracy on evaluation data: 8532 / 10000\n", + "Epoch 140 training complete\n", + "Cost on training data: 368.3553798343564\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 369.39421258287035\n", + "Accuracy on evaluation data: 8523 / 10000\n", + "Epoch 141 training complete\n", + "Cost on training data: 365.4352729027066\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 366.4707956930006\n", + "Accuracy on evaluation data: 8537 / 10000\n", + "Epoch 142 training complete\n", + "Cost on training data: 362.54990328774375\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 363.5836037193533\n", + "Accuracy on evaluation data: 8543 / 10000\n", + "Epoch 143 training complete\n", + "Cost on training data: 359.6907516071708\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 360.72026544312575\n", + "Accuracy on evaluation data: 8548 / 10000\n", + "Epoch 144 training complete\n", + "Cost on training data: 356.858459360209\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 357.89238318984405\n", + "Accuracy on evaluation data: 8554 / 10000\n", + "Epoch 145 training complete\n", + "Cost on training data: 354.0567123419969\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 355.0846343070974\n", + "Accuracy on evaluation data: 8558 / 10000\n", + "Epoch 146 training complete\n", + "Cost on training data: 351.2835573342333\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 352.30842646691156\n", + "Accuracy on evaluation data: 8540 / 10000\n", + "Epoch 147 training complete\n", + "Cost on training data: 348.53793333604204\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 349.56513256154835\n", + "Accuracy on evaluation data: 8551 / 10000\n", + "Epoch 148 training complete\n", + "Cost on training data: 345.8198659188105\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 346.84459446444373\n", + "Accuracy on evaluation data: 8559 / 10000\n", + "Epoch 149 training complete\n", + "Cost on training data: 343.12563614667414\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 344.14608389224134\n", + "Accuracy on evaluation data: 8561 / 10000\n", + "Epoch 150 training complete\n", + "Cost on training data: 340.45085397882076\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 341.4745757848774\n", + "Accuracy on evaluation data: 8552 / 10000\n", + "Epoch 151 training complete\n", + "Cost on training data: 337.81345895014556\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 338.83404652808093\n", + "Accuracy on evaluation data: 8554 / 10000\n", + "Epoch 152 training complete\n", + "Cost on training data: 335.19695725523275\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 336.2133280206937\n", + "Accuracy on evaluation data: 8559 / 10000\n", + "Epoch 153 training complete\n", + "Cost on training data: 332.6112735932789\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 333.6304041936384\n", + "Accuracy on evaluation data: 8556 / 10000\n", + "Epoch 154 training complete\n", + "Cost on training data: 330.0395353095022\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 331.05079413272006\n", + "Accuracy on evaluation data: 8571 / 10000\n", + "Epoch 155 training complete\n", + "Cost on training data: 327.49906652877127\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 328.5106834775448\n", + "Accuracy on evaluation data: 8566 / 10000\n", + "Epoch 156 training complete\n", + "Cost on training data: 324.98419411055943\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 325.9942169108906\n", + "Accuracy on evaluation data: 8575 / 10000\n", + "Epoch 157 training complete\n", + "Cost on training data: 322.48921143565127\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 323.4970947805908\n", + "Accuracy on evaluation data: 8578 / 10000\n", + "Epoch 158 training complete\n", + "Cost on training data: 320.01798918372\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 321.0312360003588\n", + "Accuracy on evaluation data: 8571 / 10000\n", + "Epoch 159 training complete\n", + "Cost on training data: 317.57872587929256\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 318.5838256912846\n", + "Accuracy on evaluation data: 8573 / 10000\n", + "Epoch 160 training complete\n", + "Cost on training data: 315.1586109738737\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 316.1645439538399\n", + "Accuracy on evaluation data: 8583 / 10000\n", + "Epoch 161 training complete\n", + "Cost on training data: 312.7646412025612\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 313.7643771571211\n", + "Accuracy on evaluation data: 8587 / 10000\n", + "Epoch 162 training complete\n", + "Cost on training data: 310.38625581200415\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 311.3964603937147\n", + "Accuracy on evaluation data: 8578 / 10000\n", + "Epoch 163 training complete\n", + "Cost on training data: 308.0398477322997\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 309.0376827690023\n", + "Accuracy on evaluation data: 8586 / 10000\n", + "Epoch 164 training complete\n", + "Cost on training data: 305.710107032002\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 306.7113002333237\n", + "Accuracy on evaluation data: 8591 / 10000\n", + "Epoch 165 training complete\n", + "Cost on training data: 303.4121414666272\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 304.4081780131596\n", + "Accuracy on evaluation data: 8583 / 10000\n", + "Epoch 166 training complete\n", + "Cost on training data: 301.1292379235853\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 302.12535961459776\n", + "Accuracy on evaluation data: 8593 / 10000\n", + "Epoch 167 training complete\n", + "Cost on training data: 298.8654691804667\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 299.8665602399167\n", + "Accuracy on evaluation data: 8595 / 10000\n", + "Epoch 168 training complete\n", + "Cost on training data: 296.6396998841399\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 297.6318128757215\n", + "Accuracy on evaluation data: 8592 / 10000\n", + "Epoch 169 training complete\n", + "Cost on training data: 294.4269930257843\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 295.4212920390459\n", + "Accuracy on evaluation data: 8595 / 10000\n", + "Epoch 170 training complete\n", + "Cost on training data: 292.23094816860225\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 293.22300295673244\n", + "Accuracy on evaluation data: 8594 / 10000\n", + "Epoch 171 training complete\n", + "Cost on training data: 290.06120834507647\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 291.050800598309\n", + "Accuracy on evaluation data: 8597 / 10000\n", + "Epoch 172 training complete\n", + "Cost on training data: 287.91379716835115\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 288.9044479060854\n", + "Accuracy on evaluation data: 8601 / 10000\n", + "Epoch 173 training complete\n", + "Cost on training data: 285.7823538151577\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 286.77581096991906\n", + "Accuracy on evaluation data: 8597 / 10000\n", + "Epoch 174 training complete\n", + "Cost on training data: 283.67349163350394\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 284.661969081252\n", + "Accuracy on evaluation data: 8601 / 10000\n", + "Epoch 175 training complete\n", + "Cost on training data: 281.5922594716126\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 282.57535969547797\n", + "Accuracy on evaluation data: 8596 / 10000\n", + "Epoch 176 training complete\n", + "Cost on training data: 279.5207276380414\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 280.50919978589343\n", + "Accuracy on evaluation data: 8597 / 10000\n", + "Epoch 177 training complete\n", + "Cost on training data: 277.4751999465301\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 278.459848325272\n", + "Accuracy on evaluation data: 8603 / 10000\n", + "Epoch 178 training complete\n", + "Cost on training data: 275.44687665881946\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 276.43097335768863\n", + "Accuracy on evaluation data: 8598 / 10000\n", + "Epoch 179 training complete\n", + "Cost on training data: 273.4412299273835\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 274.421817033263\n", + "Accuracy on evaluation data: 8604 / 10000\n", + "Epoch 180 training complete\n", + "Cost on training data: 271.45526309904886\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 272.4340215137264\n", + "Accuracy on evaluation data: 8607 / 10000\n", + "Epoch 181 training complete\n", + "Cost on training data: 269.4888238430352\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 270.4734186145994\n", + "Accuracy on evaluation data: 8595 / 10000\n", + "Epoch 182 training complete\n", + "Cost on training data: 267.5477047801853\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 268.51846626929256\n", + "Accuracy on evaluation data: 8607 / 10000\n", + "Epoch 183 training complete\n", + "Cost on training data: 265.62102478057795\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 266.59560521153884\n", + "Accuracy on evaluation data: 8609 / 10000\n", + "Epoch 184 training complete\n", + "Cost on training data: 263.7067427419016\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 264.68295896739085\n", + "Accuracy on evaluation data: 8612 / 10000\n", + "Epoch 185 training complete\n", + "Cost on training data: 261.81097326243054\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 262.7848543310213\n", + "Accuracy on evaluation data: 8613 / 10000\n", + "Epoch 186 training complete\n", + "Cost on training data: 259.9368717499867\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 260.91448265753957\n", + "Accuracy on evaluation data: 8614 / 10000\n", + "Epoch 187 training complete\n", + "Cost on training data: 258.08083343785245\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 259.0617474159273\n", + "Accuracy on evaluation data: 8611 / 10000\n", + "Epoch 188 training complete\n", + "Cost on training data: 256.24805761339815\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 257.21757723055197\n", + "Accuracy on evaluation data: 8613 / 10000\n", + "Epoch 189 training complete\n", + "Cost on training data: 254.4299194558225\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 255.40062564829273\n", + "Accuracy on evaluation data: 8614 / 10000\n", + "Epoch 190 training complete\n", + "Cost on training data: 252.62965191750462\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 253.59935326736127\n", + "Accuracy on evaluation data: 8621 / 10000\n", + "Epoch 191 training complete\n", + "Cost on training data: 250.84793235481067\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 251.81114771446647\n", + "Accuracy on evaluation data: 8616 / 10000\n", + "Epoch 192 training complete\n", + "Cost on training data: 249.07771749287798\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 250.04505824101886\n", + "Accuracy on evaluation data: 8625 / 10000\n", + "Epoch 193 training complete\n", + "Cost on training data: 247.3331671875962\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 248.2972454590133\n", + "Accuracy on evaluation data: 8629 / 10000\n", + "Epoch 194 training complete\n", + "Cost on training data: 245.6036950578122\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 246.56204777439743\n", + "Accuracy on evaluation data: 8618 / 10000\n", + "Epoch 195 training complete\n", + "Cost on training data: 243.88972440137857\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 244.8494830589509\n", + "Accuracy on evaluation data: 8625 / 10000\n", + "Epoch 196 training complete\n", + "Cost on training data: 242.18765145569887\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 243.148048655624\n", + "Accuracy on evaluation data: 8628 / 10000\n", + "Epoch 197 training complete\n", + "Cost on training data: 240.50928854411845\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 241.46652714341346\n", + "Accuracy on evaluation data: 8629 / 10000\n", + "Epoch 198 training complete\n", + "Cost on training data: 238.8369595723929\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 239.80291639042477\n", + "Accuracy on evaluation data: 8615 / 10000\n", + "Epoch 199 training complete\n", + "Cost on training data: 237.19928156931718\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 238.1593795403392\n", + "Accuracy on evaluation data: 8625 / 10000\n", + "Epoch 200 training complete\n", + "Cost on training data: 235.56292904829627\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 236.5162182831002\n", + "Accuracy on evaluation data: 8628 / 10000\n", + "Epoch 201 training complete\n", + "Cost on training data: 233.94626282633675\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 234.8990050571617\n", + "Accuracy on evaluation data: 8631 / 10000\n", + "Epoch 202 training complete\n", + "Cost on training data: 232.33961375996932\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 233.29654876209673\n", + "Accuracy on evaluation data: 8635 / 10000\n", + "Epoch 203 training complete\n", + "Cost on training data: 230.75366601467536\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 231.7021175700539\n", + "Accuracy on evaluation data: 8647 / 10000\n", + "Epoch 204 training complete\n", + "Cost on training data: 229.18785105449035\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 230.14166748103742\n", + "Accuracy on evaluation data: 8642 / 10000\n", + "Epoch 205 training complete\n", + "Cost on training data: 227.63282724599225\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 228.58242560916312\n", + "Accuracy on evaluation data: 8638 / 10000\n", + "Epoch 206 training complete\n", + "Cost on training data: 226.09428013718954\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 227.04974382081173\n", + "Accuracy on evaluation data: 8626 / 10000\n", + "Epoch 207 training complete\n", + "Cost on training data: 224.56690376581918\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 225.51972427823927\n", + "Accuracy on evaluation data: 8624 / 10000\n", + "Epoch 208 training complete\n", + "Cost on training data: 223.05935518933688\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 224.0108765153497\n", + "Accuracy on evaluation data: 8638 / 10000\n", + "Epoch 209 training complete\n", + "Cost on training data: 221.56165354341528\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 222.51857933132027\n", + "Accuracy on evaluation data: 8617 / 10000\n", + "Epoch 210 training complete\n", + "Cost on training data: 220.08391896727622\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 221.0304317591979\n", + "Accuracy on evaluation data: 8636 / 10000\n", + "Epoch 211 training complete\n", + "Cost on training data: 218.61954312189553\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 219.5689186901096\n", + "Accuracy on evaluation data: 8638 / 10000\n", + "Epoch 212 training complete\n", + "Cost on training data: 217.16965826031802\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 218.11144124625233\n", + "Accuracy on evaluation data: 8627 / 10000\n", + "Epoch 213 training complete\n", + "Cost on training data: 215.73220759143933\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 216.6758949077567\n", + "Accuracy on evaluation data: 8640 / 10000\n", + "Epoch 214 training complete\n", + "Cost on training data: 214.31087335695136\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 215.2513700030919\n", + "Accuracy on evaluation data: 8643 / 10000\n", + "Epoch 215 training complete\n", + "Cost on training data: 212.89713502548943\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 213.83807365382322\n", + "Accuracy on evaluation data: 8630 / 10000\n", + "Epoch 216 training complete\n", + "Cost on training data: 211.50455776175468\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 212.43986553298024\n", + "Accuracy on evaluation data: 8635 / 10000\n", + "Epoch 217 training complete\n", + "Cost on training data: 210.1249525277765\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 211.0595339889059\n", + "Accuracy on evaluation data: 8644 / 10000\n", + "Epoch 218 training complete\n", + "Cost on training data: 208.7622283718393\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 209.69665723725356\n", + "Accuracy on evaluation data: 8640 / 10000\n", + "Epoch 219 training complete\n", + "Cost on training data: 207.4041676163128\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 208.34381945180186\n", + "Accuracy on evaluation data: 8640 / 10000\n", + "Epoch 220 training complete\n", + "Cost on training data: 206.06151215092228\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 207.0013956126107\n", + "Accuracy on evaluation data: 8647 / 10000\n", + "Epoch 221 training complete\n", + "Cost on training data: 204.72899550778732\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 205.66995103692176\n", + "Accuracy on evaluation data: 8644 / 10000\n", + "Epoch 222 training complete\n", + "Cost on training data: 203.4148741057804\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 204.34963000485172\n", + "Accuracy on evaluation data: 8655 / 10000\n", + "Epoch 223 training complete\n", + "Cost on training data: 202.11243991703992\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 203.052069505311\n", + "Accuracy on evaluation data: 8645 / 10000\n", + "Epoch 224 training complete\n", + "Cost on training data: 200.8227599595114\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 201.75518051149888\n", + "Accuracy on evaluation data: 8639 / 10000\n", + "Epoch 225 training complete\n", + "Cost on training data: 199.53944196996383\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 200.4739575227732\n", + "Accuracy on evaluation data: 8645 / 10000\n", + "Epoch 226 training complete\n", + "Cost on training data: 198.27752212485956\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 199.21085016706672\n", + "Accuracy on evaluation data: 8649 / 10000\n", + "Epoch 227 training complete\n", + "Cost on training data: 197.02639481636746\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 197.95125565357029\n", + "Accuracy on evaluation data: 8654 / 10000\n", + "Epoch 228 training complete\n", + "Cost on training data: 195.78193752271304\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 196.71004541693966\n", + "Accuracy on evaluation data: 8662 / 10000\n", + "Epoch 229 training complete\n", + "Cost on training data: 194.55273457972157\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 195.4905483968906\n", + "Accuracy on evaluation data: 8660 / 10000\n", + "Epoch 230 training complete\n", + "Cost on training data: 193.33421740160804\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 194.27031555823748\n", + "Accuracy on evaluation data: 8655 / 10000\n", + "Epoch 231 training complete\n", + "Cost on training data: 192.13597390948422\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 193.06432004807883\n", + "Accuracy on evaluation data: 8653 / 10000\n", + "Epoch 232 training complete\n", + "Cost on training data: 190.94552071293452\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 191.8683317160994\n", + "Accuracy on evaluation data: 8655 / 10000\n", + "Epoch 233 training complete\n", + "Cost on training data: 189.76103053263805\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 190.68179423213036\n", + "Accuracy on evaluation data: 8658 / 10000\n", + "Epoch 234 training complete\n", + "Cost on training data: 188.59368127770017\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 189.51686504069605\n", + "Accuracy on evaluation data: 8663 / 10000\n", + "Epoch 235 training complete\n", + "Cost on training data: 187.4328535202134\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 188.35812683993993\n", + "Accuracy on evaluation data: 8660 / 10000\n", + "Epoch 236 training complete\n", + "Cost on training data: 186.28977843120714\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 187.2072003173373\n", + "Accuracy on evaluation data: 8665 / 10000\n", + "Epoch 237 training complete\n", + "Cost on training data: 185.15409175178246\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 186.07632172050174\n", + "Accuracy on evaluation data: 8662 / 10000\n", + "Epoch 238 training complete\n", + "Cost on training data: 184.03141460718146\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 184.95106571351653\n", + "Accuracy on evaluation data: 8659 / 10000\n", + "Epoch 239 training complete\n", + "Cost on training data: 182.91744177672672\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 183.83714019394898\n", + "Accuracy on evaluation data: 8667 / 10000\n", + "Epoch 240 training complete\n", + "Cost on training data: 181.8171436216399\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 182.73712828652603\n", + "Accuracy on evaluation data: 8671 / 10000\n", + "Epoch 241 training complete\n", + "Cost on training data: 180.72543110458759\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 181.6500173400304\n", + "Accuracy on evaluation data: 8668 / 10000\n", + "Epoch 242 training complete\n", + "Cost on training data: 179.64727287962748\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 180.56057765822743\n", + "Accuracy on evaluation data: 8672 / 10000\n", + "Epoch 243 training complete\n", + "Cost on training data: 178.57174632470438\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 179.49594153364026\n", + "Accuracy on evaluation data: 8670 / 10000\n", + "Epoch 244 training complete\n", + "Cost on training data: 177.51740563308863\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 178.43616691566348\n", + "Accuracy on evaluation data: 8674 / 10000\n", + "Epoch 245 training complete\n", + "Cost on training data: 176.46362714459403\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 177.38280606276197\n", + "Accuracy on evaluation data: 8669 / 10000\n", + "Epoch 246 training complete\n", + "Cost on training data: 175.4257382395825\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 176.34185115516706\n", + "Accuracy on evaluation data: 8676 / 10000\n", + "Epoch 247 training complete\n", + "Cost on training data: 174.3976151228819\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 175.31263328694456\n", + "Accuracy on evaluation data: 8674 / 10000\n", + "Epoch 248 training complete\n", + "Cost on training data: 173.37949375745762\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 174.29401848005543\n", + "Accuracy on evaluation data: 8677 / 10000\n", + "Epoch 249 training complete\n", + "Cost on training data: 172.37032456949137\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 173.28459064004417\n", + "Accuracy on evaluation data: 8669 / 10000\n", + "Epoch 250 training complete\n", + "Cost on training data: 171.37657336125426\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 172.29275402565054\n", + "Accuracy on evaluation data: 8679 / 10000\n", + "Epoch 251 training complete\n", + "Cost on training data: 170.3829500387815\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 171.296886256157\n", + "Accuracy on evaluation data: 8666 / 10000\n", + "Epoch 252 training complete\n", + "Cost on training data: 169.4039035823571\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 170.3164870721019\n", + "Accuracy on evaluation data: 8678 / 10000\n", + "Epoch 253 training complete\n", + "Cost on training data: 168.42934419181105\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 169.34335679895372\n", + "Accuracy on evaluation data: 8678 / 10000\n", + "Epoch 254 training complete\n", + "Cost on training data: 167.47313418933896\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 168.3860559253519\n", + "Accuracy on evaluation data: 8675 / 10000\n", + "Epoch 255 training complete\n", + "Cost on training data: 166.5251939438949\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 167.43336040490613\n", + "Accuracy on evaluation data: 8684 / 10000\n", + "Epoch 256 training complete\n", + "Cost on training data: 165.58334235276064\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 166.49412033363566\n", + "Accuracy on evaluation data: 8680 / 10000\n", + "Epoch 257 training complete\n", + "Cost on training data: 164.64762789905262\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 165.55362457984546\n", + "Accuracy on evaluation data: 8691 / 10000\n", + "Epoch 258 training complete\n", + "Cost on training data: 163.72663015635248\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 164.63590415994042\n", + "Accuracy on evaluation data: 8688 / 10000\n", + "Epoch 259 training complete\n", + "Cost on training data: 162.81663877640517\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 163.719930083709\n", + "Accuracy on evaluation data: 8689 / 10000\n", + "Epoch 260 training complete\n", + "Cost on training data: 161.90884423576173\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 162.81603158294493\n", + "Accuracy on evaluation data: 8690 / 10000\n", + "Epoch 261 training complete\n", + "Cost on training data: 161.0084809939894\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 161.914860583511\n", + "Accuracy on evaluation data: 8666 / 10000\n", + "Epoch 262 training complete\n", + "Cost on training data: 160.12517901775055\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 161.03101735181747\n", + "Accuracy on evaluation data: 8676 / 10000\n", + "Epoch 263 training complete\n", + "Cost on training data: 159.2451632268175\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 160.14934712805677\n", + "Accuracy on evaluation data: 8677 / 10000\n", + "Epoch 264 training complete\n", + "Cost on training data: 158.37676080936623\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 159.27857952719538\n", + "Accuracy on evaluation data: 8692 / 10000\n", + "Epoch 265 training complete\n", + "Cost on training data: 157.50662351672293\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 158.41625142062648\n", + "Accuracy on evaluation data: 8675 / 10000\n", + "Epoch 266 training complete\n", + "Cost on training data: 156.65343062858608\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 157.56173182285363\n", + "Accuracy on evaluation data: 8679 / 10000\n", + "Epoch 267 training complete\n", + "Cost on training data: 155.80691312319397\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 156.70730174260984\n", + "Accuracy on evaluation data: 8692 / 10000\n", + "Epoch 268 training complete\n", + "Cost on training data: 154.9674401319616\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 155.87473531347396\n", + "Accuracy on evaluation data: 8688 / 10000\n", + "Epoch 269 training complete\n", + "Cost on training data: 154.14142784866416\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 155.040656426804\n", + "Accuracy on evaluation data: 8681 / 10000\n", + "Epoch 270 training complete\n", + "Cost on training data: 153.31783874912932\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 154.21720963886844\n", + "Accuracy on evaluation data: 8676 / 10000\n", + "Epoch 271 training complete\n", + "Cost on training data: 152.5062270176732\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 153.408159637891\n", + "Accuracy on evaluation data: 8688 / 10000\n", + "Epoch 272 training complete\n", + "Cost on training data: 151.69867406747943\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 152.59981346601157\n", + "Accuracy on evaluation data: 8696 / 10000\n", + "Epoch 273 training complete\n", + "Cost on training data: 150.90316938827812\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 151.80152786045207\n", + "Accuracy on evaluation data: 8697 / 10000\n", + "Epoch 274 training complete\n", + "Cost on training data: 150.11215880727028\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 151.00612606450156\n", + "Accuracy on evaluation data: 8704 / 10000\n", + "Epoch 275 training complete\n", + "Cost on training data: 149.33287907749076\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 150.22683571817862\n", + "Accuracy on evaluation data: 8694 / 10000\n", + "Epoch 276 training complete\n", + "Cost on training data: 148.55689151351538\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 149.45330154282198\n", + "Accuracy on evaluation data: 8699 / 10000\n", + "Epoch 277 training complete\n", + "Cost on training data: 147.79290346284552\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 148.68665763751193\n", + "Accuracy on evaluation data: 8702 / 10000\n", + "Epoch 278 training complete\n", + "Cost on training data: 147.02650190246015\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 147.93148797910948\n", + "Accuracy on evaluation data: 8690 / 10000\n", + "Epoch 279 training complete\n", + "Cost on training data: 146.2803138343504\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 147.1707261371315\n", + "Accuracy on evaluation data: 8712 / 10000\n", + "Epoch 280 training complete\n", + "Cost on training data: 145.534134767814\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 146.42932997745598\n", + "Accuracy on evaluation data: 8692 / 10000\n", + "Epoch 281 training complete\n", + "Cost on training data: 144.79110810926028\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 145.68633266165716\n", + "Accuracy on evaluation data: 8699 / 10000\n", + "Epoch 282 training complete\n", + "Cost on training data: 144.0663569368176\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 144.96268133907947\n", + "Accuracy on evaluation data: 8699 / 10000\n", + "Epoch 283 training complete\n", + "Cost on training data: 143.34532651415586\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 144.23998827116023\n", + "Accuracy on evaluation data: 8697 / 10000\n", + "Epoch 284 training complete\n", + "Cost on training data: 142.62800205572586\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 143.52076173586718\n", + "Accuracy on evaluation data: 8696 / 10000\n", + "Epoch 285 training complete\n", + "Cost on training data: 141.9157453245709\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 142.80597651574308\n", + "Accuracy on evaluation data: 8704 / 10000\n", + "Epoch 286 training complete\n", + "Cost on training data: 141.21253865619656\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 142.1061250384873\n", + "Accuracy on evaluation data: 8698 / 10000\n", + "Epoch 287 training complete\n", + "Cost on training data: 140.51613157390847\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 141.40326044510223\n", + "Accuracy on evaluation data: 8712 / 10000\n", + "Epoch 288 training complete\n", + "Cost on training data: 139.81914687489768\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 140.72017982965693\n", + "Accuracy on evaluation data: 8689 / 10000\n", + "Epoch 289 training complete\n", + "Cost on training data: 139.1451078390237\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 140.03527578030744\n", + "Accuracy on evaluation data: 8701 / 10000\n", + "Epoch 290 training complete\n", + "Cost on training data: 138.46607773131637\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 139.35690917465806\n", + "Accuracy on evaluation data: 8694 / 10000\n", + "Epoch 291 training complete\n", + "Cost on training data: 137.79904759621144\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 138.68935429071928\n", + "Accuracy on evaluation data: 8706 / 10000\n", + "Epoch 292 training complete\n", + "Cost on training data: 137.13519814745504\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 138.02376218741455\n", + "Accuracy on evaluation data: 8692 / 10000\n", + "Epoch 293 training complete\n", + "Cost on training data: 136.47722198099095\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 137.3605211413804\n", + "Accuracy on evaluation data: 8712 / 10000\n", + "Epoch 294 training complete\n", + "Cost on training data: 135.82787954738038\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 136.71486909924369\n", + "Accuracy on evaluation data: 8711 / 10000\n", + "Epoch 295 training complete\n", + "Cost on training data: 135.18516369608005\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 136.07162543638694\n", + "Accuracy on evaluation data: 8708 / 10000\n", + "Epoch 296 training complete\n", + "Cost on training data: 134.54113146840547\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 135.42467607228778\n", + "Accuracy on evaluation data: 8703 / 10000\n", + "Epoch 297 training complete\n", + "Cost on training data: 133.90727982167843\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 134.79450356647376\n", + "Accuracy on evaluation data: 8704 / 10000\n", + "Epoch 298 training complete\n", + "Cost on training data: 133.28368979044492\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 134.17139375885554\n", + "Accuracy on evaluation data: 8701 / 10000\n", + "Epoch 299 training complete\n", + "Cost on training data: 132.66392811216136\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 133.55307568548562\n", + "Accuracy on evaluation data: 8699 / 10000\n", + "Epoch 300 training complete\n", + "Cost on training data: 132.05058657045532\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 132.93341819006184\n", + "Accuracy on evaluation data: 8712 / 10000\n", + "Epoch 301 training complete\n", + "Cost on training data: 131.44729495480772\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 132.32705751731527\n", + "Accuracy on evaluation data: 8712 / 10000\n", + "Epoch 302 training complete\n", + "Cost on training data: 130.8449796603363\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 131.73036189462405\n", + "Accuracy on evaluation data: 8715 / 10000\n", + "Epoch 303 training complete\n", + "Cost on training data: 130.25004571180622\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 131.1325106337243\n", + "Accuracy on evaluation data: 8713 / 10000\n", + "Epoch 304 training complete\n", + "Cost on training data: 129.65721400954104\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 130.5456368475591\n", + "Accuracy on evaluation data: 8697 / 10000\n", + "Epoch 305 training complete\n", + "Cost on training data: 129.07433520452574\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 129.95810083085203\n", + "Accuracy on evaluation data: 8698 / 10000\n", + "Epoch 306 training complete\n", + "Cost on training data: 128.4980157161894\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 129.37878616956664\n", + "Accuracy on evaluation data: 8723 / 10000\n", + "Epoch 307 training complete\n", + "Cost on training data: 127.92772267141942\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 128.81311328727887\n", + "Accuracy on evaluation data: 8698 / 10000\n", + "Epoch 308 training complete\n", + "Cost on training data: 127.36052623793111\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 128.24768721656028\n", + "Accuracy on evaluation data: 8711 / 10000\n", + "Epoch 309 training complete\n", + "Cost on training data: 126.80107933067903\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 127.68027615482468\n", + "Accuracy on evaluation data: 8709 / 10000\n", + "Epoch 310 training complete\n", + "Cost on training data: 126.23968780121014\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 127.11718715553202\n", + "Accuracy on evaluation data: 8717 / 10000\n", + "Epoch 311 training complete\n", + "Cost on training data: 125.68753642892965\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 126.56415295199116\n", + "Accuracy on evaluation data: 8726 / 10000\n", + "Epoch 312 training complete\n", + "Cost on training data: 125.14287657161243\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 126.02182239865802\n", + "Accuracy on evaluation data: 8718 / 10000\n", + "Epoch 313 training complete\n", + "Cost on training data: 124.60306114902335\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 125.48264009305342\n", + "Accuracy on evaluation data: 8718 / 10000\n", + "Epoch 314 training complete\n", + "Cost on training data: 124.0630799939732\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 124.93814018582513\n", + "Accuracy on evaluation data: 8720 / 10000\n", + "Epoch 315 training complete\n", + "Cost on training data: 123.53645558787491\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 124.41360176790063\n", + "Accuracy on evaluation data: 8714 / 10000\n", + "Epoch 316 training complete\n", + "Cost on training data: 123.01099770085493\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 123.88430888793442\n", + "Accuracy on evaluation data: 8730 / 10000\n", + "Epoch 317 training complete\n", + "Cost on training data: 122.49269891065484\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 123.36592471653726\n", + "Accuracy on evaluation data: 8728 / 10000\n", + "Epoch 318 training complete\n", + "Cost on training data: 121.98307386579589\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 122.85951896690838\n", + "Accuracy on evaluation data: 8706 / 10000\n", + "Epoch 319 training complete\n", + "Cost on training data: 121.47058097879379\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 122.34750139137397\n", + "Accuracy on evaluation data: 8722 / 10000\n", + "Epoch 320 training complete\n", + "Cost on training data: 120.97657562063837\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 121.84499009701167\n", + "Accuracy on evaluation data: 8733 / 10000\n", + "Epoch 321 training complete\n", + "Cost on training data: 120.47391993694356\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 121.34946935158412\n", + "Accuracy on evaluation data: 8722 / 10000\n", + "Epoch 322 training complete\n", + "Cost on training data: 119.97736628340792\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 120.84857898242143\n", + "Accuracy on evaluation data: 8727 / 10000\n", + "Epoch 323 training complete\n", + "Cost on training data: 119.48686095016149\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 120.3573682404226\n", + "Accuracy on evaluation data: 8731 / 10000\n", + "Epoch 324 training complete\n", + "Cost on training data: 118.99829949171993\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 119.86980515365235\n", + "Accuracy on evaluation data: 8733 / 10000\n", + "Epoch 325 training complete\n", + "Cost on training data: 118.5132831888578\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 119.39236830774443\n", + "Accuracy on evaluation data: 8721 / 10000\n", + "Epoch 326 training complete\n", + "Cost on training data: 118.03300826477859\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 118.9097973367445\n", + "Accuracy on evaluation data: 8737 / 10000\n", + "Epoch 327 training complete\n", + "Cost on training data: 117.56383533172436\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 118.44233740949386\n", + "Accuracy on evaluation data: 8727 / 10000\n", + "Epoch 328 training complete\n", + "Cost on training data: 117.09764675749541\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 117.97140024766638\n", + "Accuracy on evaluation data: 8729 / 10000\n", + "Epoch 329 training complete\n", + "Cost on training data: 116.64303163751788\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 117.51072128416818\n", + "Accuracy on evaluation data: 8728 / 10000\n", + "Epoch 330 training complete\n", + "Cost on training data: 116.18638954121458\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 117.06385083798648\n", + "Accuracy on evaluation data: 8705 / 10000\n", + "Epoch 331 training complete\n", + "Cost on training data: 115.73398069406579\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 116.6057690301439\n", + "Accuracy on evaluation data: 8726 / 10000\n", + "Epoch 332 training complete\n", + "Cost on training data: 115.28015541364582\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 116.16055469304042\n", + "Accuracy on evaluation data: 8714 / 10000\n", + "Epoch 333 training complete\n", + "Cost on training data: 114.83979059073609\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 115.71192600964213\n", + "Accuracy on evaluation data: 8722 / 10000\n", + "Epoch 334 training complete\n", + "Cost on training data: 114.4014183604674\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 115.26553621897676\n", + "Accuracy on evaluation data: 8732 / 10000\n", + "Epoch 335 training complete\n", + "Cost on training data: 113.95884919381142\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 114.83095109669779\n", + "Accuracy on evaluation data: 8730 / 10000\n", + "Epoch 336 training complete\n", + "Cost on training data: 113.52668403212395\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 114.39873651897652\n", + "Accuracy on evaluation data: 8730 / 10000\n", + "Epoch 337 training complete\n", + "Cost on training data: 113.10604465302974\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 113.9789739901931\n", + "Accuracy on evaluation data: 8715 / 10000\n", + "Epoch 338 training complete\n", + "Cost on training data: 112.68393580176843\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 113.54946963072105\n", + "Accuracy on evaluation data: 8726 / 10000\n", + "Epoch 339 training complete\n", + "Cost on training data: 112.2635296297063\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 113.13795797404451\n", + "Accuracy on evaluation data: 8717 / 10000\n", + "Epoch 340 training complete\n", + "Cost on training data: 111.84865539463958\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 112.71940663427014\n", + "Accuracy on evaluation data: 8729 / 10000\n", + "Epoch 341 training complete\n", + "Cost on training data: 111.44023970702192\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 112.30526795382791\n", + "Accuracy on evaluation data: 8746 / 10000\n", + "Epoch 342 training complete\n", + "Cost on training data: 111.03171341403386\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 111.90088190800971\n", + "Accuracy on evaluation data: 8731 / 10000\n", + "Epoch 343 training complete\n", + "Cost on training data: 110.63022181797568\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 111.49690369109585\n", + "Accuracy on evaluation data: 8731 / 10000\n", + "Epoch 344 training complete\n", + "Cost on training data: 110.23647593531734\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 111.09334426642403\n", + "Accuracy on evaluation data: 8748 / 10000\n", + "Epoch 345 training complete\n", + "Cost on training data: 109.83773742723704\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 110.70477305507984\n", + "Accuracy on evaluation data: 8729 / 10000\n", + "Epoch 346 training complete\n", + "Cost on training data: 109.44827619027838\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 110.31274817304575\n", + "Accuracy on evaluation data: 8735 / 10000\n", + "Epoch 347 training complete\n", + "Cost on training data: 109.05864589756922\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 109.92759800979896\n", + "Accuracy on evaluation data: 8732 / 10000\n", + "Epoch 348 training complete\n", + "Cost on training data: 108.6708832303437\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 109.54026548739286\n", + "Accuracy on evaluation data: 8737 / 10000\n", + "Epoch 349 training complete\n", + "Cost on training data: 108.29147239676392\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 109.16253575387928\n", + "Accuracy on evaluation data: 8726 / 10000\n", + "Epoch 350 training complete\n", + "Cost on training data: 107.9154732174377\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 108.77842254558615\n", + "Accuracy on evaluation data: 8739 / 10000\n", + "Epoch 351 training complete\n", + "Cost on training data: 107.54680346548534\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 108.40775195261295\n", + "Accuracy on evaluation data: 8746 / 10000\n", + "Epoch 352 training complete\n", + "Cost on training data: 107.17045296458343\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 108.04301818033696\n", + "Accuracy on evaluation data: 8723 / 10000\n", + "Epoch 353 training complete\n", + "Cost on training data: 106.8073478702147\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 107.67227818658797\n", + "Accuracy on evaluation data: 8741 / 10000\n", + "Epoch 354 training complete\n", + "Cost on training data: 106.44646067091344\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 107.3028765897326\n", + "Accuracy on evaluation data: 8749 / 10000\n", + "Epoch 355 training complete\n", + "Cost on training data: 106.08397702476185\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 106.94655444959778\n", + "Accuracy on evaluation data: 8739 / 10000\n", + "Epoch 356 training complete\n", + "Cost on training data: 105.73554038839815\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 106.59671074612005\n", + "Accuracy on evaluation data: 8739 / 10000\n", + "Epoch 357 training complete\n", + "Cost on training data: 105.37913259529881\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 106.24350398703528\n", + "Accuracy on evaluation data: 8733 / 10000\n", + "Epoch 358 training complete\n", + "Cost on training data: 105.03701077706633\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 105.89864331586588\n", + "Accuracy on evaluation data: 8740 / 10000\n", + "Epoch 359 training complete\n", + "Cost on training data: 104.6931944943798\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 105.55405539342767\n", + "Accuracy on evaluation data: 8738 / 10000\n", + "Epoch 360 training complete\n", + "Cost on training data: 104.34964773326813\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 105.21272553990029\n", + "Accuracy on evaluation data: 8740 / 10000\n", + "Epoch 361 training complete\n", + "Cost on training data: 104.01050921870427\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 104.86943780120198\n", + "Accuracy on evaluation data: 8743 / 10000\n", + "Epoch 362 training complete\n", + "Cost on training data: 103.67574406359458\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 104.53139990150672\n", + "Accuracy on evaluation data: 8748 / 10000\n", + "Epoch 363 training complete\n", + "Cost on training data: 103.33938027849409\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 104.20051905538996\n", + "Accuracy on evaluation data: 8742 / 10000\n", + "Epoch 364 training complete\n", + "Cost on training data: 103.01184903122954\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 103.86923438475914\n", + "Accuracy on evaluation data: 8739 / 10000\n", + "Epoch 365 training complete\n", + "Cost on training data: 102.68846046182193\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 103.54082672413065\n", + "Accuracy on evaluation data: 8753 / 10000\n", + "Epoch 366 training complete\n", + "Cost on training data: 102.35784771777288\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 103.21982971325806\n", + "Accuracy on evaluation data: 8749 / 10000\n", + "Epoch 367 training complete\n", + "Cost on training data: 102.04482658739934\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 102.89748985097862\n", + "Accuracy on evaluation data: 8739 / 10000\n", + "Epoch 368 training complete\n", + "Cost on training data: 101.7276178927609\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 102.57929348375416\n", + "Accuracy on evaluation data: 8756 / 10000\n", + "Epoch 369 training complete\n", + "Cost on training data: 101.41330512147384\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 102.27384413656318\n", + "Accuracy on evaluation data: 8737 / 10000\n", + "Epoch 370 training complete\n", + "Cost on training data: 101.11115443843724\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 101.96719885077736\n", + "Accuracy on evaluation data: 8747 / 10000\n", + "Epoch 371 training complete\n", + "Cost on training data: 100.80123586252398\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 101.65863952483006\n", + "Accuracy on evaluation data: 8734 / 10000\n", + "Epoch 372 training complete\n", + "Cost on training data: 100.49665639189158\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 101.35125935456337\n", + "Accuracy on evaluation data: 8735 / 10000\n", + "Epoch 373 training complete\n", + "Cost on training data: 100.19738035429397\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 101.04904303264655\n", + "Accuracy on evaluation data: 8759 / 10000\n", + "Epoch 374 training complete\n", + "Cost on training data: 99.89557694920671\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 100.75592799070584\n", + "Accuracy on evaluation data: 8742 / 10000\n", + "Epoch 375 training complete\n", + "Cost on training data: 99.59942574050527\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 100.45468895838482\n", + "Accuracy on evaluation data: 8755 / 10000\n", + "Epoch 376 training complete\n", + "Cost on training data: 99.30205632296797\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 100.16107502917133\n", + "Accuracy on evaluation data: 8732 / 10000\n", + "Epoch 377 training complete\n", + "Cost on training data: 99.01652042555641\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 99.87140948819584\n", + "Accuracy on evaluation data: 8742 / 10000\n", + "Epoch 378 training complete\n", + "Cost on training data: 98.72711943938577\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 99.58542527838058\n", + "Accuracy on evaluation data: 8747 / 10000\n", + "Epoch 379 training complete\n", + "Cost on training data: 98.44656680345635\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 99.30364412252871\n", + "Accuracy on evaluation data: 8744 / 10000\n", + "Epoch 380 training complete\n", + "Cost on training data: 98.1662738691718\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 99.0252085059032\n", + "Accuracy on evaluation data: 8742 / 10000\n", + "Epoch 381 training complete\n", + "Cost on training data: 97.89013107915832\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 98.74333253513667\n", + "Accuracy on evaluation data: 8756 / 10000\n", + "Epoch 382 training complete\n", + "Cost on training data: 97.61463990410955\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 98.46870164445117\n", + "Accuracy on evaluation data: 8750 / 10000\n", + "Epoch 383 training complete\n", + "Cost on training data: 97.33582717262179\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 98.20041106346012\n", + "Accuracy on evaluation data: 8734 / 10000\n", + "Epoch 384 training complete\n", + "Cost on training data: 97.06740208858464\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 97.92173670468\n", + "Accuracy on evaluation data: 8749 / 10000\n", + "Epoch 385 training complete\n", + "Cost on training data: 96.79775024216926\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 97.65682889024194\n", + "Accuracy on evaluation data: 8748 / 10000\n", + "Epoch 386 training complete\n", + "Cost on training data: 96.53302239724131\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 97.39041916318308\n", + "Accuracy on evaluation data: 8745 / 10000\n", + "Epoch 387 training complete\n", + "Cost on training data: 96.2702089974505\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 97.1303864792669\n", + "Accuracy on evaluation data: 8737 / 10000\n", + "Epoch 388 training complete\n", + "Cost on training data: 96.01246741390375\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 96.86708960403563\n", + "Accuracy on evaluation data: 8750 / 10000\n", + "Epoch 389 training complete\n", + "Cost on training data: 95.75499321842724\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 96.61168409297798\n", + "Accuracy on evaluation data: 8738 / 10000\n", + "Epoch 390 training complete\n", + "Cost on training data: 95.50381793609345\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 96.35222659156449\n", + "Accuracy on evaluation data: 8765 / 10000\n", + "Epoch 391 training complete\n", + "Cost on training data: 95.25173528813379\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 96.10381745897298\n", + "Accuracy on evaluation data: 8754 / 10000\n", + "Epoch 392 training complete\n", + "Cost on training data: 94.99597555143336\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 95.85094564648602\n", + "Accuracy on evaluation data: 8742 / 10000\n", + "Epoch 393 training complete\n", + "Cost on training data: 94.7480800086226\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 95.59945501319582\n", + "Accuracy on evaluation data: 8748 / 10000\n", + "Epoch 394 training complete\n", + "Cost on training data: 94.50371299067632\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 95.35738743272965\n", + "Accuracy on evaluation data: 8735 / 10000\n", + "Epoch 395 training complete\n", + "Cost on training data: 94.26392186740038\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 95.11843298540589\n", + "Accuracy on evaluation data: 8735 / 10000\n", + "Epoch 396 training complete\n", + "Cost on training data: 94.02594368516337\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 94.8745838454645\n", + "Accuracy on evaluation data: 8753 / 10000\n", + "Epoch 397 training complete\n", + "Cost on training data: 93.78360461539384\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 94.64456714896474\n", + "Accuracy on evaluation data: 8729 / 10000\n", + "Epoch 398 training complete\n", + "Cost on training data: 93.54867103529122\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 94.39627555487957\n", + "Accuracy on evaluation data: 8765 / 10000\n", + "Epoch 399 training complete\n", + "Cost on training data: 93.31607889277592\n", + "Accuracy on training data: 1000 / 1000\n", + "Cost on evaluation data: 94.16415429831616\n", + "Accuracy on evaluation data: 8755 / 10000\n" + ] + } + ], + "source": [ + "from elements import network2\n", + "from elements import mnist_loader\n", + "\n", + "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()\n", + "training_data = list(training_data)\n", + "test_data = list(test_data)\n", + "\n", + "net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)\n", + "net.large_weight_initializer()\n", + "evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \\\n", + " net.SGD(training_data[:1000], 400, 10, 0.5,\n", + " evaluation_data=test_data,\n", + " lmbda = 0.1, # this is a regularization parameter\n", + " monitor_evaluation_cost=True,\n", + " monitor_evaluation_accuracy=True,\n", + " monitor_training_cost=True,\n", + " monitor_training_accuracy=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 368 + }, + "id": "sR0mHDLFQXyW", + "outputId": "688698c1-a56d-47e5-ccb3-81ee70b1f867" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 0, 'Epoch')" + ] + }, + "metadata": {}, + "execution_count": 18 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFNCAYAAAAO36SFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wU9f3H8deHKxxwdI6jdxAOQeAAxUKxBRvYFRXFEiwhJlGjorFEjRqJxhixa7CjWBJUFNGAIjaKFOlVehNBTurB5/fHDvmt591xffb23s/HYx/sfOc7u++dWz47+53ZGXN3REQkflUKO4CIiJQuFXoRkTinQi8iEudU6EVE4pwKvYhInFOhFxGJcyr0ErfMbJKZXRF2jgPM7Akzu62k+xaXma0ws+PL4rkkHCr0AoCZXWBm08wsy8zWmdn7ZnZ0MR+zzAqImd1pZi+V4uMX+7W4+1XufndJ9y1LZuZm1ibsHFI4KvSCmV0HPAzcC6QDzYDHgIFh5ipPzCwx7AwieXJ33SrwDagJZAHn5NOnMpEPgrXB7WGgcjCvHvAusBXYAkwmsgHxIrAf2Bk8/o15PPavgSXBsmOBRlHzHLgKWBw8/kjAcnmM/sAeYG/wXLOC9knA3cAUYDvwIVAvarkjgM+Dx54F9M0j4y9eC9AiyHc5sBL4NOg7BlgPbAM+BTpGPc4o4J7gfl9gNXA9sBFYB1xaxL51gXeAH4GpwD3AZ/n8PQcD3wHfA7cCK4Djg3k9gS+CdbIOeBRIDuZ9Grzmn4L1cB5QO/j7bwJ+CO43Cft9rVuOv3nYAXQL+Q0QKZLZQGI+fe4CvgTqA2lBcbw7mHcf8ASQFNyOOVCMowtIHo97LLAZ6Ebkw+SfBwpmMN+DwlGLyLeMTUD/PB7rTuClHG2TgKVAO6BKMH1/MK9xUOhOJvLBdEIwnZbH4//stfD/hf4FoBpQJWi/DKjO/384zoxaJmfxzg7WbVKQYwdQuwh9Rwe3qkAGsIo8Cn0wPwvoHWR8KHjsA4U+k8gHYGLwGucDv8/xN2kTNV0XOCt47upEPuj+Hfb7Wref3zR0I3WBze6enU+fC4G73H2ju28C/kxkqxAiW9ENgebuvtfdJ3tQAQrgQuA5d5/h7ruB4UAvM2sR1ed+d9/q7iuBiUCXAr+yiH+5+yJ33wm8HrX8RcA4dx/n7vvdfQIwjUgRLYw73f2n4PFx9+fcfXvweu4EDjOzmnksu5fIet3r7uOIFOBDCtPXzBKIFNo73H2Hu88Dns8n79nAu+7+aZDxNiLfVgjyT3f3L909291XAE8CffJ6MHf/3t3fDJ57O/CX/PpLOFTo5Xug3kHGmBsR+ap/wHdBG8AIIkMvH5rZMjO7uRDP/bPHdfesIE/jqD7ro+7vAFIL8fj5Ld8cOMfMth64AUcT+dAqjFUH7phZgpndb2ZLzexHIt8CIDK8lZvvc3zA5vf68uqbRmTre1XUvOj7OTWKnu/uPxFZ5wdeQzsze9fM1gev4d588mNmVc3sSTP7Luj/KVAr+ACSGKFCL18Au4HT8+mzlkhhPKBZ0Eaw9Xq9u7cCBgDXmdlxQb+Dbdn/7HHNrBqRbxhrCvUKCvZcOa0CXnT3WlG3au5+fyEfP7r9AiI7sI8nsu+jRdBuhcxWGJuIDL00iWprmk//ddHzzawqkXV+wOPAAqCtu9cAbiH//NcT+RZyeNC/94GHLugLkNKnQl/Bufs24HZgpJmdHmyhJZnZSWb2QNDtVeBPZpZmZvWC/i8BmNmpZtbGzIzIDsh9/P9QwAagVT5P/ypwqZl1MbPKRLYevwqGDAprA9DCzAr6nn4JOM3MfhVsiaeYWV8za5JH/4O9FoiMUe8msoVclcjrKVXuvg94C7gz+Nu1By7OZ5E3gFPN7GgzSyYy7h+9zqoT2ambFTzW1TmWz7keqhPZSb3VzOoAdxTrBUmpUKEX3P1B4DrgT0S2EFcBw4B/B13uITJ+PRuYA8wI2gDaAh8RGTP+AnjM3ScG8+4j8gGx1cxuyOV5PyIyRvwmkS3N1sD5RXwZY4J/vzezGQfr7O6riGx938L/v+Y/kvf/iXxfS+AFIkNRa4B5RHZgl4VhRL5BrCdyhNCrRD5wfsHd5wK/AV4hss5/IHJEzwE3EPlmsh14Gngtx0PcCTwfrIdziexwrkJkp/qXwAcl8oqkRB04OkJE4oSZ/RVo4O6XhJ1FYoO26EXKOTNrb2adLaInkWP73w47l8QO/ZpPpPyrTmS4phGRMfQHgf+EmkhiioZuRETinIZuRETinAq9iEici7kx+nr16nmLFi2KvPxPP/1EtWrVSi5QCVGuwonVXBC72ZSrcGI1FxQt2/Tp0ze7e1quMwtyQhwiJ75aSOSn7jfnMv86IscNzwY+JnLek+j5NYgcq/vowZ4rMzPTi2PixInFWr60KFfhxGou99jNplyFE6u53IuWDZjmRT2pWXDOipHASUTOfDfIzDJydPsG6O7unYn88u6BHPPvJnIODBERKWMFGaPvCSxx92XuvofI6VB/dkEKd5/o7juCyS+JOu+GmWUSuZjFhyUTWURECqMghb4xPz8b3mp+fnbBnC4H3gcIzjvyIJGfVYuISAgOehy9mZ1N5GIPVwTTg4mcqW5YLn0vInLejT7uvtvMhgFV3f0BMxtCZHgnt+WGAkMB0tPTM0ePHl3kF5SVlUVqamHPZFv6lKtwYjUXxG425SqcWM0FRcvWr1+/6e7ePdeZeQ3e+//vSO0FjI+aHg4Mz6Xf8USuRlM/qu1lIpdZW0HkpEc/ElzhJ6+bdsaWLeUqvFjNplyFE6u53Et+Z2xBDq+cCrQ1s5ZEzsp3PpGz2/2PmXUlciWa/u6+MepD5MKoPkOIbNEX5sIUIiJSTAcdo/fIVW2GAeOJbLG/7u5zzewuMxsQdBtB5Go3Y8xsppmNLbXEIiJSKAX6wZRHrlE5Lkfb7VH3jy/AY4wicsFjEREpQzoFgohInIu5UyAU1d59+xk9dRXVd+0/eGcRkQokbrbo123dxV3vzOXtJXvDjiIiElPiptA3q1uVwUe04NPV2Sxcvz3sOCIiMSNuCj3Ab49tQ0oi3Pf+/LCjiIjEjLgq9LWrJTOgdTKTFm7is8Wbw44jIhIT4qrQAxzXLJHGtapw77j57N+vyySKiMRdoU9OMG7sfwjz1v3I29+sCTuOiEjo4q7QA5zWuRGdm9Tkbx8uZNfefWHHEREJVVwW+kqVjFtO7sC6bbt49rPlYccREQlVXBZ6gCNa1eX4Duk8Pmkpm7N2hx1HRCQ0cVvoAW4+qT079+7jkY8Xhx1FRCQ0cV3o29RPZVDPprz81UqWbsoKO46ISCjiutAD/P74dqQkVuK+cfoRlYhUTHFf6OulVmbYsW35aP5GJi/eFHYcEZEyF/eFHuCyo1vQrE5V7n53Htn7dHZLEalYKkShr5yYwC0nd2DRhixe+Xpl2HFERMpUhSj0AL/qmE6vVnV5aMIitu7YE3YcEZEyU2EKvZlx+2kZ/LhzLw9/pMMtRaTiqDCFHqBDwxqc37MZL375HUs26pz1IlIxVKhCD3D9Ce2ompzA3e/qcEsRqRgqXKGvm1qZ3x3Xlk8WbWLigo1hxxERKXUVrtADXNyrBa3qVePu9+axV4dbikicq5CFPjmxEn86tQPLNv3E85+vCDuOiEipqpCFHqDfIfXp0y6Nf3y0mI3bd4UdR0Sk1FTYQm9m3HFaBruy93H/+wvCjiMiUmoKVOjNrL+ZLTSzJWZ2cy7zrzOzeWY228w+NrPmQXsXM/vCzOYG884r6RdQHK3SUvn1Ma14a8Yapq7YEnYcEZFScdBCb2YJwEjgJCADGGRmGTm6fQN0d/fOwBvAA0H7DuBid+8I9AceNrNaJRW+JAw7tg2NaqZw27+/1XlwRCQuFWSLviewxN2XufseYDQwMLqDu0909x3B5JdAk6B9kbsvDu6vBTYCaSUVviRUTU7ktlMzWLB+Oy99+V3YcURESpy5e/4dzM4G+rv7FcH0YOBwdx+WR/9HgfXufk+O9p7A80BHd9+fY95QYChAenp65ujRo4v4ciArK4vU1NRCLePu/G3aLpZt28/9x1SlZmUr8vOXZK6yoFyFF6vZlKtwYjUXFC1bv379prt791xnunu+N+Bs4Jmo6cHAo3n0vYjIFn3lHO0NgYXAEQd7vszMTC+OiRMnFmm5JRu3e5tb3vPrX59ZrOfPS1FzlTblKrxYzaZchROrudyLlg2Y5nnU1YIM3awBmkZNNwnafsbMjgduBQa4++6o9hrAe8Ct7v5lAZ4vFK3TUrn86Fa8MX0107/7Iew4IiIlpiCFfirQ1sxamlkycD4wNrqDmXUFniRS5DdGtScDbwMvuPsbJRe7dPz22DY0DHbM7tuf/5CWiEh5cdBC7+7ZwDBgPDAfeN3d55rZXWY2IOg2AkgFxpjZTDM78EFwLtAbGBK0zzSzLiX/MkpGtcqJ3HpKB+at+5GXv9KOWRGJD4kF6eTu44BxOdpuj7p/fB7LvQS8VJyAZe2UTg15tc1KRoxfSP9DG1C/ekrYkUREiqXC/jI2L2bGXQMPZffe/dyjUxmLSBxQoc9F67RUru7bmrGz1vLpok1hxxERKRYV+jxc3bc1LetV40///pZde/eFHUdEpMhU6POQkpTAX04/lJVbdvDof5eEHUdEpMhU6PNxZJt6nNm1MU9+ulTXmBWRckuF/iBuOaUDVZMTueXtbw/8yldEpFxRoT+IeqmVGX5Se75evoUx01eHHUdEpNBU6Avg3O5N6d68NveNm8+Wn/aEHUdEpFBU6AugUiXj3jM7sX1XNn95T8fWi0j5okJfQO3SqzO0dyvenLGazxZvDjuOiEiBqdAXwrXHtaVVvWrc/NZsduzJDjuOiEiBqNAXQkpSAvef1ZnVP+zkwQ8XhR1HRKRAVOgLqWfLOgw+ojnPTVnOjJU6b72IxD4V+iK4sf8hNKiRws1vzmZPti4oLiKxTYW+CKqnJPGXMw5l0YYsHpuk0yOISGxToS+iY9unc3qXRoycuIRFG3R6BBGJXSr0xXD7aR2pnpLEjW/M1qUHRSRmqdAXQ51qydxxWgYzV21l1Ocrwo4jIpIrFfpiGnBYI45rX5+/jV/Id9//FHYcEZFfUKEvJjPjL2d0IjHB+OOY2ezXEI6IxBgV+hLQoGYKd57Wka9XbNEQjojEHBX6EnJmt8Yc36E+D4xfwLJNWWHHERH5HxX6EmJm3HtGJyonJnDDmFk6CkdEYoYKfQmqXyOFuwZ2ZMbKrTz72bKw44iIACr0JW7AYY34Vcd0/vbhIl1nVkRiQoEKvZn1N7OFZrbEzG7OZf51ZjbPzGab2cdm1jxq3iVmtji4XVKS4WORmXHP6Z2olpzA9WNmk71P58IRkXAdtNCbWQIwEjgJyAAGmVlGjm7fAN3dvTPwBvBAsGwd4A7gcKAncIeZ1S65+LEprXpl7j79UGat2spTkzWEIyLhKsgWfU9gibsvc/c9wGhgYHQHd5/o7juCyS+BJsH9XwET3H2Lu/8ATAD6l0z02HZq50ac0qkhD09YzPx1P4YdR0QqsIIU+sbAqqjp1UFbXi4H3i/isnHl7tMPpUaVJP7w2kz27NNROCISjsSSfDAzuwjoDvQp5HJDgaEA6enpTJo0qcgZsrKyirV8SRt8CPx9+nZGJzjJCZPCjvMLsba+DojVXBC72ZSrcGI1F5RCNnfP9wb0AsZHTQ8HhufS73hgPlA/qm0Q8GTU9JPAoPyeLzMz04tj4sSJxVq+NNz69mxvcdO7PmXJprCj/EIsri/32M3lHrvZlKtwYjWXe9GyAdM8j7pakKGbqUBbM2tpZsnA+cDY6A5m1jUo4gPcfWPUrPHAiWZWO9gJe2LQVqHcenIG6VWNG16fxbade8OOIyIVzEELvbtnA8OIFOj5wOvuPtfM7jKzAUG3EUAqMMbMZprZ2GDZLcDdRD4spgJ3BW0VSpXkBIYeVpmN23dz+3++DTuOiFQwBRqjd/dxwLgcbbdH3T8+n2WfA54rasB40apmAtce15aHJiziuA7pDDisUdiRRKSC0C9jy9A1fVvTtVkt/vT2HNZu3Rl2HBGpIFToy1BiQiX+fm4Xsvc7N4yZpXPXi0iZUKEvYy3qVeO2UzP4fOn3PK1fzYpIGVChD8H5PZrSv2MDRoxfyKxVW8OOIyJxToU+BGbG/Wd1on71ylw7+hu279IhlyJSelToQ1KrajL/GNSVVVt2cPt/5oYdR0TimAp9iHq0qMPvjmvH29+s4c3pq8OOIyJxSoU+ZMOObUPPlnW47T/f6lqzIlIqVOhDllDJ+Mf5XUhOrMS1o79hd/a+sCOJSJxRoY8BDWtW4a9ndebbNT8y4oOFYccRkTijQh8jftWxAYOPaM4zny3nvws2hB1HROKICn0MufWUDrRvUJ3rXp/FGp0iQURKiAp9DElJSuCxC7uRvc8Z9soM9mTrwuIiUnwq9DGmVVoq95/ViW9WbuWBDxaEHUdE4oAKfQw6tXMjLukVGa8fP3d92HFEpJxToY9Rt5zSgc5NanLDmFms/H5H2HFEpBxToY9RlRMTGHlBNwB+88oMHV8vIkWmQh/DmtapyoPnHMacNdu49735YccRkXJKhT7GndixAb8+piXPf/Ed78xaG3YcESmHVOjLgRv7tyezeW1uenM2izZsDzuOiJQzKvTlQFJCJR67sBvVKidy5YvT+VHnrxeRQlChLyfSa6Qw8oJurNqyg+te0/VmRaTgVOjLkZ4t63DrKR34aP4GHpu0JOw4IlJOqNCXM0OObMHpXRrx4IRFfLJoU9hxRKQcUKEvZ8yM+87szCHp1bn21W9YtUU/phKR/KnQl0NVkhN4cnAm7s6VL05n5x79mEpE8lagQm9m/c1soZktMbObc5nf28xmmFm2mZ2dY94DZjbXzOab2SNmZiUVviJrXrca/xjUlfnrf2T4W7Nx185ZEcndQQu9mSUAI4GTgAxgkJll5Oi2EhgCvJJj2SOBo4DOwKFAD6BPsVMLAP0Oqc/1J7Tj3zPX8tSny8KOIyIxKrEAfXoCS9x9GYCZjQYGAvMOdHD3FcG8nCdQdyAFSAYMSAJ0+aQS9Jt+bZi/fjv3f7CAdunV6de+ftiRRCTGFGTopjGwKmp6ddB2UO7+BTARWBfcxru7TtpSgsyMEWd3JqNhDa599RuWbMwKO5KIxBg72NhuMObe392vCKYHA4e7+7Bc+o4C3nX3N4LpNsA/gPOCLhOAG919co7lhgJDAdLT0zNHjx5d5BeUlZVFampqkZcvLaWd6/ud+/nzFzupkmjc3qsK1ZIKtiukoq6v4ojVbMpVOLGaC4qWrV+/ftPdvXuuM9093xvQi8iW+IHp4cDwPPqOAs6Omv4jcFvU9O1ECn2ez5eZmenFMXHixGItX1rKItfU5d97m1ve84ue+dL3Zu8r0DIVeX0VVaxmU67CidVc7kXLBkzzPOpqQYZupgJtzaylmSUD5wNjC/ghsxLoY2aJZpZEZEeshm5KSfcWdbjn9EOZvHgz97+vyxCKSMRBC727ZwPDgPFEivTr7j7XzO4yswEAZtbDzFYD5wBPmtncYPE3gKXAHGAWMMvd3ymF1yGB83o0Y8iRLXjms+WMmbbq4AuISNwryFE3uPs4YFyOttuj7k8FmuSy3D7gymJmlEL60ykdWLIxi1venkOzOlU5vFXdsCOJSIj0y9g4lJhQiZEXdqNZnapc+dJ0lm/+KexIIhIiFfo4VbNKEs8N6UElMy4bNZWtO/aEHUlEQqJCH8ea163GU4MzWfPDTq58cTp7snP+nk1EKgIV+jjXvUUdRpzTma+Wb2H4W3N0ThyRCqhAO2OlfBvYpTHLN//Ewx8tplVaNX7Tr03YkUSkDKnQVxC/O64tyzf/xIjxC2lRtxqndG4YdiQRKSMq9BWEmfHXszqz5oed/OH1mdSvUZkeLeqEHUtEyoDG6CuQlKQEnr64O01qVeGK56exZOP2sCOJSBlQoa9galdL5vnLepKUUIlLnpvKD7t0JI5IvFOhr4Ca1qnKqEt7sHXHHh6avpsfd+0NO5KIlCIV+grq0MY1efyiTNZm7ecqHWMvEtdU6Cuw3u3SuOzQZD5f+j1/fGMW+/frGHuReKSjbiq4oxonUatRS0aMX0iDmikMP6lD2JFEpISp0AvX9G3Num07efKTZTSokcKlR7UMO5KIlCAVesHM+POAQ9n4427uenceadUrc2rnRmHHEpESojF6ASChkvHIoK5kNqvNH16byaSFG8OOJCIlRIVe/iclKYFnh/Sgbf3qXPXSdKau2BJ2JBEpASr08jM1qyTxwuU9aVSzCpeNmsrctdvCjiQixaRCL79QL7UyL15xONUrJ3Lxs1+zbFNW2JFEpBhU6CVXjWtV4aUrDgfgome+Ys3WnSEnEpGiUqGXPLVKS+WFy3uyfXc2g5/5is1Zu8OOJCJFoEIv+erYqCb/GtKDtdt2cvGzX7Ntp86LI1LeqNDLQXVvUYcnB3dn8cbtXD5qKjv2ZIcdSUQKQYVeCqRPuzQeOb8rM1b+wGWjprJzz76wI4lIAanQS4Gd1Kkhfz+vC18v38KvX5jGrr0q9iLlgQq9FMrALo0ZcfZhTFm6maEvTlexFykHClTozay/mS00syVmdnMu83ub2Qwzyzazs3PMa2ZmH5rZfDObZ2YtSia6hOWszCb89czOfLpoE1e/NJ3d2Sr2IrHsoIXezBKAkcBJQAYwyMwycnRbCQwBXsnlIV4ARrh7B6AnoJOoxIFzezTl3jM6MXHhJn7z8je6cIlIDCvIFn1PYIm7L3P3PcBoYGB0B3df4e6zgZ/9bw8+EBLdfULQL8vdd5RMdAnbBYc34+7TD+Wj+Rv47asz2LtPxV4kFhWk0DcGVkVNrw7aCqIdsNXM3jKzb8xsRPANQeLE4COac+dpGYyfu4Hfj55Jtoq9SMwx9/wvHxeMufd39yuC6cHA4e4+LJe+o4B33f2NqGWfBboSGd55DRjn7s/mWG4oMBQgPT09c/To0UV+QVlZWaSmphZ5+dIS77nGr9jLqwv20LNBAkM7VyaxksVErtIQq9mUq3BiNRcULVu/fv2mu3v3XGe6e743oBcwPmp6ODA8j76jgLOjpo8APomaHgyMzO/5MjMzvTgmTpxYrOVLS0XI9eQnS7z5Te/6lS9M89179xXrsWJ1fbnHbjblKpxYzeVetGzANM+jrhZk6GYq0NbMWppZMnA+MLaAHzJTgVpmlhZMHwvMK+CyUs4M7d2a20/N4IO567n6JR16KRIrDlro3T0bGAaMB+YDr7v7XDO7y8wGAJhZDzNbDZwDPGlmc4Nl9wE3AB+b2RzAgKdL56VILLjs6Jbcc/qhfLxgo35UJRIjCnTNWHcfB4zL0XZ71P2pQJM8lp0AdC5GRilnLjqiOckJlbjprdlc+q+pPDukO1WTdXlikbDol7FSKs7t0ZSHzj2Mr5Z/zyXPfc32XTrrpUhYVOil1JzRtQmPDOrKjJVbGaxTHIuERoVeStWpnRvx2IXdmLt2Gxc+8yU//LQn7EgiFY4KvZS6X3VswFODu7NoQxbnPvkF67ftCjuSSIWiQi9lol/7+jx/aU/WbdvFWY9/rguOi5QhFXopM71a1+XVXx/Bzr37OOeJL/h2zbawI4lUCCr0UqY6NanJmKt6kZKUwKCnvuTLZd+HHUkk7qnQS5lrnZbKG1f3Ir1mChc/9zUT5m0IO5JIXFOhl1A0rFmF16/sRYcG1bnqpem8OX112JFE4pYKvYSmTrVkXv71EfRqVZfrx8zimcnLwo4kEpdU6CVUqZUTeXZId07u1IB73pvPX96bx/79+Z86W0QKRycgkdBVTkzgn4O6kZY6l6cnL2fttl0MTFexFykpKvQSExIqGXcO6Ejj2lW4d9wCFteuRM9ee6hVNTnsaCLlnoZuJGaYGUN7t+afg7qybOt+zn7iC1b/oEsMixSXCr3EnNMOa8Qfe6Sw8cddnPHY5/phlUgxqdBLTDqkTgJvXn0kyQmVOPfJL5i0cGPYkUTKLRV6iVlt06vz1jVH0qJuNS5/fhqjv14ZdiSRckmFXmJaeo0UXr+qF0e1qcfNb83hnnfnsU+HX4oUigq9xLzUyok8d0l3hhzZgmc+W84Vz0/VFatECkGFXsqFxIRK3DmgI38541AmL97MmY99zsrvdUSOSEGo0Eu5cuHhzXnh8p5s3L6bgSM/09kvRQpAhV7KnSNb1+M/vzmKOtWSueiZr7STVuQgVOilXGpRrxpvXXMURwY7ae96RztpRfKiQi/lVs0qSf/bSfvclOVcNmoqP2onrcgvqNBLuXZgJ+29Z3RiypLNnDFyCkt1PVqRn1Ghl7hwweHNePHyw9m6Yy8DH53C+Lnrw44kEjMKVOjNrL+ZLTSzJWZ2cy7ze5vZDDPLNrOzc5lfw8xWm9mjJRFaJDe9Wtflnd8eTeu0alz54nT+Nn6hxu1FKEChN7MEYCRwEpABDDKzjBzdVgJDgFfyeJi7gU+LHlOkYBrVqsJrV/bivO5NeXTiEi4bNZWtO/aEHUskVAXZou8JLHH3Ze6+BxgNDIzu4O4r3H02sD/nwmaWCaQDH5ZAXpGDSklK4P6zOnHvGZ34fOlmTnv0M+at/THsWCKhMff8v9oGQzH93f2KYHowcLi7D8ul7yjgXXd/I5iuBPwXuAg4Huiex3JDgaEA6enpmaNHjy7yC8rKyiI1NbXIy5cW5Sqcksq1ZOs+Hv1mNzv2OpceWplejYp/rZ14X2clTbkKryjZ+vXrN93du+c6093zvQFnA89ETQ8GHs2j7yjg7KjpYcCNwf0heS0XfcvMzPTimDhxYrGWLy3KVTglmWvjj7v8nMc/9+Y3veu3/3uO79qbXazHqwjrrCQpV+EVJRswzfOoqwXZvFkDNI2abhK0FUQv4BgzuwZIBZLNLMvdf7FDV6S0pFWvzMu/Ppz731/As58t55tVWxl5QTea1qkadjSRMlGQMfqpQFsza2lmycD5wNiCPLi7X+juzdy9BXAD8IKKvIQhKaESt52awRMXZbJ880+c8shkHYIpFcZBC727ZwLPZmoAABYHSURBVBMZghkPzAded/e5ZnaXmQ0AMLMeZrYaOAd40szmlmZokaLqf2gDxl17DC3qRQ7BvPvdeezJ/sUxBCJxpUB7ptx9HDAuR9vtUfenEhnSye8xRhEZwxcJVdM6VRlzVS/uGxcZypn+3Q88ekFXmtTWUI7EJ/0yViqkyokJ3DmgI49d2I2lG7M45ZHP+Hj+hrBjiZQKFXqp0E7u1JB3fns0TWpX4fLnp3Hn2Lns2rsv7FgiJUqFXiq8FvWq8ebVRzLkyBaM+nwFp4+cwsL128OOJVJiVOhFiPya9s4BHfnXpT3YnLWbAY9+xvOfrzjwexCRck2FXiRKv0Pq8/7vetOrdV3uGDuXK56fxuas3WHHEikWFXqRHNKqV+ZfQ3pw52kZTF6ymf4PT+aTRZvCjiVSZCr0IrkwM4Yc1ZKxw46iTrUkLnnua+56Z5521Eq5pEIvko/2DWowdtjRXNKrOc9NWc7pI6ewart+YCXliwq9yEGkJCXw54GH8q8hkR21d36+k5ETl5C9TwVfygcVepEC6te+PuN/35vM9ARGjF/IWY9/zuINOgxTYp8KvUgh1E2tzDVdUnj0gq6s3LKDU/75GU9+slSXLJSYpkIvUgSndm7Eh3/oQ79D0rjv/QWc/cTnLN2UFXYskVyp0IsUUVr1yjxxUSb/OL8Lyzb9xMn/mMwzk5dp615ijgq9SDGYGQO7NGbCH3pzTNt63PPefM5/6guWb/4p7Ggi/6NCL1IC6tdI4emLu/PgOYexYP12fvXwp4ycuIS9OjJHYoAKvUgJMTPOymzCR9f14bj29RkxfiGn/fMzvln5Q9jRpIJToRcpYek1Unj8okyeGpzJ1h17OfPxz7lz7FyydmeHHU0qKBV6kVJyYscGTLiuNxcf0Zznv1jBiQ99ooubSChU6EVKUfWUJP488FDeuOpIUlMSufz5afzm5Rls3L4r7GhSgajQi5SBzOa1efe3x3DDie2YMH8Dxz/4Ca98tZL9OhRTyoAKvUgZSU6sxLBj2/LB744ho1ENbnl7Dmc8/jmzV28NO5rEORV6kTLWKi2VV399BA+f14W1W3cycOQUbn17Dlt37Ak7msQpFXqREJgZp3dtzMfX9+HSI1syeuoqjn3wE16bquEcKXkq9CIhqpGSxO2nZfDub4+mdVo1bnpzDmc98TnfrtkWdjSJIyr0IjGgQ8MavH5lLx469zBWbdnBgEc/Y/hbc3S9WikRBSr0ZtbfzBaa2RIzuzmX+b3NbIaZZZvZ2VHtXczsCzOba2azzey8kgwvEk/MjDO7NeHj6/sy5MiWjJm2ir4jJvH4pKW6hKEUy0ELvZklACOBk4AMYJCZZeTothIYArySo30HcLG7dwT6Aw+bWa3ihhaJZzWrRIZzPvxDb45oVYe/frCAE/7+Ce/NXoe7xu+l8AqyRd8TWOLuy9x9DzAaGBjdwd1XuPtsYH+O9kXuvji4vxbYCKSVSHKRONcqLZVnLunBy1ccTrXkRH7zygzOffILHY4phVaQQt8YWBU1vTpoKxQz6wkkA0sLu6xIRXZUm3q8d+0x3HdmJ5Zv/okBj07hutdnsn6bfl0rBWMH+yoYjLn3d/crgunBwOHuPiyXvqOAd939jRztDYFJwCXu/mUuyw0FhgKkp6dnjh49ukgvBiArK4vU1NQiL19alKtwYjUXhJttZ7bzztK9fLhiL5UqwcktkzipRRKVEy1m15lyFV5RsvXr12+6u3fPdaa753sDegHjo6aHA8Pz6DsKODtHWw1gRs72vG6ZmZleHBMnTizW8qVFuQonVnO5x0a2ld//5Ne8NN2b3/SuH/6Xj/zVr77zjz7+b9ixchUL6ys3sZrLvWjZgGmeR10tyNDNVKCtmbU0s2TgfGBsQT5hgv5vAy94jq18ESm6pnWqMvLCboy5qhcNaqZw81tz+NOUnXzwrXbYyi8dtNC7ezYwDBgPzAded/e5ZnaXmQ0AMLMeZrYaOAd40szmBoufC/QGhpjZzODWpVReiUgF1KNFHd6+5kieuCgTgKtemsEZj33OF0u/DzmZxJLEgnRy93HAuBxtt0fdnwo0yWW5l4CXiplRRPJhZvQ/tAFJG6uwuXpr/j5hMYOe/pI+7dK4qX97MhrVCDuihEy/jBWJEwmVjPN6NGPSH/sy/KT2zFy1lVP+OZlrX/2GpZuywo4nIVKhF4kzKUkJXNmnNZ/e2I+r+7RmwrwNnPDQJ1z32kxWbP4p7HgSAhV6kThVs0oSN/Zvz+Sb+nHFMa0Y9+06jnvoE258YxartuwIO56UIRV6kThXL7Uyt5zcgU9v7MfFvZrz75lr6fe3SQx/aw5rtu4MO56UARV6kQqifvUU7jitI5/+sR8XHN6MN6evpu+Iidz2729Zt00FP56p0ItUMA1qpnDXwEOZ+Me+nNO9Ka9+vZI+D0S28Fd+ryGdeKRCL1JBNa5VhXvP6MTEG/pyTvcmvDl9Nf0enMQfXpvJog3bw44nJUiFXqSCa1qnKn85oxOTb+rHpUe24INv13Pi3z/lyhenMWe1rnQVDwr0gykRiX/pNVL406kZXNOvDaOmLGfU5ysYP3cDvdulcXWf1hzRqg5mFnZMKQJt0YvIz9Splsx1Jx7ClJuP5ab+7Zm3dhuDnv6SAY9OYeystWTv23/wB5GYokIvIrmqnpLE1X1b89lNx3LfmZ34aU821776DX1GTOLZz5aTtTs77IhSQCr0IpKvlKQEBvVsxkd/6MMzF3ence0q3P3uPHrd9zH3vT9fF0ApBzRGLyIFUqmScXxGOsdnpDNz1VaenryMpz9dxrOTlzOgSyN+fUwrOjTUCdRikQq9iBRal6a1GHlBN1Zt2cFzU5bz2tRVvDVjDce0rcdlR7WkT7s0KlXSjttYoUIvIkXWtE5V7jitI78/rh0vf/0do6as4NJRU2lZrxoX92rO2Zm/OHu5hEBj9CJSbDWrJnFN3zZ8dtOxPDKoK7WrJvHnd+ZxxL0f89K83SzTaZJDpS16ESkxyYmVGHBYIwYc1ohZq7by/OcrGDtzDR89+Al92qUx5MgW9G6XRoKGdcqUCr2IlIrDmtbiofO60KfWD3yX2IQXv/yOS0dNpXGtKpzfoynn9mhKeo2UsGNWCCr0IlKqalY2ru3blquCi6C88vV3PDhhEQ9/vJjj2tfngsOb0butdt6WJhV6ESkTyYmVOKVzQ07p3JDlm39i9NcrGTN9NR/O20CT2lUY1LMZ53RvQv3q2sovadoZKyJlrmW9agw/uQNfDD+Wfw7qStPaVRkxfiFH3vdfrn5pOpMXb2L/fg87ZtzQFr2IhKZyYgKnHdaI0w5rxLJNWbz69UremL6a979dT5PaVTg7swlndWtC0zpVw45arqnQi0hMaJWWyq2nZHD9iYcwfu56xkxbzT8+XszDHy3myNZ1Oad7E/p3bEiV5ISwo5Y7KvQiElNSkhIY2KUxA7s0ZvUPO3hrxhrGTF/FH16bxe2V53LqYQ05p3tTujatpdMmF5AKvYjErCa1q3LtcW0Z1q8NXy3fwpjpq/j3N2t59etVtE6rxtmZTRnYpRGNalUJO2pMU6EXkZhXqZLRq3VderWuy10Ds3lv9lrGTFvNXz9YwAPjF3BEy7qc0a0xJx3agOopSWHHjTkFOurGzPqb2UIzW2JmN+cyv7eZzTCzbDM7O8e8S8xscXC7pKSCi0jFlFo5kfN6NOONq4/kkz/25XfHtWXdtp3c+MZsut/zEcNemcHH8zewJ1sXSDngoFv0ZpYAjAROAFYDU81srLvPi+q2EhgC3JBj2TrAHUB3wIHpwbI/lEx8EanImtetxu+Pb8fvjmvLN6u28vaMNbwzey3vzl5HjZRETuzYgFM6N+ToNvVISqi4R5MXZOimJ7DE3ZcBmNloYCDwv0Lv7iuCeTk/Qn8FTHD3LcH8CUB/4NViJxcRCZgZ3ZrVpluz2tx2agZTlmzmndlrGf/tet6YvppaVZP4VUak6B/Zui6JFazoF6TQNwZWRU2vBg4v4OPntmzjAi4rIlJoyYmV6Ne+Pv3a12d39j4mL9rMe3PW8d6cdbw2bRW1qybR/9CGNPV9HL1vf4Uo+uae/6/PgjH3/u5+RTA9GDjc3Yfl0ncU8K67vxFM3wCkuPs9wfRtwE53/1uO5YYCQwHS09MzR48eXeQXlJWVRWpqapGXLy3KVTixmgtiN5ty5W/PPmfO5n1MXZ/NNxv3sXsf1EiG7umJ9GiQyCF1KlEpRg7XLMo669ev33R3757bvIJs0a8BmkZNNwnaCmIN0DfHspNydnL3p4CnALp37+59+/bN2aXAJk2aRHGWLy3KVTixmgtiN5tyHdyJwb+79u7j0Tcnsnx/Hf47fyP/XbWLeqmVOblTA07p1JDuLeqEeirlkl5nBSn0U4G2ZtaSSOE+H7iggI8/HrjXzGoH0ycCwwudUkSkBKUkJdC9QSI39O3Gjj3ZTFywiffmrOX1aat44YvvqFstmWPb1+eEjHSOaZtW7n+Ne9BC7+7ZZjaMSNFOAJ5z97lmdhcwzd3HmlkP4G2gNnCamf3Z3Tu6+xYzu5vIhwXAXQd2zIqIxIKqyYn/O6vmT7uzmbhwIxPmbeCDuesZM301KUmVOLpNGidmpHNsh/rUS60cduRCK9APptx9HDAuR9vtUfenEhmWyW3Z54DnipFRRKRMVKucyKmdG3Fq50bs3befr5dvYcK8DUyYt4GP5m/ADDKb1eaEjHROyEinVVr4+x4KQr+MFRHJRVJCJY5qU4+j2tTjjtMymLfuRz6cGyn6972/gPveX0DrtGqckNGA4zrUp2vTWjF7BI8KvYjIQZgZHRvVpGOjmvzhhHas/mEHH83bwIT5G3hm8jKe+GQpNVISOaZtGn0OSaNvuzTqx9BlElXoRUQKqUntqgw5qiVDjmrJtp17mbJkM5MWbuSTRZt4b846ADIa1qDvIWn0PaQ+3ZqFu7WvQi8iUgw1qyRxcqeGnNypIe7O/HXbmbRoI5MWbuLJT5fx2KSlVE9J5Ji29ejbrj59Dkkr84uiq9CLiJQQMyOjUQ0yGtXgmr5t+HHXXqYs3sykhZv4ZNEmxs1ZD0D7BtU5uk09jm5bj8Nb1i31wzdV6EVESkmNlCRO6tSQk4Kt/YUbtjNp4SYmL97EC19+xzOfLSc5oRLdmtfimLZpHNO2Hp2b1CrxHCr0IiJlwMxo36AG7RvU4Ko+rdm1dx9fL9/ClCWbmbx4MyPGL2T83PWMHXZ0iT+3Cr2ISAhSkhLo3S6N3u3SGA5sztrNpu27S+W5VOhFRGJAvdTKpfar29g8ul9EREqMCr2ISJxToRcRiXMq9CIicU6FXkQkzqnQi4jEORV6EZE4p0IvIhLnVOhFROKcCr2ISJwzdw87w8+Y2Sbgu2I8RD1gcwnFKUnKVTixmgtiN5tyFU6s5oKiZWvu7mm5zYi5Ql9cZjbN3buHnSMn5SqcWM0FsZtNuQonVnNByWfT0I2ISJxToRcRiXPxWOifCjtAHpSrcGI1F8RuNuUqnFjNBSWcLe7G6EVE5OficYteRESixE2hN7P+ZrbQzJaY2c0h5mhqZhPNbJ6ZzTWz3wXtd5rZGjObGdxODinfCjObE2SYFrTVMbMJZrY4+Ld2GWc6JGq9zDSzH83s92GsMzN7zsw2mtm3UW25rh+LeCR4z802s25lnGuEmS0InvttM6sVtLcws51R6+2J0sqVT7Y8/3ZmNjxYZwvN7FdlnOu1qEwrzGxm0F5m6yyfGlF67zN3L/c3IAFYCrQCkoFZQEZIWRoC3YL71YFFQAZwJ3BDDKyrFUC9HG0PADcH928G/hry33I90DyMdQb0BroB3x5s/QAnA+8DBhwBfFXGuU4EEoP7f43K1SK6X0jrLNe/XfB/YRZQGWgZ/L9NKKtcOeY/CNxe1ussnxpRau+zeNmi7wkscfdl7r4HGA0MDCOIu69z9xnB/e3AfKBxGFkKYSDwfHD/eeD0ELMcByx19+L8aK7I3P1TYEuO5rzWz0DgBY/4EqhlZg3LKpe7f+ju2cHkl0CT0njug8ljneVlIDDa3Xe7+3JgCZH/v2Way8wMOBd4tTSeOz/51IhSe5/FS6FvDKyKml5NDBRXM2sBdAW+CpqGBV+9nivr4ZEoDnxoZtPNbGjQlu7u64L764H0cKIBcD4//88XC+ssr/UTS++7y4hs9R3Q0sy+MbNPzOyYkDLl9reLlXV2DLDB3RdHtZX5OstRI0rtfRYvhT7mmFkq8Cbwe3f/EXgcaA10AdYR+doYhqPdvRtwEvAbM+sdPdMj3xVDORTLzJKBAcCYoClW1tn/hLl+8mJmtwLZwMtB0zqgmbt3Ba4DXjGzGmUcK+b+djkM4ucbFGW+znKpEf9T0u+zeCn0a4CmUdNNgrZQmFkSkT/gy+7+FoC7b3D3fe6+H3iaUvq6ejDuvib4dyPwdpBjw4GvgsG/G8PIRuTDZ4a7bwgyxsQ6I+/1E/r7zsyGAKcCFwbFgWBY5Pvg/nQi4+DtyjJXPn+7WFhnicCZwGsH2sp6neVWIyjF91m8FPqpQFszaxlsFZ4PjA0jSDD29yww390fimqPHlM7A/g257JlkK2amVU/cJ/IzrxviayrS4JulwD/KetsgZ9tZcXCOgvktX7GAhcHR0UcAWyL+upd6sysP3AjMMDdd0S1p5lZQnC/FdAWWFZWuYLnzetvNxY438wqm1nLINvXZZkNOB5Y4O6rDzSU5TrLq0ZQmu+zstjLXBY3InumFxH5JL41xBxHE/nKNRuYGdxOBl4E5gTtY4GGIWRrReSIh1nA3APrCagLfAwsBj4C6oSQrRrwPVAzqq3M1xmRD5p1wF4iY6GX57V+iBwFMTJ4z80BupdxriVExm4PvM+eCPqeFfx9ZwIzgNNCWGd5/u2AW4N1thA4qSxzBe2jgKty9C2zdZZPjSi195l+GSsiEufiZehGRETyoEIvIhLnVOhFROKcCr2ISJxToRcRiXMq9FIhmdk++/kZM0vsjKfBmRDDOuZf5BcSww4gEpKd7t4l7BAiZUFb9CJRgnOUP2CRc/Z/bWZtgvYWZvbf4CRdH5tZs6A93SLngp8V3I4MHirBzJ4Ozjf+oZlVCe1FSYWnQi8VVZUcQzfnRc3b5u6dgEeBh4O2fwLPu3tnIicPeyRofwT4xN0PI3Lu87lBe1tgpLt3BLYS+eWlSCj0y1ipkMwsy91Tc2lfARzr7suCE0+td/e6ZraZyM/49wbt69y9npltApq4++6ox2gBTHD3tsH0TUCSu99T+q9M5Je0RS/yS57H/cLYHXV/H9ofJiFSoRf5pfOi/v0iuP85kbOiAlwITA7ufwxcDWBmCWZWs6xCihSUtjKkoqpiwYWhAx+4+4FDLGub2WwiW+WDgrbfAv8ysz8Cm4BLg/bfAU+Z2eVEttyvJnLGRJGYoTF6kSjBGH13d98cdhaRkqKhGxGROKctehGROKctehGROKdCLyIS51ToRUTinAq9iEicU6EXEYlzKvQiInHu/wBfjVC8BEq7swAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(np.array(training_cost[200:])/1000)\n", + "plt.grid()\n", + "plt.title('Cost on the training data')\n", + "plt.xlabel('Epoch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 368 + }, + "id": "k5WngX7QQYRk", + "outputId": "222f20d0-ad64-4bb6-bed0-210a3cb282c1" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text(0.5, 0, 'Epoch')" + ] + }, + "metadata": {}, + "execution_count": 19 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAFNCAYAAAAO36SFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eXxcV333//7OPqNdliVL3h07sRNnd3YS7Cwk0BZ4WqChBUq30F+htClb2+dXQilPaZ+mTUuhQAotLSkJEFKgQMlCojhk35w48b5vsvaRZjT7zHn+uPfcuTOakUayZEvKeb9eennm3nPvPTOSP/d7P+d7vkeUUhgMBoNh4eI50x0wGAwGw+xihN5gMBgWOEboDQaDYYFjhN5gMBgWOEboDQaDYYFjhN5gMBgWOEboDfMSEQmKyA4R6ZzGsReIyFOz0a8ar79ZRI6dqevPJCKiRGTtme6HYWKM0C9ARKRbRIZFJHim+zKL3AZsVUr1AIjIr4lIj4gcEpEtupGInCUiT4mIV29TSr0KREXkl05HR2dTDEXkgyLy8xk61yERuXEmzlXh3Kvs78E3G+c3TIwR+gWGiKwCrgUU8PbTfO3T+Z/494Bvuq7718AlwEeAf3K1+wJwu1IqX3b8fwIfOg39NBjOOEboFx4fAJ4BvgH8hnuHiCwXkQdEpF9EBkXki659vysiO0UkZlsil9jbS6JREfmGiHzOfr1ZRI6JyKdE5CTwbyLSIiI/sq8xbL9e5jq+VUT+TURO2Pu/b29/zR1hi4hfRAZE5OLyDygiK4A1wLP2pkXAcTu6f8Teh4i8y97+bPk5gG7ghmpPPSLSJSI/FJEhEdknIr/r2vcZEfmOiPyH/X29LiKbqpxnq/3yFRGJi8ivuvZ9TET67CeR33RtD4rInSJyRER6ReQrIhKucO4NwFeAq+xzRyc7XkTa7N9J1P5sT4iIR0S+CawA/ts+1yerfJ5P2P09ISK/VbbvF0TkZREZFZGjIvIZ1279PUTt819lP209av8tDojIf4pIc6XrGk4RpZT5WUA/wD7g94FLgSzQYW/3Aq8AdwF1QAh4k73v3cBx4DJAgLXASnufAta6zv8N4HP2681ADvgbIAiEsUT3V4AI0AB8F/i+6/gfA98GWgA/8GZ7+yeBb7vavQPYXuUz/gLwuuu9B9gDLAN+CXjevvY2YNEE39UocEGVfVuBf7a/p4uAfuB6e99ngBTwNvt7/TzwzATXKf8O9ff2Wfs7eBuQAFrs/XcBPwRa7c/x38Dnq5z7g8DPy7ZVPd7u61fs6/qxnv7E3ncIuHGCz3EL0AtstP+GvuX+bPbnOt/+fVxgt32nvW+V3dbnOt9a4Cb7b2ex/Z3/w5n+P7QQf854B8zPDP4y4U1Y4t5mv9+FZVsAXGWLla/CcQ8Cf1jlnJMJfQYITdCni4Bh+3UnUNCCVtauC4gBjfb7+4FPVjnnr5cLK3AD1pPM4/Y1/x74bWAL8Jj9GTeWHXMcuK7C+ZcDeaDBte3zwDfs158BHnHtOxdITvAdVBL6ZJno9QFXYt1ox4CzXPuuAg5WOfcHcQn9ZMdj3Vx+4O6Pq90hJhb6fwX+2vX+7PLPVtb+H4C77NerKBP6Cu3fCbx8pv8fLcQfY90sLH4DeEgpNWC//xZF+2Y5cFgplatw3HJg/zSv2a+USuk3IhIRka+KyGERGcWK0prtwdDlwJBSarj8JEqpE8CTwK/Yj+9vxfLRKzGMFam6j/+ZUupKpdSbsQRlE9ZN6T+wxPAvga+VnacBiFY4f5fdz5hr22Fgqev9SdfrBBCa4hjFYNnvIgHUY0W2EeBF216JAj+1t9fCZMf/LdZT30MickBE/mQKfe4CjrreH3bvFJErROQx27YbwRpHaat2MhHpEJH7ROS4/bdyz0TtDdPHjIAvEGwP9j2A1/bLwXokbhaRC7H+g64QEV8FsT8KnFXl1Aks4dAsAdypgeXlTz8GnANcoZQ6KSIXAS9jRZpHgVYRaVZKVRLYfwd+B+vv8mml1PEqfXoVWF3ps4iIAF8E/gBLNLxKqcP2d3KBq91SIADsrnD+E3Y/G1xivwLrCWC2GcCK9s+b4PO7Kf/+Jzze/jwfAz4mIhuBR0XkeaXUzyqcq5werJu1ZkXZ/m9hffdvVUqlROQfKAp3pXP/lb39fKXUkIi80z7eMMOYiH7h8E4su+FcLOviImAD8ATWAO1zWP9R/1pE6kQkJCLX2Md+Dfi4iFwqFmtFZKW9bxvwayLiFZFbgDdP0o8GLKGJikgrcIfeoazB0v8B/lmsQVu/iFznOvb7WJkzf4gViVdEKXUMKyq9vMLu3wFeUkptAwaBsIici2XhHHC1ezPwqFIqXeH8R4GngM/b39MFWDbQPZN89mr0Yg8QT4ZSqgD8C3CXiLSDdVMSkZsnOPcyEQnUcryI/KL9+xVgBOtvplBjP78DfFBEzhWRCK7frU0D1pNQSkQuB37Nta/fvs6asvZxYMS+8X5igmsbToUz7R2Zn5n5wXo8/7sK29+DZTP4sCKw72MJ4ADwBVe738OKbuPAa8DF9vZNwOtY/vk3gXsp9eiPlV2vCyujJY41QPohXN4s1gDhv2OJyjDwQNnxX8PymOsn+bwfBr5ctq3N7nuja9uv25//ELDFtf3HwNsnOP8y4EfAEJat9XuufZ8B7nG9X8UE/rP93fZg2UTvqfK9HcL2x7EGgP8K68Y0CuwEPlrl3AH7swwBA5MdD9xuX2sM68nsz13negdwxO7nx6tc70/s7/ME8FuUDsa+C8vOidnf3RfLvqfPYgl+FGs84jzgRftvZRvWk8axStc1P6f2o0fbDYY5gYh8GjhbKfW+SdoFsSyhG5Q9aWoK17gA+KpS6qrp99RgmD8YoTfMGWyr52Xg/UqprZO1NxgMtWE8esOcQKwJSUeB/zEibzDMLCaiNxgMhgWOiegNBoNhgWOE3mAwGBY4c27CVFtbm1q1atW0jx8bG6Ourm7mOjRDmH5NjbnaL5i7fTP9mhpztV8wvb69+OKLA0qpyjOoz3R+Z/nPpZdeqk6Fxx577JSOny1Mv6bGXO2XUnO3b6ZfU2Ou9kup6fUNeEFV0VVj3RgMBsMCxwi9wWAwLHCM0BsMBsMCxwi9wWAwLHCM0BsMBsMCxwi9wWAwLHCM0BsMBsMCxwi9wWAwLHCM0BsMBsMCxwi9wWAwzCAjySwvHh6uub1Siq17+vUqXLOCEXqDwWCYQe597gi33v00qWy+pvYvHYnygX99jmcPDs1an4zQGwwGwwwyksySzSsGxzI1tY+ncwD0jCRnrU9G6A0Gg2EG0ZH8ULw2oc/kCgD0x9Kz1icj9AaDwTCDpG3hHhyrTbizeav9QI03hulghN5gMBhmECeir9G6mTMRvYjcLiKvi8hrInKviITK9t8lItvsnz0iEnXtWyEiD4nIThHZISKrZvYjGAwGw9whnbUj+lqtGyeinz2hn3SFKRFZCnwUOFcplRSR7wC3At/QbZRSt7va/wFwsesU/wH8H6XUwyJSDxRmqO8Gg8Ew50jnrIi+1sHYORPRY90QwiLiAyLAiQnavhe4F0BEzgV8SqmHAZRScaVU4hT6azAYDHOalB3RD03Zoz+DQq+UOg7cCRwBeoARpdRDldqKyEpgNfCovelsICoiD4jIyyLytyLinZmuGwwGw9xDe/Q1Wzc5fWPIkC/MzqQpmWw2loi0AN8DfhWIAt8F7ldK3VOh7aeAZUqpP7Dfvwv4OpaVcwT4NvATpdTXy467DbgNoKOj49L77rtv2h8oHo9TX18/7eNnC9OvqTFX+wVzt2+mXxOTzCl6xgqsabJizdnq12eeSnJotMCaJg+fvio8afsf7s/wwN4sAP+wJUxz0DOtvm3ZsuVFpdSmijurLSarf4B3A193vf8A8M9V2r4MXO16fyXwuOv9+4EvTXQ9szj46cX0a+rM1b6Zfk3Ml7v3qXV/9hOVzuaVUrPXrxv/rlut/NSP1LV/82hN7f/uwV1q5ad+pFZ+6kfq9eMj0+4bp7g4+BHgShGJiIgANwA7yxuJyHqgBXjatfl5oFlEFtvvrwd21HBNg8FgmFGiiSyZfMGZiTpbpHJTTK/MF12V/lny6Wvx6J8F7gdeArbbx9wtIp8Vkbe7mt4K3GffWfSxeeDjwM9EZDsgwL/MYP8NBoOhJrR3PjbLQq/TK+PpXE31brRHDzAwS5k3k6ZXAiil7gDuKNv86bI2n6ly7MPABdPpnMFgMMwUWnRjqVmO6LN5Qn4PqWyBobEMXc0T+/SZfJ6gz0M6VzhzEb3BYDDMF144NOSkK5aTtIW+VutGKcXT+wenXD44lSvQ1WSJey32TTanaI74Cfu9sxbRG6E3GAwLghPRJO/6ytP8z2snK+5PZrTQZ2s634uHh3nvvzzD0/sHa+6DUopMruBE8bXkxmfyBQI+D20NARPRGwwGw0To6Hm4ShSddFk3hwbG+M7uzITRup6purs3VnMfdEGzruZQSZ8mIpMvEPB6aKsPztqkKSP0BoNhQaAtmbFMZWumOBib56evn+QnB7MTRtAjSSvyP9A/VnMf9DV0RF+T0OcK+L0ezl/axJLGyfPup0NNg7EGg8Ew14nbg6zVsmpSTjZMlmjCEvFUpnrpLUfoB+I190FH9G31Qfxeqan0cDZfIOjz8Nl3bKz5OlPFRPQGg2Fecjya5MhgsXSWE9GnK6c0OoOxqRwjyUzJtkpood/fN/WIPuz30hwJEE1UFvpCQfH8IWvpQB3RzyZG6A0Gw7zkz7//Gh+650XnfcwW+kQV60YPxsbSOSein0joR1NWm5OjqZpz7/VTQ8jvpTUSqGrdbN3bz7u/8jR7e2Nk7cHY2cQIvcFgmJf0x9LsOjnqRN5F66ayeLsnTDlCn5kooi+K+8GB2qJ6XaI46PPQUud3rlOOLngWTWZNRG8wGAzVGElmUQpePjIMFNMmJxuMjadzRO2bw0QzV0eSWeoCVgG0/f21+fTuiL4lEmCoinWj+5jM5MnklYnoDQaDoRI6kn/xsCX0OpJPVIjolVIl6ZUjtgAnJozos2xc2oRI7Zk3+sYR8ntoqavu0evxhFQ2TyaXJ2AieoPBYCilUFCOh66FXpc2qDTzNZMvoEu9uyP6iTz6WDLL4oYgy1rCHJjEuhlJZtl9MuZk3QR9XloifoYTWQoVasxrzz+ZzZM1Eb3BYDCMJ57JoRQEvB62HY2Syxcc66bSYKw7jXJ4LONE8pNl3TSF/axaVMeRoYkXxrt7637e9ZWnSiP6SIB8QVWsraOfPqyIvoDfK5N84lPDCL3BYDjtbDsa5e6t+6d9/Ig9yHnFmlYSmTw7e2KuCVPjxdst6CeiKed1qop1o5RyhL4x7CdmPwE8uquXL3fvHzejtnc0TSyVY9i2arRHDzjb3Oi+JjN5k3VjMBgWJt9+/gh//T+7pr10nvbnr1yzCLAmNU00YUpH2s0RPxlX0bNqEX0ikydXUDSG/TSGfE7q5reePcrf/HQXdz2yt6S9zq7pGbFuIkGfh9Y6S+grDcgWrZuCyboxGAxzl4986yX+sUzwauVENEVB1b44x2vHR7jirx7hpC2ko7bQn7XYWm6vP5Z2ouREJj/OF9eC3lYfLNlebTBW30iawn7qgz5i9niAHhf4ws/28tiuPqe9HnTV/Qv6vbTYQh9NZHj/15/la08ccNqXDMaaiN5gMMxVXjo8zKvHotM69kQ0CdRW3RHgib0D9I6m2XbUup4W3GUtYQJeDwPxTMkgbDKb57XjI04kr4V+cZnQu9MrDw6M0RdLlZy/KeynIeQnlS2QzReIpXJsOWcxLRE/P3zlhHOsHtztGbE+l+XR++1tKX6+b8DpO5QOxuqiZrOJEXqDwTAtYulc1Zz1ydAWR3+N9dd39owCxbozOuJujvhpqw9YEX0qR8hvSdrxaJJ3fOlJvvfSMaDoxS9uKBV694Sp3/rG83ziu69a50+4hd4qCRZP5RhNZmmpC7BlfTuP7uojZ9tAbutGxBok1hH9y0eiKFXsMxQHY+Pp4qDybGKE3mAwTMqRwQR///AeZxBSKcVYOjdhHromX1B8/n920jdajJZ19F1rRO8IvZ3P7rZWFjcE6YulGMvkWdJolQfe3xcnX1D0jlrn1+u4uoW+MVCM9EdTWQ4OjPHU/gFiqaxz/saQZd2Alb45msrSGPJz04YORpJZXjg8bA/cWtZNz0iKoM+DiNAQ9OHzCC/Z6Z/uWbL68+vr+I11YzAYzjQ/fb2HL/xsL312BJ7M5imo2lZrOhor8NXHD/DYbsvT1rYN1BbRp7J5J4/9QH8xovcI1Ad9tNUHOTRo7W+3hf6QXexMT4xKZopVJQG8HqEp6HGEflePVXM+m1ds3TNQciNpCPmda8bTORpDPq49ezEBr4dHdvSSyFi58GAVKAv5rdm0IkJzJOD0PZosjkfo702PNZiI3mAwnHF0RoselNTvK81CLac/aYmgLg/Q40pvrCWi32dH5231RdEcTeZoDPsRERY3BDk+bN08OmyhPzKkxbV0YlRbvWWnNIf9BL1F60Y/MYT9Xh7ecZJR+/M12Vk3YBU3Uwoa7QHaq85axKO7+5xraIKu6Fz79FCM6PXTEOBcx0T0BoPhjBMrE6bYJIt8uOlPWgKvbZ4TrgHLWuq177BF+JaNS4gmsgyNZZwcd7CidJ1ks6TRitgPDVgRfXmVSm3dNEX8BLzF7Tt7RmmJ+Hnb+Z08uquPobE0ItAQ8lFvC71+EtGe/XldjRwZTDBU9hl0RA84Pj1Y1k8uXyCdK5CzO6zz84MmojcYDGeasTKrwZ2zPtni2f0Ja78W1Z5oCp9HOKejoSbrZmfPKGG/ly3ntAOWfeMWerfv3t5gRfSHB0sj+vLBWCuil5KIfv2SRm4+r4PRVI4fvdpDQ9CHxyOOdaOFvtF+39kcJldQ7O2zbB+xJ7eGfC6htyN6PfN1NJUryfPXFpFJrzQYDGeceFlEr8WqoIqrKlVDC71OZTwRTdLRGKK9MVSTdbOzZ5RzljSwrr0BsCpJlkf0mnY7ou+xB361R58qS69sjgQIeKzt+YJid2+MDZ2NXL++nWUtYQ4PJmi0z68j+ONa6O3tS+11YbXt09VkLQMY9BdlVU+aOq+rCbBmybrLKOs0TjNhymAwnHFiZR59zBWVTrYoh7Zuki7rprMpRFt9cNKIPpnJ8/KRKBctb2ZpS5iAz8OB/jEn+wVKI3qddaMfMtwevc8jNNkRdnPYT8ArJLN5Dg6MkcoW2NDZgM/r4bfftBrAuZHorJvjZdZNpy3sO+2B3DWL64DyiN4S+ktWtFj9SRQzjvxecQZx50RELyK3i8jrIvKaiNwrIqGy/XeJyDb7Z4+IRMv2N4rIMRH54kx23mAwnB6KWSJ2hciUW+irD8jmC4qBZJl1M5KisznM4oYgQ4mMk4teiZ/vGyCdK3DTuR14PcKqRRH29sUZTWadyFoPsEIx60YzkrSqRyazecJ+L0Gfl7A9a1V79LtPWkK9obMRgPdsWk5T2O+IdMjvJeD1jLNuuhyhtyL6VYssoXdH9IvsJ4hLV7bY/ck44xqL6oo3qNkuajbp4uAishT4KHCuUiopIt8BbgW+odsopW53tf8D4OKy0/wlsHUmOmwwGE4/Wti11eBOq5xoQLZ3NIUdtJK0SxP0RFPccl6IxfUBlF0GoVygNQ/vOElDyMflq1sBuHx1Kw+8dJxsvlDRo2+NBAh4PWTyBUSsyD6WypHK5gnZi4j88/su4eyOBv7Pt4+RyOSd2azLWizhrgv6+JcPbHImX4EVxevUUh3RN4Z91AW8DI5lCPo8LGmyPkPQFdH/8sVLaW8Icl6XdROJJrKIbea3NQQ4aVtMcyKix7ohhEXEB0SAExO0fS9wr34jIpcCHcBD0+2kwWA4s2hhj1UQ+mprtAIl5X2T2by1dF6+QEdjyBHo/io+fb6g+NnOPjaf0+542Ddu6HDy1t3Wik5prAt6qQtaQru02RLu4USGVLZA2M6G2XJOO0ubwwS9Vt57XyyN3yvO+cC6oVywrNl5Xx/yOXaQHpwVETrtazRH/I4f775BtNQF+KULu2i2LaNoIutYXe6I/ozn0SuljgN3AkeAHmBEKVVRtEVkJbAaeNR+7wH+Dvj4THXYYDCcfsZZNy6hj09g3Ry1hb69IUgyk3eeDBpCPmcQ1Z1ieWhgjDsf3E2+oNh2dJjBsQw3bmh39l911iJneb/GsBVZiwht9UFCfg8+r4dIwNq+rt0qeBZNZklm8o7QawK2XXJ0KEFbfdCJtCuho/iw31sSfXfaUXxzOOBk2ITKrgOW3SNi9UULvXsQebYj+lqsmxbgHVgCHgW+KyLvU0rdU6H5rcD9Sin9m/994CdKqWMTfYkichtwG0BHRwfd3d1T+hBu4vH4KR0/W5h+TY252i+Yu32brX4ppRyBPnS8l+7ubnYfKEbhz730CupEZSl5Ym8GQbHIn6V3cJjuJ5+2zrNvN6rPErcnnt+GOmGJ5A/2ZfivfVkKw0fZ1pcn6AVf/x66u4tVMje0wAu9cPzAXrqTBwEIqjQBUdbnz9kVJNNW6YGtz7zA8d4cmawq+X5ULg0IO470EYIJv7t80rJ3gp5CSTtJ2t9DZozDe3YAMNR3ku7u4XHniPjg9b0H6Y9Ynzs53Ovse+XllxjaVxT7mf5dTir0wI3AQaVUP4CIPABcDVQT+g+73l8FXCsivw/UAwERiSul/sR9kFLqbuBugE2bNqnNmzdP9XM4dHd3cyrHzxamX1NjrvYL5m7fZqtfY+kc6sEHAfDXNbJ589X8oHcbnqPHKShYvW49my9dVvHYH/RuoyV0ghWdizk8mGDDBRvhqae54pIL2bSqhU9ufZCvbc/waI+fh26/jh/1bweO8eBxH4cHM7z/qlX8wk3nlZxzqPEYL3znFa657CKuWdsGwL1HX4C+OJs3b6Z9x5Mcj0fZcukGfnpoOyvXbSA8dIQIsHnzVc55njj2MJBhOONh06pWNm++rOp38J9HXmDnUC9tTXVs3vxmZ/srub1sPbaHlZ2L2XLNOXz+ua2sXrmczZvPHXeOxc8/Rn1rM52L62HXHi7esJafHNwJwNVXXu6UXIaZ/13WIvRHgCtFJAIkgRuAF8obich6oAV4Wm9TSv26a/8HgU3lIm8wGOY2bptGT5iKpXIsbgjSO5qeML1ycCxDU1AI+70kMnnnXHVBH5GAjzvffSEP7zjJg6/3cjya5OhQAo/A/v4xvB5xUh3d/OIFXaSyBWeAFuCPbzrHWcmpzrZu1tp599FEllQ273jomqBrElN5+eJynAHYUKlkdtq59C2RgCtLp7IN0xQJOB59wOdxzgmz79FPKvRKqWdF5H7gJSAHvAzcLSKfBV5QSv3QbnorcJ+abJqcwWCYV+gc+pDf48q6ybKkMWQJ/QSDsSOJDHV+IRzwkszmnZuCFrl3XbqMVYsiPPh6Lwf6xzg6lOCWjUt46XCUq9cuYllLZNw5Az4Pv3bFipJt5yxpcF5HbA9/rR0ha6Ef79EXX7c1lN4EytEplXogVrPUNRjbYv902dvKaQ77idqF0eqDPsKuDpxxjx5AKXUHcEfZ5k+XtfnMJOf4Bq6UTIPBMLOMpBV///Ae/vCGdXg9M5eXrcW5qznsrKA0ls7TVh/A65Fxhc1+trOXTK7AW8/vJJrM0hmAsN9HKlMU+rpgUXrW2IK862SMntEU72pv4PP/6wJCgemJn/W04KUpYhUfiyYzTh69m6Ard32yiF5PmmoMlwq9HoxtivjxeT08/sktzhNFOc0RP4cHxxhL56gLeksGbc3MWIPBUBMv9+X4ws/2sq8vPqPn1XZLV1PYTm0sEE/naAj5iQS840oV/91De/jHn1mDp9FE1o7orZLAOkOn3iWGrXUBmiN+fr6vH6VgRWuEpoi/JB99Kly/vp33bFoOWLNbRxJZkpkCwQkj+ulZN8tbI7zl3A6uPqvN3u+vepMtRvR56gK+khvPnIjoDQbD3MfOfGQgnuYcGiZuPAW0ddNl+9GxVI5YKkdd0Ed90FeSR5/NF9jXF6c54idfUIymstT7/YT9XnIF5dSe0bnumjVtdTx/0MpUWd5S2fqolV+6sItfurALsKLoaLKadVN7RN9Qxbrxez3c/YFNNfWrKRJgJJkllspSH/SVRfSzOzPWRPQGwwIhkbOGx2pdnq9WdMSua7vEUlni6SwNIcsiGbNnl/bFUuzvj5PJFxgayzCazKIU1PnFEbX+uDWL1FdmVaxZXE/GLoWwYtF4X366NEf8RBMZS+jLrKDgFCL6etds2Gn3JexHKTg8mKAuWBbRn+nBWIPBMD9I2UJf6/J8tRK3B2B1RD80Zs00rQ/6qAv6GEvn+Mi3XsYrwnuvsCyTXEE5s2Lr/DgDjwPxtON3u9EFwQJeDx0NlcshTIfmcIAXD/eSK6iSCUrWtYqvy9eSLUdbN+UR/VRYaj+pnBxN8eazFzs3Hr9XJpysNRMYoTcYFgh6SdKZjujH7KqTOptED8jWB33UBSyh39UzSiKbZ1lr0XbRC3nX+cXJhOmPpUsGYjVr2qwB2WUtYTwzOJDcHPGTyha4YnUr7728NFNHD8YGfB4aKvTJTWMVj34qvOXcDh6+/TrSuQJr2+udG/JsR/NghN5gWDCk7Oph1WrHTJdYKkfA63Fqs+hyvfUhH3VBL9uOxpybwQ+3Fctg6YW86/3i2BQD8XTFqHhtuxXRL2udOdsG4Np1izk5kuKuWy8aV5pAp7svnqT8AcC6jgbetLbNKTc8HUSEdR3FsRPdn9leRhCMR28wzAu+9+Ix3v/1Z8dtf/D1k9x811bSuTyJ7Gx59FnqQz7Hnz5hr/naYE960rVqRCzLRpf71ULv9ugt62Z8Ns2K1jq8HmFF66kNxJZzy8YlfP2Dlzl58G58HsHvlUn9ebCyae75nStYPoM3In3zOx0RvRF6g2Ee8NzBIZ7YO0DGtZpToaC488Hd7O6N0TeadmXdTL4Oq2bHiVHyhYnnOMZTVt63ziHXZX2tiL5oCty4oQOAN61dBFgrQQHUBYoRfSpbqGjdBHwevvRrl3DbtWfV3PeZIOT3simRA1wAACAASURBVLh+4slSs3ltmP0cejBCbzDMCwbHrChdT/MH6N7Tx147Z344kSE5xcHYE9Ekb/vCE3ztiQMTtrNmcvqpD/gQKa6dWhf0OZUkIwGvU67gpnOXAHBwwI7ofZTMAq0k9GBF3zOZcVMLS5vDnNVeP3nDWcDrEQI+j1NieTYxHr3BMA8YHLMEfjCeocNepOOrjx/A6xHyBcXQWFHoB+Np8gU16exY7bV//ecH+eA1q6pOUIqlcs5C2fVBn3NzaQj6iNiivbqtjivXLOK5/30D7Q0h6oM+a1JV0IfXUxyMhdLJUmea7/zeVadFaKsR8nlMRG8wGCyGbKHX/0YTGZ49OMTb7YlBw4kMiZzl+xZUaeRfDe3l98XS/GBb9bWExjI5J4/85vOWsKg+wIX2Gq46otdlDNrt1MhFth3SVKFGe7WI/kzQGJr+DNyZIBzwzvqsWDBCbzDMKb702D5+75svUl4bcND23bWF8+JhaxbpzedZNknvaJpcwYqsobYBWW3xLG0O8yffe5Xz73iQrXv6x7WLp3JO7vud776QJz55PT/48DVEAkWPfo19XY2uFKlXVnJPDqo0GPtGJez3zvqsWDBCbzDMKZ4/NMRPXz/J84eKC1ekc8XyvlrwXzw8jM8jXHd2Gx4pruS0enFR6Pf2xsbdMNz0x9J4BL78vkv47TetJpbOsf34yLh28XSuahSuSxnoCU8anYrZHLYEvxaP/o1IyG8ieoPhDYdeyenurfudbdqucb9+4fAw5y1tIhLw0RwJOLNQdWT9xUf3cdNdW53iYpUYiKdprQtywbJm/uxtG/B7xalro1FKMZrKldROd9PVFMYjsHFpU8n2RXVl1o3PCH0luprDzpjLbGK+cYNhDqGF9pGdfezri7O2vd6J4sEalM3kCrxyNMqvX7ESsOwRLfTaunnu0BABr4d/eGQvXc1hp5qjm/5Y2pn6LyI0hvxOvXlNMpsnkys4i2qUc/nqVp750xtoLxOrVtujb7ZTMj0eIeT3OKUTDBZfeO/FzOBE4KqYiN5gmEPEUlkuXWnNvnzpiGXfDLoi+sF4mtdPjJDOFdi0ymrXGglwfNjKoFnSFHKySP723Rewtr2eH2w7XvFa/fEMba4c8sawf1xEr58g9MLX5YjIOJGHYkTf7DpO+/Qmoi9Sb086m22M0BsMc4hYOuesWpS0ywoM2QOwnU0hhsYyzkCsviG01AXI2ZOeGkN+2huDrGiN8IsXdLGkMeScp5wBV0QPVuEuvVSgJmoX0Gmpm9qkokVORF88rij0ZjD2dGNurQbDaUQpxb6+eEnNE02hoIinc7Tb4pvMWgKtrZt1HQ0cG0qw48QoHY1Bx9t1R9sNIR+ffftGWuqs1Z/CAa/zRDA0lqGgrCqOSin64+mSOuxu6+bw4BgdjSFXRD81oW+1B2ObIn6w10EJ2QOyDcHpV4A0TA8T0RsMp5Gtewe46a6t7OuLjds3lsmhVLE2esKOxAfHMvi9wsrWCINjGXb0jDr1ZKA02m4I+dmyvp2LljcDVhSdsm8Yf/bAdv7wvpcBa0HsTK5QEtE3hn3E7O1v/ccn+ObTh518/Na6qYnzytYIItZqURoT0Z85jNAbDKeRQ3ZZgP12wS83OoWyMWStyJS0V24aimdorQvQVh9kJJllf3+c9UtcQu+KtssHOq3zWELfF0txzPbydQ69u0Z7Q9DPaDLL0FiGRCbPocExhu2IvnmKEf2qtjqe+dMbuHLNImebnh1rBmNPP0boDYbTSO+oVfmxxy4/4EYPhOqVmxzrZsxKg9SZLNm8YkNn0fpptUXY7xm/9mg44HWW+ktk8o4VMxAbL/Q6otc3gd7RNMO2R98cnrrdUp42GDKDsWcMI/QGw2mkd9QS0R578Q43bqEP+b0l1s2iuoCTyQJwrsu6cWaf+sbn6YX8XlJZq+JlMpsnlsqRzRecmvUl1k3ITzKbpy9m9a0vlmI4kaEp7B+39N90CPu9iFBS98ZwejBCbzDMMCeiyaqZLlpEj1eM6K3ouSHktyJ6J+smw6L6gFNWIODzOPnyUCw3UGk500jASyZfIJcvODeOaCLriujd/r51goMDVk5+72iK4US2amrlVAkHvNQFfLO+bJ5hPEboDYYZ5h1fepJ/qVL6Vy/DN1lEX2LdOB69JcrndDSURNjaP49UiOidOvC5Agl7DCCayNAfT+P1SIm/r+vN63GE/liagVh6yqmV1VjSFHLWnTWcXozQGwwziFKK/li6alGxiTx6PRjrtm50nZtFdQEnZdHtz4Mroq8QeOuUxkQmRyJbfEIYiFl2kHt9Vr0K06FBS+gLCvb1x6ecWlmN2288m2/fdtWMnMswNWoSehG5XUReF5HXROReEQmV7b9LRLbZP3tEJGpvv0hEnraPfVVEfnU2PoTBMFdI2ytA6ZRGN8lMntFUjoDPQ28sTS5fKNlfybrRE5aaIwGaw36uX9/O287vLDmuKexHpLJHryP6kUQWXd9sOJHl5GiK9sbSJfS0daOFHqyofqaEPuT3ztjTgWFqTCr0IrIU+CiwSSm1EfACt7rbKKVuV0pdpJS6CPgn4AF7VwL4gFLqPOAW4B9EpHkmP4DBMJfQAp/KFcbt0/78xq5G8gVFX1nUH0vlrMFKv5dIwOcMnoIlwh6P8K8fvIzN57SXHOf1CM1h/4TWjbuMwnAiQ89Ikq6m0vVZtXVzfDhZUn9lpjx6w5mjVuvGB4RFxAdEgOqrFMB7gXsBlFJ7lFJ77dcngD5g8fS7azDMbbSvXmkwVmfcXLTcKl2g114dHsswEE8Ts+u+WwXArIheR/mVFrd289e/cgE3rxrfRme4uAujDScy9ERTdDVXFvqCKi4kAlMvf2CYe0ya0KqUOi4idwJHgCTwkFLqoUptRWQlsBp4tMK+y4EAsL/CvtuA2wA6Ojro7u6ewkcoJR6Pn9Lxs4Xp19SYq/2Cift2csyK5Hv6+se1ebbHis6DcStOeuSplxg94OUvnk4R9EJb2EOAPN3d3QwPpBkZy/Hkcy8CsG/nduRk9bTEINAsiXHX3DVo3XCe2fZasR+v7SeWzpEYPE53d3GhkUS2WLu+xZNEAAX0HztId/exqteejLn6u5yr/YKZ79ukQi8iLcA7sAQ8CnxXRN6nlLqnQvNbgfuVUvmyc3QC3wR+Qyk17plWKXU3cDfApk2b1ObNm6f6ORy6u7s5leNnC9OvqTFX+wUT923HiVF44gnC9U1s3nx1yb59TxyAV3byazdfzZdfeYzmrtX4uho5NPocDUEfKzsX0ZZPsHnzdTyd2MlTPYdYffZ58MJLXHvVZSWzYWvtV/PRKDz/JK2dK2GHVZt+mDpghDddspHN9lKEYNXakUd/glJw3prlHB7roS+W5sqLN7J5Y+m4wFSYq7/LudovmPm+1WLd3AgcVEr1K6WyWP771VXa3opt22hEpBH4MfC/lVLPnEpnDYa5TiqXL/nXTV8sTcDnYVlLmIagj56RFHdvtdIwY+kchwbHnAFRPdFp1DVAOx20R+9evGTXSavOztKyVEePR2iwZ60uqgu4iqYZ62a+U4vQHwGuFJGIWDMdbgB2ljcSkfVAC/C0a1sA+C/gP5RS989Mlw2GuYszGJsdPxjbO5qiozGIiNDVHOZbzx7hib0DXGXXg9nbF3eEXnvrOk1zuvVhyoW+IeQjYw8Ud5YNxlr7rRtKa32ADjsrx3j0859JhV4p9SxwP/ASsN0+5m4R+ayIvN3V9FbgPlW6SOV7gOuAD7rSLy+aue4bDHOL1ISDsSk6Gqwo+ZO3nMN7L1/Oh968hr94x3kAKAX1ttDqNVZ13v20hd4+j3shcACP4JRDdqMHZBfVBU1Ev4Co6a9HKXUHcEfZ5k+XtflMhePuASp5+QbDgkRH8ukK1k3vaNqpUXPDhg5u2NABWN64XmZPR/Q6Eu+LpakP+vBOc705LfQ6ol/aHGbXyRgdjaGK9Wv09RfVB1izuJ6GoK9klSjD/MTMjDUYZpBq1o1SihPRJJ1N40sAeDzCqkVW7ZqidWP9q4V+uoTsapZa6HVKZXlqpUancbbWBXj/lSt55GNvxj8DBc0MZxbzGzQYZhAnj75sZuxwIks6V6gqsGfZeet6MDQcsP5r9o2mHPGfDj6vh4DXw5C9gMjSFuv6lW44YJUqBmswNuDzjCs1bJifGKE3GKZBIpPjHV96ku7dfSXbdSSfLyiyrhIHJ+zaNtWKeq1ZrCN626P3W4LbH0ufktADhPwelLJ8+SW2cFe74TSF/fg8MukELcP8wgi9wTANtu4Z4JWjUf72wd248w/cNW7cUb0W+kqZLuAW+tKsm1xBOQO000XbQJGAzymAVi2if9+VK7nz3ReWFDszzH+M0BsM0+CRnb0AvH5ilKf2Dzrb3ULvfq3LEleLpM/ragKKN4Kwa3GOU43o9bnCAS/LbOtmXfv4xcnBspDeefHSU7qeYe5hhN5gmCIFpXh0Vx9v3biEtvogX3m8WNXDLe5p14DsiZEkAa+nZJUoN2d3NLD1E1u4ck0rUMy6AWg8ZevGOlck4GXN4nq2fmIL16xdNMlRhoWEWbzRYJgi+6MFhsYy/MIFnXQ2hbnnmcPOvmRV6ybFkqbQhJbIikUR57V7ub1TXUw77PfY/3rHXcfwxsBE9IZ5w8mRFO/44s+dSUSTUSgoPvhvz/H4nv4J2/3J917lL/77dUrn+o0nns7xlrse5+9eSOH3CtedvZhF9QEy+ULFtMoS6yaanNLqSqXWzal59PpcZq3WNy5G6A3zhp0nR3nl2IhTq2UyYqkc3bv7efHQ0ITtntg7wL89eYi/+enuCdsdGUywpzfOuhYvn3vnRhpDfsc/13XjkyUefYG+0RTJTJ6ekdS4+u8TEfLNoEdvZ/DUneKTgWH+YoTeMG/QZQX02qeTMZK0CoKV57SXk8jkCPu9fOXx/Ww7Gq3aLpq0ctF/YY2fX71sBVC0VfQygOky6+ZX736Gj313GydHU3ROIaL3eMSxWk7ZutGDsX4T0b9RMUJvmDcktNBXqCNTCV35cXKhz/POi7vweoSHXj9ZtZ1e1q/OX/TZta2iFwhJZQuEbE88mclzfDjJT7afJF9QVTNuqqEF+pStG7s/xrp542KE3jBv0IKdyEwxos+MrySpyeULpHMFOpvCXL6q1UmbrIQW+nqX7mpbJe6ybnQRsOFEhoxr0tRUrBsoRuCnbt3o9Epj3bxRMUJvmDckbYEfqzGi10JfaaFuTcLeFwl4uencDvb0xjnsWhzbjbZu3BG9tlVGbaFPZfM020J/0s6d1+mRU7FudJ/g1IU+ZAZj3/AYoTfMG3RkXqt1o4V+oieARFoLvY8b7WqSj+zsq9h2JJEl6PMQ8BaFvnGcdZOn2S71q7ODPnHLej5x8zmcXWWSUjVmyrqJ+Etn2xreeBihN8wbEllLsGdyMFbfBOqCXlYsinB2R33VdMxoIjuuZK9j3aR1RF+gpc5qo2fDrmuv58Nb1k65rMDMDcZqj95YN29UjNAb5g0666ZW62bUEfrqHr1+OtAieOGyZnacGKnYNprM0BwundlaX5ZemcrmabLb6Ih+uvXcZ8q6CfuNdfNGxwi9Yd7gpFdOcTA2NcGNYcyOxLUIbuhsZCCeoS82flJWNJGlqUy0/V4PIb/HsW6S2Tx1AS8Br4eTWujD01uhKWyfJ3SKaZEhf7HWjeGNiRF6w7whkZ1aeuVE1s2n7n+VB1465oroLRFc32n56Dt7xk/KGklmHf/dTUPITzydQylFKpsn5PcS8nucLJ3pR/S+U47mwcyMNRihN8wjphvRV7ox/PerJ9i6p9/Zp2eN6qX+dvWMjjumkkcP1mIho6kc2byioKz67zqKDvqmH5F/8OpV3PH286Z1rJuIEfo3PGZ0xjBvcDz6dK0Tpoq+uZtUNk8ik2ckmWXMvmloH7s5EqCzKcTOSkKfzDipk24aQj5iqZzz5GBF9Pp808+Y2bi0iY1Lm6Z9vGZxvZXW2d5gVot6o2IiesOc4sHXT7Knt3Itm0QNE6aUUnzr2SMMxNOuwdh8ScEybamMJLNOBo+7DsyGzkZ29sR4av8AT+0fAKybQypboKmadZPKOuUPQn5v8cYxTX9+Jjl/WROP/PGbZ+SmYZifGKE3zBmUUnzsO6/w1ccPVNyvJ0xN5NEfjyb5s//azrefP+pYN9ayfkWh1wtljySzJROmNBs6G9jbF+P9X3+OP7pvG4WCcs5VKUKvD5ZG9GHbowfGDd6eKda215/pLhjOIEboDXOGobEM8XSO/ni64v5kDYOxJ6JWpsv+/jgjyWyx7ozLvonaC2WPpnIk0nm8HiHoK/5X2NDZSEFZ4t8XS7P9+EhxYLVChN4Q8hFP55wSxSG/l6AT0c8NoTe8sTFCb5gzHB221lUdiJUKvS4lUPToq1s3PSPWOV47PkK+oJzFsJOum8NQohjRj2VyRPxeRIqTmTaf086Ht5zFf/3+NXg9wiM7e52bQ8XB2JCfWCrnjAWE/B6X52+E3nDmqUnoReR2EXldRF4TkXtFJFS2/y4R2Wb/7BGRqGvfb4jIXvvnN2b6AxgWDkeGEgAlEf1Dr5/kqr/+GYcHx5xIPp0rkC9UXiRER/R7++IAdGihd0X0w3Z0nskVGB7LEAmWZqPUB3184ub1rG2vZ9PKFh7e0Ut0IuvGjujdA7v6SaLS4K3BcLqZVOhFZCnwUWCTUmoj4AVudbdRSt2ulLpIKXUR8E/AA/axrcAdwBXA5cAdItIysx/BsFA4agv90FiGgj14+uPtPSgFx4aTJLN5R0CrDcjqiF6PvS5pKkb0P361h8ODYwzbHj3AiZEUdROUBrjp3A52nYzx2nFrtmwl4dZFywbj1nmDrqybSoO3BsPpplbrxgeERcQHRIATE7R9L3Cv/fpm4GGl1JBSahh4GLhlup01LGy00OcLingGsvkCj+2yCoz1jKRQCtrqg0B1n/5ENFny3rFusjn+6Nsv87UnDjqDsWDZQhPNGL35vCWIwH8+ewSo7LnrSU39tuUUdmfdGOvGMAeYVOiVUseBO4EjQA8wopR6qFJbEVkJrAYetTctBY66mhyztxkM4zg6nHBej2QUzx8acnLhtYBroa/m05+Ipji7o5hhoq2bvtE02bziyFDC8dvBEvqJIvrlrRHecm4HQ2MZ/F6pOOlIV5fUlpN7wtRcSK80GCadMGVbLe/AEvAo8F0ReZ9S6p4KzW8F7ldK1TajpXiN24DbADo6Ouju7p7K4SXE4/FTOn62MP2anD3HE7SGhKGU4mQ0wRMPvYjPA7kCvLDTTrlMWTn2Tzz9LM96hGRe0Rb20BiwBlOPDIyxqcPHAfu4wWP7Adj64msA7D42QFvEg1cgr7AW9o6PTPgdXFaf50Eg7IXHH3983Hd2YMC66WzfexiAl194jr4ea9vhvTvoHpp4LdqZYi79Lt2Yfk2dme5bLTNjbwQOKqX6AUTkAeBqoJrQf9j1/jiw2fV+GdBdfpBS6m7gboBNmzapzZs3lzepme7ubk7l+NnC9GticvkCQw/9lFvOW8KPt/eQ9YbYl/DxpnUtvHh4GBVuBgY4Z2UX2/qP4u9Yx5/+13YAVrRGePRj15HNK+I//SmXnXcWPdkT7O6Ncf1Vl/KlbU8RXrQUOMhQWmhrqWN5a45Dg9YTxPLOdjZvvqRq3zYDD/U+TSKbY/Pma8d9Z01Hhrnzhafw1rUA/Vx/3Zs4+vQhfnJwL9dduem0TVSaK7/Lcky/ps5M960Wj/4IcKWIRMTKQbsB2FneSETWAy3A067NDwJvEZEW+8ngLfY2g6GEnpEU+YLi4hXNAPQlChwcGOOi5c20RAIc19ZNg2WFbDs6DMD7r1zJkaEEP97ewwl7ILarOcRZ7XWIQHuDZfVo6yeTL7C3N86qtjrn2rVUdfzq+y/l7vdvqrhPWzd9MW3dzEwJBINhpqjFo38WuB94CdhuH3O3iHxWRN7uanorcJ9yzTVXSg0Bfwk8b/981t5mMJSgUyvP7Wwk5PewYzCPUtbkpZa6gCPUi+os4d590rJwPn7zOZy1uI67tx6gx06t7GwKc/36Dq5dt9gpbaCzccBKz1zZGnHe19Ug9C11gaqLe+vB2J09oyxrCRP0ebhgaRMXLmtisX2jMRjOJDUVNVNK3YGVJunm02VtPlPl2H8F/nU6nTO8cXjlmDX1YnlrhLb6IPtsYT+3s5GWiN+ZddpmC+ee3jjNET9NYT+3XbeGT31vO9946hAAS5vDXLlmEe+6dJkzUep4tLS+fFt9kPqglf8eOcUVnLTQN4X9fP03LsPjEa5e28YPPvKmUzqvwTBTmJmxhjPOY7v6+PuH9nDN2kUsawmzuCFIQVnlf5e1hGl15a631Vmvk9k8K+yo/J0XL+WCZU08srMXkWKmDeCUNhiIp/F6BD0BtqUu4OS/R05xYY9IwMcnbj6Hb/725ZyzZGrrwhoMpwMj9IYzilKKP/7ONs7uaOAr77sUEXFSKNd3NiAiJZOUFtUXrZDlLZbQB31e/u2Dl7FmcR1dTWECrro1Ho84Oe0tET+d9k2gtS5Ao50Tf6oRPcCHt6zlgmXNp3weg2E2MPXoDWeU4USW4USWj1y/zhnU1L72BnsRkNa64oBmS50fj0BBWTaPZlF9kP/6/65x6ti4CQe8JLN5WiIBWusCnBhJObYP1ObRGwzzGRPRG84oepB1aXPRbtERvRZ6d0QfCfichbyXt5YOjjZF/Kx2ZdNoihF9wLk5zHREbzDMZYzQG84oWug7m4qirSP69bbf3VpXFPqw3+vMTl3hiugnQtfHaanzO8e0RAJORH+qHr3BMNcxoYzhjNJjlyDudEX0N5/XwQvbd3O+PdGoxY7oAz4PXo9YKZOxtOPRT4bOk2+tC/DLlyzF6xHaG4JFoQ8aoTcsbIzQG84oJ0aSBLwe2uqKg6ztDSH+17oAPm8xEofSRa49QtW89nLc68Eua4nw4S1rAWgMaY/e/DcwLGyMdWOYUQoFxb3PHSGdK5Y72tcX57HdfRXbn4imWNIUwuORivsBJ71SC3Yk4KWzLLtmIsK2kLeWlRhuCvuc8xkMCxkj9IYZZfvxEf70ge107+53tn3l8f38/j0vkc0XxrXviSbpbAqN2+5GD8ZqC2bTqlauX99ec5/CziIgpeUILlzezNr2ejprfDIwGOYr5pnVUBN//v3XuHhFM798ybIJ2+mVmEbtfwGGxzIks3l29oyOyzXvGUlxxerWCc8Z8HmoD/qcyPtTt6yfUt/1k4B7UBfg4hUtPPLHb57SuQyG+YiJ6A01cf+Lx7h764GSbYMVFvHWAu+uF6/F/8XDwyVt8wXFydFUyUBsNZojfkewp4p+EmgpE3qD4Y2CEXrDpCQyOZLZPLtOxpxVoP72wV1c+rlHnPeamL1QSNwl9MP2JKYXyoS+L2ZVrHSnVlajqznsFDSbKiFXHr3B8EbEWDeGSdFroQL8bGcveQVfesxa0KMvli6ZoTqasqL3mEvoR+zFuF88NIxSCrELzuiFvLtqiOj/8daL8HmmF5doy6d8MNZgeKNghN4wKe41Vr/8+H56R9OsWVzHgf4xUtnSxcS0dRO3I3ulFNFklqawn5OjKU6MpFhqD37qyVK1pEnWEvVX4/ylTVy4vNmpMmkwvNEw1o1hUrTQX7G6ld7RNFetWcTf/MoFAOOEXls32qOPp3PkC4rN5ywGSn16XSP+VES8Fm7Z2MkPPnzNhCmcBsNCxgi9YVIG7EHXj998Dh/echZ3f+BSmu1ZpcnyiN62brRHH7Vtm8tXt+IRK6e+eN4MAZ/HKRdsMBhmB/M/zDApOqLf0NnIZausVEgt4HphD422bnRkP2K/X1wfpLMpzDHX4O3wWIbWSMDx7A0Gw+xgInrDpAyNZQj6PCXlfHUmSzXrpjyib44EWN4adpYMBCsbx6ypajDMPkboDZMyEM+wqK408ta56XqJP422brRHH01aTwPNET/LWyIcHXYLfXbcJCaDwTDzGKE3TMrQWJrW+lJBDtl1Zso9+qoRfdgqEdw7mnaeAobHMmYSk8FwGjBCb6hKPJ0jly8wNJYZN1nJ5/Xg98r4wdgqHn1j2O/k2x8btrJthhMZWox1YzDMOkboDRVRSnHT3z/OP3fvd6ybckJ+b8lgbC5fYCyTx+8V0rkC2XyBaCJD2O8l5Pc6Qn90KEG+YOXXm0lMBsPsY4TeUJFYOkfPSIrH9/QzNJap6KWH/d6ScsTartF58WPpHNFE1hlw1Uv/HR1OMJrMolTpMoEGg2F2MEJvqEjfqFWe4NVjUZLZ/DiPHuxFt10R/WjSEnpd0iCWyjmzYsFKsQz5PRwZTDiLeJvBWINh9qlJ6EXkdhF5XUReE5F7RWRccRIReY+I7LDbfcu1/f/a23aKyBfEJE3PC3pHrUlS2bwCKFkBShPyeUs8ep1x02VH9PF0jhFXRC8iTuZNNFHMxjEYDLPLpEIvIkuBjwKblFIbAS9wa1mbdcCfAtcopc4D/sjefjVwDXABsBG4DDAFwOcBvXZEr6kUeYcCXpKu9Eot9LrscDydI5rM0BwuHruiNcKRoSTDY9mq5zUYDDNLrdaNDwiLiA+IACfK9v8u8CWl1DCAUkqvG6eAEBAAgoAf6D3VThtmHx3RdzRakXxF68bvKZkwVbRuihF9NJF11nwFWN4a4cjgmDPb1pQONhhmn0mFXil1HLgTOAL0ACNKqYfKmp0NnC0iT4rIMyJyi33s08Bj9nE9wINKqZ0z+QEMs0PvaIqGkI9r1rYBla2bsN9bIvSxMusmlrKEvskV0Z/d0cBYJs/24yOAWQzEYDgdTFrrRkRagHcAq4Eo8F0ReZ9S6p6y86wDNgPLgK0icj7QBmywtwE8LCLXKqWeKLvGbcBtAB0dHXR3d0/7A8Xj8VM6fraYb/16bX+KBm+B5QzQERF2goyFKQAAIABJREFUbXuWA97S4ZVYNMXAWME5/qVDltAf3/saAE+//DqZfIGhniN0d58EIBm1bgyPbD+CV+D5p56oWOtmrn5fMHf7Zvo1NeZqv2Dm+1ZLUbMbgYNKqX4AEXkAuBpwC/0x4FmlVBY4KCJ7KAr/M0qpuH3s/wBXASVCr5S6G7gbYNOmTWrz5s3T/kDd3d2cyvGzxVzs10+29/CT3dv55w9tHrfvCzueZHWDl9vfcyW3Vzn+h73bOHloyPlcLz+8B3bt5ZduvJb//8mHCC3qAg5x8cb1bL58BQCXZ3J87tkH6RlTtDcE2bJlS8Vzz8XvSzNX+2b6NTXmar9g5vtWi0d/BLhSRCJ2xswNQLn98n0sUUdE2rCsnAP2sW8WEZ+I+LEGYo11M0f40asn6D5qReGFguLQwBiHB8coFBS9o2k6GiZe+SkUKLductQHfTQErfhB155fuai4AlUk4GP1ojrA+PMGw+li0oheKfWsiNwPvATkgJeBu0Xks8ALSqkfAg8CbxGRHUAe+IRSatA+7npgO9bA7E+VUv89S5/FMEUGYhkSOWuR7i937+POh/YA8KdvXU9fLEV748RCb3n0pVk3jSEfHo9QH/Sx/fgIXo9w0fLmkuPWdzZwYGCsZJDWYDDMHjXVo1dK3QHcUbb50679Cvhj+8d9XB740Cn20TCD7O2NMZLMsmlVK/32giKjySyHBhO01gVYXB/kW88dIZtXTsZNNUJ+D8ls3lkHNpbK0hCyxLs+6COeznHe0kYigdI/sw1LGvnJ9pMmojcYThNmZuwbjE9971U+/t1XABiIWUIfTWYZHsvQ2RTi7Rd1cXjQKiXcUUNEny8osnnFswcG6d7dz1ntli1Tb68adcmKlnHHbehsBEzGjcFwujBC/wbgP54+xBcf3Ut/LM3LR6OciKZIZvLEnFLCGYYSGVoiAW7c0OEcN3lEb9WkPzw4xu/8xwssawnzuXeeD1gRPcCmVRWEvssWejMr1mA4LZilBN8A3PfcUXb3xsjkFUpBJl9g18lRZ380mSWayLKsJcLZHfUsbw1zdChJ+ySDsXrxkWcPDhFL5fjGb17mzHTVQn/pyvFC39UU4qM3rONt5y+ZqY9oMBgmwET0CxyllFMW+J8e3etsf/XYiPN6NJllaMyqDS8ivOXcJQS8Htoni+h9ltAftZcHXNpczK5ZVG8tHagrWboREf74prNZv6TxlD6bwWCoDRPRL1BeORol6PewpDFELJ0j5PeQyhbYtLKFFw4P88qxqNN2IJ5hNJV1Bkdvv+ls3n5hF0Gft9rpgWJEr9eBXeQqk/Bnb9vgLCdoMBjOLCaiX6D84X0v8+fff42jQ9ZqTh/ZspbmiJ/f33IWANtdEf2RwTGUKnrm9UEfF5alRFYibHv0R4cTtET8+L3FP6eOxhBrFtfP2OcxGAzTx0T0C5D+WJpDgwkG4hkOD40BcP36Dj5y/TqUUgR9Hvb1xwEIeOHAgNVmqlkwejD26FBy0oFbg8Fw5jAR/QIincuTzRecGanxdI6n9g8CxdWdRISlzWF7dSc/TQHh0KAt9FPMaw/5rT+fkWSWtnoj9AbDXMVE9AuI93z1GVYvirC4oSi6D+/opSXidyYygVUv/sDAGIvrg2RTeQ7bi3VPtTa89uiBkmsaDIa5hRH6ec7T+wdZ1hKmPujjlaNRXj0WZWlzmHM7G9l1cpT+WJoLlzWVHKMzYdrqg4zmEyhrEakpWzfao9fnMhgMcxNj3cxjsvkCv/3vz/MX/72Dl45Ydo1ScGw4ybXr2ljVZs1SXdYaKTlOLwzS1hCkzl8sETzVCUxuoTcRvcEwdzFCP4/Z2TNKIpPn5/v6eWLvAD6P8M6LugC4ZGWLU2pgRbnQN1kToRbXB6m3hT7o85QIdy0ETURvMMwLjHUzj9GDrqlsgfueP8J5S5v45C3rCQe8XLuujX19cX78ag/LW6pF9AH6baFviQQqLgAyESaiNxjmByain8e8cHiYJY0hGoI+ZzJUV3OYz//yBUQCPjYutbz51baFo9ER/tLmsGPdTKfAmN8reD3W8W0V1pQ1GAxzAyP08xSlFC8eGuay1a28+ZzFwPi6Mteta+M/f+cKrlzTWrJ9VVsd9912JW/d2IkuCT+dAmMi4kT1i411YzDMWYx1M085MZLi5GiKTStbWLEowpP7Brhidamgi4izuHc5V65ZBEB9YPoRPVi59GOZqadmGgyG04cR+nnKC4eGACuK37i0iZc//ZZpncexbqZZMjjk99IaCeDzmodDg2GuYoR+nrK/fwwROLuj4ZTOo4W+dZqrPYX9XqckscFgmJuY/6HzlJ5oksX1QQK+U4ukGwOCCJOuD1uN5ojfCL3BMMcx/0PnCd27+7hwWbPjpfeMpJw0yVOhISB850NXsbGrafLGFfi/77oQn2dqaZkGg+H0YozVeUA0keE3v/E8//iz4sIhJ6JJupqnF4WXc9mq1pK6NVNhdVsdy8smZBkMhrmFEfp5wI6eUZSyCpQppVBKcWIkSVeF1ZsMBoOhHCP0c5hMrgDAzp4YAMejSXadjBFNZEllC3TOgHVjMBgWPkbo5yijqSyX/uXD/GDbcXb2jNIQ9P2/9u49Os76PPD499FII2tG1t2WhS+yTGywuQY5iVNCCsXZ0uwWttvGkE2aJk1DN6fdXDhhgWUP5GS355SkaXuyJcuSkgQKxASaUI43AXOxQwsYsM3NlmyDL/JNlqz7ZUaa0cyzf7zvjEZjybqNZkavns85czT6ve/MPPPT6JnfPPN7fy8i8EJTG6d7nWWFE2vWGGPM+Viiz4FYXPn1e62MxOIT7rP/VB/9wyP88q1TNJ3u48pVFVy5soLnm9s43TMEkJEvY40x3jelRC8i3xSR/SKyT0R+JiLnDCVFZIuINLn7PZ7SvkpEtotIs7t9debCn59ePdzBVx/byxO7T0y4T3Nrn7PvB5180D7AhroyNq+v5d2Tvbx9wlnMrC5DX8YaY7xt0kQvIsuBrwEbVfVSwAfckrbPWuAu4GpVvQT4RsrmR4Dvqep64KNAe4Zin7cSJ+z+x389Siyu4+7T3NqHCERicSKxOOvryvjUhloAnnjzBEU+oSZo68sYYyY31dJNIVAiIoVAADidtv0rwP2q2g2gqu0AIrIBKFTV5932AVUNZSTyeazVrbEf7Rjk+aa2cfdpPtPHpoZqykucpQnW15Wxdmkpq6oCdAxEqCsvocDmrxtjpmDSRK+qp4C/AY4DrUCvqm5P220dsE5EXhGRXSJyQ0p7j4j8QkTeEpHvicjMJmx7yKmeMLVlxaysKuHRXS3nbB+JxTnUNsBlK8q5/uKlLCoqYM2SICKSHNXX2RexxpgpEtXxSwfJHUQqgX8GbgZ6gCeBp1T10ZR9tgFRYAuwAngZuAzYDDwEfBjnjeIJ4Feq+lDaY9wK3ApQW1vbuHXr1hk/oYGBAUpLS2d8+7mSGtd9b4SJxmHl4gJ2tY7ww+sDY076cao/zt2vhLn18mIurfbRHo7zoQrn/bG5M8Z9bw7x8Qt8/Pnls0/286G/8k2+xmZxTU++xgUzi+26667bo6obx92YOABnogvwGeChlN+/APwwbZ8HgC+l/P4i8BFgE/CblPY/xinxTPh4jY2NOhs7duyY1e3nSmpcv/3dl/QvHtuj//TaMa2/Y5se7xwcs+/Tb53U+ju2aXNr7zn3Ex2J6TX3vaQ/+bcjGY8rn+RrXKr5G5vFNT35GpfqzGIDdusEeXUqNfrjwCYRCYgz7LweaE7b52ngWgARqcEp2RwB3gQqRGSJu9/vAE1TeEzPUlVO9w6xvKIkeU7XxAybhKbWPop8woVLzn1HL/QVsPNb1/LFqxuyEq8xZv6bSo3+deApYC/wnnubB0XkOyJyo7vbc0CniDQBO4DbVbVTVWPAt4AXReQ9QIAfzcHzmDc6ByNERuLUlS/i4mWLEYEDZ/rH7PPKBx1cvqKCognWeLcvYY0x0zGl1StV9V7g3rTme1K2K3Cbe0m/7fPA5bOI0VNa3YOd6ipKCBYXUl8VGDOib+0Ns+9UH3fccHGuQjTGeIwdGZtlp3qcqZXL3aNaL15WNibRv9DsHGbwqQ1Lsx+cMcaTLNFnWWIOfWJ65Pq6Mlq6QgwOjwDOWjYNNcFx6/PGGDMTlugz7FBbP/f8yz7iExzx2to7RHFhQfJk2uvrFqPqfCHbPxTltcOdbF6/dMx0S2OMmQ1L9Bm2ff8ZHnmthTN9Q+NuP9UT5oKKkmQi/8jqKooLC3hqz0meePMEkVic37/igmyGbIzxODuVYIZ1DkYAONs/PO7qkqe6w2OOaq0M+vmjxhU8ufskFYEiPtZQxeUrKrIWrzHG+2xEn2FdbqLvGBg+Z1s8rrzf1s/apWPr7392zRqi8Tjt/cP8+W+vyUqcxpiFw0b0GdY5MHGiP9EdYjASSx4oldBQE+SmKy7g8NlBrl1ns22MMZlliT7DUks36RLTKNMTPcD3t1zpHKpsB0MZYzLMEn2Gdboj+Q53ZJ+qqbWfAoGLli0+Z5uvQHAOHDbGmMyyGn0GqSrdofOP6BtqgiwqWvArNRtjssgSfQb1DY0QjTnz58+OU6Nvbu0bt2xjjDFzyRJ9BiXKNgUCHWkj+lBUOdkdtkRvjMk6S/QZlJhaubomeM6I/kR/HHCOhDXGmGyyRJ9BiRk3F9Uupn9ohKFoLLntSK+T6C9dXp6T2IwxC5cl+gxKzKFPzKpJnUv/QU+MVVUBli62c70aY7LL04m+Nxxl7/HuuX+cUJQ9Ld10DTqJ/eJkoncSv6ryfnecjfWVcx6LMcak83Sif+TVY2x54DX6h6Jz+jg/fuUon3ngVQ6c6ae0uDC5xk1iiuXxrhB9EeUqS/TGmBzwdKJv6x9iJK4cTDtVX6Yd7RgkrrC9qY2qoJ+a0mJgtHSzp8X5VLFxtSV6Y0z2eTrRdw86I/n0k29n2vGuEACRkTjVpX6qS5215hMj+t0t3ZQUwrqlNuPGGJN93k707lGqzXM8oj/ZHUperw76KS70UREoor3fWZN+b0s3F5b7bB0bY0xOeDrRJ+a1Z3pE/8bRruTUycHhEToGIlz9oWqA5JmjVlUFaOkMEYsrRzoGWVnm6a42xuQxT2efnpBTujl4pn/CU/tN19n+YW5+8DUefvUYACe7nXPA/lHjCi4oX8S6Wqc8s6YmyJGzg5zqDhMZibMsaKN5Y0xueDbRqypdoQhVQT+hSIyWrtDkNxrHs/vO8OiuluTvrb1hVOHNY13AaH2+oaaUHbdfy5c/0QDAmiWlnOoJ09TaC0Bd0LNdbYzJc57NPuFojMhInI+vcUoqMy3ffPfZAzz48pHk7219ozNpVJUTbqJfVRWguNCXPBfsmiVBAF5sbgdgmSV6Y0yOTCn7iMg3RWS/iOwTkZ+JyDmHd4rIFhFpcvd7PG1bmYicFJF/yFTgk0nU5z+2pgpfgdB0evqJ/vDZAY50DNITGl1bvs096Xd3KMrhs4Mc7woR9PuoDBSNue2aGud0gS8daKe8pIjFYzcbY0zWTJroRWQ58DVgo6peCviAW9L2WQvcBVytqpcA30i7m/8JvJyRiKcoUZ9fVraIi5ct5q0T0z9C9oWmNsBZfjjm1vjb3UQPzmyak90hVlYFkiP5hIYaZ0TfORhhzZLgOduNMSZbplpPKARKRKQQCACn07Z/BbhfVbsBVLU9sUFEGoFaYPvsw526xIi+Muinsb6St473MBKLT+s+nncTPUBf2HnjaOsbpqa0mIpAEbtbujje5ST6dCV+H8vdI2QTo3tjjMmFSRO9qp4C/gY4DrQCvaqanrTXAetE5BUR2SUiNwCISAHwfeBbmQ17cok59JUBJ9GHIjEOTGM+fefAMHuOd1Nf7STxnkSi7x+irnwRjasqeW5/G0c7Blk1TqKH0Tp94qcxxuTCpOeMFZFK4CagAegBnhSRz6vqo2n3sxa4FlgBvCwilwGfB36lqifPV7oQkVuBWwFqa2vZuXPnjJ4MwMDAADt37uSNFicxN739ZvKsT1tfeIPN9VMrlh/qjqEK60ojtHTCS/+2iwsrfBw+HaZ6kXBpYIgjRTGq/cLSSCs7d7afcx/+YeeL23DbMQZKh2b1vOZKor/yTb7GBfkbm8U1PfkaF8xBbKp63gvwGeChlN+/APwwbZ8HgC+l/P4i8BHgMZxPAseADqAP+OvzPV5jY6POxo4dO1RV9W+3H9TVd27T6EhM4/G4fuyvXtC/fHzvlO9n+/4zWn/HNn341aNaf8c2felAm6qqXvWd7XrXL96d0n088toxrb9jm77f1peMK99YXNOXr7FZXNOTr3Gpziw2YLdOkFcnHdG7iXqTiASAMHA9sDttn6eBzwI/EZEanFLOEVX9XGIHEfkizhe6d073zWgmukMRyhYVUehzqlONqyvZ4859TzhydoDS4kKWljmTiOJxZdeRTj5+YXVypk19tVN26Q1FiYzE6RyMUDvFNeX/8KrlVAf9fGjpYk42ZeqZGWPM9EylRv868BSwF3jPvc2DIvIdEbnR3e05oFNEmoAdwO2q2jlHMU9JdyiaXI4A4IoV5ZzuHRozVfKrj+7lvmcPJn9/cs8J/vM/vs7e4930ujX51YkafSiSPD1gbVnxlGII+Av59GV1s34uxhgzG1MZ0aOq9wL3pjXfk7Jdgdvcy0T38VPgp9OOcIa6ByNUpMxtT5zZqXMwQkXAeQNo7x+iO+TMjInHlf/rHhjV2jtETyiKr0CSM2d6wtHkHPraMjtLlDFm/vDs4ZrdoQhVgdERfSLpJ0b0qkrf0AihyAgALx5o58jZQcCZmtkTjlBe4pR+yhYV0hOKJufQL53iiN4YY/KBdxN9ysgdRleV7HLXqB8Ydg6CCkecVSgf3dXC8ooSRJxTAPaEolSUOG8OFQE/veFocvkDG9EbY+aTKZVu5iOnRj9auql0k35ifn2iBh9yE31rb5jLV5QTiozQNThMbzhKeSCR6IvoCUVo6xuisEDGfFIwxph858kRffdghHA0xpLFoyWWSndE3+0eMdsXdko2iUQ/OBwj4C+kKuinM21EX15SRE84ysnuMLVli+wEIsaYecWTiX7vcWddmytWVCTbgn4ffl8B3e4aOKMj+pHkz2Cxj+pgMZ1ujT7xKaAi4KcnFOXAmT4uXmanAzTGzC+eTPS7W7opLBCuWDma6EWEikBRckSfXroZHI4RLC6kutTvfBkbSindlBRxtn+Yw2cHWV9XluVnY4wxs+PJGv2eY91csrycRUW+Me1VQX+yRp9YpGx4JM5QNEYkFifo91EV9NPWN0T/0AgVJYkRfREDw87I3xK9MWa+8dyIfiSuvHOyh431ledsqwgUjSb6oWiyvdMd5Qf8hVQH/fQPjST3B6dGn7C+zko3xpj5xXOJvqUvzvBIfNxEXxX0J5cvTpRuwDkPLEBpcSHVpaNf4FYERqdXApQU+ZJLIhhjzHzhudLN+93OmvON4yT6SvdLVRg/0QeKfZT4R8s9iZF8YvbNRcsW47MZN8aYecZziX5fZ4w1NcHkQmWpKgNOjT4e1zGJvsNdwyboL6S4cPRDTkVgtEYPVp83xsxPnird9A9Fae6MsXlD7bjbK4N+4gr9QyPJL2MBOtwRfTC9dOOO5BNH1W6w+rwxZh7y1Ij+5UMdxBQ2r58g0bsj865QhN5wlEVFBQxF48lVKQPurJuExEi+oSbI3918Bb97ybI5fgbGGJN5nhrRv9DcRmkRXLWqYtztyaNj3US/zC3vnE0Z0SfeDERg8aLEdeEPPryCgN9T74vGmAXCM4l+JBbnpQPtXLGkMHmykXTJ9W4GI/SGR1hW7iT60Rq9j0JfAZWBIsoWFdkXr8YYT/DMELWtf5gLKkq4cunwhPtUJRc2i9I3FKWu3FlrPnVED05NfiSucxyxMcZkh2dG9MsrSvj1169hY61vwn0q3dUsz/SGiYzEU0b0ztz6EvdI2iWLi5Ojf2OMme88M6JPEJm43FJaXEhhgXC0IwRAnZvoB4ZHCPh9yVUp/8e/32AjemOMZ3gu0Z+PiLCsfBFvHHNOZ1sV9OP3FTjr3BSPdsWly8tzFaIxxmScZ0o3U7Vl40pOdIUBKFtUlDwSNuifuORjjDHz2YJL9H+8qT5Ziy8vKSLgJnibOmmM8aoFl+grg362bFzhXA/4kyP60mJL9MYYb1qQ2e22T13EJcvLWVlVQtAdyQeKrXRjjPGmBZnoywNFbNm4EiClRr8gu8IYswAsuNJNukSNPmgjemOMR00p0YvIN0Vkv4jsE5Gficg5awCLyBYRaXL3e9xtu1JEXnPb3hWRmzP9BGbLvow1xnjdpNlNRJYDXwM2qGpYRH4O3AL8NGWftcBdwNWq2i0iS91NIeALqvq+iFwA7BGR51S1J9NPZKZKipwusBG9McarpjqMLQRKRCQKBIDTadu/Atyvqt0Aqtru/jyU2EFVT4tIO7AEyJtEP1q6sRG9McabRHXyQ/1F5OvAXwFhYLuqfi5t+9PAIeBqwAd8W1WfTdvno8DDwCWqGk/bditwK0BtbW3j1q1bZ/yEBgYGKC0tnfL+Pz8Y4VdHo3x+vZ/N9UWT3yBLcWWLxTV9+RqbxTU9+RoXzCy26667bo+qbhx3o6qe9wJUAi/hjMSLgKeBz6ftsw34pbu9ATgBVKRsrwMOApsme7zGxkadjR07dkxr/79//pDW37FNf/7m8Vk97mSmG1e2WFzTl6+xWVzTk69xqc4sNmC3TpBXp/Jl7GbgqKqeVdUo8Avgt9L2OQk8o6pRVT2KM7pfCyAiZcD/A+5W1V1TeWfKpoAdMGWM8bipJPrjwCYRCYizNOT1QHPaPk8D1wKISA2wDjgiIn6ckf4jqvpUxqLOoMQ8+oAlemOMR02a6FX1deApYC/wnnubB0XkOyJyo7vbc0CniDQBO4DbVbUT2AJ8EviiiLztXq6ciycyUwFb1MwY43FTGsaq6r3AvWnN96RsV+A295J6u0eBR2cZ45xaURnAVyDJk5AYY4zXLPh6xUcbqnjjv19PdWlxrkMxxpg5seCXQAAsyRtjPM0SvTHGeJwlemOM8ThL9MYY43GW6I0xxuMs0RtjjMdZojfGGI+zRG+MMR5nid4YYzzOEr0xxnicJXpjjPG4KZ1hKptE5CzQMou7qAE6MhROJllc05OvcUH+xmZxTU++xgUzi61eVZeMtyHvEv1sichuneh0WjlkcU1PvsYF+RubxTU9+RoXZD42K90YY4zHWaI3xhiP82KifzDXAUzA4pqefI0L8jc2i2t68jUuyHBsnqvRG2OMGcuLI3pjjDEpPJPoReQGETkoIh+IyJ05jGOliOwQkSYR2S8iX3fbvy0ip1JOkv7pHMV3TETec2PY7bZVicjzIvK++7MyyzFdlNIvb4tIn4h8Ixd9JiI/FpF2EdmX0jZu/4jjB+5r7l0RuSrLcX1PRA64j/1LEalw21eLSDil3x6Yq7jOE9uEfzsRucvts4Mi8rtZjuuJlJiOicjbbnvW+uw8OWLuXmeqOu8vgA84DKwB/MA7wIYcxVIHXOVeXwwcAjYA3wa+lQd9dQyoSWv7LnCne/1O4L4c/y3PAPW56DPgk8BVwL7J+gf4NPBrQIBNwOtZjuvfAYXu9ftS4lqdul+O+mzcv537v/AOUAw0uP+3vmzFlbb9+8A92e6z8+SIOXudeWVE/1HgA1U9oqoRYCtwUy4CUdVWVd3rXu8HmoHluYhlGm4CHnavPwz8xxzGcj1wWFVnc9DcjKnqy0BXWvNE/XMT8Ig6dgEVIlKXrbhUdbuqjri/7gJWzMVjT2aCPpvITcBWVR1W1aPABzj/v1mNS0QE2AL8bC4e+3zOkyPm7HXmlUS/HDiR8vtJ8iC5ishq4MPA627TX7ofvX6c7fJICgW2i8geEbnVbatV1Vb3+hmgNjehAXALY//58qHPJuqffHrd/SnOqC+hQUTeEpHfiMg1OYppvL9dvvTZNUCbqr6f0pb1PkvLEXP2OvNKos87IlIK/DPwDVXtA/4PcCFwJdCK87ExFz6hqlcBvwf8hYh8MnWjOp8VczIVS0T8wI3Ak25TvvRZUi77ZyIicjcwAjzmNrUCq1T1w8BtwOMiUpblsPLub5fms4wdUGS9z8bJEUmZfp15JdGfAlam/L7CbcsJESnC+QM+pqq/AFDVNlWNqWoc+BFz9HF1Mqp6yv3ZDvzSjaMt8VHQ/dmei9hw3nz2qmqbG2Ne9BkT90/OX3ci8kXgPwCfc5MDblmk072+B6cOvi6bcZ3nb5cPfVYI/CfgiURbtvtsvBzBHL7OvJLo3wTWikiDOyq8BXgmF4G4tb+HgGZV/duU9tSa2h8A+9Jvm4XYgiKyOHEd58u8fTh99Sfubn8C/Eu2Y3ONGWXlQ5+5JuqfZ4AvuLMiNgG9KR+955yI3AD8N+BGVQ2ltC8REZ97fQ2wFjiSrbjcx53ob/cMcIuIFItIgxvbG9mMDdgMHFDVk4mGbPbZRDmCuXydZeNb5mxccL6ZPoTzTnx3DuP4BM5HrneBt93Lp4F/At5z258B6nIQ2xqcGQ/vAPsT/QRUAy8C7wMvAFU5iC0IdALlKW1Z7zOcN5pWIIpTC/3yRP2DMwvifvc19x6wMctxfYBTu028zh5w9/1D9+/7NrAX+P0c9NmEfzvgbrfPDgK/l8243PafAv8lbd+s9dl5csScvc7syFhjjPE4r5RujDHGTMASvTHGeJwlemOM8ThL9MYY43GW6I0xxuMs0ZsFSURiMnbFzIyteOquhJirOf/GnKMw1wEYkyNhVb0y10EYkw02ojcmhbtG+XfFWbP/DRH5kNu+WkRechfpelFEVrntteKsBf+Oe/kt9658IvIjd73x7SJSkrMnZRY8S/RmoSpJK93cnLKtV1UvA/4B+Hu37X8DD6vq5TiLh/3Abf8B8BshMjnuAAABD0lEQVRVvQJn7fP9bvta4H5VvQTowTny0picsCNjzYIkIgOqWjpO+zHgd1T1iLvw1BlVrRaRDpzD+KNue6uq1ojIWWCFqg6n3Mdq4HlVXev+fgdQpKr/a+6fmTHnshG9MefSCa5Px3DK9Rj2fZjJIUv0xpzr5pSfr7nXX8VZFRXgc8C/utdfBL4KICI+ESnPVpDGTJWNMsxCVSLuiaFdz6pqYoplpYi8izMq/6zb9l+Bn4jI7cBZ4Etu+9eBB0Xkyzgj96/irJhoTN6wGr0xKdwa/UZV7ch1LMZkipVujDHG42xEb4wxHmcjemOM8ThL9MYY43GW6I0xxuMs0RtjjMdZojfGGI+zRG+MMR73/wE1HimZ3C4+NwAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(np.array(evaluation_accuracy[200:])/1000)\n", + "plt.grid()\n", + "plt.title('Accuracy (%) on the test data')\n", + "plt.xlabel('Epoch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 352 + }, + "id": "GNAzP1i9QgWK", + "outputId": "ed336538-121b-4996-d9f1-e5a273dfab64" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 20 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAE9CAYAAADqP8J6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1fn48c9DVsjGHvZFBQEhCQQCoiIuFFyKG60gdakLti7V2lrxW6tWv1p/amu/WutWcRdsUZEq1qUSxY19E5AdTEJYA2Rf5/n9cSdhCCHrJJO587xfr3nN3HvPvfc8jD5zcu6554qqYowxxt3aBLoCxhhjmp8le2OMCQGW7I0xJgRYsjfGmBBgyd4YY0KAJXtjjAkB4YGuQHWdO3fWfv36NXr/goICYmJi/FehAHNbPOC+mNwWD7gvJrfFA8fGtHz58v2q2uV45Vtdsu/Xrx/Lli1r9P7p6emMHz/efxUKMLfFA+6LyW3xgPticls8cGxMIrKztvLWjWOMMSHAkr0xxoQAS/bGGBMC6kz2IjJLRPaKyHfH2S4i8qSIbBGRNSIywmfb1SKy2fu62p8VN8YYU3/1adm/DEyqZft5wADvawbwDICIdATuA0YDacB9ItKhKZU1xhjTOHUme1X9AsippchFwKvq+BZoLyLdgYnAJ6qao6oHgU+o/UfDGGNMM/FHn31PIMNnOdO77njrjTHGtLBWMc5eRGbgdAGRmJhIenp6o4+Vn5/fpP1bG7fFA+6LyW3xgPticls80PCY/JHss4DePsu9vOuygPHV1qfXdABVfR54HmDkyJHalJsf3HbzhNviAffF5LZ4wH0xuS0eaHhM/ujGmQ9c5R2VMwY4rKrZwEfAj0Skg/fC7I+864wxxgAej7Int5jlOw/y79W7+M93u5vtXHW27EVkNk4LvbOIZOKMsIkAUNVngQXA+cAWoBD4uXdbjog8CCz1HuoBVa3tQq8xxjRZhUfZeaCA73fnOa/sXLbtKuKpDV/79TwRYUJcdARx0eHER0cQHx1OXHQE8W3Dj1rfNjKMvbklZB0qJOtQMVkHi9h1qIisQ0VkHy6irOLIo2FP6RHPpKHd/FrPSnUme1WdVsd2BW4+zrZZwKzGVc0YY2p3sKDUm9Rz+T7bed+4J4/iMg8AbQT6d44hKgzaRoT59dyl5R4ycgrJKy4nt7iM/JJy6nqktwgkxkXTs0NbUnq35/xh3enZoS0920fTs307erSP9msdfbWKC7TGmNapwqPszXNao1ne1qhvyzSvuDxgdSst93CgoLRquWNMJIO7x3FFWl8GdY9jcLd4BiTGEh0R5u3fHt2s9fF4lPzScvKKy8krLnN+BIrKKCytoEtcFD3bt6VbQjQRYYGZuMCSvTEhTlXZl1fCht15bNydy8bd+Xy3vYh7Fn/G7sPFlHuObq62bxdBz/Zt6dsphvZtIxAJTL3D2gj9O8cwqFs8g7rH0SU2CglUZYA2bcTbnRMBtA1YPY7Hkr0xQepwYRmLtuxDkGP6ieOiw4muoduiuKyCTXsq+7K93R+788jxaSF3jYuifTiM7NuBHu3bersZnFeP9m2JibK0EYzsWzMmiBSXVbDw+728uzKL9I37KK3wHLdsZFgbJ/m3dZJ/fkk5O/YXUNlQbxsRxsBucfxoSCKDusVxcrd4BnWLo0NMpLfbY3gLRWVagiV7Y1o5j0dZvD2H91Zl8cHabPKKy+kSF8WVp/blgqTutIsMq+ofruwvzvVeNHSWnW3dE6L5cVIPBnWLY1D3ePp0bEdYm8B1e5iWZcnemFZq4+483l2ZxfxVWew6XExMZBgTh3bjkuE9GXtiZ0vUpkEs2RvTROUe5UB+SVUr2mlZO63rquWicgpKyvHUNTYPUGDdrlw2ZOcS1kY4c2AX7jpvEBOGJNIu0v6XNY1j/+UY11FVSso9Vd0YR7o3Krs2ju7eyPXp+qjcll/PxOycD/j401rLxESGERsdTlg9R4t0S4jmj5NP4YKk7nSOjarXPsbUxpK9CWrfbjvAi19uZ09u8VFJvbYLl+Dc3BIX5TOCpW0EPdtHEx8dR1x0eIMSc2bGTpIGDfDePekc78hdlRHOsazLxQSYJXsTlDZk5/Lof75n4cZ9dI2LYkiPePp1ivEm2sohiOE+yffIe3x0ODGR4bTxUwJOT89m/Gn9/XIsY5qLJXsTVDJyCnnik028uyqLuKhw7j5vEFeP7VfjmHJjzBGW7E1QyCko5W+fbeH1b3ciAjPGncBNZ55EQruIQFfNmKBgyd60aoWl5cz6cjvPfb6NgtJyfpLam9snDKB7Quu7Hd2Y1sySvWmVyio8vLU0g//772b25ZUwYUgiv5t4MgMS4wJdNWOCkiV706rszStmzpIM3lz8A7tzixnVrwPP/mwEqX07BrpqxgQ1S/Ym4FSVZTsP8uo3O/nPd9mUVShnDOjMw5cO5ayTuwZ0JkNj3MKSvQmYwtJy3lu1i1e/2cmG7FziosP52Zi+XDmmLyd0iQ109YxxFUv2psXtLvDwwL/X86/lGeQVlzOoWxwPXzKMi4f3sOkAjGkm9n+WaVblFR52HChgg3fu9BU7D/HNtiLC2+zgvGHduerUvozs28G6aoxpZpbsjd/szy+peiDGhuw8Nu7JZdOefErLnakLwtsIJ3aJ5ZKTIrj7p+PoGt98z9s0xhzNkr1pkvIKD3/5ZBP/XJbJ/vySqvVd4qIY1C2Oa8b2c+ZP7xbPiV1jiAp3ngdqid6YlmXJ3jTa3rxibnlzJUu25zDxlERG9evI4O7xnNwtzmZqNKaVsWRvGmXpjhxufmMFucVl/OWnyVw6olegq2SMqUWb+hQSkUkislFEtojIzBq29xWR/4rIGhFJF5FePtsqRGSV9zXfn5U3LU9VefHL7Ux7/lvaRYbx7k2nWaI3JgjU2bIXkTDgaWACkAksFZH5qrrep9jjwKuq+oqInA38CbjSu61IVVP8XG8TAPkl5dz19ho+WJPNhCGJPP6TZBLa2kRkxgSD+nTjpAFbVHUbgIjMAS4CfJP9EOAO7+eFwDx/VtIE3pa9efzi9RVs25fPXZMGceO4E/w2H7wxpvnVpxunJ5Dhs5zpXedrNXCp9/MlQJyIdPIuR4vIMhH5VkQublJtTUB8sCabi/72FQcLSnn9utH8cvyJluiNCTKidTxnU0SmAJNU9Xrv8pXAaFW9xadMD+BvQH/gC+AyYKiqHhKRnqqaJSInAJ8B56jq1mrnmAHMAEhMTEydM2dOowPKz88nNtY9t9oHMp5yj/KvjaV8tLOcExPacPPwKDpG1+syT63sO2r93BaT2+KBY2M666yzlqvqyOPuoKq1voBTgY98lu8G7q6lfCyQeZxtLwNTajtfamqqNsXChQubtH9rE4h4PB6PfrFpr17696+0713v633vfaclZRV+O759R62f22JyWzyqx8YELNNacmt9+uyXAgNEpD+QBUwFrvAtICKdgRxV9Xh/DGZ513cAClW1xFvmNODRepzTBEBucRlzl2Xy+rc72ba/gI4xkfzf1BQuSqnea2eMCTZ1JntVLReRW4CPgDBglqquE5EHcH5J5gPjgT+JiOJ049zs3X0w8JyIeHCuDzyiR4/iMa3A97tzefWbncxbmUVhaQXD+7TnicuTOX9Yd6LC7dmuxrhBvW6qUtUFwIJq6+71+TwXmFvDfl8Dw5pYR9MMSss9fLRuN699s5MlO3KICm/DRSk9uHJMP4b1Sgh09YwxfmZ30IaYA/klvPLNTmYv+YF9eSX06diO358/mJ+M7EX7dpGBrp4xpplYsg8hS7bncPObK9ifX8JZJ3flylP7cuaALjaM0pgQYMk+BKh3ioM/ffg9fTq249Vr0xjcPT7Q1TLGtCBL9i6XX1LOXXPX8MHabCaekshjP0kmPtqmODAm1Fiyd7Ete/O48bXlbN9fwN3nDWLGuBPsiVDGhChL9i71/ppd/G7uGtpFhvHG9WM49cROde9kjHEtS/YuU1bh4U8LvmfWV9tJ7duBp68YQbcEeyqUMaHOkr2L7Mkt5uY3VrBs50GuGduP/zl/MJHhTZ/LxhgT/CzZu8S32w5wy5srKSgptykOjDHHsGQf5MorPDy9cCtPfraZvh3b8eYNoxmYGBfoahljWhlL9kEs82Aht89ZxbKdB7k4pQcPXjyUOBtWaYypgSX7IDV/9S5+/+5aVOGJy5O5ZLg9B9YYc3yW7INMfkk59723jrdXZDK8T3v+7/Lh9OnULtDVMsa0cpbsg8iqjEPcNmclGTmF/Orsk/jVOQMID7PRNsaYulmyDwIVHuXZz7fyxCebSIyPZs6MU0nr3zHQ1TLGBBFL9q3cgSIPV7zwLYu353BBUncevmQYCW3tIqwxpmEs2bdi6Rv3cu/XRaiU8tiUJKak9rK5bYwxjWLJvpX6eut+Zry2nMS2bXj1xjPo3zkm0FUyxgQxS/at0JrMQ9zwyjL6dmzHbUM9luiNMU1mQzlamS1787h61hLat4vktetGExtp3TbGmKazZN+KZB4s5Gf/WEJYmza8cf1om63SGOM3luxbiX15JVz54hIKS8t57bo0+lnXjTHGj+qV7EVkkohsFJEtIjKzhu19ReS/IrJGRNJFpJfPtqtFZLP3dbU/K+8Wh4vKuHrWErIPF/HSz0fZ82GNMX5XZ7IXkTDgaeA8YAgwTUSGVCv2OPCqqiYBDwB/8u7bEbgPGA2kAfeJSAf/VT/4FZVWcP0rS9m8N4/nrhxJal+7WcoY43/1admnAVtUdZuqlgJzgIuqlRkCfOb9vNBn+0TgE1XNUdWDwCfApKZX2x1Kyz388o3lLNt5kCcuT+HMgV0CXSVjjEvVJ9n3BDJ8ljO963ytBi71fr4EiBORTvXcNyRVeJTf/Gs16Rv38dDFw7gwqUegq2SMcTF/jbP/LfA3EbkG+ALIAirqu7OIzABmACQmJpKent7oiuTn5zdp/5agqry6vpSFGeX8ZGAEPYq2kZ6+rcaywRBPQ7ktJrfFA+6LyW3xQCNiUtVaX8CpwEc+y3cDd9dSPhbI9H6eBjzns+05YFpt50tNTdWmWLhwYZP2b25FpeX68IL12veu9/XhBevrLN/a42kMt8XktnhU3ReT2+JRPTYmYJnWklvr07JfCgwQkf44LfapwBW+BUSkM5Cjqh7vj8Es76aPgId9Lsr+yLs9pHg8ypIdOcxbmcUHa7PJKy5nWlpvZk4aFOiqGWNCRJ3JXlXLReQWnMQdBsxS1XUi8gDOL8l8YDzwJxFRnG6cm7375ojIgzg/GAAPqGpOM8TRKm3cnce8VVm8tzKLXYeLaRcZxqRTunHR8J6MG9DZJjUzxrSYevXZq+oCYEG1dff6fJ4LzD3OvrM40tJ3vd2Hi5m/Oot3V+5iQ3YuYW2EcQM6c9d5g5gwJJF2kTYdkTGm5Vnm8ZN1uw7z8IINfL31AKqQ0rs99/94CBcm96BzbFSgq2eMCXGW7P3kwffX8/3uPH519gAuHt7TZqo0xrQqluz9YG9eMYu353Dr2QP49YSBga6OMcYcwyZC84OPvtuNKlyY1D3QVTHGmBpZsveD99dkc1LXWAYmxgW6KsYYUyNL9k20N7eYJTtyuGCYteqNMa2XJfsm+s86pwvnAuvCMca0Ypbsm+j9NdkMsC4cY0wrZ8m+CfbmFrN0R4616o0xrZ4l+yb40DsKx/rrjTGtnSX7JvhgTTYnJ8YxwLpwjDGtnCX7RtqTW8zSnTmcb616Y0wQsGTfSB+uzfaOwukW6KoYY0ydLNk30gdrsxnULY6TuloXjjGm9bNk3wi7DxezdMdB68IxxgQNS/aN8OF32QCW7I0xQcOSfSN8sKayCyc20FUxxph6sWTfQNmHi1i286CNrTfGBBVL9g304drdAJxvd80aY4KIJfsG+mBtNoO7x3NiF+vCMcYED0v2DbDrUBHLdx60h5QYY4KOJfsGWLDWRuEYY4JTvZK9iEwSkY0iskVEZtawvY+ILBSRlSKyRkTO967vJyJFIrLK+3rW3wG0pAVrsxnSPd4eJm6MCTp1PnBcRMKAp4EJQCawVETmq+p6n2L3AP9U1WdEZAiwAOjn3bZVVVP8W+2Wl3WoiBU/HOLOiScHuirGGNNg9WnZpwFbVHWbqpYCc4CLqpVRIN77OQHY5b8qtg4fertwbMilMSYY1SfZ9wQyfJYzvet83Q/8TEQycVr1t/ps6+/t3vlcRM5oSmUD6YO12ZzSI55+1oVjjAlCoqq1FxCZAkxS1eu9y1cCo1X1Fp8yd3iP9WcRORV4ERgKRACxqnpARFKBecApqppb7RwzgBkAiYmJqXPmzGl0QPn5+cTG+ndY5P4iD7/9vIgpAyO48IRIvx67Ls0RT6C5LSa3xQPui8lt8cCxMZ111lnLVXXkcXdQ1VpfwKnARz7LdwN3VyuzDujts7wN6FrDsdKBkbWdLzU1VZti4cKFTdq/Js9/vlX73vW+7tif7/dj16U54gk0t8XktnhU3ReT2+JRPTYmYJnWklvr042zFBggIv1FJBKYCsyvVuYH4BwAERkMRAP7RKSL9wIvInICMMD7QxBUPlibzdCe8fTtZF04xpjgVGeyV9Vy4BbgI2ADzqibdSLygIhM9hb7DXCDiKwGZgPXeH9pxgFrRGQVMBf4harmNEcgzSUjp5BVGYe4YFiPQFfFGGMarc6hlwCqugDnwqvvunt9Pq8HTqthv7eBt5tYx4CqnM7YRuEYY4KZ3UFbhw/W7mZYzwT6dGoX6KoYY0yjWbKvxfb9BazOOMQFNheOMSbIWbKvxZuLdxLeRrh0ePXbCowxJrhYsj+O4rIK/rU8kx+dkkjX+OhAV8cYY5rEkv1xLFibzaHCMn42um+gq2KMMU1myf44Xv92Jyd0juHUEzsFuirGGNNkluxrsH5XLit+OMQVo/sgIoGujjHGNJkl+xq8sXgnUeFtmJLaK9BVMcYYv6jXTVWhJL+knHkrs7gwqQft27XspGfGmEZQhbIiKM2HkjwoyYWSys/OcrfsnZARA10GQnRCy9Qpdxdkr4K9GyAqDtp1gpjOENMF2nV2lsNaLgVbsq9m3sosCkormD6mT6CrYow7VJRB4QFo2xHCm9CAKsyBXSudBLprJez+DooOOgldK2rddRDAxqechbju0HkgdDn5yHuXQU4Sbmy3bd4ep06+r4K9de/XtsOR5B/TGboNgzN/17g61MGSvQ9V5Y3FPzCkezzDe7cPdHWMCQ4l+XA4Aw5nwqEffD573/N2gXpA2kB8T+jQD9r3dd479IMO3s++ybboEGSvPjp5Htp55JwdT4QeKc4+UXFHXpFxRy97X4u/+C+jT2wP+76HfZtg/0ZYNRtK844cM7o9dB4AUfEQ0RbCo51338/h0RDRDiKiIX/fkbrlVT6vSZwfj5POgR7DnVfXIVBeAgX7oHC/816w3/kB9P28f5Pz79RMLNn7WPHDITZk5/LQJUPtwqxxP1Un0Rzc4bwOed8PZ5JyYC9sjQNPhdNq9lQ4ieio5QonKRcfOvq4bcKdpN6+D/Q/AxJ6Q2xXJ7FVnmvLp5C/++j9Ito5PwIVpZCz9cj69n2dpDnyWue9ezK0bVhjrKhddzh5PJx83tHx5+5yEv++Tc4PQc5W56+FvGyna6i82HkvK4KKkmMP3GkA9Dv9SGLvNgyiapg3PyoWYgI7ss+SvY83Fu8kNiqci1LsjlkTBDwep6ugrNDpKikvcd4rSr2vap8LD3iT+k5v0t0JZQVHHzM2ERJ6oxLmtGQlDNqE+by3OXo5Mhba93YSekJv53NsorOtLqWFzl8CVfXxvqQNpFxxJIG26+j3fzrA+SsioafzOvHsust7PE7yr/wBiIqD6Pi692slLNl7HSwo5f012fx0ZC9io+yfxbQSHo/TyszZ5rQ6D2z1fva+yosbdryImCPdJyeMP7pLpX0fiHQm/Fudns748eP9GcmxIttB10HOKxi0aePUOTI4J0W0rOb19opMSss9TLc7Zt2nvMT5cz0v2+kq6J7c+AtxjaEKebth73rnYmJFqbeFWOJ9Ffus874X7ocDlQm96MixwiKhQ3/odKLTGu3Qz2lhhkU428Iiq332eUUnOBcBrYsyJFmyBzwe58Jsat8ODO4ePH+WGa+c7XBwu5PQc7MhN8ub3Hc574UHji6f0AeGTIYhF0HPkU6LzV88HidB714N2Wtg9xrnvXB/3fuGRTkXAMOjnD7pjifCiWdBx/7O504nOn3h9ekiMaYaS/bA11sPsH1/Ab8656RAV8XUV1kxrJ8HS1+EzCVHb2vXCeJ6QHwP6JnqJMi47s5y3m5Y/x4sfg6++ZtTbshkGHIx9B5d/8Sv6vyI5GynW/YnsOADJ6nv+c4Z7w3QJsLpohg4yblw122oM9QuLMpJ6OHRzlDE8Gin5W0tbtOMLNnjXJjt0C6C84bavPWtXs52WDYLVr4ORTnQ6ST40UPQc4ST0OO6O8PiajN8OhQfho3/cRL/spdg8bMQ2w0G/9hp8fcd65TN3eW01A9uP/IXRM42yNlRNWxvEDgXKhOHOhcWuyVB9yRn7HZ4VHP+axhTbyGf7PfkFvPx+j1cd3p/oiPsz+NWyVMBmz+Gpf+ALf91RmsMugBGXQf9z2xcizg6AZIvd14lebDpIyfxr3wdlr4AUQlOX3lF6ZF92kR4x4T3hz6nOu8d+7N4aw6jJ03zb3eQMX4W8sn+raUZVHiUaWl2x2yrk78XVrwKy192btSJ6w5n3gWpVztdMv4SFQfDpjivknzY8glsXej0m3sTOh36Q0KvGvvLi7LTLdGbVi+kk315hYfZS37gjAGd6d85JtDVMZUKDsBnDzqtbE8Z9B8HEx+Ck893Rpo0p6hYOOUS52WMi4R0sl+4cR/Zh4u578dDAl0VA1BRDstfchJ9ST6M/Dmk3ehMXmWMaZKQTvavf7uTxPgozhmcGOiqmJ1fw4I7ndEs/c+E8x4NnpttjAkC9epoFJFJIrJRRLaIyMwatvcRkYUislJE1ojI+T7b7vbut1FEJvqz8k3xw4FCvti8j8tH9SEizPpbAyZ3F7x9Pbx0njPPyk9egaves0RvjJ/V2bIXkTDgaWACkAksFZH5qrrep9g9wD9V9RkRGQIsAPp5P08FTgF6AJ+KyEDVOuYjbQFvLvmBNiJMS+sd6KoEr9JC2LgAVs9xknaP4dAr1blRqeuQ2ufqLi+Fb/8Onz8KnnIY9zs4/ddBeyu6Ma1dfbpx0oAtqroNQETmABcBvslegcpbTxOAyvk+LwLmqGoJsF1EtniP940f6t5oJeUV/GtZBucM6kr3hLaBrErw8Xhg55dOgl8/3xlrntDbmdZ14wJY9bpTLqIddE85kvx7jXImnAI6HlgBz/wGDmxxLrpOfNgZ8WKMaTb1SfY9gQyf5UxgdLUy9wMfi8itQAxwrs++31bbN+BTSv7nu90cKChl+hibB6fe9n4Pa+bAmn9BbqYzb/gpF0HyNOgz1hl6qOrcdJS5HLKWQeYy507VCp+HRsT3IClruXP7//S5MGBCYOMyJkSIqtZeQGQKMElVr/cuXwmMVtVbfMrc4T3Wn0XkVOBFYCjwJPCtqr7uLfci8KGqzq12jhnADIDExMTUOXPmNDqg/Px8YmNrmE/ax7Ori9mQ4+GJ8W1p08pvUa9PPM1ClcjSQ3TZ9yXddi8kLn8rShtyOg5nd7ezONApDU9Y3XeHiqeM2PztxOduIj53E+0KM8hMGMXeE3+CtmnmYZQtJGDfUTNyW0xuiweOjemss85arqojj1e+Pi37LMC3Y7uXd52v64BJAKr6jYhEA53ruS+q+jzwPMDIkSO1KVOrptdjatZHVy8ipW8UZ5+V1ujztJT6xNNgHo/zIIk8n4nD8rK9E4ntOjJDZOUcL91T4PRHkKGX0Sm2K019BMOelpg+twU1y3cUYG6LyW3xQMNjqk+yXwoMEJH+OIl6KnBFtTI/AOcAL4vIYCAa2AfMB94Ukb/gXKAdAFSbtaplVXiUrfvyGXtiYJ8aExAl+fDBb+C7t52blXy1CXfmhonvAYlDnO6V+J5w0rk2MsYYF6gz2atquYjcAnwEhAGzVHWdiDwALFPV+cBvgBdE5Nc4F2uvUad/aJ2I/BPnYm45cHOgR+Jk5BRSUu5hYGJcIKvR8vZugH9e5VwUTb3GGS1TORNkfA/nWZ42da4xrlWvm6pUdQHOcErfdff6fF4PnHacfR8CHmpCHf1q0x5npsKTEt3Vf1er1W/B+7dDZIwzhr3/uEDXyBjTwkLuDtrNe51+6AFdQyDZlxXDf2Y6UxD0GQtTZkG8TeNsTCgKvWS/J4/uCdHERbtjJMhxHdzhdNtkr4bTboez/1D7TU7GGFcLuf/7N+/N5yS3t+q/XwDzfuF8njobBp1fe3ljjOuF1KQwFR5ly958916crSiHT+6FOdOcB1Hf+IUlemMMEGIt+8yDzkgcV/bX5+2GudfCzq9g5LUw8U91P57PGBMyQirZb97jvTjrppa9qjNu/j8zobQALnneedSeMcb4CKlkv2mvd9ilW1r2+zbBgt/A9i+cGScvfga6Dg50rYwxrVBIJfste/LpFh9NQtsgH4lTWgiLHoevnnSmBL7gL86NUnZTlDHmOEIq2W/em8+AYL+ZauOHsOB3cPgHSL4CJjwAsV0CXStjTCsXMsne4x2JMy2tT6Cr0jgHdzr98hsXQJfBcM0C6FfjTcvGGHOMkEn2WYeKKCqrCLqWvXjK4IvHnZe0gQkPwphfQliQd0UZY1pUyCT7yjlxgmbYpSps+ZRRS2+DoiwYPBkm/QkSegW6ZsaYIBQyyf7InDhBMOxy+xfw2f9CxmIkups90ckY02Qhk+w37cmja1wUCe1acffHD986SX7HIojrARf8hSV5fTjTEr0xpolCJtm36mkSslbAwodgy6cQ0xUmPQKpP4eIaDQ9PdC1M8a4QEgk+8qROD8d2bvuwi1p93ew8GHY+AG07egMoxx1vTPvvDHG+FFIJPusQ0UUlla0npb9vk2Q/jCsexeiEuCs38PoX0B0fKBrZoxxqZBI9lsqL862hmGX6+c7E5aFR8EZv4Wxt0DbDvapsfsAACAASURBVIGulTHG5UIi2beaYZfr3oW510HPVJg2G2I6B7Y+xpiQERLJfvPefLrERdG+XWTgKrF2LrwzA3qnwfR/QVQr6VIyxoSEkHh4yeY9eYFt1a/5J7xzA/QZ44yZt0RvjGlhrk/2qsrmQA67XDUb3r0R+p7mbdG3gusGxpiQU69kLyKTRGSjiGwRkZk1bH9CRFZ5X5tE5JDPtgqfbfP9Wfn62HW4mMLSisDMYb/ydZj3S+g/Dq74pw2pNMYETJ199iISBjwNTAAygaUiMl9V11eWUdVf+5S/FRjuc4giVU3xX5UbpvLibIu37Je/Av/+FZx4Nkx9EyLatuz5jTHGR31a9mnAFlXdpqqlwBzgolrKTwNm+6Ny/rCl8lGELdmyXzbLSfQnTYCpsy3RG2MCrj7JvieQ4bOc6V13DBHpC/QHPvNZHS0iy0TkWxG5uNE1baRNe/LoHBtJh5gWGomz5AV4/9cwcBJMfcMe+m2MaRVEVWsvIDIFmKSq13uXrwRGq+otNZS9C+ilqrf6rOupqlkicgLOj8A5qrq12n4zgBkAiYmJqXPmzGl0QPn5+cTGHmnFP/BNEVFhcFda87eue2b+mwFb/sH+TqNZd8qdaJumT7pWPR43cFtMbosH3BeT2+KBY2M666yzlqvqyOPuoKq1voBTgY98lu8G7j5O2ZXA2FqO9TIwpbbzpaamalMsXLiw6rPH49FT7v2P/mHe2iYds15WvK56X7zqnOmqZSV+O6xvPG7htpjcFo+q+2JyWzyqx8YELNNacmt9unGWAgNEpL+IRAJTgWNG1YjIIKAD8I3Pug4iEuX93Bk4DVhffd/mkn24mPyScgY098VZTwWk/wl6pcGUlyA8gDdvGWNMDeocjaOq5SJyC/AREAbMUtV1IvIAzi9JZeKfCszx/sJUGgw8JyIenOsDj6jPKJ7mduSBJc3859vGD+FwhvMkKXtcoDGmFarXdAmqugBYUG3dvdWW769hv6+BYU2oX5Nsbqlhl0ueh/heMPC85j2PMcY0kqvvoN28J59OMZF0bM6ROHu/h+2fw6jrICwkphoyxgQhVyf7TXvzmn9a46UvQFgUjLi6ec9jjDFN4Npkr6ps2ZPfvA8YLz7szH0zbArEdGq+8xhjTBO5Ntnvzi0mr6S8eVv2q2ZDWQGk3dB85zDGGD9wbbLfXDVNQjO17D0e58JsrzToMbzu8sYYE0DuTfbN/SjCbZ9BzlZIm9E8xzfGGD9yb7Lfk0fHmEg6x0Y1zwkWPw8xXWFIbXPCGWNM6+DeZL83v/nmsM/ZBps/hpE/t7tljTFBwZXJXlXZtCePgc3VhbP0RWgTBqk/b57jG2OMn7ky2e/NKyGvuLx5Ls6WFsDK12DwZIjv7v/jG2NMM3Blst/cnA8sWfNPZ3y9XZg1xgQRVyb7ykcR+n22S1Xn4STdhkGfMf49tjHGNCNXJvvNe/Np3y6CzrF+vni68yvYu85p1Yv499jGGNOM3Jns9+QxsGsc4u+EvPg5aNsBhv3Ev8c1xphm5rpkr6rOsEt/j8Q5nAnffwDDr7QHiBtjgo7rkv3hEuVwURkD/X1xdtksUA+Mut6/xzXGmBbgumS/q8B5UJZfL86WFcPyl+Hk86BDX/8d1xhjWojrkn1Wvgfw87DLde9C4QEbbmmMCVquTPYJbSPoEufHOXGWPA+dB8IJ4/13TGOMaUGuS/a78j0M6Brrv5E4mctg1wobbmmMCWquSvaqSla+x7/99Uueh8g4SJ7qv2MaY0wLc1Wy359fSkGZH/vrc7Od/vqUKyCqGR9vaIwxzcxVyX6zd5qEgf5q2X/5F2e45Zhf+ud4xhgTIPVK9iIySUQ2isgWEZlZw/YnRGSV97VJRA75bLtaRDZ7X1f7s/LV+fXpVIcynOGWKdOhY/+mH88YYwIovK4CIhIGPA1MADKBpSIyX1XXV5ZR1V/7lL8VGO793BG4DxgJKLDcu+9Bv0bhtXlvHu3Coas/RuIs+rMz8dm43zb9WMYYE2D1admnAVtUdZuqlgJzgNqexTcNmO39PBH4RFVzvAn+E2BSUypcm0178ukR26bpI3EO7nTmrE+9Gtr38U/ljDEmgOps2QM9gQyf5UxgdE0FRaQv0B/4rJZ9e9aw3wxgBkBiYiLp6en1qNaxNmQWMKyjNnr/Sid//xSJKiwOO5WSJh6rqfLz85scT2vjtpjcFg+4Lya3xQMNj6k+yb4hpgJzVbWiITup6vPA8wAjR47U8ePHN/jE+/NLyPvPp/TtEEVj9q9yYCt8vhDSbuDUSVMafxw/SU9Pb1o8rZDbYnJbPOC+mNwWDzQ8pvp042QBvX2We3nX1WQqR7pwGrpvk8REhvPi1SMZ3jWsaQf64jEIi4DTf113WWOMCRL1SfZLgQEi0l9EInES+vzqhURkENAB+MZn9UfAj0Skg4h0AH7kXed3bSPDOGdwIl3bNWE06f7NsOYtZ2bLuG7+q5wxxgRYnd04qlouIrfgJOkwYJaqrhORB4BlqlqZ+KcCc1RVffbNEZEHcX4wAB5Q1Rz/huBHn/8/CI+G024PdE2MMcav6tVnr6oLgAXV1t1bbfn+4+w7C5jVyPq1nL3fw9q5cNptENsl0LUxxhi/ctUdtE2S/ieIjIGxvwp0TYwxxu8s2QPs/g7Wz4PRv4CYToGujTHG+J0le3Ba9VHxMPaWQNfEGGOahSX7Xavg+/fh1JuhbYdA18YYY5qFJfv0RyA6wWa2NMa4Wmgn+8zlsOlDGHurk/CNMcalQjvZpz8MbTs6F2aNMcbFQjfZ/7AYtnzqjKu3p1AZY1wudJN9+sPQrjOk3RDomhhjTLMLzWS/4yvYlu5MdhYZE+jaGGNMswvNZL/iFaevftR1ga6JMca0iNBM9j98C/1Oh4i2ga6JMca0iNBL9nm74dBO6F3jw7aMMcaVQi/ZZyxx3i3ZG2NCSAgm+8UQFgXdkwJdE2OMaTGhmex7joDwqEDXxBhjWkxoJfuyYmfis95pga6JMca0qNBK9rtWgqfM+uuNMSEntJJ9xmLn3ZK9MSbEhFiyXwIdT4SYzoGuiTHGtKjQSfaqTsveWvXGmBAUOsk+ZxsU7oc+luyNMaGnXsleRCaJyEYR2SIiM49T5qcisl5E1onImz7rK0Rklfc1318VbzDrrzfGhLDwugqISBjwNDAByASWish8VV3vU2YAcDdwmqoeFJGuPocoUtUUP9e74TIWO0+j6nxyoGtijDEtrj4t+zRgi6puU9VSYA5wUbUyNwBPq+pBAFXd699q+sEPi6FXGrQJnZ4rY4ypVJ/M1xPI8FnO9K7zNRAYKCJfici3IjLJZ1u0iCzzrr+4ifVtnKJDsG+DdeEYY0JWnd04DTjOAGA80Av4QkSGqeohoK+qZonICcBnIrJWVbf67iwiM4AZAImJiaSnpze6Ivn5+cfs3/HACpKAVTlRHGrCsQOhpniCndticls84L6Y3BYPNDym+iT7LKC3z3Iv7zpfmcBiVS0DtovIJpzkv1RVswBUdZuIpAPDgaOSvao+DzwPMHLkSB0/fny9A6guPT2dY/b/7EuQMFLOvxaiYht97ECoMZ4g57aY3BYPuC8mt8UDDY+pPt04S4EBItJfRCKBqUD1UTXzcFr1iEhnnG6dbSLSQUSifNafBqynpWUshm5Dgy7RG2OMv9SZ7FW1HLgF+AjYAPxTVdeJyAMiMtlb7CPggIisBxYCd6rqAWAwsExEVnvXP+I7iqdFVJRD5nLrrzfGhLR69dmr6gJgQbV19/p8VuAO78u3zNfAsKZXswn2fAdlBZbsjTEhzV8XaFsvezJVSCorKyMzM5Pi4uJmP1dCQgIbNmxo9vO0JLfF5KZ4oqOj6dWrV4P3C4FkvxjiekBCw/9xTPDKzMwkLi6Ofv36ISLNeq68vDzi4uKa9RwtzW0xuSUeVeXAgQNkZmY2eF/332GUsdh5WEkz/w9vWpfi4mI6derU7InemJYkInTq1KlRf7G6O9kfzoLDGdBnTKBrYgLAEr1xo8b+d+3uZJ9Z2V9vjyE0gTFv3jxEhO+//z7QVWlxL7/8Mrt27apa7tevH/v372/UsXbs2MGbb75Zd8EajB07ts4y119/PevX+3+g4P3338/jjz9ea5l58+Y1y7mrc3ey/2ExhLeFbkmBrokJUbNnz+b0009n9uzZzXqeioqKZj1+Y1RP9k1RW7IvLy+vdd+vv/66zuP/4x//YMiQIY2qW1NZsveHjMXQcwSERQS6JiYE5efn8+WXX/Liiy8yZ86cqvUVFRX89re/ZejQoSQlJfHUU08BsHTpUsaOHUtycjJpaWnk5eXx8ssvc8stt1Tte+GFF1bdIh8bG8tvfvMbkpOT+eabb3jggQcYNWoUQ4cOZcaMGTgjomHLli2ce+65JCcnM2LECLZu3cpVV13FvHnzqo47ffp03nvvvaPqr6rceeedDB06lGHDhvHWW28BR+7cnDJlCoMGDWL69OlV56o0d+5cli1bxvTp00lJSaGoqAiAp556ihEjRjBs2LCqv3YKCgq49tprSUtLY/jw4cfUA2DmzJksWrSIlJQUnnjiCV5++WUmT57M2WefzTnnnEN+fj7nnHNO1bF9jxEbG1tnvcePH8+yZcuqyv/+978nOTmZMWPGsGfPHgC2bt3KmDFjGDZsGPfcc0/Vcat76KGHGDhwIKeffjobN26sWv/CCy8watQokpOTueyyyygsLOTrr79m/vz53HnnnaSkpLB169Yay/mDe0fjlBbC7jUw9leBrokJsD/+ex3rd+X69ZhDesRz349PqbXMe++9x6RJkxg4cCCdOnVi+fLlpKam8vzzz7Njxw5WrVpFeHg4OTk5lJaWcvnll/PWW28xatQocnNzadu2ba3HLygoYPTo0fz5z3926jRkCPfe69z+cuWVV/L+++/z4x//mOnTpzNz5kwuueQSiouL8Xg8XHfddTzxxBNcfPHFHD58mK+//ppXXnnlqOO/8847rFq1itWrV7N//35GjRrFuHHjAFi5ciXr1q2jR48enHbaaXz11VecfvrpVftOmTKFv/3tbzz++OOMHDmyan3nzp1ZsWIFf//733n88cf5xz/+wUMPPcTZZ5/NrFmzOHToEGlpaZx77rnExMRU7ffII4/w+OOP8/777wPOXw0rVqxgzZo1dOzYkfLyct59913i4+PZv38/Y8aMYfLkycf0b9dV78p/1zFjxvDQQw/xu9/9jhdeeIF77rmH2267jdtuu41p06bx7LPP1vidLF++nDlz5rBq1SrKy8sZMWIEqampAFx66aXccMMNANxzzz28+OKL3HrrrUyePJkLL7yQKVOmANC+ffsayzWVe1v2u1aCp9zG15uAmT17NlOnTgVg6tSpVV05n376KTfeeCPh4U5bq2PHjmzcuJHu3bszatQoAOLj46u2H09YWBiXXXZZ1fLChQsZPXo0w4YN47PPPmPdunXk5eWRlZXFJZdcAjhjtNu1a8eZZ57J5s2b2bdvH7Nnz+ayyy475nxffvkl06ZNIywsjMTERM4880yWLl0KQFpaGr169aJNmzakpKSwY8eOev2bXHrppQCkpqZW7fPxxx/zyCOPkJKSwvjx4ykuLuaHH36o81gTJkygY8eOgPNXyP/8z/+QlJTEueeeS1ZWVlWL3Fd96h0ZGcmFF154TD2/+eYbfvKTnwBwxRVX1FinRYsWcckll9CuXTvi4+OZPHly1bbvvvuOM844g2HDhvHGG2+wbt26Go9R33IN5d6Wfca3zrtdnA15dbXAm0NOTg6fffYZa9euRUSoqKhARHjssccadJzw8HA8Hk/Vsu+Qu+joaMLCwqrW33TTTSxbtozevXtz//331zk876qrruL1119nzpw5vPTSSw2qV1RUVNXnsLCwOvvNq+/nu4+q8vbbb3PyyQ17sJBvy/+NN95g3759LF++nIiICPr161dj/PWpd0RERNVfBA2JrS7XXHMN8+bNIzk5mZdffvm4M1bWt1xDubdln7EEOg+Edh0DXRMTgubOncuVV17Jzp072bFjBxkZGfTv359FixYxYcIEnnvuuaokkpOTw8knn0x2dnZVyzkvL4/y8nL69evHqlWr8Hg8ZGRksGTJkhrPV5nYOnfuTH5+PnPnzgUgLi6OXr16VfXPl5SUVPUBX3PNNfz1r38FqPHi5BlnnMFbb71FRUUF+/bt44svviAtrf6Np7i4OPLy8uosN3HiRJ566qmq/vOVK1c2+FiHDx+ma9euREREsHDhQnbu3FnvetbXmDFjePvttwGOugbja9y4ccybN4+ioiLy8vL497//XbUtLy+P7t27U1ZWxhtvvFG1vnpsxyvXVO5M9qpHbqYyJgBmz55d1XVS6bLLLmP27Nlcf/319OnTh6SkJJKTk3nzzTeJjIzkrbfe4tZbbyU5OZkJEyZQXFzMaaedRv/+/RkyZAi/+tWvGDFiRI3nq+znHTp0KBMnTqzqDgJ47bXXePLJJ0lKSmLs2LHs3r0bcJ4dMXjwYH7+85/XeMxLLrmkqo5nn302jz76KN26dav3v8E111zDL37xi6Mu0NbkD3/4A2VlZSQlJXHKKafwhz/84ZgySUlJhIWFkZyczBNPPHHM9unTp7Ns2TKGDRvGq6++yqBBg+pdz/r661//yl/+8heSkpLYsmULCQkJx5QZMWIEl19+OcnJyZx33nlHfQ8PPvggo0eP5rTTTjuqflOnTuWxxx5j+PDhbN269bjlmkxVW9UrNTVVm2LhwoWqezeq3hevuvyVJh2rNVi4cGGgq+B3LRHT+vXrm/0clXJzc1vsXP5UUFCgJ5xwgh46dOiYbcEa0/H4I56CggL1eDyqqjp79mydPHlyk4/ZWOvXrz/m/yNgmdaSW93ZZ5+x2HnvbXfOGlOTTz/9lOuuu45f//rXNbZQzbGWL1/OLbfcgqrSvn17Zs2aFegqNYh7k33bDtDppEDXxJhW6dxzz22Wfm03O+OMM1i9enWgq9Fo7uyzz1gMvdKgjTvDM8aYhnJdNgwvy4X9m+zirDHG+HBdso/P3eR8sJkujTGmiuuSfcLhDSBh0KPmIWrGGBOKXJjsv4fuSRDZLtBVMcamOG4FUxwDdU4zXKn6xHM1SU9Pr9dMmq2Nu5J9RRlxeZtsyKVpNWyK4+af4rg+KieL8wdL9q3B7jWEeUrt4qxpFWyK4+ab4riiooI777yTUaNGkZSUxHPPPQdAdnY248aNIyUlhaFDh7Jo0SJmzpxJUVERKSkpTJ8+/Zhjv/TSSwwcOJC0tDS++uqrqvX//ve/GT16NMOHD+fcc89lz5497Nixg2effZYnnniClJQUFi1aVGO5Vqm2O64qX8AkYCOwBZh5nDI/BdYD64A3fdZfDWz2vq6u61xNuoP2m787d84eymz8MVoZu4O2cY66g3bBXaqzzvfva8FdVYc/3t2Zr7/+ul577bWqqnrqqafqsmXLVFX173//u1522WVaVlamqqoHDhzQkpIS7d+/vy5ZskRVVQ8fPqxlZWX60ksv6c0331x1zAsuuKDq3w/Qt956q2rbgQMHqj7/7Gc/0/nz56uqalpamr7zzjuqqlpUVKQFBQWanp6uF110kaqqHjp0SPv161dVn8qY5s6dq+eee66Wl5fr7t27tXfv3rpr1y5duHChxsfHa0ZGhlZUVOiYMWN00aJFx8R/5pln6tKlS6uW+/btq08++aSqqj799NN63XXXqarq3Xffra+99pqqqh48eFAHDBig+fn5Rx1r4cKFesEFF1QtP/fcc/rggw+qqmpxcbGmpqbqtm3b9PHHH9f//d//VVXV8vLyqu8mJiamhm9IddeuXdq7d2/du3evlpSU6NixY6v+vXNycqrumH3hhRf0jjvuUFXV++67Tx977LGqYxyvXHNqljtoRSQMeBqYAGQCS0Vkvqqu9ykzALgbOE1VD4pIV+/6jsB9wEhAgeXefQ82+VeqJhmLKY7qQnRCz2Y5vDENMXv2bG677TbgyBTHqampfPrpp/ziF784aorjtWvXHjPFcV1qmuL40UcfpbCwkJycHE455RTGjx9/zBTHAGeeeSY33XQT+/bt4+23327QFMfx8fFVUwUDVVMFV58Xvia+Uxy/8847gDPF8fz586v61SunOB48ePBxj/Pxxx+zZs2aqgnfDh8+zObNmxk1ahTXXnstZWVlXHzxxaSkpNRan8WLFzN+/Hi6dOkCwOWXX86mTc6IvszMTC6//HKys7MpLS2lf//+NR6jvuUCrT530KYBW1R1G4CIzAEuwmnFV7oBeLoyiavqXu/6icAnqprj3fcTnL8S/N+BqQo/LOZwwiCi/X5wE9TOe6TFT2lTHNe+X1OnOFZVnnrqKSZOnHjMti+++IIPPviAa665hjvuuIOrrrqq3sf1deutt3LHHXcwefJk0tPTuf/++5tULtDq02ffE8jwWc70rvM1EBgoIl+JyLciMqkB+/rH4UzI20VuvP9nuzOmoWyK4+ad4njixIk888wzlJWVAbBp0yYKCgrYuXMniYmJ3HDDDVx//fWsWLECcOaoryzra/To0Xz++eccOHCAsrIy/vWvf1VtO3z4MD17OunK9yle1etyvHKtjb/mxgkHBgDjgV7AFyIyrL47i8gMYAY40642arJ+rSBm5P9xsCyCLD9N9t8a5Ofn++3hBa1FS8SUkJBQr0TjDxUVFcec6/XXX+f2228/av0FF1zAq6++ymOPPcZ3333H0KFDiYiI4Oqrr+bGG29k1qxZ3HTTTRQXFxMdHc38+fNJSkqiV69eDBo0iJNPPpnk5GQKCwurjlv5HhYWxlVXXcWQIUNITEwkJSWFkpIS8vLyeOaZZ7j99tu55557iIiI4JVXXqF///60a9eOAQMGcOGFFx5T/4qKCs4991w+//xzhg0bhojwxz/+kZiYGAoLCykvL6/ap7S0lOLi4mOOcfnllzNjxgzatm3Lp59+iqqSn59PVFQUBQUFVf9ut99+OzNnzmTo0KF4PB769u17VNIFqrpGhg0bxhVXXMFNN93Epk2bSElJQVXp3Lkzb775Jh9++CFPPvkkERERxMTE8Nxzz5GXl8dVV13F0KFDSU5O5sUXX6w6bmxsLDNnzmT06NEkJCSQlJREaWkpeXl53HXXXUyZMoX27dszbty4qvqeddZZXHXVVbz77rs89thjxy3XnIqLixv+/1FtHfreX9pTgY98lu8G7q5W5lng5z7L/wVGAdOA53zWPwdMq+18fpni2EXcFo+qTXHcWtgUx8GrMRdo69ONsxQYICL9RSQSmArMr1ZmHk6rHhHpjNOtsw34CPiRiHQQkQ7Aj7zrjDEB9OmnnzJ48GBuvfVWm+I4RNTZjaOq5SJyC06SDgNmqeo6EXkA55dkPkeS+nqgArhTVQ8AiMiDOD8YAA+o92KtMSZwbIrj0FOvPntVXQAsqLbuXp/PCtzhfVXfdxYQXLP8G2OMy7jrDlpjfGi1uzqNcYPG/ndtyd64UnR0NAcOHLCEb1xFVTlw4EDVzXEN4c7HEpqQ16tXLzIzM9m3b1+zn6tyqKSbuC0mN8UTHR1Nr169GnzNxZK9caWIiIgWu209PT2d4cOHt8i5WorbYnJbPI1h3TjGGBMCLNkbY0wIsGRvjDEhQFrbaAUR2Qc05W6PzkDjnn3WOrktHnBfTG6LB9wXk9vigWNj6quqXY5XuNUl+6YSkWWqOjLQ9fAXt8UD7ovJbfGA+2JyWzzQ8JisG8cYY0KAJXtjjAkBbkz2zwe6An7mtnjAfTG5LR5wX0xuiwcaGJPr+uyNMcYcy40te2OMMdW4JtmLyCQR2SgiW0RkZqDr4w8iskNE1orIKhFZFuj6NJSIzBKRvSLync+6jiLyiYhs9r53CGQdG+o4Md0vIlne72mViJwfyDo2hIj0FpGFIrJeRNaJyG3e9UH5PdUSTzB/R9EiskREVntj+qN3fX8RWezNeW95Hy51/OO4oRtHRMKATcAEnIeaL8V5/OH6gFasiURkBzBSVYNyfLCIjAPygVdVdah33aNAjqo+4v1R7qCqdwWyng1xnJjuB/JV9fFA1q0xRKQ70F1VV4hIHLAcuBi4hiD8nmqJ56cE73ckQIyq5otIBPAlcBvO80PeUdU5IvIssFpVnznecdzSsk8DtqjqNlUtBeYAFwW4TiFPVb8Aqj+Z7CLgFe/nV3D+Rwwax4kpaKlqtqqu8H7OAzYAPQnS76mWeIKW9xGz+d7FCO9LgbOBud71dX5Hbkn2PYEMn+VMgvwL9lLgYxFZLiIzAl0ZP0lU1Wzv591AYiAr40e3iMgabzdPUHR5VCci/YDhwGJc8D1ViweC+DsSkTARWQXsBT4BtgKHVLXcW6TOnOeWZO9Wp6vqCOA84GZvF4JreB9nGfz9iPAMcCKQAmQDfw5sdRpORGKBt4HbVTXXd1swfk81xBPU35GqVqhqCtALpydjUEOP4ZZknwX09lnu5V0X1FQ1y/u+F3gX50sOdnu8/aqV/at7A1yfJlPVPd7/GT3ACwTZ9+TtB34beENV3/GuDtrvqaZ4gv07qqSqh4CFwKlAexGpfCZJnTnPLcl+KTDAe3U6EpgKzA9wnZpERGK8F5gQkRjgR8B3te8VFOYDV3s/Xw28F8C6+EVlUvS6hCD6nrwX/14ENqjqX3w2BeX3dLx4gvw76iIi7b2f2+IMRNmAk/SneIvV+R25YjQOgHco1V+BMGCWqj4U4Co1iYicgNOaB+eJYm8GW0wiMhsYjzM73x7gPmAe8E+gD87spj9V1aC54HmcmMbjdA8osAO40ae/u1UTkdOBRcBawONd/T84/dxB9z3VEs80gvc7SsK5ABuG00D/p6o+4M0Rc4COwErgZ6pactzjuCXZG2OMOT63dOMYY4yphSV7Y4wJAZbsjTEmBFiyN8aYEGDJ4qTLuAAAAZRJREFU3hhjQoAlexOSRKTCZwbEVf6cKVVE+vnOimlMaxBedxFjXKnIe/u5MSHBWvbG+PA+Q+BR73MElojISd71/UTkM+9EWv8VkT7e9Yki8q53rvHVIjLWe6gwEXnBO//4x947H40JGEv2JlS1rdaNc7nPtsOqOgz4G85d2QBPAa+oahLwBvCkd/2TwOeqmgyMANZ51w8AnlbVU4BDwGXNHI8xtbI7aE1IEpF8VY2tYf0O4GxV3eadUGu3qnYSkf04D8Uo867PVtXOIrIP6OV7m7p3at1PVHWAd/kuIEJV/7f5IzOmZtayN+ZYepzPDeE7R0kFdn3MBJgle2OOdbnP+zfez1/jzKYKMB1nsi2A/wK/hKoHTCS0VCWNaQhrbZhQ1db75J9K/1HVyuGXHURkDU7rfJp33a3ASyJyJ7AP+Ll3/W3A8yJyHU4L/pc4D8cwplWxPntjfAT7Q96NOR7rxjHGmBBgLXtjjAkB1rI3xpgQYMneGGNCgCV7Y4wJAZbsjTEmBFiyN8aYEGDJ3hhjQsD/B7E741YqedO4AAAAAElFTkSuQmCC\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(np.array(training_accuracy[:30])/1000)\n", + "plt.plot(np.array(evaluation_accuracy[:30])/10000)\n", + "\n", + "plt.grid()\n", + "plt.xlabel('Epoch')\n", + "plt.legend(['Accuracy on the training data','Accuracy on the test data'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LQZlPUpl7_kM" + }, + "source": [ + "### chapter 3 - Early stopping implemented" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "xZPeNW0U8AUD", + "outputId": "7cb1dafd-f92f-4fbd-846c-58c90d31caec" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 0 training complete\n", + "Cost on training data: 301.2561650219777\n", + "Accuracy on evaluation data: 7455 / 10000\n", + "Epoch 1 training complete\n", + "Cost on training data: 439.232945332555\n", + "Accuracy on evaluation data: 8213 / 10000\n", + "Epoch 2 training complete\n", + "Cost on training data: 515.5930110598646\n", + "Accuracy on evaluation data: 8406 / 10000\n", + "Epoch 3 training complete\n", + "Cost on training data: 555.6494874830328\n", + "Accuracy on evaluation data: 8574 / 10000\n", + "Epoch 4 training complete\n", + "Cost on training data: 577.361869837528\n", + "Accuracy on evaluation data: 7923 / 10000\n", + "Epoch 5 training complete\n", + "Cost on training data: 594.697554039724\n", + "Accuracy on evaluation data: 8555 / 10000\n", + "Epoch 6 training complete\n", + "Cost on training data: 605.2460972335653\n", + "Accuracy on evaluation data: 8649 / 10000\n", + "Epoch 7 training complete\n", + "Cost on training data: 612.7769103897504\n", + "Accuracy on evaluation data: 8599 / 10000\n", + "Epoch 8 training complete\n", + "Cost on training data: 613.8251825233201\n", + "Accuracy on evaluation data: 8628 / 10000\n", + "Epoch 9 training complete\n", + "Cost on training data: 616.3005315785125\n", + "Accuracy on evaluation data: 8686 / 10000\n", + "Epoch 10 training complete\n", + "Cost on training data: 621.4503494271853\n", + "Accuracy on evaluation data: 8709 / 10000\n", + "Epoch 11 training complete\n", + "Cost on training data: 623.2882346192231\n", + "Accuracy on evaluation data: 8655 / 10000\n", + "Epoch 12 training complete\n", + "Cost on training data: 627.8119630862499\n", + "Accuracy on evaluation data: 8580 / 10000\n", + "Epoch 13 training complete\n", + "Cost on training data: 624.4530964651585\n", + "Accuracy on evaluation data: 8752 / 10000\n", + "Epoch 14 training complete\n", + "Cost on training data: 625.8825734578031\n", + "Accuracy on evaluation data: 8371 / 10000\n", + "Epoch 15 training complete\n", + "Cost on training data: 624.219974401629\n", + "Accuracy on evaluation data: 8641 / 10000\n", + "Epoch 16 training complete\n", + "Cost on training data: 626.7646149440405\n", + "Accuracy on evaluation data: 8472 / 10000\n", + "Epoch 17 training complete\n", + "Cost on training data: 630.3514860793732\n", + "Accuracy on evaluation data: 8675 / 10000\n", + "Epoch 18 training complete\n", + "Cost on training data: 627.4011451184954\n", + "Accuracy on evaluation data: 8525 / 10000\n", + "Epoch 19 training complete\n", + "Cost on training data: 629.1162546350955\n", + "Accuracy on evaluation data: 8675 / 10000\n", + "Epoch 20 training complete\n", + "Cost on training data: 631.3399353668173\n", + "Accuracy on evaluation data: 8805 / 10000\n", + "Epoch 21 training complete\n", + "Cost on training data: 630.2551505889224\n", + "Accuracy on evaluation data: 8607 / 10000\n", + "Epoch 22 training complete\n", + "Cost on training data: 626.9095605172745\n", + "Accuracy on evaluation data: 8663 / 10000\n", + "Epoch 23 training complete\n", + "Cost on training data: 633.5111386955494\n", + "Accuracy on evaluation data: 8771 / 10000\n", + "Epoch 24 training complete\n", + "Cost on training data: 624.6207737117185\n", + "Accuracy on evaluation data: 8784 / 10000\n", + "Epoch 25 training complete\n", + "Cost on training data: 629.9928186340273\n", + "Accuracy on evaluation data: 8846 / 10000\n", + "Epoch 26 training complete\n", + "Cost on training data: 632.7831357738781\n", + "Accuracy on evaluation data: 8398 / 10000\n", + "Epoch 27 training complete\n", + "Cost on training data: 631.0431567238287\n", + "Accuracy on evaluation data: 8631 / 10000\n", + "Epoch 28 training complete\n", + "Cost on training data: 626.6097871531382\n", + "Accuracy on evaluation data: 8621 / 10000\n", + "Epoch 29 training complete\n", + "Cost on training data: 628.812798152952\n", + "Accuracy on evaluation data: 8723 / 10000\n" + ] + } + ], + "source": [ + "from elements import network2\n", + "from elements import mnist_loader\n", + "\n", + "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()\n", + "training_data = list(training_data)\n", + "test_data = list(test_data)\n", + "\n", + "net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost)\n", + "evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \\\n", + " net.SGD(training_data[:1000], 30, 10, 0.5,\n", + " lmbda=5.0,\n", + " evaluation_data=validation_data,\n", + " monitor_evaluation_accuracy=True,\n", + " monitor_training_cost=True,\n", + " early_stopping_n=10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 352 + }, + "id": "kvzubSGEVGIq", + "outputId": "540c275f-61fd-4a0f-e423-f82c4b51c361" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 22 + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAE9CAYAAADqP8J6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXyU1b348c/Jvm8kBMjCvhoJGBYBF1yo1lbRqhWrVWor7W21/dW2t9rbKrXL7dVa79Xaam3V1raARaugtCiSKCpKwLBI2ANkw+zbZJvt/P6YmTBknUlmJskz3/frlReZZ57nmXOS8J0zZ/kepbVGCCGEsYUMdwGEEEL4nwR7IYQIAhLshRAiCEiwF0KIICDBXgghgoAEeyGECAJhw12A7lJTU/WkSZMGfX1rayuxsbG+K9AwM1p9wHh1Mlp9wHh1Mlp9oGed9uzZU6u1Tuvr/BEX7CdNmsTu3bsHfX1BQQHLly/3XYGGmdHqA8ark9HqA8ark9HqAz3rpJQ63d/50o0jhBBBQIK9EEIEAQn2QggRBEZcn31vLBYL5eXldHR0DHhuYmIihw4dCkCpAsNo9YHRX6eoqCgyMzMJDw8f7qII4bFREezLy8uJj49n0qRJKKX6PbelpYX4+PgAlcz/jFYfGN110lpTV1dHeXk5kydPHu7iCOGxUdGN09HRwZgxYwYM9EL4m1KKMWPGePQpU4iRZFQEe0ACvRgx5G9RjEajJtiPBK+++ipKKQ4fPjzcRQm4F154gcrKyq7HkyZNora2dlD3On36NH//+98HXZZf/vKXHp33wgsvcM899/R7TkFBAR988MGgyyLEaCHB3gvr1q3joosuYt26dX59HZvN5tf7D0b3YD8UpaWlAQn2npBgL4KFBHsPmUwm3nvvPf70pz+xfv36ruM2m43vf//75OTkMHfuXJ588kkACgsLWbp0Kbm5uSxatIiWlpYeLc3Pf/7zFBQUABAXF8f3vvc9cnNz2blzJw8//DALFy5k8eLFrFmzBteOYsePH+fKK68kNzeXCy64gBMnTnDHHXfw6quvdt33tttu47XXXjun/FprfvCDH5CTk8P555/Phg0bgLOr8G666SZmzZrFbbfdRvfdyzZu3Mju3bu57bbbmDdvHu3t7QA8+eSTXHDBBZx//vldn3ZaW1u56667WLRoEfPnz+9RDoCHHnqIHTt2MG/ePB5//HFsNhs/+MEPWLhwIXPnzuWZZ54B4MyZM1xyySXMmzePnJwcduzYwf333097ezvz5s3jtttu63Hv559/nhkzZrBo0SLef//9ruObN29m8eLFzJ8/nyuvvJKqqipOnTrF008/zeOPP868efPYsWNHr+eJ0e1EjYmyFvtwF2P4aa1H1FdeXp7urri4uMexvjQ3N3t8rjf++te/6rvuuktrrfWSJUv07t27tdZa/+53v9M33nijtlgsWmut6+rqdGdnp548ebLetWuX1lrrpqYmbbFY9PPPP6+/9a1vdd3zc5/7nM7Pz9daaw3oDRs2dD1XV1fXVZ/bb79db9q0SWut9aJFi/Qrr7yitda6vb1dt7a26oKCAr1y5UqttdaNjY160qRJXeVx2bhxo77yyiu11WrVn376qc7KytKVlZU6Pz9fJyQk6LKyMm2z2fSFF16od+zY0aP+l156qS4sLOx6PHHiRP3EE09orbV+6qmn9Fe/+lWttdYPPPCAfvHFF7XWWjc0NOjp06drk8l0zr3eeOMN/bnPfa7r8TPPPKN/9rOfaa217ujo0Hl5ebqkpET/+te/1j//+c+11lpbrdau321sbGwvvyGtKysrdVZWlq6urtadnZ166dKlXT/v+vp6bbfbtdZaP/vss/q+++7TWmv90EMP6UcffbTrHn2d153736Trd2gkRqnTgfJGnfPQv/VFP98y3EXxue6/I2C37ie2joqpl+5+uvkgxZXNfT5vs9kIDQ316p5zJiTw0LXn9XvOunXr+M53vgPAqlWrWLduHXl5eWzbto1vfOMbhIU5fpQpKSkcOHCA8ePHs3DhQgASEhIGLENoaCg33nhj1+P8/HweeeQRTCYTjY2NnHfeeSxfvpyKigpuuOEGwDHfG+DSSy/lm9/8JjU1Nbz88svceOONXeVxee+997j11lsJDQ0lPT2dSy+9lMLCQhISEli0aBGZmZkAzJs3j1OnTnHRRRcNWOYvfOELAOTl5fHKK68A8Oabb7Jp0yZ+/etfA46ZVKWlpcyePbvP+7z55pvs37+fjRs3AtDU1MSxY8dYuHAhd911FxaLheuvv5558+b1W56PPvqI5cuXk5bmyAV1yy23cPToUcAxffeWW27hzJkzmM3mPqdNenqeGPmOVbVwx3O7aOmw0mlxNGyDeXB91AX74VBfX8/27ds5cOAASilsNhtKKR599FGv7hMWFobdfvbjpPv0vaioqK43qY6ODr75zW+ye/dukpKSeOyxxwac6nfHHXfw17/+lfXr1/P88897Va7IyMiu70NDQ7FarV5d536N1pqXX36ZmTNnevz6WmuefPJJrrrqqh7Pvfvuu7zxxhusXr2a++67jzvuuMPj+7q79957ue+++7juuusoKChg7dq1QzpPjGyn61q57Y8fERqiWL10Ei98cIq6VjOpcZEDX2xQoy7YD9QC98eCnY0bN/LlL3+5qy8ZHK3pHTt2sGLFCp555hkuu+wywsLCqK+vZ+bMmZw5c4bCwkIWLlxIS0sL0dHRTJo0id/97nfY7XYqKirYtWtXr6/nCuypqak0NjayceNGbrrpJuLj48nMzOTVV1/l+uuvp7OzE5vNRkxMDKtXr2bRokWMGzeOOXPm9LjnxRdfzDPPPMOdd95JfX097777Lo8++qjHM4vi4+NpaWkZ8LyrrrqKJ598kieffBKlFEVFRcyfP/+cc+Li4s6511VXXcXvf/97Lr/8csLDwzl69CgZGRnU1taSmZnJ3XffTWdnJx9//DF33HEH4eHhWCyWHitYFy9ezHe+8x3q6upISEjgH//4B7m5uYDj00JGRgYAf/7zn8+pV3Pz2U+KfZ0nRo/Kxna+9OxHWGx2Nnx9CeUNbbzwwSlK69uCOtjLAK0H1q1b19V14nLjjTeybt06vva1r5Gdnc3cuXPJzc3l73//OxEREWzYsIF7772X3NxcVqxYQUdHB8uWLWPy5MnMmTOHb3/721xwwQW9vl5SUhJ33303OTk53HDDDV3dQQAvvvgiTzzxBHPnzmXp0qV8+umnAKSnpzN79my+8pWv9HrPG264oauMl19+OY888gjjxo3z+GewevVqvvGNb5wzQNubn/zkJ1gsFubOnct5553HT37ykx7n5OTkEBoaSm5uLo8//jhf+9rXmDNnDhdccAE5OTl8/etfx2q1UlBQQG5uLvPnz2fDhg1d3Whr1qxh7ty5PQZox48fz9q1a1myZAnLli07p+to7dq13HzzzeTl5ZGamtp1/Nprr+Wf//xn1wBtX+eJ0aG6pYPb/vgRze0WXvzqYmakx5OdEgNAWX3bMJdumPXXoe/6Aq4GjgDHgft7eT4byAeKgP3ANc7j4cCfgQPAIeCBgV5rpA7QDhdP69Pa2qqnTJmiGxsb/VyioTPC70gGaEeeelOn/sxv3tGzfvwvXXiyrut4u9mqJ/7wdf3EtqPDWDrf83aAdsCWvVIqFHgK+CwwB7hVKdW9n+DHwEta6/nAKuB3zuM3A5Fa6/OBPODrSqlJg3lTEn3btm0bs2fP5t577yUxMXG4iyNGuQ7LyFvnMZCWDgt3Pr+Lk3Wt/PHOBSyYlNL1XFR4KEmRirKG4G7Ze9Jnvwg4rrUuAVBKrQdWAsVu52jANeUkEah0Ox6rlAoDogEz0PdUGjEoV155JadP97tJjRAeKa5s5vqn3ufb88NZPtyF8VCb2cpdLxRSXNnM07fnsWxaz+63tGhFaZB343jSZ58BlLk9Lncec7cWuF0pVQ5sAe51Ht8ItAJngFLg11rr+qEUWAjhP8+8ewKzzc5HZ0ZH677TauPrL+5hz+kGHr9lHlfOSe/1vLSYEMrq+x5rCga+mo1zK/CC1voxpdQS4EWlVA6OTwU2YAKQDOxQSm1zfUpwUUqtAdaAY6DRtarUJTExkebmZo/myNpsNo9mjYwWRqsPjP46aa3p6Ojo+js1mUw9/mZHo7p2O5v3tROqYG+1he35+YSM4HnpVrvmqb2dFFXb+GpOBPENRykoONrruYmhFiobrWzbnk9YyMitkze8/bvzJNhXAFlujzOdx9x9FccgLlrrnUqpKCAV+BLwb621BahWSr0PLADOCfZa6z8AfwBYsGCB7r4x8MmTJzGbzR6lOR7NudJ7Y7T6wOiuk3bms09KSuqaUmqUzax/9noxSp3ivhUzeHTrEVKmzWdeVtJwF6sHu11zrNrEE9uPUVR9hp9edx53Lp3U7zXvVbyFLjMzbe4iJqXGBqagfubt350nwb4QmK6UmowjyK/CEcTdlQJXAC8opWYDUUCN8/jlOFr6scCFwP96XDqnzMxMysvLqampGfDcjo6OrpWlRmC0+sDor5NrpyojaWq3sH5XKdfOHc+XFmXz661H2H6oakQE+4ZWM3vLGvm4tIGi0kb2lTXS0ulYxPfDq2cNGOgB0qIdPdal9W2GCfbeGjDYa62tSql7gK1AKPCc1vqgUuphHFN9NgHfA55VSn0Xx6Dsaq21Vko9BTyvlDoIKOB5rfV+bwsZHh7u8bL1goKCHot4RjOj1QeMWafRbt2uUlrNNr528RSSYyOYnhzCtkPV3PcZz1dC+4LNrjl0ppmiskaKnMH9ZG0rACEKZo1LYOX8CczPSiZvYrLHgXtsjKNHIJgHaT3qs9dab8Ex8Op+7EG374uBZb1cZ8Ix/VIIMUKZrXaef/8ky6aNISfDMXV3XlooLx1t5kxTO+MTowNWlm+vL+KN/WcASI2LYH52MjcvyOSC7GTOz0gkNnJww4yJkYqIsJCgXlg16tIlCCF8a9O+SqqaO3nkptyuY/PGhvHSUQvbD1dz2+KJASnHzhN1vLH/DF9ZNom7lk0mMznaZ4nLQpQiMzk6qOfaS7oEIYKY1ppn3y1hZno8l0w/Oz99fKwiOyWGtw9VB6Qcdrvmv/91iAmJUfzw6llkpcT4PENldkpMUHfjSLAXQePtQ1X8+YNTw12MEeWdozUcqWrh7kumnBNclVJcPmss7x+vpd3s/zn3bxw4w/7yJr73mZlEhXuXotxT2SkxlNZJsBfC0LTW/HLLIR5780iPnbiC2bM7SkhPiOS63Ak9nrtydjqdVjsfnBjcXsOe6rTaeGTrYWaPT+D6+d3Xa/pOVnIMzR1WmtosfnuNkUyCvQgKh860cKKmleYOK7Um83AXZ0T4pKKJ94/X8ZVlk4kI6xkKFk1OIS4yjG1+7sr524ellNW388BnZxHqxwVPWc7sl8HalSPBXgSFzfvPbpZeUmMaxpKMHM/uKCEuMowvLc7u9fmIsBAumZHK9sNVfvs01NRu4cntx7h4eiqXzEjzy2u4dKU6DtJBWgn2wvC01mzeV8mM9DgATtS0DnOJhl9FYzuv7z/DqoVZJESF93ne5bPSqWru5GA/W4EOxdPvnKCx3cIPr57ll/u7y0pxTCGVlr0QBrW3rJHyhna+dvEUosJDOCEte5577yQAX7mo/8WKy2emoRR+mZVT2djOc++d5IZ5GV3z+/0pPiqc5JjwERvsX9pdxrpdpX77FCXBXhje5n1niAgN4eqccUxJjQv6bhz31AgZSf0vmEqNi2R+VhLbD1f5vBy/eesoGrjvMzN8fu++ZKfEjMiFVfWtZn7+ejFbDpzx22tIsBcBZbXZBz7Jh2x2zev7K1k+M42EqHCmjo0L+m4c99QInrhidjr7ypuobu5/03tvHDrTzMsfl/OVpZPITI7x2X0HkjVCg/1jbx6h1Wzjwc/P8fn6AhcJ9sLvrDY72w9X8fUXdzP7wX+z8ag5YNMfC0/VU93SybXOqYVTUmMpa2gblbsx+UJvqREGcvmssQDkH/FdV85//+swCVHhfHP5NJ/d0xPZKTGUN7Rjs4+c6bfFlc2s21XKly+cyPR0/2WDlWAv/Kasvo3H3jzCRf+Tz10v7Gb3qQYWTU7h9RIL9798ICCt/M37KokOD+WK2Y6ANXVsHFrDqbrgbN27UiOsuWSqx9fMGhdPRlK0z/rtdxyr4d2jNdx7+TQSY/oeHPaHrJQYrHbNmaaRsZGJ1pqHXz9IYnQ4373Sv91ZkhtH+FSHxcabxVVsKCzl/eN1hCi4dEYaa6+bw+Wz0gkPVXznj2+xYXcZDW1mnrh1vt9WTFpsdv71yadcOSedmAjHn/rUNEeWxJKaVmaNS+jvcsPpKzXCQFyraTfuKafDYhvS78tu1/z3lsNkJkfz5SWBybnjLtttrn0gu4/68q9PPuXDknp+dn2O39/4JNgLnzj8aTPrd5Xxz6IKmtotZCZHc9+KGdyUl8mEboOAX5gewQVzprN2czF3PreLZ+9c0O/0v8H64EQd9a1mrp07vuvYZGdK3BPVwTdI60qN8Oubc73uF75i9lhe/PA0H5bUsXzm2EGX4bV9FRSfaeb/Vs0jMsw/b/L9cQX78vp28PzDjV90WGz84o1DzBoXz5cW9b7WwZck2Ishsdrs/OfG/bxSVEFEaAhX5YzjlgVZLJ06hpB+VkOuXjaZ5NgIvvfSPm555kP+fNdCxsb7dkOTzfsqiY8K49KZZxfrxESEkZEUHZTTL/tLjTCQC6eMITo8lO2Hqwcd7DssNn699Sg5GQlcO9f7MvjC+MQoQkNGxubjz75bQkVjO3+/e7FfVw67SJ+9GDSLzc631xfxSlEF31w+lY9+dAVP3jqfi6an9hvoXVbOy+BPqxdyqraVm36/k9M+7EfvtNrY+smnXHXeuB4tyClpsZTUBlef/UCpEQYSFR7KRdNTeftQ9aAH1/+y8xQVje386LOzPfr78Iew0BAmJEUNe7A/09TO7wpO8NmccSyd6nmX2lBIsBeDYrbauefvH7PlwKf8+HOz+c+rZ5EcG+H1fS6dkcbf715Mc4eFG3+/k4OVTT4p3ztHamjptHbNwnE3NS2OE9WmEZsQ7ZdbDvHlP33k03v+6b2TxEaEcusQuguunD2WisZ2jlR5v1l8Y5uZ324/zvKZaSydFpjg1peRkOr4V/86jE1rfnTN7IC9pgR74bVOq41v/m0PWw9WsfbaOR7P1+7L/OxkNn5jCeGhilXPfMiHJXVDLuPm/WdIiY1g6dQxPZ6bmhZLq9lGVXPnkF/H1+x2zct7ytlxrJZPKnzzxtfYZuaNA2f4wgWZJEYPfmzkMmf3zWBm5TyVf5yWTiv3f9b/aREG4ph+OXzBfvepel7bW8nXL5nSlZwtECTYC690WGx8/cU9bDtUzc+vz2H1Ms/2Bh7ItLHxvPwfSxmbEMkdz+3izYOfDvpebWYr24qr+GzOOMJDe/6JT01z5MgZiStp91c0UdfqyMq5blepT+75z6IKzFY7qxZlDek+YxOiyM1M5O1D3q2mLSpt4M8fnOamCzJHxAyorJQYak1mWp2blgeS3a756eZixiVE8R/LAztCLMFeeKzDYuPuv+zmnaM1/OoL53P7hb6dOjchKZqN31jKnPEJfOOve3hpd9mg7vP2oWraLTY+38cg4NSxroRoIy/Ybz9cTYhyLGR6bW8lbeahBSStNet3lTE3M5HzJgw9/8zls9IpKmukzuTZp6IPTtRy2x8/YnxSFD+4KrCbl/clK3n4sl9u3FPOgYomHrhmVtd04ECRYG8AWmt2HKuhqd1/mzK0ma3c9UIh7x2v5ZEb57LKT1PFkmMj+Pvdi1k2LZX7X97PB8e93zhj875KxsZHsmhySq/Pj42PJDYidESmTSg4Us387GT+Y/lUTJ1WXt8/tFwpRWWNHKlqYdVC3/y+rpg9Fq0h/0jNgOduP1zFV54vJDM5mn98fQljE3w722qwuubaB3jXqpYOC49sPUzexORBzYgaKgn2BrBpXyVf/tMubvjd+z6d0eLS2mll9fOFfFhSx2++mMvNC4bWHTCQmIgwfn97HlPS4rhnXRGVjZ6vdmzusFBwpIbPzR3f53Q2pZQzR87IatlXt3Swv7yJy2amsWBiMlPTYtlQOLhPNy4bdpURExHKdfN8E1zOm5BAekLkgInRXt9fyZq/7GFGejzr14ycQA/uee0Du4r2t9uPU2sy89C1/st/0x8J9qNcU5uFn71ezLSxcTS0mrn+qff5yAcDnC6mTit3PreLPacb+N9V87lhfqbP7t2fuMgwnvlyHmarnf/46x6Pc9m8dbAKs83e6ywcd1PT4ijxQcv+/7Yd4/8+9k2CsHecreXLZo1FKcWqhdnsOd3A0UHMfgHH727z/ko+P3c8cZG+6TJwrKZN592jtZitvae7eKmwjG+vK2J+dhJ/u3sxKYOYpeVPSTHhxEeGBTQh2snaVp57/yQ352UyNzMpYK/rToL9KPerfx+ioc3C/94yj39+cxnJsRHc/qeP2LinfMj3bu6w8OU/fcTeskaevHV+wD96Tk2L47Ev5rKvvIm1mw56dM3m/ZVkJEUzP6v//1BTUmOpaGwfcp/46/srKaq2+WSwN/9INekJkcwZ7xjE/MIFGYSHKtbvGlzrftPeStrMNp93uV0xayymTiu7Ttb3eO7590/yny/vZ9m0VP581yK/rIweKqUUWUOcfmnqtFJa14bdw4RqP3+9mMiwUH5w9fCNW0iwH8V2naxn3a4y7lo2iZyMRCalxvLP/1jGoskpfP8f+3jk34c9/mPsbl9ZI7c9+xGfVDTx1G0XcM354we+yA+uOm8c37psKusLywacnVLfaua9Y7VcmzthwI/JrkHaobTum9otHHOmXdi0r3KAs/tnsdnZcbSWy2aO7Sr7mLhIPnPeOF4pKh9Uls71haXMTI8f8I3PW8umpRIZFsLbbl05Wmt+u/0YP91czFXnpfPHOxcEfADSG1kp0UMK9v9vfRGXPJrPnIf+zeef3MF9L+3l6XdOsP1wFWX1574JFByp5u3D1dx7+TSfrxL3xsj9bYh+dVpt/OifB8hIiua7K85my0uMCeeFryziwdcO8ruCE5ysbeU3X5xHdIRneUj2nG7gibeP8c7RGhKjw3n69jyumJ3ur2p45L4VM9lf3sRDrx1k9vgE5vURvP79yadY7Zprcwd+Y+qaflnbOuhdkvaXNwIQHeYI9t+5Yvqg+2J3n2qgpdPaIxXBrQuzeWP/GbYe/JSV8zI8vt/ByibHz8wP/cPREaEsm+ZYTfvg5+cA8D//PsLT75zghvkZPHrTXMJ6mfI6kmSnxFBwpAattdc/H7PVzvvH67hoWiozx8VztKqF94/X8srHFV3nxEaEMi09nhlj49h1qp5JY2JYvWySj2vhHY+CvVLqauD/gFDgj1rrX3V7Phv4M5DkPOd+rfUW53NzgWeABMAOLNRa+24XhCD1zDslHK828fzqhT1aUOGhIfzyhhympsXyiy2HqPjDTv54x4J+B8k+Kqnjye3Hee94LSmxEfzn1TP58oUTiR8BH8NDQxRPrJrPtb99j//46x4233sRqXGRPc7bvK+SKWmxXd0g/Zk4JgalhpYQrai0EaXguqkRbDjSysHK5kG/cRQcqSY8VHFRt2yUS6eOISslmvW7yrwK9ut3lRERFsIN8z2/xhtXzB7L9sPVHKs28eLO07z44WluW5zNz1bmDFsqBG9kp8TQabVT09Lp9eDxvvJG2i02br8wm6tzzjYsmtosHKtu4UhVC8eqTBytaiH/SA11rZ08t3rhsCR+czdgsFdKhQJPASuAcqBQKbVJa13sdtqPgZe01r9XSs0BtgCTlFJhwF+BL2ut9ymlxgD+mx8YJEpqTPw2/zifmzuey2b1npRKKcXXLp7CxDGxfGd9ESufep8/3rngnLnWWmt2nqjj/94+xkcn60mNi+S/rpnNbRdmj7iP4MmxETx9ex43/v4D7v17ES9+ddE5rcfq5g4+PFnHty/3rHUdFR5KVnLMkGbkFJU2MC0tjosz7Lxy3MKmfZWDDvbbD1ezaHJKj4HUkBDHQO2jW49wqraVSc6snf1pN9t4dW8F1+SMIynGP4Ojrg1NvvJ8IRWN7ay5ZAoPfHbWsMwyGYwst1TH3gb7nSfqUAoWTz53dXZiTDgLJqWwYNK5U347rbZhD/TgWZ/9IuC41rpEa20G1gMru52jcbTcARIBVwfmZ4D9Wut9AFrrOq11cG4R5CNaa/7rn58QGRbCQ9fOGfD8FXPS+cc3lgBw89M7eau4Cq017xyt4aand/KlP37EqbpWHrp2Du/98DLuvmTKiAv0LjkZifzihvPZWVLHI1uPnPPcGwfOoDUedeG4TE2LHXSfvdaaorJG5mcnERehuHRGGpv3VQ5qjKSsvo1j1aaudATd3ZSXSWiIYoOHi8y2HDhDS4eVW3w0t7434xOjmTM+gYrGdu5bMWNUBXo4N9h7a+eJOmaNS/A4F9RICPTgWTdOBuD+V1YOLO52zlrgTaXUvUAscKXz+AxAK6W2AmnAeq31I0MqcZDbuKecnSV1/OKGHI8He86bkMhr31rG1/6ymzUv7mb62DiOVpmYkBjFz67P4ea8TL9tIOJrN+Vlsr+8kT+8W0JuZhKfc+aq37yvktnjE5g21vNt3aakxbGzpA67XXvd9XCytpXGNgvzs5OhrYHr5mWw7VA1hafqWTylZz6e/hQ4t/vr61NaekIUl80cyz92l3Pfihm9poBwt76wlMmpsVw4pfdFZb7yixtyqGruOKcrY7TISIpGKe+DfYfFxselDdy2OPAbrwyVr5pwtwIvaK0fU0otAV5USuU4738RsBBoA95WSu3RWr/tfrFSag2wBiA9PZ2CgoJBF8RkMg3p+pHGvT7NZs3aHW1MTwphfFsJBQUnvbrXt2ZrnrOGcqq5la+cF8GyjBDCOk7y4fve3Weohvo7uiRe80FSCPdt+JjG09FEhMLHpe3cNCPcq/vaGix0WOy8sjWf1GjvBhTfr3D0RtqqjmNSbURajxARCr//127az+s5ntCfjXs6GBujKP2kkJSJOE4AACAASURBVLI+Wsc50Va2mTp5YuN28tL7/m9babJTeKqdL84I55133vGqHO48/R1FAQUFRwY8b7j1Vp/kSEVh8UkKwjyfSXW43kan1U58eyUFBb7bk3cwvP5/pLXu9wtYAmx1e/wA8EC3cw4CWW6PS4CxwCrgz27HfwL8oL/Xy8vL00ORn58/pOtHGvf6fHd9kZ72ozf0kU+bh69APuCL39GnTe0672dv6csezdeP/PuQnvjD13VpXatX9/jwRK2e+MPX9TtHqr1+/f/653593oP/1labvas+3173sc796VbdabF5fJ92s1XP/PEW/dBrn/R7nsVq04t/sU3f+dxH/Z73s80H9dQH3tDVzR0el6E3Rv5/5HLz0x/om3//gVf3+c2bR/Tk+1/XjW1mH5Vs8LrXCdit+4mtnjRnCoHpSqnJSqkIZwDf1O2cUuAKAKXUbBxv+DXAVuB8pVSMc7D2UqAY4bX3jtXySlEFX79kKjP8uAP9aJGeEMXvbruA0vo2nso/wbysJK/TxU5JG3xCtKLSRnKzEs9JyXBd7gQa2yy8d3zgvDEuO0vq6LDY++zCcQkLDeGLCzJ552gNFX2kj+i02nj543JWzEknLd67TxfBKCvZ+4VVO0vqOG9C4pBSRQ+XAYO91toK3IMjcB/CMevmoFLqYaXUdc7TvgfcrZTaB6wDVjvfbBqA3+B4w9gLfKy1fsMfFRlu+uynF5/rsNj48asHmDQmhnsun+aX1xiNFk1O4b8+59j8YeUgcr+kxkWQEBXmdbBvM1s5/GkL87OSzzl+8fQ0kmLC2bTX826B/MPVRIeHsriPpG3uXDmJXuojX86bB6toaLP4LUmd0WSnxPBpc4fHC9Y6LDb2ljaypJc9EkYDj/rstWPO/JZuxx50+74YWNbHtX/FMf3S0B7ZeoT3j9ey6Z6LfH7v324/zqm6Nv72tcWjZiA1UFYvncT87GTOH8SUx66EaNXezcg5UN6Eza6Zn33u4q6IsBA+mzOe1/ZW0G62DbiQTWvN9sPVLJs2xqPfa1ZKDBdNS+Ufu8v49hXTeyR621BYRkZSNBcP805Qo0X2mGgAKhrbuxbZ9WfP6QbMNjtLvByAHylG9jK3UcJis7N+VynFlc2DTk/Ql4oWO0+/c4IvXJDBMvlP3INSinlZSYPesHlqWhwltd617IvKHCtne1vJu3LeBNrMNrZ5sMHHiRoT5Q3tXm3gfeuibCqbOnj32LldRaV1bbx3vJYvLsgaFYuaRoJsL6df7jxRR2iIYsGk5IFPHoEk2PvAe8dqaWizYLVrn+aUt9s1zx/sJD4qjP8K4F6VwWRKWixVzZ20dHj+eysqbWDimBjG9LKKd9GkFMYlRPGaB105+YfPZrn01JWz0xkTG8H6bnmCNuwuJUTBFxcGJiupEXRtYuJpsC+pIycjcUSsKh8MCfY+8Nreszkxaj3cwccTG/eUc7zRzo+umd1rYBFDd3aLQs+6crTWfFza2GdysZAQxbW543nnaDVNbf2/gWw/XM3M9HgykqI9Lm9EWAg35WXy9qFqqlscWUesNjv/2F3O8pljGZ/o+b2CXVp8JJFhIR5tYtLaaWVfWeOo7cIBCfZD1ma28mZxFZOdy9hrfBjsdxyvZUyU4qY8aa35y9mEaJ515VQ2dVDT0ulYTNWH63IzsNg0//qk712mWjosFJ6q96pV73LLwiysdt2Vxnr74WqqWzpZtdC/m8oYjVKK7JQYj7Yn3H26Aatdj9rBWZBgP2TbDlXTZrZxlzOjXU2L74J9TUsHY6LVqFqGPtpkp8QQGqI8HqQtKm0A6DE46y4nI4EpqbH9pj1+71gtVrvmsplp3hUYx5TRxZNT2FBYht2uWV9Yxtj4yK58NcJz2SkxlNYPvGPVzhN1hIUoFkwcnf31IMF+yDbtrWBcQlRXvvdak9ln965p6SQhQgK9P0WEhTAxxfOEaEWljUSGhTBrXN+ZNZVSXJs7gZ0ldVQ1957gNf9INfFRYeQNMnjcuiib03VtvLq3goIj1dy8IHPEpxUeibJSYiirbxtw2vTOkjpys5KI9dGOX8NB/jqGoKHVTMGRGq7NHU9yTARhIcqnffY1LZ0kRkqw97cpXmxRWFTawPkZiUSE9f9f57p5E9CaXjcMt9s1+UdquGRG2qAD9NU540iICuNH/zyAXcMX/bwvsFFlpcRg6rTS0M/4SkuHhU8qmkZ1fz1IsB+Sfzk3y1g5L4OQEEVqXCS1PurG6bDYaO6wSrAPgKlpsZysbcU2wLTZTquNTyqb++3COXvPOHIyEtjkNnjvUnymmZqWTi73Yspld1HhoXzhgkw6LHaWTRvDxDEDpz4WPXVtPt7PjJzdpxqwjfL+epBgPySv7a1gSlos501wfKRPjY/w2QCt6xOCBHv/m5oWh9lmp3yAgbpDZ1owW+39Ds66W5mbwb7yJk7VnvupYfvhapSCSwfRX+/utsXZRISGcOeSSUO6TzDzZK79zpI6IkJDBt3lNlJIsB+kysZ2dp2qZ2VuRtcAampcpM+6cVwDvYnSZ+93U8c6WsUDdeV4Mjjr7vO541Gq5/60+UeqmZuZ1OtuW96Ynh7P3odW8Jnzxg3pPsEsM9kxVbXfYH+ijnnZSaN+9boE+0F6fX8lWjv6Zl0c3Ti+GaB1Bfskadn73ZRUzxKiFZU2Mi4hyuO57OMTo1k0KYXX9lZ0DQDWmTrZW9Y4qFk4vRmpG82MFrGRYaTGRfTZjdPUbuFg5ejvrwcJ9oP22t5KcjMTu+bXg2ORRl1rp08SotVIN07AJMdGkBIbMXCwL2vwuFXvct28CZyoaaX4TDMA7x6rQWtkmuQIktXPXPtdJ+uxa7hQgn1wOl7dwsHKZq7rtgF0alwkFptvUia4PiHESzdOQExNi+13rn1NSydl9e1eB/trcsYTFqK6unK2H64hNS6SnAmD26tW+J5jrn3vwX7niToiwkK8/r2PRBLsB2HT3kqUgmvnnrsdW2qcY09KXyysqjF1kBLrmM4p/G9Kav8J0fY6k595OjjrkhwbwSUz0ti8txKLzc67R2tYPjNNkpWNIFnJMVQ2dmCx2Xs8t7Okjrzs5FHfXw8S7L2mtea1fZUsnTqmx670ac4BN1/MyKlp6ey6n/C/qWNjqTWZaWzrfcylqLSBsBA1qBb5ynkTqGzq4NkdJTS1W/rcWFwMj+yUGGx2zZnGcxfANbSaOXSmedRPuXSRYO+lfeVNnK5rY2VuRo/nXLsD+WIVbU1Lp+w2FEBTu3at6r0rp6i0kdnjEwbMUd+bK2enExUewv9uO0ZoiOLiGZKqeiTJ6mP65Ucn6wAk2Aer1/ZWEBEawlU5Pae7uabS+WJhVY1Jgn0gnc1+2bMrx2bX7CtvHHS/bWxkGCvmjMNstbNgYjIJozRFrlFlj3EurOo2SLvzRB3R4aHkZo7+/nqQYO8Vm12zed8ZLp81ttc9KBOjwwkLUUPuxtFaS8s+wDKTowkPVb227I9WtdBmtg1pkO66XMcUXZmFM/KMS4giPFT1aNnvLKljwaTkAVNjjBbGqEWA7DxRR62ps8/9TkNCFGPiIobcsjd1Wumw2LsGfIX/hYWGMGlMbK/TL4tKnYOzWYNfQXn5rLGsvXYOty6W/WFHmtAQRUZS9DnBvtbUydEqkyGmXLrIigwvvLa3gvjIsH5zkPtiFa1rNk9afCQ0DelWwgtT0+I4Vt3S43hRaQPJMeFMdH7cH4zQEMXqZZOHUjzhR67sly4fldQDxumvB2nZe6zDYuPfn3zKVTnj+p2GlRYfOeQB2q5gHxc1wJnCl6akxXK6rq3HFLyiskbmZyfLvgIGlt0t2O8sqSU2InRQG9mPVBLsPVRwpJqWTmufXTguqXGRQ55n7+rzlz77wJqaFofVrs/5ON/UbuF4tanPbQiFMWSlxNDQZqHZuRfxzhN1LJycQriB9ggwTk387LW9laTGRQ6YIyM1bugpE87pxhEBM3Wsc/pl9dl++32DXEwlRhf3VMfVzR2cqGk1VH89SLD3SHOHhbcPV/P5ueMH3GwiNS5iyCkTalo6CQtRJPUy40f4z5Q0Z/ZLt5TERaWNKAVzs4zzcV705B7sd5Y459cbLNjLAK0Htn7yKWarfcAuHHBfWNVJUszgZtPUtHSSGhcpS+oDLCEqnLT4yHNa9kVlDUwfGydz4w0uqyvYt1NSayI+MqxrnwqjkJa9BzbtqyQ7JYZ5HvTbdqVMGEKqY1lQNXympp2dfqm1pqi0cUhTLsXokBgdTmJ0OKX1bew8UceiySmG29PXWLXxg+qWDt4/XsvKeRM8mo2RGj/0/DiyoGr4TEmL40RNK1prTta20tRuMUTGQzGwrJRodp2s51Rdm6GmXLp4FOyVUlcrpY4opY4rpe7v5flspVS+UqpIKbVfKXVNL8+blFLf91XBA+WN/WewazzqwgHfpEyoNUkStOEyNS2OpnYL9a3ms4upZHA2KGSnxHCkyrHOwmiDs+BBsFdKhQJPAZ8F5gC3KqXmdDvtx8BLWuv5wCrgd92e/w3wr6EXN/Be21vJnPEJTBsb79H5Sc6UCYNdWGW3a2pNZmnZD5OpzkHaEzWtFJU1EBcZxjTnLB1hbK5++8TocOaMN1Z/PXjWsl8EHNdal2itzcB6YGW3czTg+ukkAl2bbiqlrgdOAgeHXtzAKqtvY29Z4zlbDw6kK2XCIIN9Q5sZm11LsB8m7gnRikobyc1KJFQGyoOCa0bO4skphpwc4UmwzwDK3B6XO4+5WwvcrpQqB7YA9wIopeKAHwI/HXJJh8G2Q1UAXOXlhs5DWVglC6qG14SkaCLDQjhQ0cThT1tkcDaIZCU7gr0R++vBd1MvbwVe0Fo/ppRaAryolMrB8SbwuNba1N/gplJqDbAGID09nYKCgkEXxGQyDel6dy/tamdCrOL0J4Wc9uK6EHMHJ88Mrhyf1NoAKD9eTEHdEZ/WZ6QY6XUaGw2vfVyKzQ5hTWUUFJzp9/yRXp/BMFqdPKlPp01zZXYYqa2nKCjw5n/88PD6d6S17vcLWAJsdXv8APBAt3MOAlluj0uAscAO4JTzqxGoB+7p7/Xy8vL0UOTn5w/pepfGVrOe8sAb+lf/OuT1td97aa++8JfbBvW6L+8p0xN/+Lo+WWPSWvuuPiPJSK/TN/+6R0/84et64g9f13WmzgHPH+n1GQyj1clo9dG6Z52A3bqf2OpJy74QmK6UmgxU4BiA/VK3c0qBK4AXlFKzgSigRmt9sesEpdRawKS1/q3nb0XDp+BoNTa75srZ6V5f68p8qbX2OnmWpEoYfq5B2kljYkiJlTTTwhgG7LPXWluBe4CtwCEcs24OKqUeVkpd5zzte8DdSql9wDpgtfOdZtR6q7iK1LiIQSXAGkrKhJqWTqLDQ4mNlMXNw8WVI0emXAoj8SiiaK234Bh4dT/2oNv3xcCyAe6xdhDlGxZmq513jtRwzfnjBzUqP5SUCbJ6dvi5ZuTIYiphJLKCthe7TtbT0mnlyjned+HA2YVVg0mZIKtnh995ExL431vmcXNe1nAXRQifkWDfi7eKPyUqPISLpqUO6nr3lr23alpk9exwU0px/fwMoiP63qRGiNFGgn03Wmu2Harmomlpg/7P3pUyYTDBXrpxhBB+IMG+m+IzzVQ0trNiTt/7zA4kKTqc0BDl9cKqTquNxjaLBHshhM9JsO9mW3E1SsHlswbXXw/OlAmx3qdMqHPuXSvBXgjhaxLsu9l2qIr5WUlDDriD2Xj87EbjEuyFEL4lwd7NmaZ2DlQ0DXoWjjvXwipv1EpeHCGEn0iwd7PtUDUAn/FRsPe2z15Wzwoh/EWCvZttxVVMGhPTtahmKFLjI6gzmfFmIbEr2I+JkyX6QgjfkmDvZOq0svNEHSvmpHudz6Y3aXGRmG12mtutHl9TY+okKSacyDCZ3y2E8C0J9k7vHq3BbLMPKvFZb9IGsRetLKgSQviLBHunt4qrSIoJJ2+ib5JfDWZhlaRKEEL4iwR7wGqzs/1wNZfPGktYqG9+JGfz43gR7GX1rBDCTyTYA7tPN9DUbmGFj7pwwJHmGLxv2adKN44Qwg8k2OPowokIDeHiGWk+u2dyTAShIcrjYN/aaaXNbJOWvRDCL4I+2DsSn1WxdNoY4ny4YUhXygQP0xzL6lkhhD8FfbA/Xm3idF2bz2bhuEuNi/R4Nk6NrJ4VQvhR0Af7N4urAPwT7OM9T5kgq2eFEP4U9MF+26Eq5mYmMi4xyuf3TouLpNbD2TgS7IUQ/hTUwb66pYO9ZY1+adWDI2VCrYcpE2paOgkNUSR7uWetEEJ4IqiD/fZD1WgNK3yQ+Kw33qRMqGnpZEysYwaPEEL4WlAH+22HqshIimbWuHi/3L9rYZUH/fayoEoI4U9BG+zbzFZ2HKv1WeKz3niz8XitBHshhB8FbbB/71gtnVa737pwwLv8OJIETQjhT0Eb7LcdqiI+KoxFk1P89hpdKRMGmJFjt2tp2Qsh/Coog73Nrnn7UDXLZ44l3EeJz3rjSpkwUJ99U7sFi01LsBdC+I1HkU4pdbVS6ohS6rhS6v5ens9WSuUrpYqUUvuVUtc4j69QSu1RSh1w/nu5ryswGHvLGqhrNfu1Cwc8T5kgq2eFEP42YDIYpVQo8BSwAigHCpVSm7TWxW6n/Rh4SWv9e6XUHGALMAmoBa7VWlcqpXKArUCGj+vgte2HqwkLUVzqw8RnffFk43HJiyOE8DdPWvaLgONa6xKttRlYD6zsdo4GEpzfJwKVAFrrIq11pfP4QSBaKTXsEa2svp2M5GgSo8P9/lqepEyQ1bNCCH/zJNhnAGVuj8vp2TpfC9yulCrH0aq/t5f73Ah8rLX2PMG7nzS0mQO2UjU1LmLADUxcz6dKsBdC+ImvcvreCrygtX5MKbUEeFEplaO1tgMopc4D/gf4TG8XK6XWAGsA0tPTKSgoGHRBTCbTgNeXVrWTFKmG9Dqeam8wU91sIT8/v8/5/HsOmwkPgT073+txjif1GW2MViej1QeMVyej1QcGUSetdb9fwBJgq9vjB4AHup1zEMhye1wCjHV+nwkcBZYN9Fpaa/Ly8vRQ5OfnD3jOkl9u0/dt2Duk1/HUs++e0BN/+LpubDP3ec7/W1+kl/3q7V6f86Q+o43R6mS0+mhtvDoZrT5a96wTsFv3E1s96cYpBKYrpSYrpSKAVcCmbueUAlcAKKVmA1FAjVIqCXgDuF9r/b7nb0H+Vd9mJiXW//314NnCKtloXAjhbwMGe621FbgHx0yaQzhm3RxUSj2slLrOedr3gLuVUvuAdcBq5zvNPcA04EGl1F7n11i/1MRD7WYbHRY7SQHrsx9443FZPSuE8DeP+uy11ltwDLy6H3vQ7ftiYFkv1/0c+PkQy+hTDW2OOe8psQEK9vEDbzxeY+pkwaTkgJRHCBGcgm4FbX2rI9gHajaOq8XeV8oEi81OfatZunGEEH4VdME+0C17V8qEWlPvq2jrnMcl2Ash/Cnogr2rZR+oAdqQEEVKbESf3Tiu49JnL4Twp6AL9g0B7sYBxyBtXwO0snpWCBEIQRfs69ssKEVAUiW4pPWTMkGCvRAiEIIu2De0mkmMDifMj6mNu0uNi+izz96V8TJVunGEEH4UdMG+vs1MSgC7cMDRH19j6nStLj5HTUsnCVFhRIWHBrRMQojgEnTBvqHVTHKAZuK4pMZFYrbaae6w9nhOVs8KIQIh6IJ9fWvgMl669LfxuAR7IUQgBF2wbwhgXhyX1H4WVtWYOkmLjwpoeYQQwSeogr3WmoZWS+C7cbpSJvQcpK1p6ezamFwIIfwlqIJ9q9mG2WYP+ADt2WRoHeccbzNbMXVapRtHCOF3QRXsuxZUBbhlnxwTQYjq2bJ3bUQuq2eFEP4WVMG+K1VCgFv2oSGKMb1sPF5jcrT0pWUvhPC34Ar2bcPTsgdHV06PYC+rZ4UQARJUwb6hNbAZL92lxkVQ060bR4K9ECJQgirYD1c3Djj65btPvaxp6SREwZhYCfZCCP8KqmDf2GYhNEQRH+XRBl0+lRbfM2VCjclMSmwkoSEq4OURQgSXoAr29W1mkmPCCRmG4OpKmdDSeTZlgqyeFUIESlAF+4ZWc8A2Gu+ua2GVW1eOY/WsBHshhP8FVbCvbw18xkuXswurzgb72pZOmWMvhAiIoAr2DW1mkgOcF8flbDI0xyCx1lq6cYQQARNUwb6+1TIs0y7BLRmac659c7sVs80uwV4IERBBE+y11o6W/TB145xNmeAI9rJ6VggRSEET7Js7rNjsetha9qEhipTYsxuPV7sWVEmfvRAiAIIm2HclQRumlj249qJ1tuy7Vs9KemMhhP8FTbB35cUZrpY9uBZWOcrRFezjZOMSIYT/eRTslVJXK6WOKKWOK6Xu7+X5bKVUvlKqSCm1Xyl1jdtzDzivO6KUusqXhffGcKU3dueeMqHG1ElEaAgJ0YFfzSuECD4DRhqlVCjwFLACKAcKlVKbtNbFbqf9GHhJa/17pdQcYAswyfn9KuA8YAKwTSk1Q2tt83VFBjKceXFcUuMdmS/dp10qJakShBD+50nLfhFwXGtdorU2A+uBld3O0UCC8/tEoNL5/Upgvda6U2t9EjjuvF/ANXSlNx6eefbg6LPvdKZMqGnpJFVm4gghAsSTYJ8BlLk9Lncec7cWuF0pVY6jVX+vF9cGRH2rhfBQRVzk8HWbdC2saul0tOxlJo4QIkB8FfluBV7QWj+mlFoCvKiUyvH0YqXUGmANQHp6OgUFBYMuiMlk6vX64hOdxIbBO++8M+h7D1VFraP36q0dH1FZ38G4sPYB69pXfUYzo9XJaPUB49XJaPUB7+vkSbCvALLcHmc6j7n7KnA1gNZ6p1IqCkj18Fq01n8A/gCwYMECvXz5cg+L31NBQQG9Xf+30t2Ms7axfPklg773UKWfaebXu3cwYepsTIVFzJ05meXLZ/R7TV/1Gc2MViej1QeMVyej1Qe8r5Mn3TiFwHSl1GSlVASOAddN3c4pBa4AUErNBqKAGud5q5RSkUqpycB0YJfHpfOhhtbhWz3r4kqZcOTTFuxaVs8KIQJnwJa91tqqlLoH2AqEAs9prQ8qpR4GdmutNwHfA55VSn0Xx2Dtau3YpeOgUuoloBiwAt8ajpk44JhnP3tcwsAn+lFKrCNlwqEzzYCsnhVCBI5HffZa6y04Bl7djz3o9n0xsKyPa38B/GIIZfSJhtbhy3jp4kqZ0BXspWUvhAiQoFhBa7NrGtstwzrH3iU1LoLKJkcStLES7IUQARIUwb653YLWw7t61sW9NZ8q3ThCiAAJimA/EvLiuLj66eMjw4iOCB3m0gghgkVQBPuRkPHSxbVqVvrrhRCBFBTBvisvzgho2afGOcogqRKEEIEUFMHelRcnKWZ4Z+PA2X56mXYphAikoAj29a0WYKS07KUbRwgReEER7BvazESGhRAdPvwDomnSZy+EGAZBEezrW82kxEaMiNzxmcnRpMZFkJORONxFEUIEkaDYJmkk5MVxiY8KZ/ePVwx3MYQQQSY4WvZt5hHRXy+EEMMlKIK9Iy+OBHshRPAKimBf32omZQRMuxRCiOFi+GBvsdlp7rBKy14IEdQMH+wb20bOHHshhBguhg/2rtWzI2U2jhBCDAfDB/uRlBdHCCGGi+GD/UjKeCmEEMPF8MF+JOWyF0KI4WL4YO9q2Y+EjJdCCDFcjB/s2yzERoQSNQKSoAkhxHAxfrCX1bNCCGH8YC95cYQQIgiC/UjKeCmEEMPF8MFeWvZCCBEEwb6h1SIteyFE0PMo2CulrlZKHVFKHVdK3d/L848rpfY6v44qpRrdnntEKXVQKXVIKfWECuB2UZ1WG6ZOKymxMu1SCBHcBtypSikVCjwFrADKgUKl1CatdbHrHK31d93OvxeY7/x+KbAMmOt8+j3gUqDAR+XvlysJWpK07IUQQc6Tlv0i4LjWukRrbQbWAyv7Of9WYJ3zew1EARFAJBAOVA2+uN6RvDhCCOHgSbDPAMrcHpc7j/WglJoITAa2A2itdwL5wBnn11at9aGhFNgbkhdHCCEcfL3h+Cpgo9baBqCUmgbMBjKdz7+llLpYa73D/SKl1BpgDUB6ejoFBQWDLoDJZOq6ftcZKwAnivfRWTY6x6Ld62MURquT0eoDxquT0eoD3tfJk2BfAWS5Pc50HuvNKuBbbo9vAD7UWpsAlFL/ApYA5wR7rfUfgD8ALFiwQC9fvtyTsveqoKAA1/VlO0/BvoN8ZvlSxsZHDfqew8m9PkZhtDoZrT5gvDoZrT7gfZ08ae4WAtOVUpOVUhE4Avqm7icppWYBycBOt8OlwKVKqTClVDiOwdmAdePUtzoGaKUbRwgR7AYM9lprK3APsBVHoH5Ja31QKfWwUuo6t1NXAeu11trt2EbgBHAA2Afs01pv9lnpB9DQZiY+Kozw0NHZhSOEEL7iUZ+91noLsKXbsQe7PV7by3U24OtDKN+Q1LfK6lkhhACDr6BtaJO8OEIIAQYP9tKyF0IIB0MHe8l4KYQQDoYO9o6Ml5IXRwghDBvs2802Oix22aVKCCEwcLBvaHPmxZFuHCGEMG6wdyVBk5a9EEIYONh3tewl2AshhHGDfb1kvBRCiC6GDfYNksteCCG6GDbY17dZUAoSo2XqpRBCGDbYN7SaSYoOJzQkYFveCiHEiGXYYF/fZpaZOEII4WTYYC+pEoQQ4izDBvt6CfZCCNHFsMG+QfLiCCFEF0MGe601Da0W6bMXQggnQwb7VrMNs80ueXGEEMLJkMG+QfLiCCHEOQwZ7F2pEqRlL4QQDsYM9m3SshdCCHeGDPaSF0cIIc5lyGAv3ThCCHEuQwb7hjYzoSGK+Kiw4S6KEEKMCAYN9haSY8IJkSRoQggBGDXYS6oEIYQ4hyGDfX2rZLwUQgh3HgV7pdTVSqkjntBnAAAACiNJREFUSqnjSqn7e3n+caXUXufXUaVUo9tz2UqpN5VSh5RSxUqpSb4rfu8a2swyOCuEEG4GHMFUSoUCTwErgHKgUCm1SWtd7DpHa/1dt/PvBea73eIvwC+01m8ppeIAu68K35f6Vgt5EyXYCyGEiyct+0XAca11idbaDKwHVvZz/q3AOgCl1BwgTGv9FoDW2qS1bhtimfultZaMl0II0Y0nwT4DKHN7XO481oNSaiIwGdjuPDQDaFRKvaKUKlJKPer8pOA3bVaw2bUM0AohhBtfT0RfBWzUWtvc7n8xjm6dUmADsBr4k/tFSqk1wBqA9PR0CgoKBl2A6sZWQFFVeoKCgtJB32ekMJlMQ/p5jERGq5PR6gPGq5PR6gPe18mTYF8BZLk9znQe680q4Ftuj8uBvVrrEgCl1KvAhXQL9lrrPwB/AFiwYIFevny5J2Xv1fFX3wY6WLogl+Uzxw76PiNFQUEBQ/l5jERGq5PR6gPGq5PR6gPe18mTbpxCYLpSarJSKgJHQN/U/SSl1CwgGdjZ7dokpVSa8/HlQHH3a33JZNaApEoQQgh3AwZ7rbUVuAfYChwCXtJaH1RKPayUus7t1FXAeq21drvWBnwfeFspdQBQwLO+rEB3La5gL/PshRCii0d99lrrLcCWbsce7PZ4bR/XvgXMHWT5vGayOP5NipHZOEII4WK4FbQtZk14qCIuUpKgCSGEi+GCvcnimHaplCRBE0IIF8MF+xazlv56IYToxnDB3mSWBVVCCNGd4YJ9i0Va9kII0Z3hgr3JrEmWvDhCCHEOQwV7m13TapEFVUII0Z2hgn1zuwUNsnGJEEJ0Y6hgX99mBmT1rBBCdGeoYN/Q6gj2MhtHCCHOZahgX98qLXshhOiNoYJ9g7MbR/rshRDiXIYK9vWtjixoMhtHCCHOZahg39BmJiIEoiP8uvOhEEKMOoYK9vWtZuIiJAGaEEJ0Z6hg39BqJl6CvRBC9GCoYF/fZiZOMiUIIUQPhgr20rIXQojeGSrY17eaiQuXYC+EEN0ZJthbbHaaO6zSshdCiF4YJtg3tjnm2MtsHCGE6MkwwT4pJpwt376YBemy0bgQQnRnmGAfHhrCnAkJJEZKy14IIbozTLAXQgjRNwn2QggRBCTYCyFEEPAo2CulrlZKHVFKHVdK3d/L848rpfY6v44qpRq7PZ+glCpXSv3WVwUXQgjhuQGnriilQoGngBVAOVColNqktS52naO1/q7b+fcC87vd5mfAuz4psRBCCK950rJfBBzXWpdorc3AemBlP+ffCqxzPVBK5QHpwJtDKagQQojB8yTYZwBlbo/Lncd6UEpNBCYD252PQ4DHgO8PrZhCCCGGwtcrkFYBG7XWNufjbwJbtNblSvU9/10ptQZYA5Cenk5BQcGgC2AymYZ0/UhjtPqA8epktPqA8epktPrAIOqkte73C1gCbHV7/ADwQB/nFgFL3R7/DSgFTgG1QDPwq/5eLy8vTw9Ffn7+kK4faYxWH62NVyej1Udr49XJaPXRumedgN26n9jqScu+EJiulJoMVOBovX+p+0lKqVlAMrDT7Y3kNrfnVwMLtNY9ZvMIIYTwrwGDvdbaqpS6B9gKhALPaa0PKqUexvFOssl56ipgvfMdZtD27NlTq5Q6PYRbpOL4FGEURqsPGK9ORqsPGK9ORqsP9KzTxP5OVkOMzSOOUmq31nrBcJfDV4xWHzBenYxWHzBenYxWH/C+TrKCVgghgoAEeyGECAJGDPZ/GO4C+JjR6gPGq5PR6gPGq5PR6gNe1slwffZCCCF6MmLLXgghRDeGCfYDZeYcjZRSp5RSB5zZRHcPd3m8pZR6TilVrZT6xO1YilLqLaXUMee/ycNZRm/1Uae1SqkKt8yv1wxnGb2hlMpSSuUrpYqVUgeVUt9xHh+Vv6d+6jOaf0dRSqldSql9zjr91Hl8slLqI2fM26CUiuj3PkboxnFm5jyKW2ZO4FbtlplzNFJKncKxEG1Uzg9WSl0CmIC/aK1znMceAeq1/v/t3V+IVVUUx/HvYpqHQSPtDyKYSCUEkYmP4YP4EPRUUWhSYNFDSIURhNBLEQUREWGFkUT4YImklk+hWPSHokBRK+apqAcZZ4yYaiCi9NfDXtcu49yZJv/c2ef8PjDcc/c993A2i7tms885a+uF/Ke8UNKWfp7nbPTo0zPAhKSX+nlu/0dELAYWSzoSEZcDh4E7gQeoME7T9Gcd9cYogHmSJiJiEPgc2Aw8AeyVtCsi3gCOSdrW6zhNGdnPtjKnXQKSPgV+mdR8B7Ajt3dQfojV6NGnakkakXQkt38HhimFDquM0zT9qVZWQ5jIt4P5J2At8F62zxijpiT7/1yZszICDkTE4SwW1wSLJI3k9klK+esmeDQijuc0TxVTHpNFxDLKWhRf0YA4TeoPVByjiBiIiKPAGHAQ+B4Yl/R37jJjzmtKsm+q1ZJWAbcDj+QUQmNkaY365xFhG3A9sBIYoZT1rkpEzAf2AI9L+q37sxrjNEV/qo6RpNOSVgJLKDMZN872GE1J9ieAa7veL8m2qkk6ka9jwD5KkGs3mvOqnfnVsT6fz3mTNJo/xjPAdiqLU84D7wF2StqbzdXGaar+1B6jDknjwMeUasQLIqJT32zGnNeUZH+2Mmdekb4X2D/Dd+a0iJiXF5iIiHnAbcC303+rCvuBjbm9Efigj+dyQXSSYrqLiuKUF//eAoYlvdz1UZVx6tWfymN0TUQsyO0hyo0ow5Skf0/uNmOMGnE3DkDeSvUK/1bmfL7Pp3ReIuI6ymgeSnXSd2rrU0S8C6yhVOcbBZ4G3gd2A0uBn4B1kqq54NmjT2so0wOirN3wcNd895wWEauBz4BvgDPZ/BRlnru6OE3Tnw3UG6MVlAuwA5QB+m5Jz2aO2AVcSVlL5H5Jf/Y8TlOSvZmZ9daUaRwzM5uGk72ZWQs42ZuZtYCTvZlZCzjZm5m1gJO9tVJEnO6qgHj0QlZKjYhl3VUxzeaCy2bexayR/sjHz81awSN7sy65hsCLuY7A1xFxQ7Yvi4iPspDWoYhYmu2LImJf1ho/FhG35qEGImJ71h8/kE8+mvWNk7211dCkaZz1XZ/9Kulm4DXKU9kArwI7JK0AdgJbs30r8ImkW4BVwHfZvhx4XdJNwDhw90Xuj9m0/ASttVJETEiaP0X7j8BaST9kQa2Tkq6KiJ8pi2L8le0jkq6OiFPAku7H1LO07kFJy/P9FmBQ0nMXv2dmU/PI3uxc6rE9G901Sk7j62PWZ072Zuda3/X6ZW5/QammCnAfpdgWwCFgE5xdYOKKS3WSZrPh0Ya11VCu/NPxoaTO7ZcLI+I4ZXS+IdseA96OiCeBU8CD2b4ZeDMiHqKM4DdRFscwm1M8Z2/WpfZF3s168TSOmVkLeGRvZtYCHtmbmbWAk72ZWQs42ZuZtYCTvZlZCzjZm5m1gJO9mVkL/AMgrwzaDEKb6AAAAABJRU5ErkJggg==\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "plt.figure(figsize=(6,5))\n", + "plt.plot(np.array(evaluation_accuracy)/10000)\n", + "\n", + "plt.grid()\n", + "plt.xlabel('Epoch')\n", + "plt.legend(['Accuracy on the test data'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OoTHrABq8LK3" + }, + "source": [ + "### chapter 5 - The vanishing gradient problem - deep networks are hard to train with simple SGD algorithm this network learns much slower than a shallow one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oY5zITz6-KmS", + "outputId": "dcfba3d3-3a63-4c2f-e65a-f640529c9f09" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch 0 training complete\n", + "Accuracy on evaluation data: 2584 / 10000\n", + "Epoch 1 training complete\n", + "Accuracy on evaluation data: 7714 / 10000\n", + "Epoch 2 training complete\n", + "Accuracy on evaluation data: 9120 / 10000\n", + "Epoch 3 training complete\n", + "Accuracy on evaluation data: 9188 / 10000\n", + "Epoch 4 training complete\n", + "Accuracy on evaluation data: 9379 / 10000\n", + "Epoch 5 training complete\n", + "Accuracy on evaluation data: 9417 / 10000\n", + "Epoch 6 training complete\n", + "Accuracy on evaluation data: 9473 / 10000\n", + "Epoch 7 training complete\n", + "Accuracy on evaluation data: 9381 / 10000\n", + "Epoch 8 training complete\n", + "Accuracy on evaluation data: 9520 / 10000\n", + "Epoch 9 training complete\n", + "Accuracy on evaluation data: 9518 / 10000\n", + "Epoch 10 training complete\n", + "Accuracy on evaluation data: 9570 / 10000\n", + "Epoch 11 training complete\n", + "Accuracy on evaluation data: 9540 / 10000\n", + "Epoch 12 training complete\n", + "Accuracy on evaluation data: 9523 / 10000\n", + "Epoch 13 training complete\n", + "Accuracy on evaluation data: 9572 / 10000\n", + "Epoch 14 training complete\n", + "Accuracy on evaluation data: 9620 / 10000\n", + "Epoch 15 training complete\n", + "Accuracy on evaluation data: 9615 / 10000\n", + "Epoch 16 training complete\n", + "Accuracy on evaluation data: 9594 / 10000\n", + "Epoch 17 training complete\n", + "Accuracy on evaluation data: 9584 / 10000\n", + "Epoch 18 training complete\n", + "Accuracy on evaluation data: 9549 / 10000\n", + "Epoch 19 training complete\n", + "Accuracy on evaluation data: 9573 / 10000\n", + "Epoch 20 training complete\n", + "Accuracy on evaluation data: 9598 / 10000\n", + "Epoch 21 training complete\n", + "Accuracy on evaluation data: 9542 / 10000\n", + "Epoch 22 training complete\n", + "Accuracy on evaluation data: 9534 / 10000\n", + "Epoch 23 training complete\n", + "Accuracy on evaluation data: 9569 / 10000\n", + "Epoch 24 training complete\n", + "Accuracy on evaluation data: 9621 / 10000\n", + "Epoch 25 training complete\n", + "Accuracy on evaluation data: 9633 / 10000\n", + "Epoch 26 training complete\n", + "Accuracy on evaluation data: 9551 / 10000\n", + "Epoch 27 training complete\n", + "Accuracy on evaluation data: 9572 / 10000\n", + "Epoch 28 training complete\n", + "Accuracy on evaluation data: 9629 / 10000\n", + "Epoch 29 training complete\n", + "Accuracy on evaluation data: 9633 / 10000\n" + ] + } + ], + "source": [ + "from elements import network2\n", + "from elements import mnist_loader\n", + "\n", + "training_data, validation_data, test_data = mnist_loader.load_data_wrapper()\n", + "training_data = list(training_data)\n", + "test_data = list(test_data)\n", + "\n", + "net = network2.Network([784, 30, 30, 30, 30, 10], cost=network2.CrossEntropyCost)\n", + "evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \\\n", + " net.SGD(training_data, 30, 10, 0.1,\n", + " lmbda=5.0,\n", + " evaluation_data=validation_data,\n", + " monitor_evaluation_accuracy=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vrw4ysHS_uWM" + }, + "source": [ + "### chapter 6 - shallow architecture using just a single hidden layer, containing 100 hidden neurons.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "XCOof1c0_wkj", + "outputId": "e777b2da-75b2-437d-b38a-3e4bd3b1cfeb" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Trying to run under a GPU. If this is not desired, then modify network3.py\n", + "to set the GPU flag to False.\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "WARNING:theano.tensor.blas:We did not find a dynamic library in the library_dir of the library we use for blas. If you use ATLAS, make sure to compile it with dynamics library.\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Training mini-batch number 0\n", + "Training mini-batch number 1000\n", + "Training mini-batch number 2000\n", + "Training mini-batch number 3000\n", + "Training mini-batch number 4000\n", + "Epoch 0: validation accuracy 92.56%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 91.93%\n", + "Training mini-batch number 5000\n", + "Training mini-batch number 6000\n", + "Training mini-batch number 7000\n", + "Training mini-batch number 8000\n", + "Training mini-batch number 9000\n", + "Epoch 1: validation accuracy 94.43%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 94.13%\n", + "Training mini-batch number 10000\n", + "Training mini-batch number 11000\n", + "Training mini-batch number 12000\n", + "Training mini-batch number 13000\n", + "Training mini-batch number 14000\n", + "Epoch 2: validation accuracy 95.55%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 95.12%\n", + "Training mini-batch number 15000\n", + "Training mini-batch number 16000\n", + "Training mini-batch number 17000\n", + "Training mini-batch number 18000\n", + "Training mini-batch number 19000\n", + "Epoch 3: validation accuracy 96.20%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 95.79%\n", + "Training mini-batch number 20000\n", + "Training mini-batch number 21000\n", + "Training mini-batch number 22000\n", + "Training mini-batch number 23000\n", + "Training mini-batch number 24000\n", + "Epoch 4: validation accuracy 96.64%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 96.21%\n", + "Training mini-batch number 25000\n", + "Training mini-batch number 26000\n", + "Training mini-batch number 27000\n", + "Training mini-batch number 28000\n", + "Training mini-batch number 29000\n", + "Epoch 5: validation accuracy 96.92%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 96.58%\n", + "Training mini-batch number 30000\n", + "Training mini-batch number 31000\n", + "Training mini-batch number 32000\n", + "Training mini-batch number 33000\n", + "Training mini-batch number 34000\n", + "Epoch 6: validation accuracy 97.11%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 96.84%\n", + "Training mini-batch number 35000\n", + "Training mini-batch number 36000\n", + "Training mini-batch number 37000\n", + "Training mini-batch number 38000\n", + "Training mini-batch number 39000\n", + "Epoch 7: validation accuracy 97.23%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.04%\n", + "Training mini-batch number 40000\n", + "Training mini-batch number 41000\n", + "Training mini-batch number 42000\n", + "Training mini-batch number 43000\n", + "Training mini-batch number 44000\n", + "Epoch 8: validation accuracy 97.30%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.19%\n", + "Training mini-batch number 45000\n", + "Training mini-batch number 46000\n", + "Training mini-batch number 47000\n", + "Training mini-batch number 48000\n", + "Training mini-batch number 49000\n", + "Epoch 9: validation accuracy 97.33%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.24%\n", + "Training mini-batch number 50000\n", + "Training mini-batch number 51000\n", + "Training mini-batch number 52000\n", + "Training mini-batch number 53000\n", + "Training mini-batch number 54000\n", + "Epoch 10: validation accuracy 97.36%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.31%\n", + "Training mini-batch number 55000\n", + "Training mini-batch number 56000\n", + "Training mini-batch number 57000\n", + "Training mini-batch number 58000\n", + "Training mini-batch number 59000\n", + "Epoch 11: validation accuracy 97.44%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.39%\n", + "Training mini-batch number 60000\n", + "Training mini-batch number 61000\n", + "Training mini-batch number 62000\n", + "Training mini-batch number 63000\n", + "Training mini-batch number 64000\n", + "Epoch 12: validation accuracy 97.43%\n", + "Training mini-batch number 65000\n", + "Training mini-batch number 66000\n", + "Training mini-batch number 67000\n", + "Training mini-batch number 68000\n", + "Training mini-batch number 69000\n", + "Epoch 13: validation accuracy 97.48%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.47%\n", + "Training mini-batch number 70000\n", + "Training mini-batch number 71000\n", + "Training mini-batch number 72000\n", + "Training mini-batch number 73000\n", + "Training mini-batch number 74000\n", + "Epoch 14: validation accuracy 97.53%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.53%\n", + "Training mini-batch number 75000\n", + "Training mini-batch number 76000\n", + "Training mini-batch number 77000\n", + "Training mini-batch number 78000\n", + "Training mini-batch number 79000\n", + "Epoch 15: validation accuracy 97.56%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.53%\n", + "Training mini-batch number 80000\n", + "Training mini-batch number 81000\n", + "Training mini-batch number 82000\n", + "Training mini-batch number 83000\n", + "Training mini-batch number 84000\n", + "Epoch 16: validation accuracy 97.59%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.54%\n", + "Training mini-batch number 85000\n", + "Training mini-batch number 86000\n", + "Training mini-batch number 87000\n", + "Training mini-batch number 88000\n", + "Training mini-batch number 89000\n", + "Epoch 17: validation accuracy 97.64%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.57%\n", + "Training mini-batch number 90000\n", + "Training mini-batch number 91000\n", + "Training mini-batch number 92000\n", + "Training mini-batch number 93000\n", + "Training mini-batch number 94000\n", + "Epoch 18: validation accuracy 97.64%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.59%\n", + "Training mini-batch number 95000\n", + "Training mini-batch number 96000\n", + "Training mini-batch number 97000\n", + "Training mini-batch number 98000\n", + "Training mini-batch number 99000\n", + "Epoch 19: validation accuracy 97.67%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.59%\n", + "Training mini-batch number 100000\n", + "Training mini-batch number 101000\n", + "Training mini-batch number 102000\n", + "Training mini-batch number 103000\n", + "Training mini-batch number 104000\n", + "Epoch 20: validation accuracy 97.66%\n", + "Training mini-batch number 105000\n", + "Training mini-batch number 106000\n", + "Training mini-batch number 107000\n", + "Training mini-batch number 108000\n", + "Training mini-batch number 109000\n", + "Epoch 21: validation accuracy 97.63%\n", + "Training mini-batch number 110000\n", + "Training mini-batch number 111000\n", + "Training mini-batch number 112000\n", + "Training mini-batch number 113000\n", + "Training mini-batch number 114000\n", + "Epoch 22: validation accuracy 97.59%\n", + "Training mini-batch number 115000\n", + "Training mini-batch number 116000\n", + "Training mini-batch number 117000\n", + "Training mini-batch number 118000\n", + "Training mini-batch number 119000\n", + "Epoch 23: validation accuracy 97.57%\n", + "Training mini-batch number 120000\n", + "Training mini-batch number 121000\n", + "Training mini-batch number 122000\n", + "Training mini-batch number 123000\n", + "Training mini-batch number 124000\n", + "Epoch 24: validation accuracy 97.57%\n", + "Training mini-batch number 125000\n", + "Training mini-batch number 126000\n", + "Training mini-batch number 127000\n", + "Training mini-batch number 128000\n", + "Training mini-batch number 129000\n", + "Epoch 25: validation accuracy 97.58%\n", + "Training mini-batch number 130000\n", + "Training mini-batch number 131000\n", + "Training mini-batch number 132000\n", + "Training mini-batch number 133000\n", + "Training mini-batch number 134000\n", + "Epoch 26: validation accuracy 97.58%\n", + "Training mini-batch number 135000\n", + "Training mini-batch number 136000\n", + "Training mini-batch number 137000\n", + "Training mini-batch number 138000\n", + "Training mini-batch number 139000\n", + "Epoch 27: validation accuracy 97.55%\n", + "Training mini-batch number 140000\n", + "Training mini-batch number 141000\n", + "Training mini-batch number 142000\n", + "Training mini-batch number 143000\n", + "Training mini-batch number 144000\n", + "Epoch 28: validation accuracy 97.55%\n", + "Training mini-batch number 145000\n", + "Training mini-batch number 146000\n", + "Training mini-batch number 147000\n", + "Training mini-batch number 148000\n", + "Training mini-batch number 149000\n", + "Epoch 29: validation accuracy 97.53%\n", + "Training mini-batch number 150000\n", + "Training mini-batch number 151000\n", + "Training mini-batch number 152000\n", + "Training mini-batch number 153000\n", + "Training mini-batch number 154000\n", + "Epoch 30: validation accuracy 97.52%\n", + "Training mini-batch number 155000\n", + "Training mini-batch number 156000\n", + "Training mini-batch number 157000\n", + "Training mini-batch number 158000\n", + "Training mini-batch number 159000\n", + "Epoch 31: validation accuracy 97.52%\n", + "Training mini-batch number 160000\n", + "Training mini-batch number 161000\n", + "Training mini-batch number 162000\n", + "Training mini-batch number 163000\n", + "Training mini-batch number 164000\n", + "Epoch 32: validation accuracy 97.54%\n", + "Training mini-batch number 165000\n", + "Training mini-batch number 166000\n", + "Training mini-batch number 167000\n", + "Training mini-batch number 168000\n", + "Training mini-batch number 169000\n", + "Epoch 33: validation accuracy 97.54%\n", + "Training mini-batch number 170000\n", + "Training mini-batch number 171000\n", + "Training mini-batch number 172000\n", + "Training mini-batch number 173000\n", + "Training mini-batch number 174000\n", + "Epoch 34: validation accuracy 97.56%\n", + "Training mini-batch number 175000\n", + "Training mini-batch number 176000\n", + "Training mini-batch number 177000\n", + "Training mini-batch number 178000\n", + "Training mini-batch number 179000\n", + "Epoch 35: validation accuracy 97.56%\n", + "Training mini-batch number 180000\n", + "Training mini-batch number 181000\n", + "Training mini-batch number 182000\n", + "Training mini-batch number 183000\n", + "Training mini-batch number 184000\n", + "Epoch 36: validation accuracy 97.57%\n", + "Training mini-batch number 185000\n", + "Training mini-batch number 186000\n", + "Training mini-batch number 187000\n", + "Training mini-batch number 188000\n", + "Training mini-batch number 189000\n", + "Epoch 37: validation accuracy 97.59%\n", + "Training mini-batch number 190000\n", + "Training mini-batch number 191000\n", + "Training mini-batch number 192000\n", + "Training mini-batch number 193000\n", + "Training mini-batch number 194000\n", + "Epoch 38: validation accuracy 97.59%\n", + "Training mini-batch number 195000\n", + "Training mini-batch number 196000\n", + "Training mini-batch number 197000\n", + "Training mini-batch number 198000\n", + "Training mini-batch number 199000\n", + "Epoch 39: validation accuracy 97.61%\n", + "Training mini-batch number 200000\n", + "Training mini-batch number 201000\n", + "Training mini-batch number 202000\n", + "Training mini-batch number 203000\n", + "Training mini-batch number 204000\n", + "Epoch 40: validation accuracy 97.63%\n", + "Training mini-batch number 205000\n", + "Training mini-batch number 206000\n", + "Training mini-batch number 207000\n", + "Training mini-batch number 208000\n", + "Training mini-batch number 209000\n", + "Epoch 41: validation accuracy 97.63%\n", + "Training mini-batch number 210000\n", + "Training mini-batch number 211000\n", + "Training mini-batch number 212000\n", + "Training mini-batch number 213000\n", + "Training mini-batch number 214000\n", + "Epoch 42: validation accuracy 97.63%\n", + "Training mini-batch number 215000\n", + "Training mini-batch number 216000\n", + "Training mini-batch number 217000\n", + "Training mini-batch number 218000\n", + "Training mini-batch number 219000\n", + "Epoch 43: validation accuracy 97.65%\n", + "Training mini-batch number 220000\n", + "Training mini-batch number 221000\n", + "Training mini-batch number 222000\n", + "Training mini-batch number 223000\n", + "Training mini-batch number 224000\n", + "Epoch 44: validation accuracy 97.65%\n", + "Training mini-batch number 225000\n", + "Training mini-batch number 226000\n", + "Training mini-batch number 227000\n", + "Training mini-batch number 228000\n", + "Training mini-batch number 229000\n", + "Epoch 45: validation accuracy 97.66%\n", + "Training mini-batch number 230000\n", + "Training mini-batch number 231000\n", + "Training mini-batch number 232000\n", + "Training mini-batch number 233000\n", + "Training mini-batch number 234000\n", + "Epoch 46: validation accuracy 97.66%\n", + "Training mini-batch number 235000\n", + "Training mini-batch number 236000\n", + "Training mini-batch number 237000\n", + "Training mini-batch number 238000\n", + "Training mini-batch number 239000\n", + "Epoch 47: validation accuracy 97.66%\n", + "Training mini-batch number 240000\n", + "Training mini-batch number 241000\n", + "Training mini-batch number 242000\n", + "Training mini-batch number 243000\n", + "Training mini-batch number 244000\n", + "Epoch 48: validation accuracy 97.66%\n", + "Training mini-batch number 245000\n", + "Training mini-batch number 246000\n", + "Training mini-batch number 247000\n", + "Training mini-batch number 248000\n", + "Training mini-batch number 249000\n", + "Epoch 49: validation accuracy 97.69%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.88%\n", + "Training mini-batch number 250000\n", + "Training mini-batch number 251000\n", + "Training mini-batch number 252000\n", + "Training mini-batch number 253000\n", + "Training mini-batch number 254000\n", + "Epoch 50: validation accuracy 97.68%\n", + "Training mini-batch number 255000\n", + "Training mini-batch number 256000\n", + "Training mini-batch number 257000\n", + "Training mini-batch number 258000\n", + "Training mini-batch number 259000\n", + "Epoch 51: validation accuracy 97.69%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.88%\n", + "Training mini-batch number 260000\n", + "Training mini-batch number 261000\n", + "Training mini-batch number 262000\n", + "Training mini-batch number 263000\n", + "Training mini-batch number 264000\n", + "Epoch 52: validation accuracy 97.68%\n", + "Training mini-batch number 265000\n", + "Training mini-batch number 266000\n", + "Training mini-batch number 267000\n", + "Training mini-batch number 268000\n", + "Training mini-batch number 269000\n", + "Epoch 53: validation accuracy 97.68%\n", + "Training mini-batch number 270000\n", + "Training mini-batch number 271000\n", + "Training mini-batch number 272000\n", + "Training mini-batch number 273000\n", + "Training mini-batch number 274000\n", + "Epoch 54: validation accuracy 97.68%\n", + "Training mini-batch number 275000\n", + "Training mini-batch number 276000\n", + "Training mini-batch number 277000\n", + "Training mini-batch number 278000\n", + "Training mini-batch number 279000\n", + "Epoch 55: validation accuracy 97.68%\n", + "Training mini-batch number 280000\n", + "Training mini-batch number 281000\n", + "Training mini-batch number 282000\n", + "Training mini-batch number 283000\n", + "Training mini-batch number 284000\n", + "Epoch 56: validation accuracy 97.68%\n", + "Training mini-batch number 285000\n", + "Training mini-batch number 286000\n", + "Training mini-batch number 287000\n", + "Training mini-batch number 288000\n", + "Training mini-batch number 289000\n", + "Epoch 57: validation accuracy 97.68%\n", + "Training mini-batch number 290000\n", + "Training mini-batch number 291000\n", + "Training mini-batch number 292000\n", + "Training mini-batch number 293000\n", + "Training mini-batch number 294000\n", + "Epoch 58: validation accuracy 97.68%\n", + "Training mini-batch number 295000\n", + "Training mini-batch number 296000\n", + "Training mini-batch number 297000\n", + "Training mini-batch number 298000\n", + "Training mini-batch number 299000\n", + "Epoch 59: validation accuracy 97.68%\n", + "Finished training network.\n", + "Best validation accuracy of 97.69% obtained at iteration 259999\n", + "Corresponding test accuracy of 97.88%\n" + ] + } + ], + "source": [ + "from elements import network3\n", + "from elements.network3 import Network\n", + "from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer\n", + "\n", + "# read data:\n", + "training_data, validation_data, test_data = network3.load_data_shared()\n", + "\n", + "# mini-batch size:\n", + "mini_batch_size = 10\n", + "\n", + "net = Network([\n", + " FullyConnectedLayer(n_in=784, n_out=100),\n", + " SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n", + "\n", + "net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QdhCMM6TABTN" + }, + "source": [ + "### chapter 6 - 5x5 local receptive fields, 20 feature maps, max-pooling layer 2x2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "L1vNy4PcAE9G", + "outputId": "cf3c2cb2-def9-41c1-fcb7-3fcf3f734788" + }, + "outputs": [ + { + "metadata": { + "tags": null + }, + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.8/dist-packages/theano/tensor/nnet/conv.py:98: UserWarning: theano.tensor.nnet.conv.conv2d is deprecated. Use theano.tensor.nnet.conv2d instead.\n", + " warnings.warn(\"theano.tensor.nnet.conv.conv2d is deprecated.\"\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Training mini-batch number 0\n", + "Training mini-batch number 1000\n", + "Training mini-batch number 2000\n", + "Training mini-batch number 3000\n", + "Training mini-batch number 4000\n", + "Epoch 0: validation accuracy 93.40%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 93.01%\n", + "Training mini-batch number 5000\n", + "Training mini-batch number 6000\n", + "Training mini-batch number 7000\n", + "Training mini-batch number 8000\n", + "Training mini-batch number 9000\n", + "Epoch 1: validation accuracy 95.84%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 95.64%\n", + "Training mini-batch number 10000\n", + "Training mini-batch number 11000\n", + "Training mini-batch number 12000\n", + "Training mini-batch number 13000\n", + "Training mini-batch number 14000\n", + "Epoch 2: validation accuracy 97.14%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 96.97%\n", + "Training mini-batch number 15000\n", + "Training mini-batch number 16000\n", + "Training mini-batch number 17000\n", + "Training mini-batch number 18000\n", + "Training mini-batch number 19000\n", + "Epoch 3: validation accuracy 97.75%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.65%\n", + "Training mini-batch number 20000\n", + "Training mini-batch number 21000\n", + "Training mini-batch number 22000\n", + "Training mini-batch number 23000\n", + "Training mini-batch number 24000\n", + "Epoch 4: validation accuracy 97.94%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 97.85%\n", + "Training mini-batch number 25000\n", + "Training mini-batch number 26000\n", + "Training mini-batch number 27000\n", + "Training mini-batch number 28000\n", + "Training mini-batch number 29000\n", + "Epoch 5: validation accuracy 98.08%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.04%\n", + "Training mini-batch number 30000\n", + "Training mini-batch number 31000\n", + "Training mini-batch number 32000\n", + "Training mini-batch number 33000\n", + "Training mini-batch number 34000\n", + "Epoch 6: validation accuracy 98.19%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.24%\n", + "Training mini-batch number 35000\n", + "Training mini-batch number 36000\n", + "Training mini-batch number 37000\n", + "Training mini-batch number 38000\n", + "Training mini-batch number 39000\n", + "Epoch 7: validation accuracy 98.22%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.32%\n", + "Training mini-batch number 40000\n", + "Training mini-batch number 41000\n", + "Training mini-batch number 42000\n", + "Training mini-batch number 43000\n", + "Training mini-batch number 44000\n", + "Epoch 8: validation accuracy 98.32%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.41%\n", + "Training mini-batch number 45000\n", + "Training mini-batch number 46000\n", + "Training mini-batch number 47000\n", + "Training mini-batch number 48000\n", + "Training mini-batch number 49000\n", + "Epoch 9: validation accuracy 98.36%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.44%\n", + "Training mini-batch number 50000\n", + "Training mini-batch number 51000\n", + "Training mini-batch number 52000\n", + "Training mini-batch number 53000\n", + "Training mini-batch number 54000\n", + "Epoch 10: validation accuracy 98.35%\n", + "Training mini-batch number 55000\n", + "Training mini-batch number 56000\n", + "Training mini-batch number 57000\n", + "Training mini-batch number 58000\n", + "Training mini-batch number 59000\n", + "Epoch 11: validation accuracy 98.41%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.45%\n", + "Training mini-batch number 60000\n", + "Training mini-batch number 61000\n", + "Training mini-batch number 62000\n", + "Training mini-batch number 63000\n", + "Training mini-batch number 64000\n", + "Epoch 12: validation accuracy 98.49%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.48%\n", + "Training mini-batch number 65000\n", + "Training mini-batch number 66000\n", + "Training mini-batch number 67000\n", + "Training mini-batch number 68000\n", + "Training mini-batch number 69000\n", + "Epoch 13: validation accuracy 98.52%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.49%\n", + "Training mini-batch number 70000\n", + "Training mini-batch number 71000\n", + "Training mini-batch number 72000\n", + "Training mini-batch number 73000\n", + "Training mini-batch number 74000\n", + "Epoch 14: validation accuracy 98.54%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.53%\n", + "Training mini-batch number 75000\n", + "Training mini-batch number 76000\n", + "Training mini-batch number 77000\n", + "Training mini-batch number 78000\n", + "Training mini-batch number 79000\n", + "Epoch 15: validation accuracy 98.56%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.53%\n", + "Training mini-batch number 80000\n", + "Training mini-batch number 81000\n", + "Training mini-batch number 82000\n", + "Training mini-batch number 83000\n", + "Training mini-batch number 84000\n", + "Epoch 16: validation accuracy 98.59%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.57%\n", + "Training mini-batch number 85000\n", + "Training mini-batch number 86000\n", + "Training mini-batch number 87000\n", + "Training mini-batch number 88000\n", + "Training mini-batch number 89000\n", + "Epoch 17: validation accuracy 98.60%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.56%\n", + "Training mini-batch number 90000\n", + "Training mini-batch number 91000\n", + "Training mini-batch number 92000\n", + "Training mini-batch number 93000\n", + "Training mini-batch number 94000\n", + "Epoch 18: validation accuracy 98.61%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.57%\n", + "Training mini-batch number 95000\n", + "Training mini-batch number 96000\n", + "Training mini-batch number 97000\n", + "Training mini-batch number 98000\n", + "Training mini-batch number 99000\n", + "Epoch 19: validation accuracy 98.60%\n", + "Training mini-batch number 100000\n", + "Training mini-batch number 101000\n", + "Training mini-batch number 102000\n", + "Training mini-batch number 103000\n", + "Training mini-batch number 104000\n", + "Epoch 20: validation accuracy 98.60%\n", + "Training mini-batch number 105000\n", + "Training mini-batch number 106000\n", + "Training mini-batch number 107000\n", + "Training mini-batch number 108000\n", + "Training mini-batch number 109000\n", + "Epoch 21: validation accuracy 98.62%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.67%\n", + "Training mini-batch number 110000\n", + "Training mini-batch number 111000\n", + "Training mini-batch number 112000\n", + "Training mini-batch number 113000\n", + "Training mini-batch number 114000\n", + "Epoch 22: validation accuracy 98.65%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.72%\n", + "Training mini-batch number 115000\n", + "Training mini-batch number 116000\n", + "Training mini-batch number 117000\n", + "Training mini-batch number 118000\n", + "Training mini-batch number 119000\n", + "Epoch 23: validation accuracy 98.66%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.73%\n", + "Training mini-batch number 120000\n", + "Training mini-batch number 121000\n", + "Training mini-batch number 122000\n", + "Training mini-batch number 123000\n", + "Training mini-batch number 124000\n", + "Epoch 24: validation accuracy 98.67%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.72%\n", + "Training mini-batch number 125000\n", + "Training mini-batch number 126000\n", + "Training mini-batch number 127000\n", + "Training mini-batch number 128000\n", + "Training mini-batch number 129000\n", + "Epoch 25: validation accuracy 98.69%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.72%\n", + "Training mini-batch number 130000\n", + "Training mini-batch number 131000\n", + "Training mini-batch number 132000\n", + "Training mini-batch number 133000\n", + "Training mini-batch number 134000\n", + "Epoch 26: validation accuracy 98.68%\n", + "Training mini-batch number 135000\n", + "Training mini-batch number 136000\n", + "Training mini-batch number 137000\n", + "Training mini-batch number 138000\n", + "Training mini-batch number 139000\n", + "Epoch 27: validation accuracy 98.67%\n", + "Training mini-batch number 140000\n", + "Training mini-batch number 141000\n", + "Training mini-batch number 142000\n", + "Training mini-batch number 143000\n", + "Training mini-batch number 144000\n", + "Epoch 28: validation accuracy 98.69%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.76%\n", + "Training mini-batch number 145000\n", + "Training mini-batch number 146000\n", + "Training mini-batch number 147000\n", + "Training mini-batch number 148000\n", + "Training mini-batch number 149000\n", + "Epoch 29: validation accuracy 98.70%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.76%\n", + "Training mini-batch number 150000\n", + "Training mini-batch number 151000\n", + "Training mini-batch number 152000\n", + "Training mini-batch number 153000\n", + "Training mini-batch number 154000\n", + "Epoch 30: validation accuracy 98.71%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.74%\n", + "Training mini-batch number 155000\n", + "Training mini-batch number 156000\n", + "Training mini-batch number 157000\n", + "Training mini-batch number 158000\n", + "Training mini-batch number 159000\n", + "Epoch 31: validation accuracy 98.71%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.76%\n", + "Training mini-batch number 160000\n", + "Training mini-batch number 161000\n", + "Training mini-batch number 162000\n", + "Training mini-batch number 163000\n", + "Training mini-batch number 164000\n", + "Epoch 32: validation accuracy 98.72%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.76%\n", + "Training mini-batch number 165000\n", + "Training mini-batch number 166000\n", + "Training mini-batch number 167000\n", + "Training mini-batch number 168000\n", + "Training mini-batch number 169000\n", + "Epoch 33: validation accuracy 98.71%\n", + "Training mini-batch number 170000\n", + "Training mini-batch number 171000\n", + "Training mini-batch number 172000\n", + "Training mini-batch number 173000\n", + "Training mini-batch number 174000\n", + "Epoch 34: validation accuracy 98.71%\n", + "Training mini-batch number 175000\n", + "Training mini-batch number 176000\n", + "Training mini-batch number 177000\n", + "Training mini-batch number 178000\n", + "Training mini-batch number 179000\n", + "Epoch 35: validation accuracy 98.71%\n", + "Training mini-batch number 180000\n", + "Training mini-batch number 181000\n", + "Training mini-batch number 182000\n", + "Training mini-batch number 183000\n", + "Training mini-batch number 184000\n", + "Epoch 36: validation accuracy 98.70%\n", + "Training mini-batch number 185000\n", + "Training mini-batch number 186000\n", + "Training mini-batch number 187000\n", + "Training mini-batch number 188000\n", + "Training mini-batch number 189000\n", + "Epoch 37: validation accuracy 98.70%\n", + "Training mini-batch number 190000\n", + "Training mini-batch number 191000\n", + "Training mini-batch number 192000\n", + "Training mini-batch number 193000\n", + "Training mini-batch number 194000\n", + "Epoch 38: validation accuracy 98.69%\n", + "Training mini-batch number 195000\n", + "Training mini-batch number 196000\n", + "Training mini-batch number 197000\n", + "Training mini-batch number 198000\n", + "Training mini-batch number 199000\n", + "Epoch 39: validation accuracy 98.71%\n", + "Training mini-batch number 200000\n", + "Training mini-batch number 201000\n", + "Training mini-batch number 202000\n", + "Training mini-batch number 203000\n", + "Training mini-batch number 204000\n", + "Epoch 40: validation accuracy 98.70%\n", + "Training mini-batch number 205000\n", + "Training mini-batch number 206000\n", + "Training mini-batch number 207000\n", + "Training mini-batch number 208000\n", + "Training mini-batch number 209000\n", + "Epoch 41: validation accuracy 98.70%\n", + "Training mini-batch number 210000\n", + "Training mini-batch number 211000\n", + "Training mini-batch number 212000\n", + "Training mini-batch number 213000\n", + "Training mini-batch number 214000\n", + "Epoch 42: validation accuracy 98.70%\n", + "Training mini-batch number 215000\n", + "Training mini-batch number 216000\n", + "Training mini-batch number 217000\n", + "Training mini-batch number 218000\n", + "Training mini-batch number 219000\n", + "Epoch 43: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.81%\n", + "Training mini-batch number 220000\n", + "Training mini-batch number 221000\n", + "Training mini-batch number 222000\n", + "Training mini-batch number 223000\n", + "Training mini-batch number 224000\n", + "Epoch 44: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.80%\n", + "Training mini-batch number 225000\n", + "Training mini-batch number 226000\n", + "Training mini-batch number 227000\n", + "Training mini-batch number 228000\n", + "Training mini-batch number 229000\n", + "Epoch 45: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.80%\n", + "Training mini-batch number 230000\n", + "Training mini-batch number 231000\n", + "Training mini-batch number 232000\n", + "Training mini-batch number 233000\n", + "Training mini-batch number 234000\n", + "Epoch 46: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.80%\n", + "Training mini-batch number 235000\n", + "Training mini-batch number 236000\n", + "Training mini-batch number 237000\n", + "Training mini-batch number 238000\n", + "Training mini-batch number 239000\n", + "Epoch 47: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.80%\n", + "Training mini-batch number 240000\n", + "Training mini-batch number 241000\n", + "Training mini-batch number 242000\n", + "Training mini-batch number 243000\n", + "Training mini-batch number 244000\n", + "Epoch 48: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.80%\n", + "Training mini-batch number 245000\n", + "Training mini-batch number 246000\n", + "Training mini-batch number 247000\n", + "Training mini-batch number 248000\n", + "Training mini-batch number 249000\n", + "Epoch 49: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.81%\n", + "Training mini-batch number 250000\n", + "Training mini-batch number 251000\n", + "Training mini-batch number 252000\n", + "Training mini-batch number 253000\n", + "Training mini-batch number 254000\n", + "Epoch 50: validation accuracy 98.72%\n", + "Training mini-batch number 255000\n", + "Training mini-batch number 256000\n", + "Training mini-batch number 257000\n", + "Training mini-batch number 258000\n", + "Training mini-batch number 259000\n", + "Epoch 51: validation accuracy 98.72%\n", + "Training mini-batch number 260000\n", + "Training mini-batch number 261000\n", + "Training mini-batch number 262000\n", + "Training mini-batch number 263000\n", + "Training mini-batch number 264000\n", + "Epoch 52: validation accuracy 98.72%\n", + "Training mini-batch number 265000\n", + "Training mini-batch number 266000\n", + "Training mini-batch number 267000\n", + "Training mini-batch number 268000\n", + "Training mini-batch number 269000\n", + "Epoch 53: validation accuracy 98.73%\n", + "This is the best validation accuracy to date.\n", + "The corresponding test accuracy is 98.83%\n", + "Training mini-batch number 270000\n", + "Training mini-batch number 271000\n", + "Training mini-batch number 272000\n", + "Training mini-batch number 273000\n", + "Training mini-batch number 274000\n", + "Epoch 54: validation accuracy 98.72%\n", + "Training mini-batch number 275000\n", + "Training mini-batch number 276000\n", + "Training mini-batch number 277000\n", + "Training mini-batch number 278000\n", + "Training mini-batch number 279000\n", + "Epoch 55: validation accuracy 98.72%\n", + "Training mini-batch number 280000\n", + "Training mini-batch number 281000\n", + "Training mini-batch number 282000\n", + "Training mini-batch number 283000\n", + "Training mini-batch number 284000\n", + "Epoch 56: validation accuracy 98.72%\n", + "Training mini-batch number 285000\n", + "Training mini-batch number 286000\n", + "Training mini-batch number 287000\n", + "Training mini-batch number 288000\n", + "Training mini-batch number 289000\n", + "Epoch 57: validation accuracy 98.71%\n", + "Training mini-batch number 290000\n", + "Training mini-batch number 291000\n", + "Training mini-batch number 292000\n", + "Training mini-batch number 293000\n", + "Training mini-batch number 294000\n", + "Epoch 58: validation accuracy 98.71%\n", + "Training mini-batch number 295000\n", + "Training mini-batch number 296000\n", + "Training mini-batch number 297000\n", + "Training mini-batch number 298000\n", + "Training mini-batch number 299000\n", + "Epoch 59: validation accuracy 98.71%\n", + "Finished training network.\n", + "Best validation accuracy of 98.73% obtained at iteration 269999\n", + "Corresponding test accuracy of 98.83%\n" + ] + } + ], + "source": [ + "from elements import network3\n", + "from elements.network3 import Network\n", + "from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer\n", + "\n", + "# read data:\n", + "training_data, validation_data, test_data = network3.load_data_shared()\n", + "\n", + "# mini-batch size:\n", + "mini_batch_size = 10\n", + "\n", + "net = Network([\n", + " ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),\n", + " filter_shape=(20, 1, 5, 5),\n", + " poolsize=(2, 2)),\n", + " FullyConnectedLayer(n_in=20*12*12, n_out=100),\n", + " SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n", + "\n", + "net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rrtNnqAeAOA8" + }, + "source": [ + "### chapter 6 - inserting a second convolutional-pooling layer to the previous example => better accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "s9S8-DZoAQQk" + }, + "outputs": [], + "source": [ + "from elements import network3\n", + "from elements.network3 import Network\n", + "from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer\n", + "\n", + "# read data:\n", + "training_data, validation_data, test_data = network3.load_data_shared()\n", + "\n", + "# mini-batch size:\n", + "mini_batch_size = 10\n", + "\n", + "net = Network([\n", + " ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),\n", + " filter_shape=(20, 1, 5, 5),\n", + " poolsize=(2, 2)),\n", + " ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),\n", + " filter_shape=(40, 20, 5, 5),\n", + " poolsize=(2, 2)),\n", + " FullyConnectedLayer(n_in=40*4*4, n_out=100),\n", + " SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n", + "\n", + "net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ry674HoFAWoo" + }, + "source": [ + "### chapter 6 - rectified linear units and some l2 regularization (lmbda=0.1) => even better accuracy\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yf_pLQRU_Yt2" + }, + "outputs": [], + "source": [ + "from elements import network3\n", + "from elements.network3 import Network\n", + "from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer, ReLU\n", + "\n", + "# read data:\n", + "training_data, validation_data, test_data = network3.load_data_shared()\n", + "\n", + "# mini-batch size:\n", + "mini_batch_size = 10\n", + "\n", + "net = Network([\n", + " ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),\n", + " filter_shape=(20, 1, 5, 5),\n", + " poolsize=(2, 2),\n", + " activation_fn=ReLU),\n", + " ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),\n", + " filter_shape=(40, 20, 5, 5),\n", + " poolsize=(2, 2),\n", + " activation_fn=ReLU),\n", + " FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU),\n", + " SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)\n", + "\n", + "net.SGD(training_data, 60, mini_batch_size, 0.03, validation_data, test_data, lmbda=0.1)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "gpuClass": "standard", + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/README.md b/README.md index aa618b0..fc05f8e 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,14 @@ These scrips are updated ones from the **neuralnetworksanddeeplearning.com** git The testing file (**test.py**) contains all three networks (network.py, network2.py, network3.py) from the book and it is the starting point to run (i.e. *train and evaluate*) them. -## Just type at shell: **python3.5 test.py** +## Google Colab +The best way to run the code is to use the colab notebook provided in bellow link. -In test.py there are examples of networks configurations with proper comments. I did that to relate with particular chapters from the book. +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1OAzGo1E5uKuKw1R2Cbce2KKxpvHPaLyY?usp=sharing) +## Scripts +To run the scripts, go to the chapter_codes folder, open the terminal, and type: +```bash +python chapter1-data.py +``` diff --git a/chapter_codes/chapter1-data.py b/chapter_codes/chapter1-data.py new file mode 100644 index 0000000..c60882a --- /dev/null +++ b/chapter_codes/chapter1-data.py @@ -0,0 +1,17 @@ +import sys +sys.path.append('..') + +from elements import mnist_loader +import numpy as np +import matplotlib.pyplot as plt + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz') +training_data = list(training_data) +test_data = list(test_data) + +plt.figure(figsize=(15,5)) +for i in range(1,17): + plt.subplot(2,8,i) + plt.imshow(training_data[i][0].reshape(28,28), cmap='Greys') + plt.xlabel(f'label: {np.argmax(training_data[i][1])}') +plt.show() \ No newline at end of file diff --git a/chapter_codes/chapter1-network.py b/chapter_codes/chapter1-network.py new file mode 100644 index 0000000..c2b461f --- /dev/null +++ b/chapter_codes/chapter1-network.py @@ -0,0 +1,12 @@ +import sys +sys.path.append('..') + +from elements import network +from elements import mnist_loader + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz') +training_data = list(training_data) +test_data = list(test_data) + +net = network.Network([784, 30, 10]) +net.SGD(training_data, 30, 10, 3.0, test_data=test_data) \ No newline at end of file diff --git a/chapter_codes/chapter2-network2.py b/chapter_codes/chapter2-network2.py new file mode 100644 index 0000000..218d9aa --- /dev/null +++ b/chapter_codes/chapter2-network2.py @@ -0,0 +1,14 @@ +import sys +sys.path.append('..') + +from elements import network2 +from elements import mnist_loader + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz') +training_data = list(training_data) +test_data = list(test_data) + +net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) +#net.large_weight_initializer() +evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \ + net.SGD(training_data, 30, 10, 0.1, lmbda = 5.0, evaluation_data=validation_data, monitor_evaluation_accuracy=True) \ No newline at end of file diff --git a/chapter_codes/chapter3-earlystopping.py b/chapter_codes/chapter3-earlystopping.py new file mode 100644 index 0000000..80887fb --- /dev/null +++ b/chapter_codes/chapter3-earlystopping.py @@ -0,0 +1,26 @@ +import sys +sys.path.append('..') + +from elements import network2 +from elements import mnist_loader + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz') +training_data = list(training_data) +test_data = list(test_data) + +net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) +evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \ + net.SGD(training_data[:1000], 30, 10, 0.5, + lmbda=5.0, + evaluation_data=validation_data, + monitor_evaluation_accuracy=True, + monitor_training_cost=True, + early_stopping_n=10) + + +plt.figure(figsize=(6,5)) +plt.plot(np.array(evaluation_accuracy)/10000) +plt.grid() +plt.xlabel('Epoch') +plt.legend(['Accuracy on the test data']) +plt.show() \ No newline at end of file diff --git a/chapter_codes/chapter3-overfitting.py b/chapter_codes/chapter3-overfitting.py new file mode 100644 index 0000000..0920724 --- /dev/null +++ b/chapter_codes/chapter3-overfitting.py @@ -0,0 +1,56 @@ +import sys +sys.path.append('..') + +from elements import network2 +from elements import mnist_loader +import matplotlib.pyplot as plt + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz') +training_data = list(training_data) +test_data = list(test_data) + +net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) +net.large_weight_initializer() +evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \ + net.SGD(training_data[:1000], 400, 10, 0.5, evaluation_data=test_data, + monitor_evaluation_cost=True, + monitor_evaluation_accuracy=True, + monitor_training_cost=True, + monitor_training_accuracy=True) + + +plt.figure(figsize=(6,5)) +plt.plot(training_cost[200:]) +plt.grid() +plt.title('Cost on the training data') +plt.xlabel('Epoch') +plt.show() + +plt.figure(figsize=(6,5)) +plt.plot(evaluation_accuracy[200:]) +plt.grid() +plt.title('Accuracy (%) on the test data') +plt.xlabel('Epoch') +plt.show() + +plt.figure(figsize=(6,5)) +plt.plot(evaluation_cost) +plt.grid() +plt.title('Cost on the test data') +plt.xlabel('Epoch') +plt.show() + +plt.figure(figsize=(6,5)) +plt.plot(training_accuracy) +plt.grid() +plt.title('Accuracy (%) on the training data') +plt.xlabel('Epoch') +plt.show() + +plt.figure(figsize=(6,5)) +plt.plot(np.array(training_accuracy[:30])/1000) +plt.plot(np.array(evaluation_accuracy[:30])/10000) +plt.grid() +plt.xlabel('Epoch') +plt.legend(['Accuracy on the training data','Accuracy on the test data']) +plt.show() diff --git a/chapter_codes/chapter3-regularization.py b/chapter_codes/chapter3-regularization.py new file mode 100644 index 0000000..f9b4bb2 --- /dev/null +++ b/chapter_codes/chapter3-regularization.py @@ -0,0 +1,44 @@ +import sys +sys.path.append('..') + +from elements import network2 +from elements import mnist_loader +import matplotlib.pyplot as plt + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz') +training_data = list(training_data) +test_data = list(test_data) + +net = network2.Network([784, 30, 10], cost=network2.CrossEntropyCost) +net.large_weight_initializer() +evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \ + net.SGD(training_data[:1000], 400, 10, 0.5, + evaluation_data=test_data, + lmbda = 0.1, # this is a regularization parameter + monitor_evaluation_cost=True, + monitor_evaluation_accuracy=True, + monitor_training_cost=True, + monitor_training_accuracy=True) + + +plt.figure(figsize=(6,5)) +plt.plot(np.array(training_cost[200:])/1000) +plt.grid() +plt.title('Cost on the training data') +plt.xlabel('Epoch') +plt.show() + +plt.figure(figsize=(6,5)) +plt.plot(np.array(evaluation_accuracy[200:])/1000) +plt.grid() +plt.title('Accuracy (%) on the test data') +plt.xlabel('Epoch') +plt.show() + +plt.figure(figsize=(6,5)) +plt.plot(np.array(training_accuracy[:30])/1000) +plt.plot(np.array(evaluation_accuracy[:30])/10000) +plt.grid() +plt.xlabel('Epoch') +plt.legend(['Accuracy on the training data','Accuracy on the test data']) +plt.show() \ No newline at end of file diff --git a/chapter_codes/chapter5-vanishing.py b/chapter_codes/chapter5-vanishing.py new file mode 100644 index 0000000..c5f7017 --- /dev/null +++ b/chapter_codes/chapter5-vanishing.py @@ -0,0 +1,16 @@ +import sys +sys.path.append('..') + +from elements import network2 +from elements import mnist_loader + +training_data, validation_data, test_data = mnist_loader.load_data_wrapper('../mnist.pkl.gz') +training_data = list(training_data) +test_data = list(test_data) + +net = network2.Network([784, 30, 30, 30, 30, 10], cost=network2.CrossEntropyCost) +evaluation_cost, evaluation_accuracy, training_cost, training_accuracy = \ + net.SGD(training_data, 30, 10, 0.1, + lmbda=5.0, + evaluation_data=validation_data, + monitor_evaluation_accuracy=True) \ No newline at end of file diff --git a/chapter_codes/chapter6-1conv.py b/chapter_codes/chapter6-1conv.py new file mode 100644 index 0000000..c86e568 --- /dev/null +++ b/chapter_codes/chapter6-1conv.py @@ -0,0 +1,21 @@ +import sys +sys.path.append('..') + +from elements import network3 +from elements.network3 import Network +from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer + +# read data: +training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz') + +# mini-batch size: +mini_batch_size = 10 + +net = Network([ + ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), + filter_shape=(20, 1, 5, 5), + poolsize=(2, 2)), + FullyConnectedLayer(n_in=20*12*12, n_out=100), + SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) + +net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) \ No newline at end of file diff --git a/chapter_codes/chapter6-2conv.py b/chapter_codes/chapter6-2conv.py new file mode 100644 index 0000000..5dd0db4 --- /dev/null +++ b/chapter_codes/chapter6-2conv.py @@ -0,0 +1,27 @@ +import sys +sys.path.append('..') + +import warnings +warnings.filterwarnings('ignore') + +from elements import network3 +from elements.network3 import Network +from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer + +# read data: +training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz') + +# mini-batch size: +mini_batch_size = 10 + +net = Network([ + ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), + filter_shape=(20, 1, 5, 5), + poolsize=(2, 2)), + ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12), + filter_shape=(40, 20, 5, 5), + poolsize=(2, 2)), + FullyConnectedLayer(n_in=40*4*4, n_out=100), + SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) + +net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) \ No newline at end of file diff --git a/chapter_codes/chapter6-relu.py b/chapter_codes/chapter6-relu.py new file mode 100644 index 0000000..e749e15 --- /dev/null +++ b/chapter_codes/chapter6-relu.py @@ -0,0 +1,29 @@ +import sys +sys.path.append('..') + +import warnings +warnings.filterwarnings('ignore') + +from elements import network3 +from elements.network3 import Network +from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer, ReLU + +# read data: +training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz') + +# mini-batch size: +mini_batch_size = 10 + +net = Network([ + ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28), + filter_shape=(20, 1, 5, 5), + poolsize=(2, 2), + activation_fn=ReLU), + ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12), + filter_shape=(40, 20, 5, 5), + poolsize=(2, 2), + activation_fn=ReLU), + FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU), + SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) + +net.SGD(training_data, 60, mini_batch_size, 0.03, validation_data, test_data, lmbda=0.1) \ No newline at end of file diff --git a/chapter_codes/chapter6-shallow.py b/chapter_codes/chapter6-shallow.py new file mode 100644 index 0000000..2d1aac4 --- /dev/null +++ b/chapter_codes/chapter6-shallow.py @@ -0,0 +1,18 @@ +import sys +sys.path.append('..') + +from elements import network3 +from elements.network3 import Network +from elements.network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer + +# read data: +training_data, validation_data, test_data = network3.load_data_shared('../mnist.pkl.gz') + +# mini-batch size: +mini_batch_size = 10 + +net = Network([ + FullyConnectedLayer(n_in=784, n_out=100), + SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) + +net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data) \ No newline at end of file diff --git a/expand_mnist.py b/elements/expand_mnist.py similarity index 100% rename from expand_mnist.py rename to elements/expand_mnist.py diff --git a/mnist_average_darkness.py b/elements/mnist_average_darkness.py similarity index 97% rename from mnist_average_darkness.py rename to elements/mnist_average_darkness.py index 2f5be7c..815f812 100644 --- a/mnist_average_darkness.py +++ b/elements/mnist_average_darkness.py @@ -1,64 +1,64 @@ -""" -mnist_average_darkness -~~~~~~~~~~~~~~~~~~~~~~ - -A naive classifier for recognizing handwritten digits from the MNIST -data set. The program classifies digits based on how dark they are ---- the idea is that digits like "1" tend to be less dark than digits -like "8", simply because the latter has a more complex shape. When -shown an image the classifier returns whichever digit in the training -data had the closest average darkness. - -The program works in two steps: first it trains the classifier, and -then it applies the classifier to the MNIST test data to see how many -digits are correctly classified. - -Needless to say, this isn't a very good way of recognizing handwritten -digits! Still, it's useful to show what sort of performance we get -from naive ideas.""" - -#### Libraries -# Standard library -from collections import defaultdict - -# My libraries -import mnist_loader - -def main(): - training_data, validation_data, test_data = mnist_loader.load_data() - # training phase: compute the average darknesses for each digit, - # based on the training data - avgs = avg_darknesses(training_data) - # testing phase: see how many of the test images are classified - # correctly - num_correct = sum(int(guess_digit(image, avgs) == digit) - for image, digit in zip(test_data[0], test_data[1])) - print("Baseline classifier using average darkness of image.") - print("{0} of {1} values correct.".format(num_correct, len(test_data[1]))) - -def avg_darknesses(training_data): - """ Return a defaultdict whose keys are the digits 0 through 9. - For each digit we compute a value which is the average darkness of - training images containing that digit. The darkness for any - particular image is just the sum of the darknesses for each pixel.""" - digit_counts = defaultdict(int) - darknesses = defaultdict(float) - for image, digit in zip(training_data[0], training_data[1]): - digit_counts[digit] += 1 - darknesses[digit] += sum(image) - avgs = defaultdict(float) - for digit, n in digit_counts.items(): - avgs[digit] = darknesses[digit] / n - return avgs - -def guess_digit(image, avgs): - """Return the digit whose average darkness in the training data is - closest to the darkness of ``image``. Note that ``avgs`` is - assumed to be a defaultdict whose keys are 0...9, and whose values - are the corresponding average darknesses across the training data.""" - darkness = sum(image) - distances = {k: abs(v-darkness) for k, v in avgs.items()} - return min(distances, key=distances.get) - -if __name__ == "__main__": - main() +""" +mnist_average_darkness +~~~~~~~~~~~~~~~~~~~~~~ + +A naive classifier for recognizing handwritten digits from the MNIST +data set. The program classifies digits based on how dark they are +--- the idea is that digits like "1" tend to be less dark than digits +like "8", simply because the latter has a more complex shape. When +shown an image the classifier returns whichever digit in the training +data had the closest average darkness. + +The program works in two steps: first it trains the classifier, and +then it applies the classifier to the MNIST test data to see how many +digits are correctly classified. + +Needless to say, this isn't a very good way of recognizing handwritten +digits! Still, it's useful to show what sort of performance we get +from naive ideas.""" + +#### Libraries +# Standard library +from collections import defaultdict + +# My libraries +import mnist_loader + +def main(): + training_data, validation_data, test_data = mnist_loader.load_data() + # training phase: compute the average darknesses for each digit, + # based on the training data + avgs = avg_darknesses(training_data) + # testing phase: see how many of the test images are classified + # correctly + num_correct = sum(int(guess_digit(image, avgs) == digit) + for image, digit in zip(test_data[0], test_data[1])) + print("Baseline classifier using average darkness of image.") + print("{0} of {1} values correct.".format(num_correct, len(test_data[1]))) + +def avg_darknesses(training_data): + """ Return a defaultdict whose keys are the digits 0 through 9. + For each digit we compute a value which is the average darkness of + training images containing that digit. The darkness for any + particular image is just the sum of the darknesses for each pixel.""" + digit_counts = defaultdict(int) + darknesses = defaultdict(float) + for image, digit in zip(training_data[0], training_data[1]): + digit_counts[digit] += 1 + darknesses[digit] += sum(image) + avgs = defaultdict(float) + for digit, n in digit_counts.items(): + avgs[digit] = darknesses[digit] / n + return avgs + +def guess_digit(image, avgs): + """Return the digit whose average darkness in the training data is + closest to the darkness of ``image``. Note that ``avgs`` is + assumed to be a defaultdict whose keys are 0...9, and whose values + are the corresponding average darknesses across the training data.""" + darkness = sum(image) + distances = {k: abs(v-darkness) for k, v in avgs.items()} + return min(distances, key=distances.get) + +if __name__ == "__main__": + main() diff --git a/mnist_loader.py b/elements/mnist_loader.py similarity index 96% rename from mnist_loader.py rename to elements/mnist_loader.py index 320c112..12e68fd 100644 --- a/mnist_loader.py +++ b/elements/mnist_loader.py @@ -16,7 +16,7 @@ # Third-party libraries import numpy as np -def load_data(): +def load_data(path): """Return the MNIST data as a tuple containing the training data, the validation data, and the test data. The ``training_data`` is returned as a tuple with two entries. @@ -35,12 +35,12 @@ def load_data(): That's done in the wrapper function ``load_data_wrapper()``, see below. """ - f = gzip.open('mnist.pkl.gz', 'rb') + f = gzip.open(path, 'rb') training_data, validation_data, test_data = pickle.load(f, encoding="latin1") f.close() return (training_data, validation_data, test_data) -def load_data_wrapper(): +def load_data_wrapper(path = 'mnist.pkl.gz'): """Return a tuple containing ``(training_data, validation_data, test_data)``. Based on ``load_data``, but the format is more convenient for use in our implementation of neural networks. @@ -58,7 +58,7 @@ def load_data_wrapper(): the training data and the validation / test data. These formats turn out to be the most convenient for use in our neural network code.""" - tr_d, va_d, te_d = load_data() + tr_d, va_d, te_d = load_data(path) training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]] training_results = [vectorized_result(y) for y in tr_d[1]] training_data = zip(training_inputs, training_results) diff --git a/mnist_svm.py b/elements/mnist_svm.py similarity index 96% rename from mnist_svm.py rename to elements/mnist_svm.py index c2650e9..67b6a15 100644 --- a/mnist_svm.py +++ b/elements/mnist_svm.py @@ -1,28 +1,28 @@ -""" -mnist_svm -~~~~~~~~~ - -A classifier program for recognizing handwritten digits from the MNIST -data set, using an SVM classifier.""" - -#### Libraries -# My libraries -import mnist_loader - -# Third-party libraries -from sklearn import svm - -def svm_baseline(): - training_data, validation_data, test_data = mnist_loader.load_data() - # train - clf = svm.SVC() - clf.fit(training_data[0], training_data[1]) - # test - predictions = [int(a) for a in clf.predict(test_data[0])] - num_correct = sum(int(a == y) for a, y in zip(predictions, test_data[1])) - print("Baseline classifier using an SVM.") - print(str(num_correct) + " of " + str(len(test_data[1])) + " values correct.") - -if __name__ == "__main__": - svm_baseline() - +""" +mnist_svm +~~~~~~~~~ + +A classifier program for recognizing handwritten digits from the MNIST +data set, using an SVM classifier.""" + +#### Libraries +# My libraries +import mnist_loader + +# Third-party libraries +from sklearn import svm + +def svm_baseline(): + training_data, validation_data, test_data = mnist_loader.load_data() + # train + clf = svm.SVC() + clf.fit(training_data[0], training_data[1]) + # test + predictions = [int(a) for a in clf.predict(test_data[0])] + num_correct = sum(int(a == y) for a, y in zip(predictions, test_data[1])) + print("Baseline classifier using an SVM.") + print(str(num_correct) + " of " + str(len(test_data[1])) + " values correct.") + +if __name__ == "__main__": + svm_baseline() + diff --git a/network.py b/elements/network.py similarity index 100% rename from network.py rename to elements/network.py diff --git a/network2.py b/elements/network2.py similarity index 97% rename from network2.py rename to elements/network2.py index 9ecaa5b..019b289 100644 --- a/network2.py +++ b/elements/network2.py @@ -1,360 +1,360 @@ -"""network2.py -~~~~~~~~~~~~~~ - -An improved version of network.py, implementing the stochastic -gradient descent learning algorithm for a feedforward neural network. -Improvements include the addition of the cross-entropy cost function, -regularization, and better initialization of network weights. Note -that I have focused on making the code simple, easily readable, and -easily modifiable. It is not optimized, and omits many desirable -features. - -""" - -#### Libraries -# Standard library -import json -import random -import sys - -# Third-party libraries -import numpy as np - - -#### Define the quadratic and cross-entropy cost functions - -class QuadraticCost(object): - - @staticmethod - def fn(a, y): - """Return the cost associated with an output ``a`` and desired output - ``y``. - - """ - return 0.5*np.linalg.norm(a-y)**2 - - @staticmethod - def delta(z, a, y): - """Return the error delta from the output layer.""" - return (a-y) * sigmoid_prime(z) - - -class CrossEntropyCost(object): - - @staticmethod - def fn(a, y): - """Return the cost associated with an output ``a`` and desired output - ``y``. Note that np.nan_to_num is used to ensure numerical - stability. In particular, if both ``a`` and ``y`` have a 1.0 - in the same slot, then the expression (1-y)*np.log(1-a) - returns nan. The np.nan_to_num ensures that that is converted - to the correct value (0.0). - - """ - return np.sum(np.nan_to_num(-y*np.log(a)-(1-y)*np.log(1-a))) - - @staticmethod - def delta(z, a, y): - """Return the error delta from the output layer. Note that the - parameter ``z`` is not used by the method. It is included in - the method's parameters in order to make the interface - consistent with the delta method for other cost classes. - - """ - return (a-y) - - -#### Main Network class -class Network(object): - - def __init__(self, sizes, cost=CrossEntropyCost): - """The list ``sizes`` contains the number of neurons in the respective - layers of the network. For example, if the list was [2, 3, 1] - then it would be a three-layer network, with the first layer - containing 2 neurons, the second layer 3 neurons, and the - third layer 1 neuron. The biases and weights for the network - are initialized randomly, using - ``self.default_weight_initializer`` (see docstring for that - method). - - """ - self.num_layers = len(sizes) - self.sizes = sizes - self.default_weight_initializer() - self.cost=cost - - def default_weight_initializer(self): - """Initialize each weight using a Gaussian distribution with mean 0 - and standard deviation 1 over the square root of the number of - weights connecting to the same neuron. Initialize the biases - using a Gaussian distribution with mean 0 and standard - deviation 1. - - Note that the first layer is assumed to be an input layer, and - by convention we won't set any biases for those neurons, since - biases are only ever used in computing the outputs from later - layers. - - """ - self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]] - self.weights = [np.random.randn(y, x)/np.sqrt(x) - for x, y in zip(self.sizes[:-1], self.sizes[1:])] - - def large_weight_initializer(self): - """Initialize the weights using a Gaussian distribution with mean 0 - and standard deviation 1. Initialize the biases using a - Gaussian distribution with mean 0 and standard deviation 1. - - Note that the first layer is assumed to be an input layer, and - by convention we won't set any biases for those neurons, since - biases are only ever used in computing the outputs from later - layers. - - This weight and bias initializer uses the same approach as in - Chapter 1, and is included for purposes of comparison. It - will usually be better to use the default weight initializer - instead. - - """ - self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]] - self.weights = [np.random.randn(y, x) - for x, y in zip(self.sizes[:-1], self.sizes[1:])] - - def feedforward(self, a): - """Return the output of the network if ``a`` is input.""" - for b, w in zip(self.biases, self.weights): - a = sigmoid(np.dot(w, a)+b) - return a - - def SGD(self, training_data, epochs, mini_batch_size, eta, - lmbda = 0.0, - evaluation_data=None, - monitor_evaluation_cost=False, - monitor_evaluation_accuracy=False, - monitor_training_cost=False, - monitor_training_accuracy=False, - early_stopping_n = 0): - """Train the neural network using mini-batch stochastic gradient - descent. The ``training_data`` is a list of tuples ``(x, y)`` - representing the training inputs and the desired outputs. The - other non-optional parameters are self-explanatory, as is the - regularization parameter ``lmbda``. The method also accepts - ``evaluation_data``, usually either the validation or test - data. We can monitor the cost and accuracy on either the - evaluation data or the training data, by setting the - appropriate flags. The method returns a tuple containing four - lists: the (per-epoch) costs on the evaluation data, the - accuracies on the evaluation data, the costs on the training - data, and the accuracies on the training data. All values are - evaluated at the end of each training epoch. So, for example, - if we train for 30 epochs, then the first element of the tuple - will be a 30-element list containing the cost on the - evaluation data at the end of each epoch. Note that the lists - are empty if the corresponding flag is not set. - - """ - - # early stopping functionality: - best_accuracy=1 - - training_data = list(training_data) - n = len(training_data) - - if evaluation_data: - evaluation_data = list(evaluation_data) - n_data = len(evaluation_data) - - # early stopping functionality: - best_accuracy=0 - no_accuracy_change=0 - - evaluation_cost, evaluation_accuracy = [], [] - training_cost, training_accuracy = [], [] - for j in range(epochs): - random.shuffle(training_data) - mini_batches = [ - training_data[k:k+mini_batch_size] - for k in range(0, n, mini_batch_size)] - for mini_batch in mini_batches: - self.update_mini_batch( - mini_batch, eta, lmbda, len(training_data)) - - print("Epoch %s training complete" % j) - - if monitor_training_cost: - cost = self.total_cost(training_data, lmbda) - training_cost.append(cost) - print("Cost on training data: {}".format(cost)) - if monitor_training_accuracy: - accuracy = self.accuracy(training_data, convert=True) - training_accuracy.append(accuracy) - print("Accuracy on training data: {} / {}".format(accuracy, n)) - if monitor_evaluation_cost: - cost = self.total_cost(evaluation_data, lmbda, convert=True) - evaluation_cost.append(cost) - print("Cost on evaluation data: {}".format(cost)) - if monitor_evaluation_accuracy: - accuracy = self.accuracy(evaluation_data) - evaluation_accuracy.append(accuracy) - print("Accuracy on evaluation data: {} / {}".format(self.accuracy(evaluation_data), n_data)) - - # Early stopping: - if early_stopping_n > 0: - if accuracy > best_accuracy: - best_accuracy = accuracy - no_accuracy_change = 0 - #print("Early-stopping: Best so far {}".format(best_accuracy)) - else: - no_accuracy_change += 1 - - if (no_accuracy_change == early_stopping_n): - #print("Early-stopping: No accuracy change in last epochs: {}".format(early_stopping_n)) - return evaluation_cost, evaluation_accuracy, training_cost, training_accuracy - - return evaluation_cost, evaluation_accuracy, \ - training_cost, training_accuracy - - def update_mini_batch(self, mini_batch, eta, lmbda, n): - """Update the network's weights and biases by applying gradient - descent using backpropagation to a single mini batch. The - ``mini_batch`` is a list of tuples ``(x, y)``, ``eta`` is the - learning rate, ``lmbda`` is the regularization parameter, and - ``n`` is the total size of the training data set. - - """ - nabla_b = [np.zeros(b.shape) for b in self.biases] - nabla_w = [np.zeros(w.shape) for w in self.weights] - for x, y in mini_batch: - delta_nabla_b, delta_nabla_w = self.backprop(x, y) - nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)] - nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)] - self.weights = [(1-eta*(lmbda/n))*w-(eta/len(mini_batch))*nw - for w, nw in zip(self.weights, nabla_w)] - self.biases = [b-(eta/len(mini_batch))*nb - for b, nb in zip(self.biases, nabla_b)] - - def backprop(self, x, y): - """Return a tuple ``(nabla_b, nabla_w)`` representing the - gradient for the cost function C_x. ``nabla_b`` and - ``nabla_w`` are layer-by-layer lists of numpy arrays, similar - to ``self.biases`` and ``self.weights``.""" - nabla_b = [np.zeros(b.shape) for b in self.biases] - nabla_w = [np.zeros(w.shape) for w in self.weights] - # feedforward - activation = x - activations = [x] # list to store all the activations, layer by layer - zs = [] # list to store all the z vectors, layer by layer - for b, w in zip(self.biases, self.weights): - z = np.dot(w, activation)+b - zs.append(z) - activation = sigmoid(z) - activations.append(activation) - # backward pass - delta = (self.cost).delta(zs[-1], activations[-1], y) - nabla_b[-1] = delta - nabla_w[-1] = np.dot(delta, activations[-2].transpose()) - # Note that the variable l in the loop below is used a little - # differently to the notation in Chapter 2 of the book. Here, - # l = 1 means the last layer of neurons, l = 2 is the - # second-last layer, and so on. It's a renumbering of the - # scheme in the book, used here to take advantage of the fact - # that Python can use negative indices in lists. - for l in range(2, self.num_layers): - z = zs[-l] - sp = sigmoid_prime(z) - delta = np.dot(self.weights[-l+1].transpose(), delta) * sp - nabla_b[-l] = delta - nabla_w[-l] = np.dot(delta, activations[-l-1].transpose()) - return (nabla_b, nabla_w) - - def accuracy(self, data, convert=False): - """Return the number of inputs in ``data`` for which the neural - network outputs the correct result. The neural network's - output is assumed to be the index of whichever neuron in the - final layer has the highest activation. - - The flag ``convert`` should be set to False if the data set is - validation or test data (the usual case), and to True if the - data set is the training data. The need for this flag arises - due to differences in the way the results ``y`` are - represented in the different data sets. In particular, it - flags whether we need to convert between the different - representations. It may seem strange to use different - representations for the different data sets. Why not use the - same representation for all three data sets? It's done for - efficiency reasons -- the program usually evaluates the cost - on the training data and the accuracy on other data sets. - These are different types of computations, and using different - representations speeds things up. More details on the - representations can be found in - mnist_loader.load_data_wrapper. - - """ - if convert: - results = [(np.argmax(self.feedforward(x)), np.argmax(y)) - for (x, y) in data] - else: - results = [(np.argmax(self.feedforward(x)), y) - for (x, y) in data] - - result_accuracy = sum(int(x == y) for (x, y) in results) - return result_accuracy - - def total_cost(self, data, lmbda, convert=False): - """Return the total cost for the data set ``data``. The flag - ``convert`` should be set to False if the data set is the - training data (the usual case), and to True if the data set is - the validation or test data. See comments on the similar (but - reversed) convention for the ``accuracy`` method, above. - """ - cost = 0.0 - for x, y in data: - a = self.feedforward(x) - if convert: y = vectorized_result(y) - cost += self.cost.fn(a, y)/len(data) - cost += 0.5*(lmbda/len(data))*sum(np.linalg.norm(w)**2 for w in self.weights) # '**' - to the power of. - return cost - - def save(self, filename): - """Save the neural network to the file ``filename``.""" - data = {"sizes": self.sizes, - "weights": [w.tolist() for w in self.weights], - "biases": [b.tolist() for b in self.biases], - "cost": str(self.cost.__name__)} - f = open(filename, "w") - json.dump(data, f) - f.close() - -#### Loading a Network -def load(filename): - """Load a neural network from the file ``filename``. Returns an - instance of Network. - - """ - f = open(filename, "r") - data = json.load(f) - f.close() - cost = getattr(sys.modules[__name__], data["cost"]) - net = Network(data["sizes"], cost=cost) - net.weights = [np.array(w) for w in data["weights"]] - net.biases = [np.array(b) for b in data["biases"]] - return net - -#### Miscellaneous functions -def vectorized_result(j): - """Return a 10-dimensional unit vector with a 1.0 in the j'th position - and zeroes elsewhere. This is used to convert a digit (0...9) - into a corresponding desired output from the neural network. - - """ - e = np.zeros((10, 1)) - e[j] = 1.0 - return e - -def sigmoid(z): - """The sigmoid function.""" - return 1.0/(1.0+np.exp(-z)) - -def sigmoid_prime(z): - """Derivative of the sigmoid function.""" - return sigmoid(z)*(1-sigmoid(z)) +"""network2.py +~~~~~~~~~~~~~~ + +An improved version of network.py, implementing the stochastic +gradient descent learning algorithm for a feedforward neural network. +Improvements include the addition of the cross-entropy cost function, +regularization, and better initialization of network weights. Note +that I have focused on making the code simple, easily readable, and +easily modifiable. It is not optimized, and omits many desirable +features. + +""" + +#### Libraries +# Standard library +import json +import random +import sys + +# Third-party libraries +import numpy as np + + +#### Define the quadratic and cross-entropy cost functions + +class QuadraticCost(object): + + @staticmethod + def fn(a, y): + """Return the cost associated with an output ``a`` and desired output + ``y``. + + """ + return 0.5*np.linalg.norm(a-y)**2 + + @staticmethod + def delta(z, a, y): + """Return the error delta from the output layer.""" + return (a-y) * sigmoid_prime(z) + + +class CrossEntropyCost(object): + + @staticmethod + def fn(a, y): + """Return the cost associated with an output ``a`` and desired output + ``y``. Note that np.nan_to_num is used to ensure numerical + stability. In particular, if both ``a`` and ``y`` have a 1.0 + in the same slot, then the expression (1-y)*np.log(1-a) + returns nan. The np.nan_to_num ensures that that is converted + to the correct value (0.0). + + """ + return np.sum(np.nan_to_num(-y*np.log(a)-(1-y)*np.log(1-a))) + + @staticmethod + def delta(z, a, y): + """Return the error delta from the output layer. Note that the + parameter ``z`` is not used by the method. It is included in + the method's parameters in order to make the interface + consistent with the delta method for other cost classes. + + """ + return (a-y) + + +#### Main Network class +class Network(object): + + def __init__(self, sizes, cost=CrossEntropyCost): + """The list ``sizes`` contains the number of neurons in the respective + layers of the network. For example, if the list was [2, 3, 1] + then it would be a three-layer network, with the first layer + containing 2 neurons, the second layer 3 neurons, and the + third layer 1 neuron. The biases and weights for the network + are initialized randomly, using + ``self.default_weight_initializer`` (see docstring for that + method). + + """ + self.num_layers = len(sizes) + self.sizes = sizes + self.default_weight_initializer() + self.cost=cost + + def default_weight_initializer(self): + """Initialize each weight using a Gaussian distribution with mean 0 + and standard deviation 1 over the square root of the number of + weights connecting to the same neuron. Initialize the biases + using a Gaussian distribution with mean 0 and standard + deviation 1. + + Note that the first layer is assumed to be an input layer, and + by convention we won't set any biases for those neurons, since + biases are only ever used in computing the outputs from later + layers. + + """ + self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]] + self.weights = [np.random.randn(y, x)/np.sqrt(x) + for x, y in zip(self.sizes[:-1], self.sizes[1:])] + + def large_weight_initializer(self): + """Initialize the weights using a Gaussian distribution with mean 0 + and standard deviation 1. Initialize the biases using a + Gaussian distribution with mean 0 and standard deviation 1. + + Note that the first layer is assumed to be an input layer, and + by convention we won't set any biases for those neurons, since + biases are only ever used in computing the outputs from later + layers. + + This weight and bias initializer uses the same approach as in + Chapter 1, and is included for purposes of comparison. It + will usually be better to use the default weight initializer + instead. + + """ + self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]] + self.weights = [np.random.randn(y, x) + for x, y in zip(self.sizes[:-1], self.sizes[1:])] + + def feedforward(self, a): + """Return the output of the network if ``a`` is input.""" + for b, w in zip(self.biases, self.weights): + a = sigmoid(np.dot(w, a)+b) + return a + + def SGD(self, training_data, epochs, mini_batch_size, eta, + lmbda = 0.0, + evaluation_data=None, + monitor_evaluation_cost=False, + monitor_evaluation_accuracy=False, + monitor_training_cost=False, + monitor_training_accuracy=False, + early_stopping_n = 0): + """Train the neural network using mini-batch stochastic gradient + descent. The ``training_data`` is a list of tuples ``(x, y)`` + representing the training inputs and the desired outputs. The + other non-optional parameters are self-explanatory, as is the + regularization parameter ``lmbda``. The method also accepts + ``evaluation_data``, usually either the validation or test + data. We can monitor the cost and accuracy on either the + evaluation data or the training data, by setting the + appropriate flags. The method returns a tuple containing four + lists: the (per-epoch) costs on the evaluation data, the + accuracies on the evaluation data, the costs on the training + data, and the accuracies on the training data. All values are + evaluated at the end of each training epoch. So, for example, + if we train for 30 epochs, then the first element of the tuple + will be a 30-element list containing the cost on the + evaluation data at the end of each epoch. Note that the lists + are empty if the corresponding flag is not set. + + """ + + # early stopping functionality: + best_accuracy=1 + + training_data = list(training_data) + n = len(training_data) + + if evaluation_data: + evaluation_data = list(evaluation_data) + n_data = len(evaluation_data) + + # early stopping functionality: + best_accuracy=0 + no_accuracy_change=0 + + evaluation_cost, evaluation_accuracy = [], [] + training_cost, training_accuracy = [], [] + for j in range(epochs): + random.shuffle(training_data) + mini_batches = [ + training_data[k:k+mini_batch_size] + for k in range(0, n, mini_batch_size)] + for mini_batch in mini_batches: + self.update_mini_batch( + mini_batch, eta, lmbda, len(training_data)) + + print("Epoch %s training complete" % j) + + if monitor_training_cost: + cost = self.total_cost(training_data, lmbda) + training_cost.append(cost) + print("Cost on training data: {}".format(cost)) + if monitor_training_accuracy: + accuracy = self.accuracy(training_data, convert=True) + training_accuracy.append(accuracy) + print("Accuracy on training data: {} / {}".format(accuracy, n)) + if monitor_evaluation_cost: + cost = self.total_cost(evaluation_data, lmbda, convert=True) + evaluation_cost.append(cost) + print("Cost on evaluation data: {}".format(cost)) + if monitor_evaluation_accuracy: + accuracy = self.accuracy(evaluation_data) + evaluation_accuracy.append(accuracy) + print("Accuracy on evaluation data: {} / {}".format(self.accuracy(evaluation_data), n_data)) + + # Early stopping: + if early_stopping_n > 0: + if accuracy > best_accuracy: + best_accuracy = accuracy + no_accuracy_change = 0 + #print("Early-stopping: Best so far {}".format(best_accuracy)) + else: + no_accuracy_change += 1 + + if (no_accuracy_change == early_stopping_n): + #print("Early-stopping: No accuracy change in last epochs: {}".format(early_stopping_n)) + return evaluation_cost, evaluation_accuracy, training_cost, training_accuracy + + return evaluation_cost, evaluation_accuracy, \ + training_cost, training_accuracy + + def update_mini_batch(self, mini_batch, eta, lmbda, n): + """Update the network's weights and biases by applying gradient + descent using backpropagation to a single mini batch. The + ``mini_batch`` is a list of tuples ``(x, y)``, ``eta`` is the + learning rate, ``lmbda`` is the regularization parameter, and + ``n`` is the total size of the training data set. + + """ + nabla_b = [np.zeros(b.shape) for b in self.biases] + nabla_w = [np.zeros(w.shape) for w in self.weights] + for x, y in mini_batch: + delta_nabla_b, delta_nabla_w = self.backprop(x, y) + nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)] + nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)] + self.weights = [(1-eta*(lmbda/n))*w-(eta/len(mini_batch))*nw + for w, nw in zip(self.weights, nabla_w)] + self.biases = [b-(eta/len(mini_batch))*nb + for b, nb in zip(self.biases, nabla_b)] + + def backprop(self, x, y): + """Return a tuple ``(nabla_b, nabla_w)`` representing the + gradient for the cost function C_x. ``nabla_b`` and + ``nabla_w`` are layer-by-layer lists of numpy arrays, similar + to ``self.biases`` and ``self.weights``.""" + nabla_b = [np.zeros(b.shape) for b in self.biases] + nabla_w = [np.zeros(w.shape) for w in self.weights] + # feedforward + activation = x + activations = [x] # list to store all the activations, layer by layer + zs = [] # list to store all the z vectors, layer by layer + for b, w in zip(self.biases, self.weights): + z = np.dot(w, activation)+b + zs.append(z) + activation = sigmoid(z) + activations.append(activation) + # backward pass + delta = (self.cost).delta(zs[-1], activations[-1], y) + nabla_b[-1] = delta + nabla_w[-1] = np.dot(delta, activations[-2].transpose()) + # Note that the variable l in the loop below is used a little + # differently to the notation in Chapter 2 of the book. Here, + # l = 1 means the last layer of neurons, l = 2 is the + # second-last layer, and so on. It's a renumbering of the + # scheme in the book, used here to take advantage of the fact + # that Python can use negative indices in lists. + for l in range(2, self.num_layers): + z = zs[-l] + sp = sigmoid_prime(z) + delta = np.dot(self.weights[-l+1].transpose(), delta) * sp + nabla_b[-l] = delta + nabla_w[-l] = np.dot(delta, activations[-l-1].transpose()) + return (nabla_b, nabla_w) + + def accuracy(self, data, convert=False): + """Return the number of inputs in ``data`` for which the neural + network outputs the correct result. The neural network's + output is assumed to be the index of whichever neuron in the + final layer has the highest activation. + + The flag ``convert`` should be set to False if the data set is + validation or test data (the usual case), and to True if the + data set is the training data. The need for this flag arises + due to differences in the way the results ``y`` are + represented in the different data sets. In particular, it + flags whether we need to convert between the different + representations. It may seem strange to use different + representations for the different data sets. Why not use the + same representation for all three data sets? It's done for + efficiency reasons -- the program usually evaluates the cost + on the training data and the accuracy on other data sets. + These are different types of computations, and using different + representations speeds things up. More details on the + representations can be found in + mnist_loader.load_data_wrapper. + + """ + if convert: + results = [(np.argmax(self.feedforward(x)), np.argmax(y)) + for (x, y) in data] + else: + results = [(np.argmax(self.feedforward(x)), y) + for (x, y) in data] + + result_accuracy = sum(int(x == y) for (x, y) in results) + return result_accuracy + + def total_cost(self, data, lmbda, convert=False): + """Return the total cost for the data set ``data``. The flag + ``convert`` should be set to False if the data set is the + training data (the usual case), and to True if the data set is + the validation or test data. See comments on the similar (but + reversed) convention for the ``accuracy`` method, above. + """ + cost = 0.0 + for x, y in data: + a = self.feedforward(x) + if convert: y = vectorized_result(y) + cost += self.cost.fn(a, y)/len(data) + cost += 0.5*(lmbda/len(data))*sum(np.linalg.norm(w)**2 for w in self.weights) # '**' - to the power of. + return cost + + def save(self, filename): + """Save the neural network to the file ``filename``.""" + data = {"sizes": self.sizes, + "weights": [w.tolist() for w in self.weights], + "biases": [b.tolist() for b in self.biases], + "cost": str(self.cost.__name__)} + f = open(filename, "w") + json.dump(data, f) + f.close() + +#### Loading a Network +def load(filename): + """Load a neural network from the file ``filename``. Returns an + instance of Network. + + """ + f = open(filename, "r") + data = json.load(f) + f.close() + cost = getattr(sys.modules[__name__], data["cost"]) + net = Network(data["sizes"], cost=cost) + net.weights = [np.array(w) for w in data["weights"]] + net.biases = [np.array(b) for b in data["biases"]] + return net + +#### Miscellaneous functions +def vectorized_result(j): + """Return a 10-dimensional unit vector with a 1.0 in the j'th position + and zeroes elsewhere. This is used to convert a digit (0...9) + into a corresponding desired output from the neural network. + + """ + e = np.zeros((10, 1)) + e[j] = 1.0 + return e + +def sigmoid(z): + """The sigmoid function.""" + return 1.0/(1.0+np.exp(-z)) + +def sigmoid_prime(z): + """Derivative of the sigmoid function.""" + return sigmoid(z)*(1-sigmoid(z)) diff --git a/network3.py b/elements/network3.py similarity index 100% rename from network3.py rename to elements/network3.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e8c6f00 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +# pip install -r requirements.txt + +# base ---------------------------------------- + +os-sys +numpy +matplotlib +theano +