diff --git a/stl/notebooks/STL Offline Plots.ipynb b/stl/notebooks/STL Offline Plots.ipynb new file mode 100644 index 0000000..7ed30d7 --- /dev/null +++ b/stl/notebooks/STL Offline Plots.ipynb @@ -0,0 +1,925 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 351, + "id": "642f67ca", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "import os\n", + "import pandas as pd\n", + "import sys\n", + "import json\n", + "\n", + "import seaborn as sns\n", + "import numpy as np\n", + "sns.set(style=\"whitegrid\", palette=\"muted\")\n", + "\n", + "sys.path.insert(1, \"/home/eecs/wooders/experiments/stl/offline\")" + ] + }, + { + "cell_type": "code", + "execution_count": 412, + "id": "0df714c8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Finishing last run (ID:1r9i8p8d) before initializing another..." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
Waiting for W&B process to finish, PID 30648... (success)." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Label(value=' 0.66MB of 0.66MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "
\n", + "Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 1 other file(s)\n", + "
Synced daily-waterfall-26: https://wandb.ai/ucb-ralf/experiments-stl_notebooks/runs/1r9i8p8d
\n", + "Find logs at: ./wandb/run-20211015_043830-1r9i8p8d/logs
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Successfully finished last run (ID:1r9i8p8d). Initializing new run:
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: wandb version 0.12.4 is available! To upgrade, please run:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: $ pip install wandb --upgrade\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " Syncing run rural-voice-27 to Weights & Biases (docs).
\n", + "\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact results:v12, 2349.20MB. 11474 files... Done. 0:0:0\n" + ] + } + ], + "source": [ + "run = wandb.init()\n", + "results_dir = run.use_artifact('ucb-ralf/stl/results:v12', type='dataset').download()\n", + "yahoo_train_dir = run.use_artifact('ucb-ralf/stl/yahoo_train_data:v0', type='dataset').download()\n", + "yahoo_eval_dir = run.use_artifact('ucb-ralf/stl/yahoo_eval_data:v0', type='dataset').download()\n", + "oracle_dir = run.use_artifact('ucb-ralf/stl/oracle:v0', type='dataset').download()" + ] + }, + { + "cell_type": "markdown", + "id": "b5a8aea6", + "metadata": {}, + "source": [ + "# Check Train / Eval Data" + ] + }, + { + "cell_type": "code", + "execution_count": 353, + "id": "0eedb687", + "metadata": {}, + "outputs": [], + "source": [ + "key = 3" + ] + }, + { + "cell_type": "code", + "execution_count": 354, + "id": "975d3b68", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 354, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAD7CAYAAACmJ9mYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABRgklEQVR4nO2deWAU5f3/33vmPiEJSbjvQDxw/Wo960XBirXUtnhWrW3t8bVF67fFys8q0ir2UGlVvla+ra3UehVFFBGVVi2eAYUAcgdC7vvOZo/5/THzzPHM7OzsleyGz+ufZHauZyeb572f87EJgiCAIAiCIOKIfaQHQBAEQYw+SFwIgiCIuEPiQhAEQcQdEheCIAgi7pC4EARBEHHHOdIDSATBYBB9fX1wuVyw2WwjPRyCIIiUQBAE+Hw+ZGVlwW6PzfYYleLS19eH/fv3j/QwCIIgUpKZM2ciJycnpmuMSnFxuVwAxAfkdrsjPr+6uhqVlZXxHlZCScUxA6k5bhrz8JGK407lMQ8NDWH//v3yHBoLo1JcmCvM7XYjLS0tqmtEe95IkopjBlJz3DTm4SMVx53qY45HOIEC+gRBEETcIXEhCIIg4k7cxGXVqlW46KKLMGvWLE0w/ciRI1iyZAkWLFiAJUuWoKamJuZ9BEEQRHITN3G5+OKLsW7dOpSXl2te/+Uvf4lrrrkGmzdvxjXXXIO777475n0EQRBEchM3cTn99NNRWlqqea2trQ179uzBokWLAACLFi3Cnj170N7eHvU+giAIIvlJaLZYQ0MDSkpK4HA4AAAOhwPFxcVoaGiAIAhR7SssLLR8/+rq6qjHXlVVFfW5I0UqjhlIzXHTmIePVBw3jXmUpiIzKisro0oJrKqqgsfjCbn/00M9uPPJw3jq5xUozo+8jiYRhBtzspKK46YxDx+pOO5UHrPX643pS7mahGaLlZaWoqmpCYFAAAAQCATQ3NyM0tLSqPclA29WdQAAPj3YO8IjIQiCSE4SKi5jxoxBRUUFNm7cCADYuHEjKioqUFhYGPW+ZCAzXXxsfYOBER4JQRBEchI3t9jKlSvxxhtvoLW1FTfddBPy8/Px6quv4p577sGyZcvw2GOPITc3F6tWrZLPiXbfcLPjYA+6+/344skFAACnXaxeDdIK0QRBEIbETVyWL1+O5cuX616fNm0ann/+ecNzot033Pxi7WEAkMWFtUYgbSEIgjCGKvQjIBCU1IS6+BMEQZhC4hIBPr8oLkxb/AHFdPnOb/fitY/aRmBUBEEQyQeJSwT4AkEAAGsYOuQTtwVBQF3bEP6w/vhIDY0gCCKpIHGJAGa5MO+YT7JcgsGRGhFBEERyQuISAUxcApKoMLeYHIshCIIgAJC4RIRiuWhFhsSFIAhCC4lLBLCYi/QD/qCg2SYIgiBESFw4AkEBT/7LgX/v7NDtG/KxGAu5xQiCIMwgceEYHAqiptWGB545ptvHLBemJVLrMxIXgiAIDhIXDhZXMdsnB/SDsVkuTR1DaO4ciupcgiCIZIbEhUNdGMnDB/Rlt5jBOU0dQ9hX2296rxsf3IsbVu2NdqgEQRBJC4kLh98kOu8LaAP4/kDogP6Dzx7F0scOYMhP0X6CIE48SFw4TN1iUkU+C+ibpSLvOSpaLf2DJC4EQZx4kLhwmLnFWIyFaYmVmAtZLgRBnIiQuHD4TMSlpz+A7j6/LCZWiihZ/7HOXh8uvfMzvL1Dn+JMEAQx2iBx4eAtF0G1aMsTr9Zjycrd+jqXgPHxAOCVamMO1A0AAN7a0R73MRMEQSQbJC4cQc4KMWpKGeDcYepz+OA+s1x6+kUFyslwGt6HIAhiNEHiwsGLQ8BguUkmOD6/gJrGAY1bjLd8fJzrzOmwabbjxaEmGz7c2x3XaxIEQUQLiQsHP+kbWRhMQGqaBvGDR/Zj+8HekOezbaZRdjt7PV4jFvnzuw7c89cj8b0oQRBElJC4cOjFQX+M16d9cU9Nn3I8Z7kwIWKFlzZbbJbLseZBdPb6ojqXIAhiuCBx4eBjLEaWyyAnLmrXmZ8XJ9YqRqqfsdv01zUTmvYen0awbnloH276zecm74AgCGLkIXHhCHCTvqHlMqR90UwomNgMyeKit1yM2scAQEO7F9f+eg82vN+qeX1wiGpnCIJIbkhcOPhJ34rlorZ2QrnFWEdl2NjrqmOYAPmC6Orzy68fbhgEAHy8r1u6j34sR5sG8dvn9B2cCYIgRhISFw61uNzw4F5s/Uxf9OgdMs4IA7SiAagKLfnXDaydp7Y04qqVu9E3KB7c3i3GVtwu8c+kFjV2zq/W1eCtGAszBUHA2zs65PsSBEHECokLh9oK6ez148nXGnTH8AH9Qa8yKYdyi/E1MUZusX++2wIAaO0SRaVfui6L16j7lPVLQqC2dKLleIsXv3nuGFb946j82r7afmypooJPgiCig8SFI5osrgFVDGTX4V4caRhQrsctKGZUeDnkFzT37ekXBWPAK16XicyASsRYUaZDqpthdPb68a0H9uCjz0PXvHT2+vGn1+rlAs+2HlHM9h5TlghY+tgB/P6F2tBvmiAIwgTnSA8g2Yimcp4tfwwAj26o0+zjl0I2atN/w6q9GFfglrd7BpiYiAc1d/rwp9fqMXdSlu4YliDAqGv1oqXLh6feaMAZs3MBAOvfa0FBjhMXnFIgjvHl43ivugunTc+BZ2aOLFShanqcnIARBEGEg8SFI5riRq9J52PFUhG3ebFhNHYMwW4Xj6tpHERdqxcDQ+Kk39btwz/fbcG/P+uUj3/4xVqcNTcPDs72ZFaO+vJPvFoPALK4sGNYDIeJGDtHLTLd/X4U5rgQCAho7BhC+di00A+CIAhCgsSFIxrLxWwNGCYm/E8j95vLYYc3GMRftzQCACaPS9fsb+tWiidrmgZR0zSI/Gztn1AWF7kzgHIfQRBgs9nglBSJxWuYu42NSR1T6u4TxWXD+6144tV6/O770zFHZUGNJj7c243ufj/mewpHeigEkfKQuHDEu+cXH9A3WxrZznmfapsHw16/n8vw6h3QWi5elcuubzCI7AwHXJKbq5uJixQzCgQFNHUMyd0EAKBbcpntONgDAGho845acWHtc0hcCCJ2SFw44t3zi1/zRbZgrFT+WxjLEGc17T8+oLmvWny6+/3IznDIqdPVR/rwDzQpbrEgcOODe3FOZZ58DhMrh6R8Hb2xZ6elCkcaB/DpwV4sPrdopIdCECkHiQuHVcvFbtPGNUIRKqBv1MrfoAEz0lx2XeqzGUcbRWtnYCiIJzbW4ZzKfHnf3qN96Oz1y66zqgM9qDrQg1kTMjXX+E91l/z7R593ozBH+ZjwltJo5p6njqC504dLPAXyUgkEQViD/mM4gkYzvAEup7VJnxcTf0DA3qN9lkXspClZ+PxYP3oHmQVhbtHUt3kBiOnG6//Tip1HlKaav31eTC2eWqqN5eyr7UcoNn/Sjs2ftOOUadkARNcag8VwRivsT9TW5SNxIYgIoToXDr6SPhRpLmuTanPnEDZ/0iaLyd5j/bh9zUFs290V5kyRkgI3nrt7Lq6+qAQAkOF2YEyuS3OMXfVXZCnKjJbOId011cWYVmFBf2b1BAIClty3G8/9qynia6UK7HsGretGEJFD4sJh1aJgLVnC8e6uLjz84nFda5XDqkJLMzLS7LDZbMhOd8ivrVk6Cytvmipv52aG/lbdO6BXy56ByOMmLFNNbk3T40PPQAB/3twY8bVSh9CZfQRBmEPiwmFZXJyRuYN4FxpvYTAunleAWRMykZkm/mky00RRycoQt4OCgOwMB8bmKdbL9LKMkPc1ejt9BpZL2Ri3/kAVbd1+6dwAuvr8sptuNMMsl1BdqwmCCA2JC4fVmItVy4WhTgkGtFX9as6YnYuHfzgDBVL9SrpbvE+25PNn4peZrtz/hgXjcMO5ARTni4KjtnJCwbv1xhel47QZ2cjNMj935+E+XLVyt2mcJlUIV9MkiwutcEAQEZPU4nLkyBEsWbIECxYswJIlS1BTU5Pwe1qdSBx8UUoYeMvFF6Kqn8VP7FItCrNg8jLFSZ+lEWepBCQ304kZ40SLBoClKnpmETGy0x341ben4eoLS5TrmghNtSpRIBWpaRzAZXftxCfScgZ8sSnAnGLkFiOIaEhqcfnlL3+Ja665Bps3b8Y111yDu+++O+H3DOUCWXJBMfKyos8YGtKJi/F9WK+wdBezWMQJvihfdFudf1K+Zj+gCAXLaJpSmo6rLyzG2XPFehW7wV/5zIpczb4cSbzU8ZsHvjMNN19aajjO2pbwBZ7JzLFmMavutY/aAGjrhZiWKJYLiQtBRErSiktbWxv27NmDRYsWAQAWLVqEPXv2oL09sW3gQ7lKvnhKPv6xfK68PbU0HZlpdnlyznCbP0qd5RJCxJhF9KXTxSrxMskKKSlwY+VNU/Hdy8oAAHaV5ZSVrhWi7AwHvvWlUlRMVOpX7rtxCr7zZUUozqzIxbpfzMElp4n3UcRFsVamjMvQVKurLZnGdiULjT2zTw/14OVtLaEeQVLBnhlb1dOv+nswMRGg72BNEIQ1klZcGhoaUFJSAodDnNAcDgeKi4vR0KBfXyWeeGbmwjNZ77Li3WATitLxwi8r8YUK0TpQx0CM4GMu4dxil505Bn/9eQWmjFOC9Z6ZOSjMUQL535o/Dl87t0iuNWHxmUzJZcYEIxgETp+VKwsJIFo7hTkuueUME6YcLvNM7X4rLVTcbawtDKC0j7nzycNY80q9xsWUrMhNOg2adfIFrhRzIYjIGdWVYdXV1RGf4wRw2alAVY1WLPbu2Y2WWsAGBwTYUF9/HNu316K7yw7AjoDfB3kNYwMGvX7N/lAT1qGDB2HvVSY6swWMZ+YByAOqqkTBbW9vA2BHW3MdqqqOo7neBkAUh6qqKume4p/86OF98HUAra3i+I/X1qKq6hja+5RjqqqqVE8FcAm9MPo+8tdXPsPJEwT5uG0fbke6S3dYSJT7DB/7pGfT1dWDqqoq9HkB+X1v34F0FxDwOwDYsP/AQTj6tII5EmOOlVQcM5Ca46YxJ7G4lJaWoqmpCYFAAA6HA4FAAM3NzSgtNY4BGFFZWYm0tMhbxH/0sf4hn3xyJUoL02D752cQgsCkiRPg8RThrYNHgeOdyMxIQ2e/vmCR4Q9aSwCYNWsG5k3PiXjMVVVVKC4aCxxtx7zKafDMzYO7sBd/f/8QAMDj8YgH/vMzcXteJcrGpMGX0YVPjtTgwi/MwpxJWRgcCuD3m6oxtTQdHs8s8ZwXxHOWXDINp9UN4K0dHahr9cr33rDdgWNd2QB6AQDTZlZiXKG1515VVaWMbRjpd3cC244iKzsbHs90dPT4gFf2AABOPvkU5GQ64Xp9N+D1Y/KUqfBU5qOh3Ytv/+ZzXH1WAN/6ymnDPuZYGKnnHCupOO5UHrPX643qS7kRSesWGzNmDCoqKrBx40YAwMaNG1FRUYHCwsR3rDVKBJMX5ZK+wLIFtNjPSLPHQhHLda67ZBy++cVieGaI4pSTETrbiwXuvzAnD8/dPVfudJzuduBX356KO6+epDtnUkk6rr6oBGNz9WbJgTolNZnV8Az5g9h7LDmzylhCBXOHqcMqzKq0ydviziMNYhLDjpr4trzxBwSsf68lpKuUIFKRpBUXALjnnnvw9NNPY8GCBXj66adx7733Dst9jdpl8ZM+LyrxEhejzC6rjM1z4aaFpXLshcVnLjglX3dslipGxPfNOm1GDsYXpfOnyG1nWHyGpUmL11OErLtPFJcX323B7Y8fxO6a5BMYP5dQoY25iL+zzwFrCZQmZej54lw/+uqH4lo5L/2nNb4XJogRJGndYgAwbdo0PP/88yM9DAD6SV9nuRgsBWy1c7IaRxwbQeZmObH2jtkaS+OB70zDvtq+iBpOFue70NzpkzPUWKLAuEI3Dkvf5tNV2XI9/WI1f7vUMmb/8X7MnZxca8AwcWGPQWu5cOIibbulwlNfIL6WCxPjSLpfE0Syk9TiMpI8+dPZ2Hm4F6vXHwegWCZsDtJbLvprRJMzZY+TBcQoG6ONfZwyLVvucGyVNUtnaRIQmOUyrjBNFheW0gso/cyYW667X+ll9vJ/WnDq9BxMKtFbRsMJn16s7syg7LJpjpUbWcZ7zR/p+uovKG9tb0f52DTMnphcokwQVklqt9hIUj42DQWqdUycDmO3mFpknvzpbFx3cQliwUikRpqMNIcsKIAqbTnDIfc1a+5UlmD+x7+a8dJ/WmTBYc0u+wYDWLOxHj95dL98bF078MzW4e+szFuUasEIBHjLRfwpL58Q50xrVvPkUn3Gfvt8LW57/GB8b0QQw0gSTmXJg1NlRbB/fMEkoF8+Ng2TS5W6lHDlHkZCEq/YTSJRt475w60zcf0l4zT727p9+N+N9eiTVrhklkyXtIqluubn8bed+OsbjRgcUlr6//6FY2ho9yKRMEuF/Y3UtTmyW4zbTpS4MDFjf3s+HkQQqQiJiwlqNwUfU3Fx4sJiMk4TcQhl/YS6Z7LikjpCK4WXxllpx5pElxkTF7V7jKdLijtsqWrHlqoObJLasiQKRVTEn2q3HxMPPubCJv1414jylgtbO4cgUhkSFxPUkz9vUThCBPLN3Fp8J2K73YZrLy7BWXNylXumgOVy0akF+Nq5RbjyPHFt+VApz03SQmXMLdbvDR2sYMLD3FPqGE4iUFq8iAQNLBd+O2CQthyXsUji4pREmzoCEKMBEhcTjCwLfh8TAzY3mVkefJt+h92G6y4Zp3ErpYLl4nbZ8d3LylAgpTpnhRCXjh5RMFo6fXh/T5e8iiWgb3/DXGbyBK7a/Y+tTaiu6Y3b+AGVQBgE6ZVsMe1knyjLRZe5Rr3MiFEAZYuZYCYuLslEYcew6cAsZpLm1IsLoBUUs3smK+HWj2np8mHF32pwTmWe/NrOw70oVWWyNbYPobPXL1s3bMINBAU89Ya42uWm+0+J25iVtvqCZhtQhIaPucRr0t97tA92uw2zJoiNRZX3Kt0vBXqzEUQ4SFxMMHNRSf00FTHhAv1GuF3GLjS1IKVCQJ8nm7NcJo9LR02jviX/sWblteV/PqIpwnx0Qx0e3VCHK84eCwDySpf9CYo/MAExirnIbjFmSXCTf6zcvkbMAmNi6WfiFdBbbQSRqpBbzAQzF5UVyyUjTft40wzcYupr8L+nCmq32IzyDPzqpqmGxzW0aXuvGcVgWrvElGZWiNlvsCRzPOBXHFVv/+nVevz2uWPyH1WJtSTGouDXjUnUfQhiOCFxMcFK5hezYJhbxaH6En/m7FzNOXxRPBMXdfW/M/wKxUmH2i0WFLTLD6hrhayk2NZIGWb7agdw1/8dwv7jiVlOmVkH/MJgAPB5bT/e2tEhT/KyuypBsRC+niZAyWLEKIDExQQzy4UP6Muvq7ZvXFCKNUtnydt93Ldwh8E1UtEt5nbZMb5IjJ8IgqCx0IryIui9D8j1LV5fENsP9OLVD5WU5CEpCeCDvV34v9frYxoz+zIQNImn8KKSKHeVLrZjYLnsONiDbbu7EjMAgkgAJC4mWMkWYwIhZ4upCy+dNk2bE7Yy5NTSdOlY/Tmp6BYDgMd/MgvnVubhp9+YqOlbNqM8U3NcuKWi+Qm8pVNxpbF6mT++VIfn/92Czt7QdTPhCOpcUUbHcCnICbdcQt/nF2sP476naxJyf4JIBCQuJhhZETPKM6R94raux5hKHFhRHLtMaWEaNt1/Ck6TWuIbnRNJQ8lkwumw4a5rJ2OqqkMBIPYy++vPK3DSFLFHVn52ZDkkjR2KuLD4S5vUELN3IBZxCe/yCnL7ElV/ordcEnMfghhOSFxMMLIi7rtpKm7/+gTkZ4vuHubSYlXrRsF5phcsHsMLUqpaK1bITHOgKN8trx+T4bbjjm9MwOVnjQl5jvp5qC2ZxzYcx64jSr0LWzcmGuSYC9eUUo3ACZDcMkZ1zN5jfTjapM+MszYGvljT+HWjMRFEskPiYoJRcD0vy4n5HmXBMvav7pJqWDRpxbK4MAtGKyZMdFyjUFwKpUB+Xpb4EFlGWXaGAxefVoiTpyqdmf/nmxNx0akF8nbpGLfhNXcc7MXPnjgkb/f0Ry8ubI72BwQ8+vJxHKzTJw7wAmQ06d/++EF8/+F9UY1Brm+RCznDW0hUvU+kClTnYoKV4Do7hq2Zoomf2LVuMbaPiQmbq+LdZj8ZuOObE/Heri5MkdxkLKOsUHpOzJIBgLPn5uGieQX48PMu9A0GMaEoHbXN4RtX9sZguTAXVEuXDxs/MO5jFuBiLvGe2H0BAW6XWlTCx3b8ASGulu7691pw0pQsTOdiYwQRK2S5mGDln/jkqVm4ccE43LhQbOGiTkVmosEsFJZyrCQBjF4Xx7zpObh18XhZUNlKlUbNLtlCY8yymzctG7cuHh923ZkeKebiDwj412cdco8uK1h59MxyOdbsxds7OqIO6G/+pE3ur6ZGXmpZZUUB5hX68U6HfuLVetz6xwPy9sf7unHbYweoMzMRMyQuJlgJrqe7HVhyQYm8VLBxbYxxjGUUa4sOG/ukSe+ZX1oZUEQ3M92BL58xBpOKxay6UOnMvZJb7L3qTqz6xzH8870Wy+Ph032NGo6yiXxfbT9+89wxuXtxJH+3I40DePjF42JRJofcq0xnuYS+XiQCGg2/fOoIPq/tN+1gTRBWIHGxwJhc695Do9oY3nJh1f0nkLbgtOlihhzrL5Zr0Kaf6TJv5YwNIS4soN/eLU6ErLrfCvwE7jRQF/7be1ef1LmZi5GYwayx2mZ90J8JnJIWzX6auMUSnErGdwsgiGihmEsYfnvLdJSFCDAbYRSnsXH7ZNfZCfT/O2tCJjb+6mT5GbAO0RluVcNI6VcmPExcbDbAMyMHVQd6NNd8eVsrmjqGMHO8GC+wR/BVibdcrLgoWV2NLC6qUwRBgM1mQ3PnEN7e0YGvn18Mp8Mmf7EwanUTCAroHQigvUcUxeAwWy7hYjsEEQtkuYRh7uQsubW8FQzFhVkuNhbQFx+7eoI7ZVp2xNXsqQb/bFb/9wzctlCJRTA3WL6UacbERRCAld+eiruunaS75gd7u9HRK07OkawBw8+rVr6ps+sLBpYLi5+s3dSAp95oxOe1fZrrGl09EACu/tVuHG4Y1Bxr1lssnpaL2aX8fhIXIjbIchkGeLeYUczlge9MG+ZRjTwzyjPR3ahs/+KaSfhgbzdKC8VWMkxcWGIE332ZUd8qFlqq617ae3zITLMj3W18Dm+pWMkEY+JiZLmwzK/OXmaFSNc1sQACQUFjIbCeYuaWS/hxWoUsFyKRkOUyDNi4gL6cLTZiI0pOcrOc+NLpSg1Rcb7ojjxPitOoxaU4X7Hy6trEtOVeVd3L937/OX78qJIFxRNNn7BBHy8uapee+Dv7W/MdAIzHwAtceMsllliI1xfE2zs6LN3HR+JCxAhZLsOAPqCvrX8hjJlWloE/3T4L45glo+q+rHaxNbYzy8Uv/+wbDKJvMHStTDRt7WXLRS6uVF2P/c76hEmTs5kFwKcc88spGxGLRfHvzzrx0Iu16OrzY/G5RabCR5YLEStkuQwDcvsXLhXZnqJ9xIaT8UXp8vPKVqUvjyvUJ1n0DgQQDAro7tP7jjZ/0oZlfzokT/rRWC5ek5gLEwSmeXzml5GW8S6u7n4//rO7y3RssUz6rNamSerXZuYW81HMhYgREpcEMHtCJs6emydvs75jfPuXSLKbCGhWrvyfb07Ed79cptnf3OnDZXftxOsfKxX3XsmVtXr9cXx2uBfHW0VrJpoCVn3MRS8uDPUyzaHgJ/edh/uw8ukaw7TlUOdEAvvc8csqG97nRCrCIhICTW8J4KEfzsD/u26yvM36jvHNLVO1A/JIoW6TU5DjwuJzxxoKtHrdE9YihlkDzVIL/2gMALaejFLnouxjv7MvELKLK0xA3wh1J2heBFm2WH2rF6v+cRSDQ9Yj/CzWZym2Q24xIkZIXIYBFmNJk2s7xJ+RpM4SItPLMnCq1BbGZrMhK02fDaaOxzBxcUvC3i0VQkbzxVzpjmzD2zs65AC/uE8K6Eu35q0DAfq+YaEsB6/quryLjF33pW0t+NdnnXhze4fl8bM6USudANgwB7wBPPlavWH7GoIwgwL6wwCzVJjlMqE4HROL03DzpWVmpxEGPPzDGZrtrHQHegYCKMh2okMqcmxSffM/UNeP4nwXXE4bhvwCuvuZJRPbN/PfPHcMF5ySL2+ziZpf+Eu2TgzSl0OtRzPk17rb1NYZGzdrMzTks/4+WCabnPJsISvtg73dePHdFnh9QfzoivGW70UQJC7DABMV9n3a6bDhf2+bPXIDSmH49jrpUhymON8ti4t6cv7d87V47t/NsjUju8ni4PVp7VbazciWC9uW+5BpW7yoRW3luqOG1/UOaS0ih6C8Zz9nQAgRJLSzMfqDeisq1LGMWJY3IE5MSFyGAWa5UO1A/GFfvsvHpmHfcf2aLABQ2+yV3WIbP2zF3mN9cbm3QxUzk+tcJBGTJ3BmuMgiY70TADtfLQLsuuw+kbj35HuzsZiuG6MVS4KIFIq5DAM/uLwccyZl6taTJ2KHTZjzZmTjxgXjMLkk3fA4Zs109wWw42AvGtqGDI/jMVt2QW016C0X7et8c0oz1LGcF99tQYuqISeziKKZ9INy/Ec7NnGfYHgsg74WEZFClsswML08E7/7/ozwBxIRkykF9EsL03DJaYWoa/WipmkQ2ekO9JoEoTv7rLWUdzltIWtL1PEOJiZ8zEWepA3Sl0MxqGpy+czbTXivulO5TwT+vEBQwIA3KHc20AmIoP3dzp2rOYZSk4kIIcuFSGmWXFCMM2blYHq5uOIlWycmM938ox2PTD2Wmgyo3GI2bS0J7w6z0htMfV0AaFfFdth1mYip5/x3d3Wi+kivvP3Hl47jGyuqlcJRrVfMsAiU36ZKfSJayHIhUpqz5uThrDlKwWq21K6f1RYlEj7wDijuKj7mYlR4GQp+Quezx9Sot379dzFBYNP9pwAAXv+4XTMWvkOzuv2Mvs+Z8VgIwipkuRCjihyp/1ggKOChH0zH9xdFlu79P9+ciKmlxnEbHq9fPTlLv0jqEpTbzGjbv1hpO8O3XlFv85YLu7CZu0xx0YWOq/CxIHasT9pBEkNECokLMapglksgIGD2xCzMmZwl72OrYIbCbgMumleA274+QXnRZFY1slzY8bK1EEW2mFlWYSjLxbRBJjcWhhW3GHPjUciFiBQSF2JUwWIugrytVPBfe1EJ1v1iDgqkxcj4pZZZexl1hb/ZnKqJuXCNKv0hLZfI3WJq2GTP2sywy5mfw1kuBplrbB+ziOTOz4L2dYKwSszi8vLLL+Pyyy/HnDlz8PTTT2v2DQwMYOnSpZg/fz4WLlyIrVu3xryPIMxgwnHajBwAQE6mElbMznSgMMeFXOm1sdzKn3z36nAENG4l7QQup/2qYi6/fe4YjreEXgbACvxKlEw4zLoYy21rOMHQNt4Uf8qp1EFjcSQIq8Qc0K+oqMBDDz2EJ554Qrdv7dq1yMrKwpYtW1BTU4Nrr70Wb7zxBrKysqLeRxBmTBmXjvtunIKTp4r9x9SdlFnaco5ksYzNc8tLDAOqrtUqcTHTmaBBQJy3JNSWyls7OvDh3u6I35OaANdtOcDFRgzP4RpVGjXe/PRQDyYUKbGmYAhXGkFYJWbLZebMmZg+fTrsBu1pN23ahKuuugoAMHnyZFRWVuKdd96JaR9BmGGz2XD6rFy4pSah6s7TrGFourSviLNc2EdY3WLm4nkFIe+lCYjLribtpM97wcxqb6ygL8oMnzKsCJ10TkBvufzu+VosfeyAqk4H0jmkLkR0JDQVub6+HuXl5fJ2aWkpGhsbY9oXCdXV1dEOHVVVVVGfO1Kk4piB4Ri3+DHfsWM7AKC3xw7AjqG+ZgBK3EUIBlBVVYWufuWcInczVlwp4MFXHegdDG3G/O/LR1C1M4jOLhsAO5qaW1BV1YRjx2yae8TK8boGVFXVoaFBfA/19U2oqmpAS7cyZuV5its7d1WjKAeob2gEYEdnVxeqqqpwoNFgbIIAwIba4/WoqjqOujrxPu0dHaiqasNIkIqfaxqzBXFZvHgx6uvrDfdt27YNDkf8/nHiTWVlJdLS0iI+r6qqCh6PJwEjShypOGZgeMb967wedPb64TlVtELeOngUnzd04oIzZmBL9WH5OJfTCY/nVLT3+IDX9gAAZs+eiZOmZCPjzT3oHfTBbjNuetnSY8OmnQ5UTskCWvpQOGYsPJ4JONzbDOxqiNt7KSougcdThg/rjgMH2zC2qBgeTzmONAwAb+wHAOV5vvAZAKCiYi5aj+9GUVExsL8VWdm58HimIfB5N/DeEc31BSnqUjJuHDyeUlS3NQB7m5GXlw+PZwoA4NI7P8OFp+bjZ0smxe19hSIVP9epPGav1xvTl3I1YcVl/fr1UV+8rKwMdXV1KCwsBAA0NDTgzDPPjGkfQUTKvOk5mu1rLx6HGeWZmDc9W/O6UbYYi8Ool6geMgmey7EKA9dTPOgdCKBvMKBbRpkP9KsJ5Uozcnnx8RglC0573NZPO4dFXIjUJaGpyAsXLsSzzz4LAKipqcGuXbtw3nnnxbSPIGKlfGwaFp9bBJvNhlXfnYYvnzEGgHG2mByHkX6y5RNCwU/6VoomI+H1j9vx9XurZWHg4ylG8MkFVoL1uiQAir0QERKzuGzcuBHnn38+Xn/9dTzyyCM4//zzcfDgQQDAzTffjO7ubsyfPx+33HILVqxYgezs7Jj2EUQ8OXlqNsbmi4F9JhsO1X8Fs1zsDma5mP/L8FljVidls+7LhvexYIUw+CLKQFAUHLNzjjYNYvuBHl39DkFYJeaA/qJFi7Bo0SLDfZmZmVi9enVc9xFEvOHrWtTbDrv2NXcYy4XNwf2DATyztcnyGj4uR+juy0YE+AwwC6tKMjGpa/Hi8uU7cdac3JDnfLK/B5/s78EVZ48Vz41QLAmCGlcSJzyylhi6xbS1LyzFORRsImeTc/lYawklLqcNA9aWmAGg6rJswf2mLGss/mTp0B9+Hr7mRnahWYjtqM95c3sHvnhKPtLCPC9i9EJ/eeKEh4kJkxS7xi2mPSbNFcZy4SbfcJYOI1wsh4fvWmyWOKD0CdMeY2Xi72HLQkfQgn/vsX489GItHttQF/ZYYvRC4kKc8PBuMXXhJdvHMu7DtfLnv9hbjaW4wsRyeHxSXzO5jX4EwXmGFReX18fuI277TTLlGKxJ5/GWwTBHEqMZEhfihMeguYRqn03zM5zlwlsHXp+1dDFeW9Rta4xgvcSsVNLz8Rn5dQtDY6ttBk3cYlX7u/HYy8d1r8c7U45ILUhciBMe2S1moBt2XUDfWsyFYVYTo4afs7PSzYuTmbgoMZHwY+ItFysZYF7JQjLqR8ZY/ucjeOWDNtmaimRhNGL0QuJCnPCYWi5cM8twcQq+O/GQRcslYnEJaN1i5jEX6R7cUKzM/UO+8Pdh8aL2Hr/22DBvnbfyiNEFiQtxwmPWYp8voszOMJ/01Wu8AIpbKRwCN2k7HTbMmpCpe42/rtFEzl/LigCFgneLGbnfXNK4ZMuFa+1vxPr3WrBo+U60dfsiHhORGpC4ECc8Dj4XWb2PtX+RJtAMi7EQhtcfneXisNvw8A9n4FtfGie/5lKJiy+gdYdp2v9z1/IHwrvOQsFiRmZuMfb8lDiQcUdoNW9ubwcANHcMyWP8y+YGsa8bMSogcSFOeJh1YhZzUQf2b7/Ujy+dXmh4Lb5o0mwRLzWLzynCtReXYM6kTM19NVaV6lfZXRXQZ4vxsZS/bWnEH7c4oiqAZDEjo15p7Hrs2bBj2TFmhhKLXfV7xffx+bE+PPuvZjzyYm3EYySSExIX4oTH1C3GNa6024DCLP0SyYxoY9i5WQ5cd8k4OaajNNFUjlGPkk3kPf1+vPphq0ZQeOuipcuHxi4bBi3Gf9QMcZaL0eqbbIwsDhS0kMHmlrLu+r1iHQ2L11hNgCCSHxIX4oSHBeuNhIEFq52yuOg7J8cDIxFTvw5oLSsW36hrG8IfX6rDrsN98r5QWWDeoSjERVdPo1ybCQ3vFpMtF9V1apsHNS4vdg5zuzHXHUnL6IHavxAnPA6TQkcWRGdFlLybLF7IYsLV1ahFzKayXXh3W1efX/49VCB9MApx8fIBfQP3m05cpNuoh/G9h/YBADbdf4rm+n5+2WbKIBs1kOVCnPAoE7h+YmPiIndHtundVdEwZVy6Zpt3g8k/QySn8Vlpfarlk0OlAA9EIS7yNQVx4le7upgQMIuKb+1vhnyOX9tWJpLmnfGirduHpo4IGrsRliBxIU54mIAYfeFnrWAUi0X8Gatb7I5vTsQjP5ohb+sSB8K43/jML/WkHMotNuiNXlw6e/1YtHwnth/o0d2HCYW89gu7vUkAillhctYbZ8EAwO+eP4Zn3k/8FHXd/Xtw44N7E36fEw0SF+KEh03gZt+ZbZzlEolbzMjKyc92omyM0jFZF3MxcIsZZErLaLO4jI8ZGAoY74iAN6o65N95gZNdWxbWlmHwbjG1SL65vQO76+I7RQWDAh589ij2Hu0LfzAREyQuxAmPXJxooi58x+RI3GKZaXrflt1m03ZfZpaL3IVZux0OdQwmEBR0hZTi69auZYZaHGTLhW2z1v4m4sLGyawdts16lllp6R8LfYMBbP20E7evOajbR/Ge+EIBfeKEh8U1zC0X8Wc02WIZaXa5db18Tzu3bowFy8Xsjuo045+uOYBz5uZbGluay265uSagFQ5ZaKSx8yteCvxxENOV02GXLS2/3MZGOjbBE7xZTMfrDyIzVJCLiBgSF+KEx2kgFH+6fZYmyCuLSxQxF9Fy0Vae2+024xUvWXaaTbutHoMR6jTjtm4/NrzfamlskSYmqK2fIB9z4VrEMONJbVX5uAA+H3Ph3WTxxswy8g4FDa1MIjrILUac8BgF9McXpcMzU1kGmM8WM2t2yWO0EJjdpnV5KYF8btuiiEVTIBnJ9Rlqy4UJgewWC9Eg06fKbOOzwnSpyAYpybFYMwfr+7HsyUNo6VTazIQiEguOCA+JC3HCk5ctGvCRfIs3s1zS3doLGU3gdrtN87q+iDKylGerDTL144jseG3MxXgfX0RpZrlsP9CD1etrVWIjHacSAV8MwaLNH7fjs0O9cpab2WJnrKbH5w9i5+HeqO9JiJC4ECc8ORlOfO+yMtx745SQxzCrhrmA1OJy+9cn4Oy5efK2iyvKNBIIXm94d5sS4LeWLRZtfYjDzNdmQNCg/Qsbl65Ds7RbXZPDrBg23mPNXmz6qB2N7UOG+8XXlEm/uTOyehQ2piEuccAIJmLP/7sFP//TobgLzMG6fvz67zUnTOIAiQtBAFh8bhGml2WG3M++jRsF9Od7CvH184tCnms3mMB5a4Z3tzkMA/qhhcCquPBDidgtpmn/wrvFBN0xgLZfmI+LtTA6esWYlFExJTv/xXdbcMOqvahv81oeL3uurIDU7DkxEeuUxnKofsDyfaywct1RvLurC40nSMEmiQtBWIB94VViItr9Zm4ymw1YedMUXH7WGPk1q5ZLvJOX+JU0HXZg+XWT8YWK3BBnhEYpomTZYtrXGUZuMf7be/+geLIvIKZRa8VF3McsibpW6+Ii18/4tT+NYGNzS81D490toF/qosD+toGAgP9Ud+q6LYwWSFwIwgLMtcX301L2h87qsttt8MzMxYzyTNUx5paLkYV0/sl5iJU0t/6+58zNw6VnjAlxhha1UcLcXzrLRSUyP3hkH97d1Smf87ctjfjjS8d1EzfrjiwIwIZtrRoBYZM+i2WxwLsgCFj3ZqNmwbEjDQN4S1orRj1eK3U0fA1OvJdpZu+R3WfnkV6sXHcUf3+rKa73SRYoFZkgLOCS1IVNTrw7SR1XcTntAJS6FqUwMrR1w0TF6bBrttXnfO+yclw8rxBLHzsQzVsAAKTxY5Pva809pnZn8RZKkOst1t3vR0uXDzWNg/Ixnx7qxaeHelGQo516+lWtadZsrNf0XmPf7OU1YCQrp7bFi6ffakLVgR78/gdiK50frt4PALj4NHG9Hb6extQtJh3Dnnm4ZZojhVl2ivtNbDba0D463WRkuRCEBZh7irlz+CC9WgS+5Ckw3Gfm4mKWSrpL62JS65HTYUOhalK2Kghq0kJkslmt21FPzh/u7cYHe7vkCZwvovSaZLANcWm/A15tkWlti95yYe+XWS7smZnFMOQYjpwUEN5ysSfIcmHwyxjE2gQ1WRmlb4sg4svF88RvwmdLle9mbrGJxenYdP8pmDk+A4AyWRkVazLYxMkWC2NZU/xyAOr7GNXPhIMXJL7djNmxPC9va8W9f63RLSRmpSsy36HZXIi0+5QCTMkaMRAMQRY86RyDvmeCoG2To7jF4mO5rHurEZ8e6tG9zu7DlkCI5ktCKkDiQhAWmFQiCsb4IrHZJJ8BphYB3hqwWbAOmFAwy4JNQDoRc8QmLhOK0jTbRllp37+8DAtOL9SlVIciyE3cVr7xh5u4tanIQcN97BkZ3U5OHOAaYqqv+/aODnxe26+cE9BaFLEG2p9+swl3PnnYYGxBzRhHKyQuBBEFvIvLsAlliKp7I2Rx4TKVzCwkPvPLCsX5bqy/txJfPnOMZkxq0fLMyMXSKydYjscEuSyxeMcq+LRlXlyMYOnLQRNx+e3ztbj9caWBJV/gqbaYhnxBSxaZcq3QD0Gu2wnon9fjG+qw5pU6y/dJZkhcCCIKzCZ93VLFJpZLVrr4L8gSBuRWNPK1+Psqv1u1LNTY7UC626Fattlg/GzVTck6C2ch8TGXSCZhK/AxCjYps9cFg5ajQ1wxppXFyHTiIl0jGBRwxd27Ipr0B0zWzhni0rHVlt6G91vx8jZrfeGSHRIXgogCXijU8RQ+jVhpQqm/jlJbwU3gcl2NieXiivzfl28rw6+Aqb4H2xfOQmKTfv9gEE0dQ3Fp7a9m4/ttePK1epVAaN1KRlrGu8UCAQGN7V6dFaSGb6bJLBe2yucrH7RZHrPZqp98F4LRWrFPqcgEEQW8EaKNufA/Q1subIJxObSWBPs2axbQd0cRc+HdYMqyysq1ZKtG2pfhtqPLZG0t5tZ5a0cH3trRgUtOKwh9cBRU1/ShuqZPLvT0c8JhHHPRtvJ/Z1cX3tnVhXMrQ9cKbfygFQKUv4lXukZ3f/hF1g43DOBHq/dj7U9nS/c3S3nWjt9M8FIZslwIIgr0k776d5vhT6NssfNPzhf3sevxxZXcf6h6O5qAvmxF8Q0yDRIS2L6MtDCWCze7q4saIyHc+wnl4uJb/QN61xPjuEl1f3OnD39+vUFJW5ZSnq2s4PledRcA4E2pgNNoyQD2p2UWEROV0dqNmcSFIKKAt0I0HY55i4VZJQbi8tNvTMQ/ls+V018nStlcp0vt/vn7qCv7XVEE9O0OreAZFWsy9x0TzHDiwgfwB01cQmbkZJj3uuED4bK4cFX4gFJHw0/yVkpX2GSvrDWjP2b9ey1Y+XSNvJ2dLo6ducOMxIU9YjkrTY7tMAtsdFkw5BYjiCgwC+jzMRenLDL66zgdNuRlKf+GE4rT8ezyucjJdBjeR3vPKMbNWS6sGaZhQoL0M9wCWnwA3yzeoKYwx4n2Hr+8nZOp3ebxBYyD9MGg3koZ4oLzDK+FsXX3+6X7hU4CeOLVevEYfxAup122SuR2M6pz/AEBToe4xEIgKOjGpgihcv1AUIhoQbpkhCwXgogCXeNJ1bbSH0z8aRTXMCM3yylbKJGst2Ll8nZO6OTYjpFbT7peWLcYb7mYZEqp+cYXi3HLojIU57sAWLdcdOLCFXECwLHmQTS2e3UWBAvOm9EjxVj42I4RLCvMzomL+hxmCbE/z4Hj/fj9C8fkbDTlfSlvgAnl8RYv3t7REXbMyQhZLgQRBWbuKj7WEmmLFbP7qHFypovbZQ/rkuKtKjYFqmtZ+NhQRjjLRYjOchmT68J5J+Vj00diFlZhjvl0xK7LL43MUFsLj74spg1PLNYWjfZ7LYjLQEBzH7P05YGhIHKzlOcodwZQneMdCiIr3SG77d7f0w0AOHVaNgDFytIukCbA7QL++NJxfHa4FydNzUJRnjvs2JOJmC2Xe++9FwsXLsRXvvIVXHXVVdi1a5e8b2BgAEuXLsX8+fOxcOFCbN26NeZ9BJEM8AF9Nbq1WRyhA/ph72Nyjrq5I6AUYJqhi7GwlGeDmBEjwx0u5qKdfK3GXJigsdgR3/eMh/Uf8w4F0drl02VZGVkYvABZSZMelAL4Vmpj2HuVU58NrCi2jECo2JRRxT675/46sYMAa9aZSsQsLueffz5eeeUVbNiwAbfccgtuu+02ed/atWuRlZWFLVu2YM2aNVi+fDn6+vpi2kcQyYCZu4p3gzk4kYkEM3GZVJKO1f89A/Omi9+A0/haGQNCWS7aJQO018kMF9Dn5l6ryxIzcWEp1U6HDRtXnoyrLyw2PJ51Tv5kfw+uf2AP2nuUrDSfP2goAtGk+TJX19GmQdz22AH09IeOAzHBkws6Bb0g/d/rDdhxUL+qJbOiBn1BvLytBU2q7sh8DYxZK5regUDUSRSJJGZxufDCC+FyiT7TU089FY2NjQhKEr1p0yZcddVVAIDJkyejsrIS77zzTkz7CCKZyDaIE4Ryh0W6Xj1gHkdx2G2YUZ4pu60islykYbPJ0KzFS3oYtxif5BSq/cvpM3NQOTlL3lYsF0VcHA4bXCHeB1/1rm7l/5X/twsPPntUdw5rax8JarH8vLZfdmMBeitNXlmTiweprah3dnbirv/T9xhjYtndF8CaV+rxzFZlXRf+OnzzTjXfWFGNWx76PPwbG2biGtBft24dLrjgAtilT3B9fT3Ky8vl/aWlpWhsbIxpH0EkA2NyXLj0jEL87vvTdfv4nmLOGGIuZv3IFBeXdB8Dgfjvr5bL3ZnVY9ClUpsMzeWw4WvnFaF8TGw+/59+YyJ+tmSivK1YLlLrG2kQVtva8PGTPUf7dccMGRQzpodxv/GwJZgBvSWkr1kJnwTA4JcZaFItHcAXYapb0by8rQXdfVrRbO6MrrYokYQN6C9evBj19fWG+7Zt2waH9BXo1VdfxSuvvIJ169bFd4QxUF1dHfW5VVVVcRzJ8JCKYwZSc9w7dmzHOROBltpmtNSyV8V/p717dqP1ONDSagdgR2NjPaqq6tDnVY6J5D0X5zowcYygOke8xpHDh5HhFdDVKd5ncHAASk6SiL3/KL71BWD5C+I5tceOospRg9pjNgAO9PT06a7Ltge9DgA21NXV4qzpAmpq7ahrs8MGAQIiF8pduz6TrBrxPocO7MNQO9DXK46/taUZVVWNaGwQxxaOts4+3fu1gtsRwGAE57Wr7vPxJzuQ4QbYe/h83wGgR0B9vfgemlvagZnAgQOHEO499HFxlP6BQfk+O3dVoykPsEH8G+zZewDBLgF1HcCat5x4d/txXH0WOz/yz5QR8f4/DCsu69evD3uRLVu24KGHHsJf/vIXjB07Vn69rKwMdXV1KCwU18JoaGjAmWeeGdO+SKisrERaWlr4Azmqqqrg8XgiPm8kScUxA6k57pBjfuEzAMBJJ1WifGwa3jtaCxxpx4QJ4+HxFItpsK+IX3giec9P8YdK95k5Yzo8FbnYeugYUNuB7KxMoHNAc+hJlZXiMgHSOVOmTIbHU4guewfwyTFkZmXB45mhuS4bW9rWvUDvECZPmgiPZyz+c6wWqGmHw2GPan350+adKmaWbdwNAKicW4Hp5Zl4Y/9R7K3vRHn5OHg8pWgKtAGfHg97PcHmBhD5N/a87HR0D4Su1OcJQLnPnMqTUZjjkp/VxMlT4TkpX/xbH2pHRlYugE5MnDwF+OBYRONKc6cBEK2XmbPEZ+PcsAv+oaB8n7QjvcBbh+C3ZcPjmSG6NV/YCQCYN+80U0vXDPaZ9nq9MX0pVxOzW2zr1q24//77sXbtWowfP16zb+HChXj22WcBADU1Ndi1axfOO++8mPYRRLLDx1YcnPsq3vdR6mpsyM92Gh6jjEVb5xJJbYycBRdlbZ/dpnR/FsegzaKL2C0WZQZVpItzqWM9vLtK7hog/TzSOIiHNztQ3xr50sVqvWZuNrmq36+t5uePM9o30sT8cb/zzjvh8/nw4x//GFdccQWuuOIKdHSIRT8333wzuru7MX/+fNxyyy1YsWIFsrOzY9pHEMkOmyRZ0hWb0FnAnbUKiRW5A4DcXgb427I5+PlVE3XHMPgW+zYL86yuH1mU6mK32zTCwX5nY3A6tQF+AJhSFFpArNSsAPpUaqfDhu8vKpPrTMIx6FOLi3Y8Shqx+LOz14/WHhtefr/F0rXVqBMjmGXInjWLufDLQ6tTraOxJhNJzEWUH3zwQch9mZmZWL16dVz3EUSy45QnTW3w3Gaz4Q+3zkBORnxql/maFZtNvLd6Atc3vtRaCXzasRo22fFt+WOxXNTjUYuieH1t3QsA3HBeEMGsqVjxtxrd9awuG6NbHsFhwxXnFKEw14VPD+lThHnUk/Z3f78P3/lyqbwtt3LhNNAsuysU6gaW/GJxbJ9snUhvSd1PTRS4+HxxiQfU/oUg4gz7Bm40B08vy0RJQXwqrXVdl7mmlADgsLGxaNu82COxXDiLhV9jxip2u03byYBzT7GsMZemWwBQJLWHiZZB7ts+nwINAFecPZZrgRP6ek++1iD/zqcgK69H7rLrV7Wm+cXaw1jzSp0svIpbTLouK9ZMYsuFxIUg4gy/ymOimt3qFiUzcFvxLiy+9sZMJuRuzjq3WLTj1W4zEWHpu/KqnFzrfVeMwaqTpoi1NQVSexlZXFTXvXFBKZ65a668nWXRdfnWjnY8+68my50A+DV41IWvfMzk5W2tOsuFFxBNJ4DRFnMhCEIL7xZLVCv1UG4l9STOz8t8axozI0QJ/nNuMQt+MaPGnvrKf3ECZ9/yWbEmLy7qVTojrVEBxOULNtx3EuZMEkXGyHJxOmyaQL9VcTncMIi/bG603JUgnet2EK5vG/vsyBaSvPqm3mJKtkXHSFwIIs44uUB1ov7leTeYvG2w8BcbSzRWh7xKZgSuNL5PmJEgscmdrfTIEh14S0W9zS8HbSXzy2EX4zjsfRiJCzuGYVVcGHzNSigy3A5u2/wP4pc8ZXITTW71zYBGXJKrBQx1RSaIOMO+obNJOFQ7lFjhxYVfXVL8XfzJJiN2Dov7lI9V6sD+tmyOZoJiV2GTcCQdBtLddk0Kr9Gp7DktOL0Qu2v6MK1M7CTAu440FgZ3ocw0e9hliB2cmBiJi81mg8OuTNRZYdr/81hp5a++NyPscgZyexmtW4xZLupOAP4kc4uRuBBEgmAuD77NR7xwct/EmRoYrc0icIuClY1Jw++/Px2TVJ2Vx+YZB87ZN3r5ugLwh/+egdc+asOmj9qNz+EmUbXlcs1FJRoRm+8pxHxPoep+fKBf675Sk5nmCCsufPdlo5gLoHXbRZoubjUtuo5bZjkzzH2YePCJA+wLiyZbjNxiBDE6+cU1k3DpGcokOTZXnKytpsxGipOLhbCp0Sygr570KyZlhV1lUn2OXRYqYHp5Js6akxfyHN5Dox7F9fPH4dsLy0Lfj1u+Wb190bwCTRDc6Jv/mFztd2YmqG4Ty4Vn7uQsTCi23t3DqKDTyFqrnJKl2Q63QJpfJS6CIOgtF5WgrH+vBX/bkjw9GElcCCJOnHdSPn68eIK8veD0QtzwpXH4xheLEnI/JzfpM7RLLmvPiaTtP4tv8G35bdy2EZNLRIto9oRMzTWswFs9agOjYmIW1t97kuxCM4qNPPCdafjDf8+Qt426L6u3jcjOcOCJ22bDMyMHQPhYlVEasFGcadb4TM32uDBp6Uw83trRgSX37ZYbVhqlQL+/pxt/f7tJf5ERgsSFIBKEw2HDVReWIN2dmMI2PuZiFLQ3s1zCkZ8lWgAsDTZUKxkjcrMc2HT/KTh7rmjdRJIwx1sjaneVyynWyrD3ke62o3JylsZ1lpXuwPRyZRKXizMdfPfl0NOfixOgcIF3I4yej86tl+7ArYvHY86kTN2xgNbq7RkIoEnqfiyvxplcMXwNJC4EkaIYFU0C5kWOkVguP7lyAq44eyxOniK2SXHYQlsUPGwMkfbxAszTc/lJ3+mw4Te3TMf188epxsWNk7X25ywxM8vFyVk54VKGxfvqt/mlm3VWpsOGL58xBlPGZRheg4fF7+SVLw18rolKfY8UCugTRIrCVoh0cPOe2QQVyWRfmOPC9y9X1lWy6xIHQl+Lb6oZaVH/dReXiNZH/wHN6yFdXKr3xQsob6mwsZiKC3cfKwuxZaY50DOgBPadDhvW3lGB/cf78fM/HQKgz4RzcvEgt8uuWxRNDVtgzOcP4kjDAHoH9IkE/oBg+t6GCxIXgkgx1iydhbZun753mbTfrMgxGkuCoSvINLlPqEXJrHLtJaIlwi8xojS3FAcjt4xRBf359+iUxJcFwZnImLkIdTUxFp5bRppdIy4Ouw3pbjtyMhX150WKz2RzOWzQLpqghbWI6fcG8cPV+zHRIOnAFxDgSoKZPQmGQBBEJEwqScekEiWFWOcGMpnQI4m58NhtWhHjLSajY620mYkEXhhcTu1PQF8LwywZ5kJi42bizLuuAMDp0IqWFXeimHmnrC9jN4jt8BYFbyGFu08/Z9Uca9avS+PzC0Dky1jFHRIXgkhxZDHhWrsYEa6uwsp9+KUEAODmS0tR1+rF9gM9aO70mS69HAv8ZMzcSWZuMTZOFqdQi8+q705DZrr+gbm46/KCZQR/HaOsNLczzDEC8JefVWD9ey14eVur7h5W6mmSpccYBfQJIsWRtUXaNrNceJ9/JOhjO8q1Zk/IxE++NkGXHh2tWywUSnNL6afLuNmlGidnPam7Epw8NRvTy/SZWixjjV2XrZWz8L8KdccyZk/Mwq1fHY9pxaJ1wQRWKy7GlossOjaxe8Lcydp6GIYV4UiWNjAkLgSR4vDTjdG37LxM8Siz9VvCwWehaVr761rQSOewGSZOGpMtrYXjkidlFhsJPZUxS+bK84vwo6+U48JTC8LehxWXqvvEjc1zYd70nJDnpDlt+PKZY1Ag6YKyPo3KZec0foZyoapBskGk/eCSxXIhtxhBpDgs89QsC+qHFwcwYeqcmO6jiAkTEmUfb6kkynLh06/lGIyJ5cIEKN3twKKzxppef1yBG40dQ3LgnV2fPWOz+/Ddo+WeZqoHxZ/NW0jsCLX7LNPtQK/F3mUAiQtBEHHGbALMSoNcSxEtilAI3LY+9dge55jL/d+ZqlndUZDGwGpXzCZ9KzUqjAdvmYZtu7uQny2t/SJdl1XCq2M708oycKheye3i2/G4DZINeDLlZQa0adJq91l6mj1CcUkOtxiJC0GkOKy9PaukN3MRxYIsHDa9VcK/Fu9ssVOnce4oJqQWUoUzw3QeVlOU58YVZyvteth1ZXFRWRTLrpqIvGwnrrpvN4KCaukDOdbCGn6ajC3UAmmq+0Sa4ZcsDSwp5kIQKQ5rkOl2cd2L44xuuWNDy4V3i4mvxxDqMcVm04+FMVnq+ByJ5cIji0uQiYtyn4w0B3IynPJ74y0XQ6tN9SCmjEtHaWGa5rpsr9pyiaSrAkBuMYIg4sTM8RlY+rXxcpfiWIL2ZvDBenX2mG6fHNBPzFj46dPoLd9/8zQcqu+PyTXHJv1AQC8ubqcibIGgICdSsHVhzN76WXNycff1U5T7cN0D1ILCt+l3OmyGjTIZySIuZLkQRIpjs9mw4L/GIDcrsd8VeevEyC0mb/NrzcQZPonBSFzys53wzMyN6T5O3i2mej9ymjJzAco1MZHfR1dcaVeLmPaC6op/IygVmSCIhPG9y8rw48Xj43pNxS2m3Va/ZuMsFyXmEl+RGRxi8aXEXJ/BYh9GMRdmbbDHoCwzbXwts7VbmBVk44QKENcJuu3KCfL5eZnmXyJ+te4ojrfoK/eHG3KLEcQoZPG58V9DRq5lMQjo6zozJygVmfHVc8fiUMMAzpnLXIHi64nqCMBiLkaxEF5snQbi+9dlFchwO7DzcK/hfeRsMXZt1bnjCt04Y3Yu/rK5AYCx5eJy2jTusJauIYwvGtkeMCQuBEFEhJI1pnpNdk/x2WLSAXHWmOllmXj8J7Pk7QTlC8juqu6+gGZbDb+8QLrBatFFeeaLghVI/c3YImhOlX4oFpH4s3xsGqaXZeBg/QB2HemDww489bMKdPb58cNH9ovjTJA7MhJIXAiCsARr7142RvxGbDe1XKTXh2mSS1Q2WlGeVin4JZgBlZUmdw0QXw9GsK5KYY4Lq747Te5yrH5ufIfmNJcd31tUjnVvNmLXkT7Y7TYU5LjkbEH1sSMJiQtBEJY4dVo2llxQjCvP07vc5JgL2zZwnSWSRGXIFeRw4mIwacstW6R9BVmiqLBVOI0w0p2Tp2bLv6sD+k5OXPhO0Oxa6nb+fJuZkYDEhSAISzgcNty4oNRwn5wtJqcgS+fEueX+cMOP28giCEizO5vci3OBfyyfizyD7D2rGqi2XPhVMfk1YJiFpK5vMlvCebgY+REQBJHy2DkRYaLCuhifUxn6W3xc7p+oIk0LC6QJUuZvuls52EhYIkFtufBZaaH6qamtN3KLEQQxKuA797Lt/GwX1v50NkoKzAPasZKomAtfv2MEsxzU4hLzfVXikubSur+YJSPXvxi42JJBXMhyIQgiZtgkLDfPVLllysamDUNgn9WJJPg2BrC4DBMBK0RSQ2/TPVttzMUIyhYjCGJUwCYz2f8ffTuvqEiUW8zour++eapme8WNU/D29g4U54e3zmIZJhMkfi0bI6FKi6MVFS0kLgRBxAzvFktUZ+aQJCrmYmAK8QuGlRam4dpLxsX93mfMztUsaywI2v5mRmnRjHQXiQtBEKMANgkHpeC2lTXn43v/1LquFe751mRNyjL7lU9JNkprHq76IjNIXAiCiBv8BDhc2Lifcbtugt6GlfpKm82mvb+8GJw2WyzSZZCHCxIXgiDiBnPdDLu4JEgFrGSLRUQMlwtyK42ypAl1oeqapbNQ2zIY/U3iCIkLQRBxQ06XHXZxSdB143y9k6Zko3xsGq6fXxLxufwiZGyJZ7W4TCpJx6SS9NgHGgdIXAiCiBvBkRKXRF03zhfOSnfgyZ/OjupcN+cOUyyX+Iwt3sQ8rMcffxyXX345vvrVr+KKK67Aa6+9Ju8bGBjA0qVLMX/+fCxcuBBbt26NeR9BEMlL36CY3ZRtsnZJIkiUW2wkA/o87JmyZImMNHH6vvDUghEbkxkxWy7XXXcdfvCDHwAAmpqacOmll+Kcc85BXl4e1q5di6ysLGzZsgU1NTW49tpr8cYbbyArKyvqfQRBJA/XXlyCww0D8nbZGDca24eQnz28TpHEZYslj7pc9oUxONw4gPFFottrTK4La5bOGvF1W0IRs+WSk6PkfPf398NmsyEo5SNu2rQJV111FQBg8uTJqKysxDvvvBPTPoIgkofrLhmnWQt+2dWT8OD3piErfZgtF/YzebQg7pxbmY9nl1dqhHtSSfqwdZ6OlLh8vXjmmWfw1FNPobGxEb/+9a9RUCCaafX19SgvL5ePKy0tRWNjY0z7CIJIXnIynDhpSnb4A+NNcs6vJzRhxWXx4sWor6833Ldt2zY4HA5cffXVuPrqq7Fv3z7ccccdOOuss2SBGUmqq6ujPreqqiqOIxkeUnHMQGqOm8Y8fFgZd+8gADgRDAYT8D6dlsfBSMVnHe8xhxWX9evXW77YrFmzUFxcjI8++ggLFixAWVkZ6urqUFhYCABoaGjAmWeeCQBR74uEyspKpKVF7o+sqqqCx+OJ+LyRJBXHDKTmuGnMw4fVcXf2+oGNu2G32+HxnBrfQbzwGQBYfn6p+KzZmL1eb0xfytXEHHM5dOiQ/HttbS327t2L6dOnAwAWLlyIZ599FgBQU1ODXbt24bzzzotpH0EQBI8SayH/WLIQc8xl9erVOHjwIJxOJxwOB5YvX45p06YBAG6++WYsW7YM8+fPh91ux4oVK5CdnR3TPoIgCJ4kjWmf0MQsLo888kjIfZmZmVi9enVc9xEEQeggcUk6krS2kyAIwjo2Upekg8SFIIiUZzTXt6QqJC4EQaQ8JC7JB4kLQRApz4lQoZ9qkLgQBJHyWFh7ixhmSFwIgiCIuEPruRAEQZiw8L8KcfJUqrOLFBIXgiBSHpe0ONl8T/x7Gv7kaxPifs0TARIXgiBSHpfTjn/eUwm3izz9yQKJC0EQo4KMtOFdQ4Ywh2SeIAiCiDskLgRBEETcIXEhCIIg4g6JC0EQBBF3SFwIgiCIuEPiQhAEQcSdUZmKLAhip6GhoaGor+H1euM1nGEjFccMpOa4aczDRyqOO1XHzOZMNofGgk2Ix1WSjJ6eHuzfv3+kh0EQBJGSzJw5Ezk5OTFdY1SKSzAYRF9fH1wuF2zUg5sgCMISgiDA5/MhKysLdntsUZNRKS4EQRDEyEIBfYIgCCLukLgQBEEQcYfEhSAIgog7JC4EQRBE3CFxIQiCIOIOiQtBEAQRd0hcCIIgiLgzKtu/RMuRI0ewbNkydHZ2Ij8/H6tWrcLkyZNHeljo6OjAz372Mxw7dgxutxuTJk3CihUrUFhYiIsuughutxtpaWkAgDvuuAPnnXcegJF/P6HGZjaukRzz8ePH8aMf/Uje7unpQW9vLz766KOke86rVq3C5s2bUVdXh1deeQUzZ84MO5aRfu5GYzb7bAOhP0MjOeZYxjVcnxWjcZt9vmN5TyERCJnrr79eeOmllwRBEISXXnpJuP7660d4RCIdHR3CBx98IG8/8MADwp133ikIgiBceOGFwr59+wzPG+n3E2psZuMa6TGrWblypXDvvfcKgpB8z/njjz8W6uvrdeOK9tkOx3swGrPZZ1sQRv65h3rO0Y5ruD4rocatRv35FoT4P2sSF4nW1lbB4/EIfr9fEARB8Pv9gsfjEdra2kZ4ZHpef/114YYbbhAEIfQHIhnej9HYzMaVDGNmeL1e4cwzzxSqq6sFQUje56weV7TPdrjfg9kkpv5smx070mOOZlwj8VkJNU7+8212bLTjJreYRENDA0pKSuBwOAAADocDxcXFaGhokE30ZCAYDOKZZ57BRRddJL92xx13QBAEeDwe3H777cjNzU2a98OPzWxcgiAkxZgB4O2330ZJSQnmzp0b8r0k03MGzD/DZs82WZ670WcbSN7nHum4kuU5A8af72jek9m4KaCfYtx3333IzMzEddddBwBYt24dNmzYgBdffBGCIGDFihUjPEKFZB5bOF588UVceeWV8nYqv5dUgf9sA8n73JN1XFbhP99A/N8TiYtEaWkpmpqaEAgEAACBQADNzc0oLS0d4ZEprFq1CkePHsXDDz8sdyxl43O73bjmmmuwfft2+fWRfj9GYzMbVzKMGQCamprw8ccf4/LLLzd9L+z1ZBhzuLEk+3M3+myz9wQk33OPZlwjPWaG0eebjR2I37MmcZEYM2YMKioqsHHjRgDAxo0bUVFRkTQusYceegjV1dV49NFH4Xa7AQD9/f3o6ekBILbKfu2111BRUQFg5N9PqLGZjWukx8xYv349vvjFL6KgoMD0vQAj/5zVRPtsR/o9GH22geR97tGOa6SfM4P/fMfynsyglvsqDh06hGXLlqG7uxu5ublYtWoVpk6dOtLDwoEDB7Bo0SJMnjwZ6enpAIDx48dj2bJluPXWWxEIBBAMBjFt2jQsX74cxcXFAEb2/dTW1oYcm9m4kuFvsGDBAtx11104//zzw76XkRrzypUr8cYbb6C1tRUFBQXIz8/Hq6++GvWzHY73YDTmhx9+2PCz/eijjybFczca85o1a6Ie13B9VkJ9PgD95xtIzGecxIUgCIKIO+QWIwiCIOIOiQtBEAQRd0hcCIIgiLhD4kIQBEHEHRIXgiAIIu6QuBAEQRBxh8SFIAiCiDskLgRBEETc+f8Tjc+Gcms6XwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_train = pd.read_csv(f\"{yahoo_train_dir}/{key}.csv\")\n", + "plt.plot(np.arange(len(df_train)), df_train[\"value\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 355, + "id": "c37f7834", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 355, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAD7CAYAAACmJ9mYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABgD0lEQVR4nO2deWAV5bn/vzNnS3KyQxKSsATCIhAFPVqutWCxUmNvqdDtYsH2tljv1Wor1Suhte2V2qK9bVV6LV5vKbTVeulPC65QQKlQUcSjQsJOIASyQUL25azz+2POO2fmPXPmrCE54fn8c5jzzvLM5PA+86yvIEmSBIIgCIJIIuJQC0AQBEGMPEi5EARBEEmHlAtBEASRdEi5EARBEEmHlAtBEASRdMxDLcBg4Pf70dvbC4vFAkEQhlocgiCIlECSJHg8HtjtdohiYrbHiFQuvb29OH78+FCLQRAEkZJMnToVWVlZCZ1jRCoXi8UCQH5AVqs15uNrampQUVGRbLEGlVSUGUhNuUnmS0cqyp3KMrvdbhw/flyZQxNhRCoX5gqzWq2w2WxxnSPe44aSVJQZSE25SeZLRyrKneoyJyOcQAF9giAIIumQciEIgiCSDikXgiAIIumQciEIgiCSDikXgiAIIumQciEIgiCSDimXOGhpd+PWVQdw8FTPUItCEAQxLElYubz88stYuHAhZsyYgeeee04zVlVVhXnz5uG2227DbbfdhnXr1ilj/f39uP/++7FgwQJUVlZi165dUY0NB/5R0yF/VncMqRwEQRDDlYSLKKdPn44nnngCzz77rO74XXfdhWXLloV8v379etjtduzYsQN1dXVYunQptm/fDrvdbjg2FBw81YPdBztw76KxAICGVhcAoHR06hVKEQRBXAoStlymTp2KyZMnx9zkbOvWrViyZAkAoKysDBUVFdi9e3fEsaFg5f/W4vV9bWArQrvc8qfVQl5FgiAIPQZ9dtywYQMWLlyIe+65B7W1tcr3jY2NKC0tVbaLi4vR3NwccWwo8fpkpcI6I/gC2wRBEISWiG6xxYsXo7GxUXds7969MJlMYY9dsWIFCgoKIIoitmzZgjvvvBM7d+40PCaZ1NTUxH2s0+lUbcmPab/zI9jMwMWLIgARp8/Uw2k9AwD47ZsmXD/Zj6snDJ3C0cqcOqSi3CTzpSMV5SaZo1AumzdvjvvkRUVFyr8XLVqENWvWoLm5GaWlpSgpKUFDQwPy8/MBAE1NTZgzZw4AGI7FQkVFRVwN5JxOJxwOR/CLFw8AAK68ahay0s14+3Q9cKYdxSVj4XAUwu+X0PjiQby034Q7vzgr5uslgxCZU4RUlJtkvnSkotypLLPL5UropVzNoLrFWlpalH/v2bMHoigqCqeyshKbNm0CANTV1aG6uhpz586NODaUeL2yVSIG/GJ+v7ztIfcYQRCEhoSzxV577TX84he/QFdXF9588008++yz+P3vf4/Jkydj5cqVaGtrgyAIyMzMxLp162A2y5dcvnw5qqqqsGDBAoiiiNWrVyMzMzPi2FDi9WtjLl6f/OnxknIhCIJQk7By+fznP4/Pf/7zumMbN24Me1xGRgbWrl0b89hQ4uUsFKZsPF7/UIhDEAQxbKFc2hhg2WEBnaJsk1uMIAhCCymXGGBKhMVafIrlQsqFIAhCDSkXHZ7dZcIftjeFfK9YLgEvGHOT8e4ygiCIyx1SLhySJKG+TcD/7TofMqZYLpLWcmGfBEEQhAwpF44Bd/jgvI9zizGLJV7l0t7tgZuSAQiCGIGQcuEwip8wy4V5wZhS8evohwudbpxu6je81td+fhgrfnsiPkEJgiCGMaRcOIziJ7zl4gvUuehZLt/97xO4Z+1xpdllOE41DcQpKUEQxPCFlAuHUVqxl0tF9vq1ykZNR48XANDnIrcXQRCXH6RcOIzcYrwy4ete9HAFYjgujx+vvdcKt4eUDUEQIx9SLhweX/jJn/UWU1KRo8gWGwgok/eOdOLplxvwf7tawu5LEAQxUiDlwmEUc+ns9eLY2b5gKrIvvFuM4QooF5aF1trliXgMQRBEqpNwb7GRhtfALfbMa/K6NrMmyU00g3Uu4c/HlEqwkzI0xxIEQYxEyHLhiGbSZ7UpXh3LhbdImHVjEgXN+Y0UEkEQRKpDyoUjGoOCubr0KvR55cQrEb4vGUEQxEiElAtHNJM+c3Xxqcny8frnC7aMgWabIAhiJELKhSOaQDuzXM60DODB/zmJ7n6fMhZquWgzzHxcAWas/P3jdhyt743vYIIgiEtEwsrlkUceQWVlJb7whS9gyZIlqK6uVsb6+/tx//33Y8GCBaisrMSuXbsSHhtseMtDr8J+QKldkXCorhfvH+0KHu/TVy7sk41Go8Q8Xj9e2nNe03/s8U31WLHuZMRjCYIghpKEs8XmzZuHH/zgB7BYLNi1axdWrFiBnTt3AgDWr18Pu92OHTt2oK6uDkuXLsX27dtht9vjHhts+ElfTwfwzS37BowsF+33TFfxcRoW8Ffz8t5WrN/ahI9P9uCn35ykK29rp4fiNwRBDDsStlzmz58Pi8UCAJg9ezaam5vhD/iAtm7diiVLlgAAysrKUFFRgd27dyc0NtioDQ9JknQnbt66USubiAF8KTROw2I3p5v78d9bzin7Nl90AwA+ON6tPYfq33c8dhj/+osjcHmiubvw3P/0Cbyxry2xkxAEQQRIaszl+eefx6c//WmIonzaxsZGlJaWKuPFxcVobm5OaGywUVsu615tQG2DcWdjgA/o66ci+/3afdXuM6ZcHv79Kby+rw3nO2Slwnvk1ErMw7Xqb+kMXv+FXS3oHYg+qOPx+nHsXB9+s+Vc1McQBEEYEdEttnjxYjQ2NuqO7d27FyaTCQDw+uuv49VXX8Xzzz+fXAkToKamJuZjTp4VAMj39Oq7bXj9vVYAoS4rNV3dfco+z24+hOsm+sEebe2p08jynUJ9vXzerq5OOJ1OXOiCso/zw49htwG9/SYAAt531mBsPtByQQTT/06nE519wWPe/0A+RvkTCvI+hxsE/PldE46cbMRt1+gX0/gl4NR5AeWFEgQBmvM6nU4AwF/2ydf96pzBL8hh10wlSOZLRyrKTTJHoVw2b94c8SQ7duzAE088gY0bN2L06NHK9yUlJWhoaEB+fj4AoKmpCXPmzEloLBYqKipgs9liOqZTbAf21SvbfslYsQCAJFoByH6pvSdEfHjGDECelMdPKIPDkY/a7haguhnZ2TlwOCahrrkf2H4cAPDG4VzMn50Li7kJLq8PJeOnwDEtG3vPngNOy64qh8OB+vMDwBvHAADTZ1yJglwr8OIBAIDbK8DhuAb9tg7g3TMwp+XC4ZgIANh3pAutXW788xz5b7P5HxewcU8jfnJHGf5pRg6On+sDcEK5DgA8HDjv4/dcHdPzixWn06lcM1UgmS8dqSh3KsvscrnieinXI2G32K5du7BmzRqsX78eY8eO1YxVVlZi06ZNAIC6ujpUV1dj7ty5CY0NNvEEx/s4F5Q2BqOtb2GuLvVlDtT24MmXzim1L+x4ptYy02RLql/Vvt/l8WtceAOBmAtLDHB7gmP/+cfT+O8tDcp200VX4FN2v7V3y8sDCDp6lF2jo8eLtz5qD92BIAhCh4SzxVatWgWLxYLvfve7yncbN25EXl4eli9fjqqqKixYsACiKGL16tXIzJT7csU7NtjE01BywKCNPp+KHCymDL0O+4opEVZP0zPgw8a/NaEwz6rs++1fyxZMZroJPf0+tHTKmoEpJqPlk4WA2mKydPTImslqDtUuXX0+5Gaa8av/V48PjnfjivEZKBkVmzVIEMTlR8LK5b333gs7lpGRgbVr1yZ1bLAxaIocFr1ljpXz+dg+XDGlzoXYPjV1PbjY7VGUCwBs+vt5pFvDG5rMculnysUTen6vT4LZJKCt26NsA0B7YGEzi0k+v1rxtfd4kJtpVhY/6+7zAaPC3y9BEARAXZFD8MejXQzg3WJGjSsFQQAgYYdTdj+NybdqxvvdoQcxl5zLG9jHxbblfdVFoD39shXCFizrCXQWYNaOx+fHP2o6MEZlITGlkmaTFU9nrzfSLacszRddGHD7UTYmfahFIYiUh5QLhy/JPb94t1iwH1nodfhuAKzOxQhmZHxYJ+LfnzyGT1XkAAiugKleWZMpF6akevp98PklxUJyeST87PkzmvMzC4hZTUzZjES++V9HAQBb18waYkkIIvUh5cJh5OKK73xa5aLXSZmRaKX9mZYBzCyTuxj0ufx4bmezsg0A7x7uxKhsCwYCMZ1t+y9i2/6LKC8J/6b+l7db8PbBdphNcjxmJFsuPD6/hI4eL0ZlW4ZaFIJIOUi5cCS7lcq5VheO1vcGl0b2hW9cGc0aL6JgvCzAh4Fq/s5eL55/U7uk8u+3NQEAxhZoA/K1jeELRQ+f6cPhM324ZoqcUMHcbpcDv9/WhL/uuYA/Vc3A6BxSMAQRC9QVmSPZbbp2ONuxYt1JRWl19Hjx1F/Poi/KSfr6GdmYd2WOohDSDIL6ANDcHtmVpk5pjpbuPlnePtWxfz/Qju7+kWvJ7D7YAQDo7hu590gQgwUpF45oLRedPpNRnbe9x4tt+y9i6/vR9fHKsJmw6mtluGl2HgDAYg79kxllkekRj/XR2iVnmDGl2NblweP/V49Hn6uL+VypAotXUWNQgogdUi4c0da56E3yRni5LLSefuMJnhVOWi2yFstIk68nIVS+SQYxEz36dCyXSAqKFVr2DcjH7gssM3Dw1MhdW4b1b6MlqQkidki5cEQ7kbDJPlr4CV1vggeA228qwpo7JyEvWw6HWQNKzB5QNn4/8LsHrsB9i4PdEGK1XPTIy4ou/NbR68W+I134zeaR3+TS4yPLhSDihQL6HNFaLulWEbE0Qxlway0VvqsxIy/TjNnlWUizaJVKZjpTLhJKR9tgUVXTi5yPbkyeNWLshU8MGJVtQWNb5HjNobpeHKo7rWybLoPXE97qJAgiMpfB1BAb0b6lstTcaOEXGFPXn6hhvcFY7UlhrpylxJSLGGgApm7V8pmr8zTn4LPB9LhifEbgevJ2tt2MNXdOwh03j1H2GZUd+d0jNzO4T2ObC+8c6ox4zHCgu9+LH204hcZWV9h92LMmy4UgYoeUC0e08wib7KMleuUifzK32egcuVo+P0tWMtdMzQKgjfnMLLPj0S97Ma5QViqlo2149JsTscChVTpqFt1QgN9+dypuvEreJ90qYnZ5FiYUBRXTr++egqsmBetk9KwUFoMBgHvXHsejz9XpLg093Dhypg8fHO/Gb19pCLsPa+QZT785grjcIeXCodfzS49rp2bjga+Mw6xyuf4jknvIxfX6CtdY0hSwiFhbF2a5lIyy4eFlZfj250oAaC2X9EBrlvEFaQCADJsIx9RswwaTGTYRE4vTlWMLAnUcGWlBpVmYa8VD/zJB2WaKTnNfqvtglf8ug0aewwVmeep1SmDKhCkXCugTROyQcuHQm2wAYOKYNM22xSzg5mvylboTa4TsMd5y8YZ5G2ZusWy77G4qyA0W790wM0cp5lO75TJsskKwB6ypNLatUhRzpmdrrpMe2IcpAqY42LkYdlXiQnZGqLXm94fGJLojZMINd9ifhoWyKOZCELFDAX2O4nwbMqwS+tzamMp//Mt4lBWl4XM/OAgg6I9nysBqEXUbSzJc3Fi4NjPsfD/71iTUnO5FmlXf/SYIAr67eKxmOWNmPfHJAADw8NIyXOz24BuPHwEQzDBj7VyyMrSJAwybJfL7R3u3R164LEBPnw8FOREPG1J415162++XAJMQaCSqjbmQoiGI6CDlwrH4UwWw++rxxDbtozGJwckGAESRfS9/pttEdBqUfBit+aK5TsAiKRlli7huyq2f0Pa+Z4qJ1cKolYvZJGh6ZDF32JL5RWhodeHqyXIshykX9qm+53ChlK8/fgTfqixWtlPBcuFdXWplz5SJKGi3PV4/vvCjatw0Q0CKLTRIEJcccovpoGcsmLh0X3Ngm1kwkdqyRPvGm0hqL5OFTZR2rhZHfQ85AbfbjAl2rH9wuqJMsjNM+MqNBfj+l8eFnP+2G+RlkvmlAIBg3zIgcoHocID9PZjCVHfDZoaKwD1PlqrtrKP/NgQRCfpfooNFx57jJ32zWesW45VPvCRyHiYje9O2K+nLwX1YzCbdFt7d9q3KElw/I9Sv9amKHGxdMwvjI6Q6qwP6bV2eYdmbi08vVlsuoQF97XIJMTZnIIjLkoTdYo888gjeffddWK1WZGRk4Ic//CGuvPJKAEBVVRX27t2LvDw53bWyshJ33303AKC/vx+rVq3CoUOHYDKZsHLlSsyfPz/i2KXAEoXloiiVwL5J0i0x18+o+ey1+dj8Tis+GVAMWenyn3fauAxln2fun6a0comWK8Zl4OjZPiX+wqy0vExzcBVLs6CkV7PFyI6f68P3nj4BYPitkeLjFIg6kYMfM1qDhyAIfRJWLvPmzcMPfvADWCwW7Nq1CytWrMDOnTuV8bvuugvLli0LOW79+vWw2+3YsWMH6urqsHTpUmzfvh12u91w7FKg55oKcYuZQi2X2+cXYt/RLpxqGkjg2vErl7Ix6ZpJfHSOBd//8jjNmi72NJMmFhMNP1s+CRc6PIqbyBZQLsWjrIpyUdftsPjSWx/F0sPg0sInVGgtF/mTj7mw75OtY7r7vPjoZA/mXZWb3BMTxBCSsIE/f/58WCyyq2X27Nlobm6GP4oVt7Zu3YolS5YAAMrKylBRUYHdu3dHHBsqRN4tFlAurPWKKAJf/2wxFt1QkNB1kuVeYyxw5EdMDIhEhs2ECUXBVGxmwbDCTp4Xd1/Ay+9c0FVikiRpMrM6+uTW/ZeaULeYynKRmOWizRYbrGLKjX9rxpoXzuDwmWBGyLuHO3G6Kfw6OwQx3Elqttjzzz+PT3/60xBVM/GGDRuwadMmjBs3Dg888ADKy8sBAI2NjSgtLVX2Ky4uRnNzc8SxWKipqYn3VvD9W7042ijgjQPyBFldfRAZVoA9sjN1p5DhlnCxVQQgoq+3F06nE/X1AoDoLANRkOCXtMrkxPGjGIiuG38ITqczvgNjpL5BvmfR0w6995PWTg+eea0RN17hByBibJ4Ep9MJvwT8+CUzbGYJP1okB/1/s90El7ce6a5TMJvkYPq+kwJmT5CQHpo3kDROnZb/Tl1dXXA6neh1Aexve+BANfLsgMdjAiDgzJmzcNrqUdcq7yMhuc/6bJP8PN/78Bj6WyX4JWD1S7Isj345efGqS/X7SDapKDfJHIVyWbx4MRobG3XH9u7dC1Mg6PD666/j1VdfxfPPP6+Mr1ixAgUFBRBFEVu2bMGdd96JnTt3KscMNhUVFbDZYn9rdzqdWDDPgazDnXjjQB0A4JqrZ8tv4i8eAABMnTIZjiuy8UFTA1DbipycLDgc5eixtAPv10d1HbNJhJtrAzNz5nRMKc0Ic4SxzI5LlB+7+cApAN2Yd90k7D5WF3a/7NwCAG2wZ9rhcExB80UXgKNweQVFVlfgeU6bcRVyMy14/2gXXj9wGkL6KHzntrFhz50ozd5WwNmArKxsOBzluNjtAV49DACYMbMCJaNssO08DPR5UFwyFg5HISyneoC/1wJAUp/1myfPAGc7UFY2EY6r8+QEiJcOJfU6l/L3kUxSUe5UltnlciX0Uq4monLZvHlzxJPs2LEDTzzxBDZu3IjRo0cr3xcVFSn/XrRoEdasWYPm5maUlpaipKQEDQ0NyM/PBwA0NTVhzpw5AGA4dilRdxvm3VWsK7GoxFz091NjErX1FfK+WuViTrJbbDD49j+XwPtKA66enIkfLp2AfUe6sPPDUNdWRy+Lx8g3XX8+fJPInn4fcjMtaAssSsZiOYOFEj/httX/5htXDlZAn/1m2HX4Fw6CSEUSjrns2rULa9aswfr16zF2rPZNs6UluIb7nj17IIqionAqKyuxadMmAEBdXR2qq6sxd+7ciGOXEnXmFh/kN3FKha/Y18PKVbuLIjBrUqamRoZvnz8cmVCUhse+XY4MmwmfqsgN24W5+aJcF8LWRWnt9ChjfMyD1cawCfydmmB35V+/WI/n34zdLWqEEj8JfPg1dS6BIkoutdsXKN9Jto4ROeUSbjkGgkglEo65rFq1ChaLBd/97neV7zZu3Ii8vDysXLkSbW1tEAQBmZmZWLduHcxm+ZLLly9HVVUVFixYAFEUsXr1amRmZkYcu5SYDCyXkGwxU2TlYrOImvXrTaKAny2fhNPN/bjvNyciHj9cCdci5kSDHJDu6fPh/aNdmmad/S6/ptUMUy7s+WTYgufc4ZStoqWfCS4HkCh+TqloAvp+rsDSr9032bAXFGYxkeVCjAQSVi7vvfde2LGNGzeGHcvIyMDatWtjHruUqK0V3qLgiyfZsKHlYg49h0kUNK4w86UJRyWVSOudtPd48ZM/nMaU0uByzL0DPl3lwnqljcm3Yu+hTjgCSwwkG6YomL7Qc4spyoXVuQySQcG738Itx0AQqQTVGhtgpCgsZn1LxShXIdQtFnqOVLRcLFH2rGGWDAAcPNWDcxeC9UAnG/vx8t4LinI51TSAnz5Xp3GPJZOQmIuOWyzcZ7Jhf3NWrEluMWIkQI0rDTCKf/AxF/57AJhSmq6ZUEMtl9BjUlG53FCRg3WvyotuzZqUiQOneiIe8+sXz2q2X9pzAQBwLWepnFOtFClJkqaRZiIELZdAPEXHcgm6ztj3gWMSvPYDz5zAgNuPp787DUBomxkWo+JrqwgilSDlYoBRKxa+iJKhVg7/8dXxaO/x4gfra+Hzh675InLxGv7fqYK6QebPlk+C1ydh0Y+r4zpXXYu2u4G6L5nHK8FqSZZy4beDX9Q190MQgorHr2SLBXZIULscPtOn2eZXvFTazyR2GYIYUki5GGDk7eED+mziUSsXq0XEVZMyYRIF+PyS0uY+eP5gdT//XSrBAvrzZ+cqcaR4udjt0WyrU5L7XH5YLSKaLrrw9wMdWPLpwrgtGb9BS5cn/3oOgNw7Tb1PpNhSorDG2UwWirwQqQwZ3gYYTZJMubCJlQV9TTqKgn2OL9SuZsn2TfWAviAI+Ot/VuD+L4W26R8XSFNm6cqlo42LWvmgeUsgnRkABtxyPOahZ2vxx+3NykJn8cArFb3WLqEZZYFj4r5qBJl8WkuJ+mQSqQwpFwP0XFRswS2mXJibhr3Vqo8RuZhKuk3E6z+7CosC66LoBfRToc5Fj3SbSXep569/dgz+8qOZyjLRmTE2zWxuDyoXlqbM6mX4paNjQeKC9PziYZp9DFKR36npwMEoYkxGMgTXlGHXgeZ7gkhFSLkYoNd48Rd3leOufy5R1pNnloteESXv9jKJAkRRCFsjwx8/Esi1m5GVYVbSjtNtoqYJZiTUC481t7s12+qaoVhhE7hR9T2vVPTcYo8+fwYr/7c2Lhn4+hneVafmnUOd2PnhxbiuQxBDAcVcDNBTLiWjbFj8qWDnY/a2zlz/RplfvAIKZosh7DGpTnZgxUu1cvn58qnY/sFFJbYRLav/VKesoAnIMZh4YYqiodWFW1cdwL8vLAnZh1cqya5z8Xj9MJtMITEdSUfRPfpcHQDg5mvykysEQQwSZLkYEM3CXUwxsOpzraII/IMpHhP7nhVeJncVy+FIdgZTLvKnKAoQBAFptvA/PaNECnWcpd8V/3LKzAhxeeR/bP5Ha9h9PjzRg1tXHVAWWUuWu4pfhMzPuceGAo/Xj9PN1OqfSBxSLhFY+S/j8ZOvl4UdnzZe7mB86ydGATC2XNg2U1psDklk9cnhys++NQnzrsxR3IfMcmGuH7tqmeXHv12Oh/5lvLLNFFEk+gMxF4/Xj1f2tipJFdEgcS6uLp3kAGZBsGaap5K8vgqrxFcsF194y4WXabB44qWzuOep48NyaWoitSC3WAQ+PTvPcDwr3axZ/dEofhISYwnME8kqDBxOXDMlC9dMCRZEZgWUC3tbVy9gNrPMDpMo4Beb5KUKbr4mDy/tuYDRORa0dnpgswiKhaFm35EujMm34uOTPfj9tiaYzQI+F1DykQitc4m8D59Knii+cJ0ADNxvXp+kdIcYDN493KVchyASgSyXJGPWZH5px9gQ20e6jCoZ0gOdn9mklZURtFyY0rUFMu9ml2di65pZuHqy3Kw03IqXb37Uju/+9wmlZUxnDG36+dRjPYuA30eJiUR9Ffm82/a3oZ2r35HPH+bTwDpJ9qT/P6814KU955VtloGnlz1HELFAyiXJqN8qeYskaLnI25dTqqk58FxYa5N0W2iyBHseSvDfKn/mZhob2JaAJdjvjj4GE43lwv99lIaSBrUxPBc6PXjqr+fwyB/rQsaYG8zHVeZHslySyZZ3WvG7N5pCZRvkglFi5EPKJclYdGo9GKJJG3O5nCgLpB/fep3sttJ7Bn5euQTcUOr1bvSwBNLB1evFRIK3DnQtF165cBN7NPOv2xPMSgsnA9/DzCiuwsswWMQSvyIIPUi5JBmjTCe+Ip+fnNRptiON3EwLtq6ZhZuuDh/DuuVaOc02J2CpsAD66eYBrPveVHz1xkLd4/oCbrFYUoX5faOxIt0e+SC9TspsMm5sc+Gvey4oCsLL/Es67xNKnYuyGBnXw0yHZFouekqMGdteslyIBBm5s9kQoRucD/w/VVKPWbaY6j/3n38wI6Ql/0hn5ZLxqK87rWzf84VSfOXGQmQFssUmFadj/7FudPR4UTYmHVdP8eIvb58POc+R+l4AwewxQJ6EBSF8mjdvuUQzl/LrrKgVlMfnh8lkwn9tqsfRs3345MxsjMm3KcpATwy+IWY0rf2TudaLnjJmC2+T5UIkSsKz2bp167Bw4UIsWrQIt912G9544w1lrL+/H/fffz8WLFiAyspK7Nq1K+GxVEZJRdaZafKyLLpFmyOZT8/Kw8yxwUlMFAUU5VmVbVasOm2snO6dFkb5nj0vu5zUdS9ff/wwqlSV836/hOaLLtV27PKGKBe15eJn9yB/tgbSl9kxei8dwRgL12bGQLZkxkL0zsW3/yeIeEnYclm2bBnuvvtuAEBLSwtuvfVW3HDDDcjJycH69etht9uxY8cO1NXVYenSpdi+fTvsdnvcYykJX0TJ2vWPwBTkZJJjN+NHy8pw5ST5766OvZhNgmIVsM7JLNOpvduD9m6vUvQIAH/a2Yz/23Uef1g5HYW51rgW/vIENIhe7y+fsgZLwOXpZ8cw5RJ6vhC3WBQNK6NJIjDC7fUrXSX0lYtsu1AqMpEoCVsuWVnBWoa+vj4IggB/4H/W1q1bsWTJEgBAWVkZKioqsHv37oTGUoHS0TaMKwzt/st3SSbdEplPzsxR3GRq5aI3+fUN+PCXt8/j7IXQ4Pm+I3L9BiuWTMRy8XNKAQhaMSZBu6qk10C5hLrFQs/Lk0iK8Ds1HbjtR9U4HYhl6Z0raLnEfx2CAJIUc3nhhRfwhz/8Ac3Nzfj5z3+OvDw5aNvY2IjS0lJlv+LiYjQ3Nyc0lgr8z4ppmvitAK2lwjeuJKIjzRp0Gy79TBGef7NFM97Q5saGbU3KOixq3F7tZB9PpbuiXFibFtUEzLvFeOWi96fmU5B9nLLRI5Gllt8/1g0AOHauDxOL03UVtEhuMSJJRFQuixcvRmNjo+7Y3r17YTKZcPvtt+P222/HsWPH8OCDD+L6669XFMxQUlNTE/exTqczaXJ4vSYAAupO18I2IOFUswDAhL6+3qReJ5nnupREK7c8r8o/2Wm5DVhRCTyxLfQn3NHjAfNFfvCBE4IA9PTJf4MDNcfQe0FC20URsRruvf0uAAIkSZa5ZyAoz8cHDiI3A+jpls977PhJmHolHG+U/9Zut0d1n/IxR44eh+uihM4u+Zj6pk5876mPUJwLRTYmPzvm0KEj6IrzPav9YisAEadOn4FTrEN3f/C8TDa/X35OR44eh6d9eCiYVPxdk8xRKJfNmzdHfbJp06ahsLAQ77//Pm655RaUlJSgoaEB+flyimlTUxPmzJkDAHGPxUJFRQVsNuPFqfRwOp1wOBwxHxcOy7ZDgMuL6VdMwdWTs2Cu7QH+UYvsrEw4HJOTco1ky3ypiFnulw4AAK67Vj7mNzsOhryBC6IAKWBJVFx1NdKsIjJ3HUFnnxtZoyfA4RiF1w+fBhq6YhNWMAPwwS8BDodD7jn22mEAwIyZFSjOt+GV6lNASzfGTZgIx6w8DKR1AnvrYDZb4HDMls/zonwPk6fIv4e/fHgSON+Lpg4BTRDQ67UBkF17V19zjWzhBo6ZOu0KzJhgx4Dbj/3HujD3ytyoRHc6nRhTVACcakNp6Tg4HAW40OEGXj8CAMrfwPxKNTw+PyaVT4ZjWjZcHj9e3H0eX5lXOCTZjKn4u05lmV0uV0Iv5WoS/rXU1gYzcs6ePYsjR45g8mR5wqysrMSmTZsAAHV1daiursbcuXMTGktlWPW+4p8nt1jMPLx0Ar7/5eCKl+kRCix3Oi/i7PkB2AOFme2BLK54vD5uxS0mQJIkTWzEF0hUY8kaSsdjrmWMpFMbw8d/1Pfk80vaYwLn2/i3Jvz8z2diWqiMZSl6uM4AGji32Lb9bXhuZwte3H0h6usQBJCEmMvatWtx8uRJmM1mmEwmPPzwwygvLwcALF++HFVVVViwYAFEUcTq1auRmZmZ0FgqYwlUUbJeWEX5VqPdCR1uqMjVbGdlmNDd70N+lhkXu0OD9U+/0gAAKAushDnACiHj0C4eVZDFL2kVFHthYHE1Fp9RllHWaUrJ17cw1Ft+vwS/KhuAHc8WTWtqc+GqSdH932CKL5idFrqPEnMJKCCWWdbcHpokQRBGJKxcnnrqqbBjGRkZWLt2bVLHUhlrwHKZc0U2vnpjIb76af2KcyJ6cuxmNLa5cd20bPztg/ArNboCacps9cp4LBe1YnhjX5tmRU3FCghMzvzqksxN51MpkpMN/ciwiSGTvMTVzwhCqOUSXF47evnNilXFyRyQUxQFJfmEVeizDD23TldqgjCCKvQvAey9k/3ntphFfLOyeOgEGkGw/mOFuVbct2gsXtx9Hk0X3SH7uQIWy+v72tDW5Uk4G+q3rzRg2rgMZZudj/2tmZuMr4lRK6g/7ZQj81NK0zXnViuM+pYBlI4Oxg0VZcV0WQyeVeYW8/pD3WI+plwUy0X+NHHHEES0XF79RoaIL82TK83VbeaJ5MDcUFaLgM/NGYWpYzN09xtQtYZ570hXXJYLT13zgPJvH9eQOZhWrP2MpkEm62EGAN9/5iRWB5Y4ls+LwHnkTyPlcvb8AF7cHdouh/U70yqXwPlErfzKJxVVEjFClssl4EtzC/G5T4zSbTNPJAariGexARa45+lzaf1HB2qjD4SHQ92kNOzaL7zlojNH88fynY8P1fUG9w1RVuHlu/+3J9Dn8uMLnxytuTaLB6kVYtDyEjTbfL0OQUQLWS6XCFIsgwOb4FlrHXua/IV1EFdrZKiXDVAmZyXmEvjkssX0qu9DWvsbNKfk3XlGFgVTqPwiZLodBvzaYk9SKkSikHIhUhq2fk5DoOVLZqD5Z2YYCyaZiKpUci6eH1JtzywMvQB8iOXiDR+l59OWo4mF8FYUUzJ8zAVASCqyz6B9DUEYQcqFSGm+NDfQOXm8HGvJCCiXS/2+HZycA+m+ytos2liLXvsW3g3mjsJyUQLtqmMbW1242KW3nLJ+mxnd9jVcQN/DNeQkiGihmAuR0kwdm4H/e3gmsgPJEsxyiacxZazoLRYmcVaBpLikEFYu3lIxckUF3VehymX5r47CJAKv/WyW5hg+CYBt61ouXCoyu0dSLUSskOVCpDw5drOyXgqzXKJJNc6waX/+sbp+JJ03f79fu+2LwnKJpYZEyeriYiP8uBo+fZm3ZNT/5htXXgolTYxMSLkQIwoWa/H7JfzfwzPxzP3TlLGKMnldGDYx84uz6Xl+jBSOelrnU4/DZovpKD0jNxgPnyUWTeA9XPdljXIJuMGCdS7hlSFBRAMpF2JEUTJKbqlz/Ywc5NjNGJ1jUca+8dkxeOPnV8EeyNzj64704gqGoQbVvMvHMUJWl4zBLWZEOCXmNjhHSECfs6605+W2yXIh4oRiLsSIIjfTgnXfm6osl6xeYCzdZoIgCOhhvd3yrDjVFCyEFAW2gnwQkyiEdbGp3+r5di8+f+g+8rbeeaK4MbYvp6xY2rKRay0Yc9EqJL3Fzth3fONNgogVslyIEUfZmHSlrkjdebpAZcUAcssYNaIIPHH3ZHz1xmDPN3X/MB71vBts98JN4BJ/TGKTddAK0V7HY9BkTLGqOKtErTT3HOzAsbN9IRaWkkqdkNTE5QgpF+KygbnBmOtsDNeVWhSAK8bbcct1+cp3X/tMEVZ8aRyy7aF1M+o4B+9G4pUAALy4+zwudISmCseCn0scYDJ4DCwXPycLX3sDAC/sOo/VfzodarlIoRbMy+9cQPXpxDscECMbcosRI56pY9PR5/IrGWXZGXInZd4qYem9amvHYhLx2Wtz8McdzQC0DcTUyuVvH1xEdoY51HJRGRTrtzYlfC98zEVRLoYBfWhk8YVxeV3s9iqp3Lz86vt45jV5Zdqta7QpzwShhpQLMeJ5/NvlUFdqrFwyHgdqezB2tHaVUhbQF0X1d/InazNjMQu67VkO1fXiUN1pJSONb7mSLPjgvKJcogjoR3LZAcHU6XCNNwkiWki5ECOeNKvWpTUm34Yx+bawgXq15cKsGfZpMekrF4aR5ZIM+L5gXmVVyfDHhLZ/0cqqd/6gW0z7PUFES8LKZd26dXjjjTdgMpkgSRL+7d/+DZ/73OcAAFVVVdi7dy/y8vIAyMsX33333QCA/v5+rFq1CocOHYLJZMLKlSsxf/78iGMEkSxMooB135uKnR+246U9FzTfB/8tf4qK5SICLiMrgX1KaLro0m2xnwhhU4YNrhMuHqSnMEKVI0sGIMuFiI2ElcuyZcsUhdHS0oJbb70VN9xwA3JycgAAd911F5YtWxZy3Pr162G327Fjxw7U1dVh6dKl2L59O+x2u+EYQSSTsjHpyLF3AwCkQE6UVrkwV5n8qe6ErAebjPdUd2JPdSdmTNBfXyZeXn23Fe09HpX1wa5rJBPbV6sw9PSFokx8WgXEJwUQRCQSzhbLyspS/t3X1wdBEOCPwobeunUrlixZAgAoKytDRUUFdu/eHXGMIJKNokwC86Y25sLcYvK2JUIrf97TdqZlQH9HDv684ZRYd78PW9+/GMz8imLS591ijW1ufPO/juDo2d6QfZn8u6s78ehzdSEdAWJZVpm4vElKKvILL7yAyspKLF68GD/96U8VNxgAbNiwAQsXLsQ999yD2tpa5fvGxkaUlpYq28XFxWhubo44RhDJhikTNoHqu8XYEtXRWS6MaF/0bRbtf0VLBAuJzwAzqnX0c66zjh4vmi+68fcDHSH7quV951Bn2NTnSByt70XvgC/yjsSIJaJbbPHixWhsbNQd27t3L0wmE26//XbcfvvtOHbsGB588EFcf/31yMvLw4oVK1BQUABRFLFlyxbceeed2LlzJ0ymS7NwVk1NTdzHOp3OJEpyaUhFmYGhl7vhnADABK/XC6fTGZi45f8aR48eQWcz4BowARDgdQ/AqEdwb1+/Ztzn8xnuzxAkr2Y/AcbHtV3sACCio7MTTqcTtS3yPQDq5ynfw7HjJzGtGLh4sR3q90mfzx9RtrNNFwCI6O7thdPpxIAneN5wf7cBD/Doy2ZcUezHshsSN3WG+vcRDyRzFMpl8+bNUZ9s2rRpKCwsxPvvv49bbrkFRUVFytiiRYuwZs0aNDc3o7S0FCUlJWhoaEB+vlyw1tTUhDlz5gCA4VgsVFRUwGazRd6Rw+l0wuFwxHzcUJKKMgPDQ+4Wbxvw8TmYTGY4HLPlN/2/HgQAXFkxExOK0pD57nGgox/ZWXY0dfSFPZfNlgbApWyLJhMQRe8wq9WKwMwNAEhPs6DP7Q27f1pGFoBeZGZmw+Eoh3CiG9hzCgCCz/PFAwCAiZPKgf6TyM7JBRq6lHP4pchKz56ZC6ALaWnpcDimobPXC7x8SHOdv+1vw5sfteMXd00GAJy7MADgGDpdaXA4pke8hhHD4fcRK6kss8vlSuilXE3CbjG1q+vs2bM4cuQIJk+Wf2QtLS3K2J49eyCKoqJwKisrsWnTJgBAXV0dqqurMXfu3IhjBJFsmOuLBfRFTSoy+4zPLRZt3xS++7LZZPxf0+2RFZZPp4I+nEzxxOJZQ0wlO03HLfbkX8+h+nSvEvfp6ZfdYRlptLT35UzC2WJr167FyZMnYTabYTKZ8PDDD6O8vBwAsHLlSrS1tUEQBGRmZmLdunUwm+VLLl++HFVVVViwYAFEUcTq1auRmZkZcYwgko3RKoss/sI8uRFjIfHplhDnVCQlxmpt/Fxqsq5MUcRlor2OUUqyyyMhzSoocRlaYOzyJmHl8tRTT4Ud27hxY9ixjIwMrF27NuYxgkg2TIHozZss2B+0XIwtCj7gHW1X4bEFNlzoDLrFIikxxaKIohOAXhfkaGFrzUST8tw34EOaVTRsRaOmu9+LrHSq4x6pUONK4rJHyRbTGTNxLWHULfz14Kv3o10I7PoZOai6fTzKS9IBRLZcXB6+sWT4fS92e9DRF59bjLWVkaIo1hxgrroolMu2/W346upDgfgMMRIh5UJc9vB1LnpjzHLJsZuw8Gofpo3VL46MZeEvNaII3HhVHtIDSy9HKtZkLfaZUjGqc/ndG0345RvmuPqDMSvEp2O58NdkijWadOW/7b8IQG6WyXjt3dao64KI4Q8pF+KyxyjmInLtX0RBwJxyCVeMD6NconQJhVyHdWRmvcwMl8AEvF7OclFdNpz7K66Yi0frUtMsMMbpUabwvFFciLn12HNtaXfj6Vca8MRLZ2MXkhiWkHIhLntMBm4xvnFlMMCvP/kbNbU0gldiJkHAH6umY9nNRbr7e/n2LCorItzkHk/MRbGQuHYzAEIafyqWSxTPgMnNjmkNxJs6esKnXxOpBSkX4rLHKKDPYh98O/4ImcIxw5QXS0kWBKAgxxpSuc9QuiHrxFz8fn03WTyKLzQrTWW5BP7NjCy2L7Pe1OnVvQM+uUYmAJObLc/c1SePRXIHEqkDKRfiskdUZsHQyZdNdiZFqQjcMcmBP6/IXY9HsSR8Ek419WusCL8k6brA3DrxID0lqf7OY5AtxlKcBU65MFnUj+gbjx/GkkcPBWX0a2UacAcSB6gx5oiBlAtx2WPUjYgpl6B7LHBMhJhIJCYHssIYQbeY/vXC0dDmxnfWHsd7h4OV9z6/pOsC04sH6VkK6nTrYMpzqOXCssLYs+CTDNT0Dmi/DKeQ1GJXn+7Bh3VkyaQqpFyIyx4ljqIzkwucJSEa7BsLP1w6Af/7/St0ZNBeJ1Jgn9HQFmw5I7vFQvfRc4vpWWBWVRo0X4Cpcb8xt1hARh8X/zEyQtj9MuWlpC+rjnno2Vr89QOq8k9VSLkQlz18sF4PgdtXTPB/jtUiojDXomzzbjF+kbJIqC0Vn1/STTvWS5PWO79eoajb48fL71zQnENxi3EyRLNqJXvULObCznUpnGKH6nrRqFLGuz5ux388e/ISXPnygspjicseJZ4SRTA5UixED7NJCKn9EAVBY5Ww8zFDItbYjlqX+P36665Ea7nodQdweyU881ojPlWRo3ynxHkE7Tav2PRjPVrLhT2fSxFzefB/ZEWydc0sAMAvNtUr1xaSHEu7nCHLhbjsMZmid3XF4xbTq7Y3idp4Cu8O4zsDREIdq/jvl8/haH3oQmB63QIEnfMbXbPpolv5N1Mm7D6CBZ3aT2ad6F2Dj7kMZTw/2rVqiOggy4W47OHjHQDwwFfGaavFuQB7LG4xi0lAP/edwFsuJu35+USCSKjdYvuOdGHfka6QffTcYiad8xtdUa+IUgCLuXCKIrCf16d2pUkwiUJQ8bCYi0EDzmRaFEaWkccnwUIzYtKgR0lc9rBpy6bqG3bzNfmaffgssUQtF1458XGfWN1v0XQG0HOV6c7ZBhO5eoh3i4Xr0Kx2x3m8EkxWQTnWHdIyJtQ95vMD5jjj+j6/hK5eL/Ky5PiW0XPyeCUg9uWfiDCQW4y47GE1FhnWyDNYNKnIvDKx6BST8BYJH8APxlwiigQAcLnj7WmmlyEXfn+1BaHUsyjb8qckaRWEekL3KJYKv611i6kVYbz92gDglb2t+NrPD6OhVQ7gdxp0ACC3WHIh5UJc9jCLZerY9Ah7Amaz1rLQ3YcLiJujsFyCAX3egkme5aKH3umNlAuflQYE4za+MNli6knbzcVYlJgL6zjgDz0m3pY6AJTY07Gz8uqh5zs8YfdNRIkRoZBbjLjsmVScjp99axKunGiPuK9ZJ/hfOsqKhrZgoJuPsehlX/GxDpGzVJSYS5Svf/G+devFdIzUmTeKmAtfwKlWDnx2WGunBwdqe5TzsviMuuNAvIoTCKZVK8pMzzfIySlJEvrdfmTYqMYmEchyIQgA10zJirgQGACYdWIuT9wzBfcuKg1/jI5yCbFcTOx7znKJMpAdt3IRgRx79O+YHo+O5cJli/FNLjWWi0drqXxwvBtVv6vFxS6vZl+fjittwO3DyYa+qGUFgs+ZpUf7fOH3Zdfed7QLX/rPGhzRybgjoidpymXfvn2YPn06nnvuOeW7/v5+3H///ViwYAEqKyuxa9euhMcIYihgsQDewgCArAwzJpfot+AHoNt8ks9+YpYM00N8l+RkwSs6QRCw8aEr8NC/jFe+M0oHdnm1mV+AOuairXPhs8eA0BgLgxU1MivFqzlG/vf/vt6E+/77BC50uBGO8x1uHKjtUbaZkmYGC39dNcxlV3NaVioHT/WE3Tce3B6/RjkOuP14fmczunpHZifopPx0e3p68Mtf/hLz5s3TfL9+/XrY7Xbs2LEDzzzzDB5++GH09vYmNEYQQwGbkthEyhdcqvuT8ZOzzSLijpvH4BNXZIc9P9/uRc9yUVf0x4vVzLvjgDSrCZnpwRswas2vtlwUtxhfROkH9xk8pt/th9vrD1kWgLXa9/tlq0XPLXY4YEmoa214vvmLI6j6XW3w/gLC+X2hii7k3gIaiCngaDoNxMK6Vxtk5dgpy19zugfPvdmC321tTO6FhglJUS6PPfYYli9fjry8PM33W7duxZIlSwAAZWVlqKiowO7duxMaI4ghgdMYfKBd7Sbjpy9BAL72mSJNdTsPX6FvNoVaSGuWl+P7Xx4Xo+BarJwVpSxOpskCC3+8Ov7Bu8V4y8Xvl+D2+jWpyQ89W4sljx6Cn3NPdaje3j//8EFs+Udr8Jpc0L/PFd63xesOpViTWUSRUpGhLgpNbvbY0UBSQVevLD87PXMJjjQSDui//fbb6OrqQmVlJf7+979rxhobG1FaGvRFFxcXo7m5OaGxWKipqYn5GIbT6Yz72KEiFWUGUkPu3m4RgIgjJ07jE5OA06dqAchv+06nE+e7APbfyev1Qh0W7+7uhNPpRP0ZQXOMjHzMkcM1aMkEWlrk6zQ3N8LpbEBtS/CY+lM1sHqCx8SD5PNoZBsY6IfT6dRcJy99AM3t+u+dHq9POf7YsRPwd0pwu00ABDQ0NMHpbEBTk3wPTRfduO1H1fjkZD/U77H9Lj+sJkkrh8un2f7rPy4o/645dAQ95wGvR77OoaO1sPTJM/PpC8C4fHUdjPxs2PNtvSDLcvrMWTjT63HybPi/wdGj8v00BuQ/G7ifeHA6nejokxVVdiAJ0T0gy/9x9WF0NAHHG2VZ2tq7hsX/gWTLEPFXunjxYjQ26ptt27Ztw69+9Sts2LAhqUIli4qKCthssVdFOZ1OOByOQZBo8EhFmYHUkbvH0g5nXT0+MWsy0H0CU6dMBvaeBgA4HA65jmL7UQCAyWQCPMHX//y8XDgcE9Ftbgf21yvHAABePAAAmDXrShTmWrG/sQE42YrxY0vhcBTBVNsN7DklH3PN1ehz+YHXguuixIpPMgMIvvnb7RlwOKbCcqoH2CO7k+754jR4fBK+vy60maNfCioAj7UEZVNGwWw+DsCLwqIxcDiK8UFTA3AiaHl8VG8GoDWHvJLW8pMMctQmTZ6KqydnIWffCTR19KG4ZAIcjlG40OnGwy8ewbyrcrHq9gnyzoHnOfvqa2ASBbzf0ADUtqKgsAQOxxh0CBeBfWc1+7BjJkwsh2NmDj5qaQSOXkBBYREcjhLjB6oD+03fuko+L+thlvbOMaBjAGUTp8AxNRt91g5g7xnY0u1wOKbEfJ1kwmR2uVwJvZSriahcNm/eHHbsgw8+wIULF/CVr3wFANDe3o5du3aho6MD9957L0pKStDQ0ID8fLnauampCXPmzAGAuMcIYii48ao8TC7JQOloG5zO0CJKdZ3kLdeNwqHTvWhodaFnwBdSu6IHvxomv64L20d9DkGIvRfX6BwLegZ8SjNNpZ2NSjSzScDk0vAJCow/v9WCl/acV+qEwvUH69cp8IwlnqG4q5R+ZPLBPf2ykvy4tjvkGJ9PbjMT7ASgLd4EZBeZ+i/CYi7eKFxo8cAKOZmLTq9+Z++hTpSMsqJsTOSaq+FOQjGXa6+9Fu+++y7eeustvPXWW7jllltw33334d577wUAVFZWYtOmTQCAuro6VFdXY+7cuQmNEcRQUTo6aAWHK4IEgCml6XjyO1MwtkDeP5qqfjYU0hVZdR1RFDTbevUzkZhVnonfPXAFPnttfuC6ocWawaQCrWx6uDwSJIMAfjJgyoTJyibn3oBysemkkHu4AD6byNWJBH/dcwEfnggqJi+3pLN68r/Y5TFMBogGXmmxT/Xj/elzdbj7qeMJXWe4MKhFlMuXL0dVVRUWLFgAURSxevVqZGZmJjRGEMOBUMtFCPk334wyGsuFHcMmMr7YUn0Oi1nQ7XRshCDISjJcuxlZ3qDC8fklmE3G1wlJPU5yFxVlkg+I6FEC+7LS0VsqwcP1LHMF3JR+lXB/3KGN43q5NGi27fdLWLrmMK6Zkomffas87vuQLVhJUWJGBZ0jgaQql8cee0yznZGRgbVr1+ruG+8YQQwHeEWhefPnXE28JaB7vsAEyYo02cTGZ6Vp3Vci+FhGJEQuO0xQZFTtI6r3lWTZDJVL4FNxiyXZcuG0lTeKNv28daBnuYRcRzmv9hwDAcX04YnE6l4ETjl6OXffSGOE3hZBDC7hKuzlMf1aFaPFyPgFy5S1Ugzcb3rdlqOVW7GqwsR21PtGcr8xd5jHJ+FMy0DS60MO1Pbg4KmeoLuKaxGj54bj28gMuP14cfd53fgPQ3GlMYUU+Ox3hR5zsqEPv33lnEaRHj7Tq8R29GRifzovd/6RCvUWI4g4MHKL8e3zFevAoJULb90olkuYHmSAfluZSARjOkwmdl7VNdg+gZ3ltjjha0vYBLtt/0Vs238R107NilkuI978qB1vftSuNBb1ci4v/SWdtfu8c6gT7xzqRLo1/Pv0gMcPt8evKCSvolxC7/2hZ2vR7/bjG58thj3NhGNn+/DAMydx+/xCzBwVzkIKuMU45TJSV78ky4Ug4sBQuYj6n4ZuscDxhblWAOqssfDHxGW5cG4x/lO+ptZySbNEsFyiyA7TI9Y5lXd18Z0AjPaNRrY/bm/G4p9UB49lPc08OtlunCuwrlluVcq6Lvt0rBLFLcYUn6pRJmDcPSAVIcuFIOIgdLEv9b+1lovRAmPWQFCeDd00Ow+dvV7ccp2czWXWOYYRT7YYn1zAWzLqfZiSsRm87QOhloNbZzKOBptFVALvevhU7jd5myUQhMZ6oqnI18MvqVKFFTdZ6H78WjWCoFXIepYL+2t5uZRnvWUGRgKkXAgiDqKxXELcYjqK4ncPXoGj9X3BWhiTgC/PK1TGjayTkKQCQX+ZYDX8JKjnsuNddHqNN9XwlkO066/ITqIgadZIyoULzjMLJjD5qydnry9+a4C5wYzOwb5isvCdob16z4AP6HMxHlIuBEEYZovxAfFgQD/0PAU5VhRcaQ17HbOBL40PNVgtorKqZjh4xafnfuPljqRceOJddMsWwf3Gx1p4y0W74mX8E3bvQORiSmat8WPK95p1bySIoqA862AGW/jrSJKU8rEYirkQRBwYxUJ411M0dS7h0Ava840iGdYIk7MsS2jRJL/NpylHcovxRFt7M/fKXM12WoTrsNgH7xbTsxY2v3MB/7Wp3nD9lnCw4kxDy4VzZbFPZZlmAyuKD+jzdTXqsfePduHWVQfQmYJt+Um5EEQcGFbb86nIBjGXSOi5xdg3/PmsUSx2xsdYFMtFk4rM9pW/y0yLbUXGqJXLVbl4efWVmFScBiCyhaS4q1grl4DikHSsiH1HuvDWx+2GdS3hYF2Xo+ukzGTRpkXrKQq2T1ObG+8d6QxbsQ8EF1Xb9PcWAED9+YGY72OoIeVCEHEQTSuXoHuJfcahXPSC9oGvbr4mDyuXjIdjipz6a9NRRLxXjZeFTWfaVGRBfRmMzoltHZlo3WImUXblMesskuXiCky4Fzo9WPPCGXT3y2/zwSWMQ5WAXtZWJJhrMZq4DR/811t9c8Dth9cnKWPvHOrEI3+sQ1uXNrNMo1wCz5ApmXiSN4YaUi4EEQdG/9lZgaIS0Fdau8R+HaNaFrNZwKdn5cFsDl+k+eM7JuL2m4pgEgN1M7wVpWO58C6zDJuIb1YWo2SUNTBuLHO0lgvLhGP3GG1s53TzAHYf7MDbBzoAyLdQ29ivm0hglCAQDjeXKmykoPjYjhJXUSmkZWsO456njoUc2xeI7bg8frzxfpsmy47928XFZfRY/afTWD8MFxwj5UIQcZCVIWuK62eEri7JxzUSibnoZZgxhWDm3W86llFBjgVfXzAmxJXGFIQUOJleKrJahq/eWIjJJelR3Ue0QXSmDC0Bd16GLbbpKD2wv98P3Pub4/jtK+dC9unVKYCM9s/g8flxtL5XsZjka2nvLaT2RtIqJsbZC66wsnX1+fCbzeew+Z3gGjZMabncTAGFf6bvHu7Ci7svhB0fKihbjCDiQBAEvPiTCt23bSUji1Mq8SgXPdg0wzfG1LMowq2YyWeLqa2e0H5mvLLkk4ijI80qyitTBl7QWTyJWYE5djOeuX8adh/swJ/faol4Pn7C/ehkaO8vZh2oSbeJSkaYEW1dXqxYdxKfnJmjfOf1S7Cqng9vufi4IL0RfQNaxXe+3aO5jvr87jgz8IYSslwIIk7saSZdtxU/GesVKiaCohA4RaGXusrrM+aaC1ouMrrFmoGvjJYBiIU/PDQdv39wurJtCTwQs6rP2YSiNMUiiQRbz8UIPSsq3Rabf5KtwwKEusj4gki34kqLfF5ewamXb/YaxHBSBbJcCCLJKE0o+caVcVouv/3uVF0LyczFdnT1A9+bTNA/Ri9ew74JaWcTpwEmiNrsN6tZG3Nhn9EGr/V6fkWDUX8xPdTS8JM8b7nopRWHg48HqS0Z/nj19tb32zB7ciaK82NfZfdSQpYLQSSZkHb2itsqvll5YnE6SkaHTiRKcN4U3ObThvlKfF7hCYis+EwhCim++xAFQWMhKW4xs/b8lihSqoHoEwd4ImWl8ehZFAw+XVkvFTn66wSVDbOQ2KNm5+vu82Lt5nP48cbTMZ//UkPKhSCSTEhrFS4rqjg/fEV+LJi5yV4UgD//cAZ+8LUJQVl4t5jScl8roxF8W/54laQoQMlsA4JKRLHATFplAwCfmJR4rIF3s+kpF6NbUrvfIimXzh4vth4Q47Kq1KnUfMyGnb+xzQ0A6OiW07DV/dSGW+PLpCmXffv2Yfr06XjuueeU76qqqjBv3jzcdtttuO2227Bu3TplrL+/H/fffz8WLFiAyspK7Nq1K6oxghjuhLaGkT9tFhHf++JYzeSf2HW01xNFARazqHGhRQroGykXNhauTX+siCJnuXBuPT41GQAWXu3Hb+6dEt8FA6RxLkXdfm0G96S2KPhJP7j2i7xPQ5sb75wQ8ep7rTHLqa4PYtlmTCylAPOiHP/JsctvB+q+btH2dLtUJCXm0tPTg1/+8peYN29eyNhdd92FZcuWhXy/fv162O127NixA3V1dVi6dCm2b98Ou91uOEYQwx02ObJ5SK1sKq8blbTrBOtpwH0GrxeuBxrvujMiGHNJLKAvCMaLnTFrSv29IGjdZNY4lnbOz7agvSfYPsViFvHrf5+M2sZ+PP1KAwA5U629O3KLlar/rcU/TVdlj4Wp4m9RZX5Fizr7jVdizKphBZ6sJY96qWSPz4+0YeSMSookjz32GJYvX468vLyoj9m6dSuWLFkCACgrK0NFRQV2794dcYwghjtMuUh+rd882fDtZfQaZLK5PFjnwh0bhXChSQDx3ZBJEDjlIuqe38Kl1akD/Gkx1sIAci3SspuLlDYzZpOA6RPsyM8Odh748bIy3D4/cjfq8x0evPJu0CpRmmdyyiWe4k01yvkE7XX4Yk21tTLcLJeElcvbb7+Nrq4uVFZW6o5v2LABCxcuxD333IPa2lrl+8bGRpSWlirbxcXFaG5ujjhGEMMdRbkEtuOdjKO9Du/q0luyONw2L5p6OKiQtBaLXswlpPBSN3NNu61M4IEPs07MBdDGaXgXVzRYzSKWfmYMCgILsVm47DQAGJNvw+03FcV8HY9Pgt8vgS9DSVS5PPXXc1i7+ayyzdfRsN+WXifo4UJEt9jixYvR2KjfWmDbtm341a9+hQ0bNuiOr1ixAgUFBRBFEVu2bMGdd96JnTt3whRPH4w4qKmpiftYp9OZREkuDakoM5CacuvLLP93OnjgY5hNQHOLCEDEuXNn4XTWJ/Hq8nVOHD+K/lbgwnn5Oj3dnXA6nTjTGtynuvoA0iwAIP+fO3WqFpZ+CScaBQAmdHV2Kvdy3wIgwxa8N7fbBEBAXd1pOP2ncD5wP263C3yQQoCk+c4sSnD7tPt89NGHGvk/Dmy3BORvapCf0+kLwX0A4PCh6uC23x1y7Ug0NZ6D03kWPV3ydTo7LsLpbMXpFkF5LodrDsBqDl5XgDeq6/xxezM2725CSa4E9bt6uIneLErw+iOft7vfh63vX0Rmmvxcz55rgNN5DmfqZZn7+/rhdDrR3huU+eMD1RidwArTyf5/GFG5bN68OezYBx98gAsXLuArX/kKAKC9vR27du1CR0cH7r33XhQVBd8EFi1ahDVr1qC5uRmlpaUoKSlBQ0MD8vPlFfeampowZ84cADAci4WKigrYbLHngjudTjgcjpiPG0pSUWYgNeUOK/OLBwAA1117DURRwL5z54CTbZgwfjwcjtHJEyBwnZkzp2NySQaqW5uA4+eRn58Lh2Mi7PW9wN9PAgAc18xGmtUEbP4YADBt6mQ4pmajz9oB7D2jHKNHxp5jQPcAyssnwnFVHg61NQHHziMjzYb2XrdmX0EUAdVbdFqaGe7eYMaUKCD4zALys+23T9cDp9sxaeIEOByjkFHfC7x9UjnWcfUs4LVDAIC8HDvOd/XF9LgmlsnPf+eJM0BDB4oKR8PhGAfLqR5gj+xNmXPdNTCZBJg2H4DPD2RnpqGrP7Rlix7dAwIkcwaA/oj7ZqSZ0dUXfC5mk2CYtmwyWQB4UVBYDIejGLU954GDTUhLT4fDMU0u8Nx6FAAw9YoZmDgmPSqZedhv2uVyJfRSriYht9i1116Ld999F2+99Rbeeust3HLLLbjvvvtw7733AgBaWoItHPbs2QNRFBWFU1lZiU2bNgEA6urqUF1djblz50YcI4jhDnMbsZhskrq+hBA+5hK8oMgFXdg2Wx8kxx7+/ZK5sczcdQRRUAogletw92jjalWMPIOsmp2dn3eLaQL84U8TFhNfpMkVb6r3YTLE6n6LJhkAAGaWaZOSWNZXOJgbTInt+FnMhbWHUQX0U80tlggrV65EW1sbBEFAZmYm1q1bB7NZvuTy5ctRVVWFBQsWQBRFrF69GpmZmRHHCCJVUOoOBkm5BBtXQnMdo2wxtn3tVNl/svD6yBYVWw1T3Y/szz+cibcPduA3m+VmkQLXbyxE+ajkuG5aFmobg2/5uZlmzfn5gL56NU6+x1Y02WN8ijPfdkYjp6KEDU8ZQrSLeS2ZX4Ql84vw442n0dnrRX6WBW1d4Y8NtyQyu2NvKsdcYuGxxx7TbG/cuDHsvhkZGVi7dm3MYwQxXFn2mSK88X6bsl1eLLsoxhemDcr1QmpWuO8BvSJK+Ysx+TZsXTPL+AKs+7LyVh8csqeZUJQbzLbir8NPc+rh//z6RM34HQuKkJEm4oYKOcWXb/+i1jU3XZ2H9u4LSmpxRpoJ7h7jiT2cxaK/yqesJLt6YyuCjDY92mIWMHFMunLtvCwLjNxpLDGAX/PFr5MCrU5LHg4Mn6Rogkhxlt48Bs//YKayfct1+Vj3vam4atLgWN38ujHK96r/1ay3GNsjnhoVvj0Ly4PVZI1x87RRsbgoalOS06wmLP3MGKX4k3eLqfujTSnNwJ9/OBNTx8qK266Tmnz/F8fivkVjQ+TnLRhd5RL4vNmRh89cnYfp4zPC30gE9M5v4RRdflZ07/cenwRJkkIWRlPrk1W/O4X//OPwaQtDyoUgBgmLWURZnAHWaAhdmwWB7fB+uHiaZyqWC7d6pfpcfFB6bKAX2rgC+TOWziThakwA9eQcWANGZwnm2ZOzUHldfsj5mJJiqc28604t56hsCx786njDmFQk9J610uIm8JmZrt8Ljmfb/ov43A8OKu43PgbD2HekK255kw0pF4JIUcyBeSlSLUu0Yzx8s0tmEbECPvVEyCuXUdkWbF0zC5+9Nh+xkpUefkLnXVt6ykUUtAo2XFGmXo8xFii3BpQXnwwQC2YT8F//Vo7vfCFYs6fEyVRKcuuaWYoy5BMheM53yJX/Hp++chlOkHIhiBQlmphL6DHRn59NviwjiU3YepaLxM1xvGLy8zsYYDIJmFKajm/eMiZkjFksypu/jnLh758pJKYMWedivWUM2Fxts2hdafEUb5pNIirKMjFhTJrqO+3fil8wLZISGwj0Oevq9eGLP6nGyYbI6c9DBSkXgkgxPjFNzvSKJubCE0tH46I8uaKd+fWDMRdu2+A6/DHRsvbeqfjqp4tCvueD82lWEb/8t8lYfmtxyLUZvLXD7seqozAUy8WizZCLdgEzrayBT5U8fPeGkDVtDFyCgLb9f7/bj5f3Dr/ljRm0WBhBpBirvlaGMy0DyAisqMjHXPQmfd6SiIZ7F41FeUk6rppk11yHoT7XTbPz8NbH7cjLkhtAKks8M8MiSenYehlfM8vsSrdgWS79Y5i8zIWn9ywkxXJhbjF5OyvDhAudxs0o7Wna5ZOVbs8qhaHUIAWuo6RFcxZMOPq4Vv4enQJMr0+Ky42XbMhyIYgUI80qYtq4YBZTiOWim6Ukf8aSLWZPM+HL8wqVQLiJc4upz3XrJ/Kxdc0sxX3EWy7Jmup495FeTCTEcglM4MzNx1xeALDmzkl48p5gS3/mFrNyCimaBcbsnItOz9XFWy5BZam1lMLRxy2N7NVJgU60r1myIMuFIFIcfgLXm59sZqAb+pNRtAQzkUPf/AWuO0BwFc7kvkErMRcu9qKptg9juSxw5KOz14svfLJAGZtdrt+Mi1ku6uagJaOsymJdevBLSiuKT+0WC+OyY/cRybDsd2sVh15A3+Xxhyi6oYAsF4JIcYzcVYwrx8mTkD09/knHKCuN/ZNvScMmen7ijRe+RkWvZoVXaGwCz7GbsfzWkqisED7mYjYJeOb+abhHlfnFc+VEO66enIkriv1a2VRuMZF7EeDdfBKAb3x2DPIyo3vv13N/DRfLhZQLQaQ4RvUnjPkz/HjhhzOQn2UJGYv6Oqr2L/J1Qsf41SrjqasxIlx2lTpWEdKbLJYUuQBWi9YSM4nyKp96tTGMHLsZP19ejsJsaGTjrRUgaNnptaJZMr8o6tVK/Tp6xOUm5UIQxCCgF1cRBSA3M37Foj6vXrEmC3wzRcc30UyWd0xtSWg/gzfNW0lGRZk8xflyhhyrc+HrUowC5UFZAh0MhPDHKMqFU5ZMcUcbkOfdZIB2RcuhhGIuBJHiKGvdK9uDkylk5BYLWbCMW1gsUYlKR1nRoIp3KB2bo5j0Y1Euj99Vjr2HOpXKfKazmJJRL7mclW5Cd38weyuk2DTwvVF6seIWC+kEHf97v4dfuWyIIOVCECOEwX5fVdxvOm6xEKUiCCH7JMLa+6aiVScVWBSjUC4xpOUW5FhxmyrgH2zFHziXSgn85OsTsf9YF17e24oBt1/lopPHlWJNHUXBnk+4Ds2JpBLrpScPBeQWI4gUh01Eg91y3SieErK2jKj9PlFjKsNm0nSXVs6nFD2Gv0Ailhxvrakn/cI8C/71lmLluQTjMxrRdIs1WVIBy/ZiSktS6l8SUC7DpPU+KReCSHFYfMA9yFlCYogCCU3/DaZDa91kg42edXDFuPg7GjPMBrEQs6I4tUqFVeYbtbxh3Q9YQSdLfWbKJlKlvhHkFiMIIimwokB+Ia3cKNNZo0WxThTLJXSMwVsuyV4xTYA2Q86mk178q3+fHNLzLFbYpM+UgLaeRvs8lJb+nOWix91fKEXZmDTMLs/UXsevdx1ta/1IDBfLJeFfX1VVFfbu3Yu8vDwA8hLFd999NwCgv78fq1atwqFDh2AymbBy5UrMnz8/oTGCILSUjJLb2t8wM1f57i8/nqlMwMmCD9KLOm4xZTnlJMdceJinK+h6Cr3XWPqohYP1FFOC85oMOc5yUZIL5HGjhsWs+wFDcZMFlJiFU2LqYsl0q6ibJcZ451An5l2Vm5T7T4SkvNrcddddWLZsWcj369evh91ux44dO1BXV4elS5di+/btsNvtcY8RBKElP9uCzY9UaLr8GrWtjxc+aK8pouTe4kOyxZI8z/FxJnbvMyYk7gpTw86rWGs6lktI5hrnIgSA9Q9cgQEDt2WagYVUkGPRZMpl283od4fvFPCPmk4cOtOLKycO7dLwgxpz2bp1K5YsWQIAKCsrQ0VFBXbv3p3QGEEQoaRZTYOWgswQOatE/WLM6lyYtcTHXJItGWtv0hNIBbaaRfzXv5XjkW9MSup1mHJhloVemxm+KagS/lHddMloGyYVh184LhhzQch1vv3Ppbjzc8UYG1h4jV9gTO/Pnqg7MBkkRbls2LABCxcuxD333IPa2lrl+8bGRpSWBtslFBcXo7m5OaExgiCGBt46USuzkJgLH5dJsnb55MwcAPJyxIyKssyQiTdRrpmShS9+qgDLPiOvLWMUcwkWeMY+s9u47DH1dUpH2/CluYWKq6wwVy6GHZVtUa5bplozhj9+qIhoOy9evBiNjY26Y3v37sWKFStQUFAAURSxZcsW3Hnnndi5cydMpqFvnFZTUxP3sU6nM4mSXBpSUWYgNeW+HGU+0SwAMAG+PtW55Cnk4IGPYTEB3d0iABHnzp2F01mP5g55H7/PF/f1wx336JeBruYjcA7ye+c1Y4CmM01oOgO09QDsnj/66EMAgMdjAiCg/sxpOP2nYAvMqj29/VHfc79bPq/XJ4U82yNHatBSD7hd8nXc/R149Mt+vHXYh7cOmyBJftwxpwe9buDXW+VjTh4/iv7W2O4z2b/piMpl8+bNhuNFRcEFfRYtWoQ1a9agubkZpaWlKCkpQUNDA/Lz5SU8m5qaMGfOHACIeywWKioqYLPZYj7O6XTC4XDEfNxQkooyA6kp9+Uq8wyXDx831eE7XyjFOFZz8uIBAMC1jmtgNgl46eNa4EIPJkwYD4djNM60DAA7j8FkMsHhmD0kcieTlnY3sO0IAChype08jK5+D6ZMKYdjZg7e3CNP0pJghcMxI6rzen0S8MpBzXnZs5191ZUoyLUiZ/8JNLT3oaiwAA7HWNT2nAcON0GCgE/+k3zMr7fKx1xZMQMTDdxwPOw5u1yuhF7K1STsFmtpaVH+vWfPHoiiqCicyspKbNq0CQBQV1eH6upqzJ07N6ExgiCGhnSbCY/dWR5ULCr4xKSQ9VyG3kuTFIxaybBMMnvgffbWT+RHfV7mxvryvIKQMZNS+a9dZoA10dRrXpkSbrFIrFy5Em1tbRAEAZmZmVi3bh3MZvm0y5cvR1VVFRYsWABRFLF69WpkZmYmNEYQxPCDT3vlV6L0+jAiyNKJ6bB2KxlpbPIHXv/ZVTEr1K1rZul+r9TPcEs8Gyq6BIowk0XCymXjxo1hxzIyMrB27dqkjhEEMfxhE+HobAtEEfhWZXGEI1IDvYaSnkAXYvUCXcmsMVHqZ7i1X6wGzS1HhOVCEATBYLV+yprwZhGvPXrVoKdJDyW9gXXtB2v1xzRu4TLePaZ7TBQLog02Qy8BQRAjBr+SSht+fZWRxrcqi5FmFVGQk9h6OeEwKf3NtH3HLDoNMRlpBmOXCrJcCIJIGsno6jvc+cQV2YoSBYAvzyvUtHIZbCxKgD/8M05kPZhkQcqFIIikwToBDwef/2DxyDcmXpLrPHnPFLR2hrZ5UbLFDJYZGA6QciEIImkolsswyFZKdaaNy8A0nWUDFLeYTlfQaeMy0NYVuqjaUEDKhSCIpOHX6Y1FJBezgeXy6yQsM5AsSLkQBJE0+gKZU8MhW2mkotS56CjwoW6zr4Z+AQRBxM1ErmHi/NlyI0m2xgyRfFhRpVGdy3CALBeCIOLmqe9M0SyKtWR+Ib56Y6Fm3RMiyQRSu9NspFwIghih8CmvgiBgGDREH5EEF0iTA1s59uE9fQ9v6QiCIAgAwTjWQGCJY5MoYNnNRagoG559F4e3XUUQBEEAAP5purxA2sQxwVb6Sz8zBrPKh6dyIcuFIAgiBfjkzBy8+JOKQethlmzIciEIgkgRUkWxAKRcCIIgiEGAlAtBEASRdEi5EARBEEkn4YB+VVUV9u7di7w8uTK3srISd999d8Sx/v5+rFq1CocOHYLJZMLKlSsxf/78iGMEQRDE8Ccp2WJ33XUXli1bFtPY+vXrYbfbsWPHDtTV1WHp0qXYvn077Ha74RhBEAQx/Bkyt9jWrVuxZMkSAEBZWRkqKiqwe/fuiGMEQRDE8CcplsuGDRuwadMmjBs3Dg888ADKy8sjjjU2NqK0tFTZr7i4GM3NzRHHooEtB1pTUxP3PTmdzriPHSpSUWYgNeUmmS8dqSh3qsssJaFvf0TlsnjxYjQ2NuqO7d27FytWrEBBQQFEUcSWLVtw5513YufOnTCZTIZjg4nHMzwWyyEIgkhFPB4P0tLSIu9oQETlsnnzZsPxoqIi5d+LFi3CmjVr0NzcjNLSUsOxkpISNDQ0ID8/HwDQ1NSEOXPmAIDhWDTY7XZMnToVFosFgkDdWQmCIKJBkiR4PJ6kxLcTdou1tLQoSmTPnj0QRVHZNhqrrKzEpk2bcOWVV6Kurg7V1dX41a9+FXEsGkRRRFZWVqK3RhAEcdmRqMXCEKQEnWv/+q//ira2NgiCgMzMTDz00EOYPXt2xLG+vj5UVVXhyJEjEEUR//Ef/4Gbb7454hhBEAQx/ElYuRAEQRAED1XoEwRBEEmHlAtBEASRdEi5EARBEEmHlAtBEASRdEi5EARBEEmHljlWcfr0aVRVVaGjowO5ubl4/PHHUVZWNtRiob29HQ899BDq6+thtVoxYcIErF69Gvn5+bjppptgtVphs9kAAA8++CDmzp0LYOjvJ5xsRnINpcznzp3Dd77zHWW7u7sbPT09eP/994fdc3788cfxt7/9DQ0NDXj11VcxderUiLIM9XPXk9notw2E/w0NpcyJyHWpfit6chv9vhO5p7BIhMIdd9whbdmyRZIkSdqyZYt0xx13DLFEMu3t7dJ7772nbD/22GPSqlWrJEmSpPnz50vHjh3TPW6o7yecbEZyDbXMah599FHpkUcekSRp+D3n/fv3S42NjSFyxftsL8U96Mls9NuWpKF/7uGec7xyXarfSji51ah/35KU/GdNyiVAa2ur5HA4JK/XK0mSJHm9XsnhcEhtbW1DLFko27Ztk77xjW9IkhT+BzEc7kdPNiO5hoPMDJfLJc2ZM0eqqamRJGn4Pme1XPE+20t9D0aTmPq3bbTvUMscj1xD8VsJJyf/+zbaN165yS0WoKmpCUVFRUpTTZPJhMLCQjQ1NSkm+nDA7/fjhRdewE033aR89+CDD0KSJDgcDnz/+99Hdnb2sLkfXjYjuSRJGhYyA8Bbb72FoqIizJw5M+y9DKfnDBj/ho2e7XB57nq/bWD4PvdY5RouzxnQ/33Hc09GclNAP8X46U9/ioyMDGUBtueffx6vvPIKXnrpJUiShNWrVw+xhEGGs2yReOmll/ClL31J2U7le0kV+N82MHyf+3CVK1r43zeQ/Hsi5RKguLgYLS0t8Pl8AACfz4fz58+juLh4iCUL8vjjj+PMmTN48sknIYryn47JZ7Va8bWvfQ0ffvih8v1Q34+ebEZyDQeZAbnh6v79+7Fw4ULDe2HfDweZI8ky3J+73m+b3RMw/J57PHINtcwMvd83kx1I3rMm5RJg1KhRmD59Ol577TUAwGuvvYbp06cPG5fYE088gZqaGjz99NOwWq0A5Aaf3d3dAORW2W+88QamT58OYOjvJ5xsRnINtcyMzZs348Ybb0ReXp7hvQBD/5zVxPtsh/oe9H7bwPB97vHKNdTPmcH/vhO5JyOocaWK2tpaVFVVoaurC9nZ2Xj88ccxadKkoRYLJ06cwOc//3mUlZUp7bDHjh2Lqqoq3HffffD5fPD7/SgvL8fDDz+MwsJCAEN7P2fPng0rm5Fcw+FvcMstt+CHP/wh5s2bF/FehkrmRx99FNu3b0drayvy8vKQm5uL119/Pe5neynuQU/mJ598Uve3/fTTTw+L564n8zPPPBO3XJfqtxLu9wGE/r6BwfmNk3IhCIIgkg65xQiCIIikQ8qFIAiCSDqkXAiCIIikQ8qFIAiCSDqkXAiCIIikQ8qFIAiCSDqkXAiCIIikQ8qFIAiCSDr/HzObEE0WX2bVAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_eval = pd.read_csv(f\"{yahoo_eval_dir}/{key}.csv\")\n", + "plt.plot(np.arange(len(df_eval)), df_eval[\"value\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 356, + "id": "b3c30d2e", + "metadata": {}, + "outputs": [], + "source": [ + "df_all = pd.concat([df_train[\"value\"], df_eval[\"value\"]], axis = 0)" + ] + }, + { + "cell_type": "code", + "execution_count": 357, + "id": "93a15c67", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 357, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAD7CAYAAACmJ9mYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABDKklEQVR4nO2deWAU5f3/37ub3U2yOTeQkyMcApEItUGxWtBqU0KLCq1WBM/S1mq1RWs1KhVFq2K1Kv2KR6WilfpDa/HgEFBQrIpgtJLIDQlHTnInm81mj/n9MfvMbvaYnZmdvcLn9U/YfeaZ+eyQzHs/x/N5NBzHcSAIgiAIFdHG2gCCIAhi6EHiQhAEQagOiQtBEAShOiQuBEEQhOqQuBAEQRCqkxRrAyKBy+WCxWKBXq+HRqOJtTkEQRAJAcdxsNvtMJlM0GrD8z2GpLhYLBYcPHgw1mYQBEEkJBMmTEB6enpY5xiS4qLX6wHwN8hgMMieX1NTg9LSUrXNiiiJaDOQmHaTzdEjEe1OZJsHBgZw8OBB4RkaDkNSXFgozGAwwGg0KjqH0nmxJBFtBhLTbrI5eiSi3YlusxrpBEroEwRBEKpD4kIQBEGojmrisnz5clx88cWYOHHioGR6bW0trrrqKsyaNQtXXXUV6urqwh4jCIIg4hvVxOWSSy7BmjVrUFRUNOj9pUuXYsGCBdi8eTMWLFiA+++/P+wxgiAIIr5RTVymTZuGgoKCQe+1tbVh7969mDNnDgBgzpw52Lt3L9rb2xWPEQRBEPFPRKvFGhsbkZeXB51OBwDQ6XTIzc1FY2MjOI5TNGY2myNpMkEQBKECQ7IUmVFTU6N4blVVlfDvB/6jw/cncvjhZJcaZkUMb5sTiUS0m2yOHoloN9kcYXEpKChAc3MznE4ndDodnE4nWlpaUFBQAI7jFI3JobS0VFG9eVVVFcrKyoTXjn9/g4/2aXD3dWfLPle08LU5UUhEu8nm6JGIdieyzTabLawv5d5EtBQ5JycHJSUlWL9+PQBg/fr1KCkpgdlsVjwWLRxODs/85wSaOwaidk2CIIihgmqey8MPP4wtW7agtbUVN954I7KysrBhwwY88MADqKysxMqVK5GRkYHly5cLc5SORYPaJive392O2sb+qF6XIAhiKKCauCxZsgRLlizxe3/cuHF48803A85ROhYNkrR8+4N+uyfP8pe1x2FK1uKWy0fEyiyCIIiEgFboB8PdWsfh5IS3tv2vA+/tbIuRQQRBEIkDiUsw3JriLS4EQRCENEhcguDkeFEhcSEIgpAPiYsPTe02/H27Dq1ddgAkLgRBEEogcfFh0652HGvTYGtVBwDAKVFcbPb4XmBJEAQRTUhcguBw8GLhcPmLy8lT/dh9oFt4Xddkxdz7q7FjT2e0zCMIgohrSFx8YBuwMVEJFBZ74NU63L+6Vnj97TELAODrwz2RN5AgCCIBGNK9xZTAxMXpjnIFEpf6VhsAwOXicKrLDpf7WJ02/K1BCYIghgLkufjA9o52BgiH+fLvHS244fF9ON7Cr+LnKPdPEAQBgMTFD+Z8SEnkf3mQD4OxyjKCIAiCh8TFB02AlfnBGLDzxyTp+EkcxOes2tSAe1cdCc9AgiCIBIByLj5oZYTFWPmxIC4BptS32sBxHEYMT8a/d5xSz1CCIIg4hsTFB9+Evhi+4uLNtq87YM5Iwj0vHQUAbHp0qmo2EgRBxDskLj4IYTFHaHXxXTjJcXyr/qIcI/7yxvFBY3VNVtVsJAiCiHdIXHxg1WL1baE3CWORMxYO67U6ccszB3HJ2dl+x975wuGQ53vtgyYUDTPi/MmZsA24kGGi/x6CIBITenqFgcutLiw/09DGr3/5b02n37GWfo+Xs/z/HcPvfzoCyQYdAD6EdkZRCtZ82AwAGFuQjKON/RRKIwgiYSFx8YGTsVjF6SMutU38ehebXfwcH33TicmjTfj7xgb8edFY/OWN48jLNgjjR2n3S4IgEhwqRfbBJaP/JAuL2R3yV09+dbgHAw4O/3J7K80d/mG4j/d04LcrDsAuIf+jBhzH4e1PT8HS74zK9QiCGLqQuPjgklCC7DmW/6lEXFhup9ca/EH+f2/X42hjPzotDtnnV0LVwR68sL4BL25oQJfFIeteEARBeEPi4oOc7Vtc7hCaXUrdsg+ffdsFADhUH7yKjAmP1RYdz8Vi46/X1mXH/Ie/xapNjVG5LkEQQw8SFx/kfFtnq/j3HLVEyhwAvMh8Ut0paWFnODjdThQTzfd3t0X0egRBDF1IXHyQ8wAPlfvPydCHaQ3Pmg+b8Mi/jmH3/u7QB4eBy+cDUVCMIAilxLW41NbW4qqrrsKsWbNw1VVXoa6uLuLXVNM5SE/lS42njk0L6zxfHeoFALR0RrZBJuszQKkWgiDCJa7FZenSpViwYAE2b96MBQsW4P7774/4NdVMYqca+dtr1Kuzz0uoxphKqWuy4q//Pi6IinAPSGQIglBI3IpLW1sb9u7dizlz5gAA5syZg71796K9vT2i15VTihyKsQUpAICiYUYAQE5G6GVF50xMDz7IAQMOl6y1OFL429snsbWqA4dO9gHw6jyg6lUIgjidiFtxaWxsRF5eHnQ6PrSk0+mQm5uLxsbIVjAZVPAyfjZjOMYXpmDR7EIsvbYY3z8rCwAwvSQz5NxUI/95880Gv7HG9gFc/qdqfLynM2wbvWGNN23ukmq2lw1tfkYQhFKG9Ar9mpoa2XMmZAGBbkuqgUPfgDThyTc24uzzgW+rv4YeQF8f8MsLAXNaMzYGueXJeg79dg0sPe0AtOCcNniyIDzvfNYKAHh96zGkO476naOqqkqSfb5YerUAtDh1qhWAFr29FgAauFxOrP+wCmlGID1F0aklodTuWEI2R49EtJtsjmNxKSgoQHNzM5xOJ3Q6HZxOJ1paWlBQUCD5HKWlpTAajfIv/u43fm8ZDXr0DUhbzHhW6WSMzkse9N408KXLj2/Ygznn5WD9zsFlvpVXj8WaD5tw5Q8L8eXfj+CSsnz8a1tzwPNrk1JQfMZYZKTqoE/inc+qqiqUlZVJss+Xt/ccBVp6kJVtBo51IiU1FeiwQqvV4tkPNNAnafDuQ1NCnudooxXmdD2y0qT/WoVjd6wgm6NHItqdyDbbbDZFX8oDEbdhsZycHJSUlGD9+vUAgPXr16OkpARmszkm9ugC7NkS9Fht4GOTdBqsvqsEi2YXINOUNKhUeXpJBlbcOgFTxqbhnWVnoXSMCQBgTvd/UPfbXbjm0b149PVjMj9FYLRue1m+iZVjczLb2/x2xUHc+reDqthEEERiE7eeCwA88MADqKysxMqVK5GRkYHly5dH5bpnjXSh+sRg3U0KIhiBSNIFH2MNKv9683jodRp0Whyo9WlUadBrMSafj0Mtml3otzcMW7n/+d5uHGvuD7saje1hw34ykVGScmnrtuPkKRs0Gk8hA0EQpx9xLS7jxo3Dm2++GfXrXnmuv7jI8lwkHFuYwz94h2cZcEZRqt94VloSNj06FVabf+8x735kt6w4AJcLuP77Gux5vxE3VhSgf8AptPOXArOWeSpOLryE/q/+uh8A7b5JEKczcRsWiyWBnBRWUTVxpL8Q+B0rw8sJRbJB/L+IeRmv/FeHNz5uwbr/nsK8pTXo7JXe7JJ5LKzEWc1ybIIgTk9IXCSic98pvQSvJEmGlxMKjUaD0XnJuOaHeZKOf9ddUdbaPSB7QajTJ+dCEAShFBIXiTDBkCIcckJoUnh+8UQsvCRf0rEsZFZ91IKf3LcH+45Jb6rpcDHPhYXFpInMseZ+1Rd2EgSR2JC4SISJij5JgrioGBaTS697o69tX3cAAKoO9YSco3XHxRzuqrAOd0hNilzU1PbiN08f8CutJgji9IbERSJMMKR4LmrmXJRyuIHfJ0aSQ+E2l3kuQumxhLmsmWZNba9cEwmCGMKQuEiEiQv7li+GNo7uqsPJ4a//Po7apuCbkjGcPjulSdElVgbdP+BfBfD14R588FVke8ERBBGfxNFjML5hHgvTloIAvb8YGgkCpIQXb5+IcydlyJrT2G7D1qoOPL72eNBjmLV2OdtwCpPdCzADTL131VE8+eYJ+eckCCLhIXEJwW1zRyA7LUnYSEsIj0nIvajNyNxkPHj9GOF1hin0WpZPqvntlPv6/dfLMJgWOnw9F6+X+45b8M0R/9AXuwtUYUYQhDckLiH48fQc/Ou+yUIehYmLlJLkSDHJvdYmPYVfA1tabAo5xymydoV9kiMNwUNndzx3GJUvHfGfq2HnD09cWjoH0N4T2c3QCIKIHiQuErns/GFIS9FhdB6/sl7NtSxyuXfhaCz/1ThB4NiOl1PGBhcZF8dh594uNLTZ/AcVhPF6rU6893mrl+ci+xSDuH75Pix8ZC927e/GyndOhncygiBiDomLRMomZODN+0uRaeK9hViKy/BMA6aMTUOmu/twirvVC/NkAuF0cXjwn3VY9MR+vzEln+S5d+ux8t161NTx62h8iwGUsvSVWrxHZc0EkfDEdW+xWFJelh1QQNgjNJbiwvjtZUX48OsO5JsN2Pa/DhTkBC8y6LYEz7nIoa3bjqqDPbAO8Ofr7uPXxLR2U0iLIAgPJC5BuOOKUQHfZ0nuQOJyydnZwjf5aDAyNxk3zCqAze7CoSPHcPXFefj3jlM4Z2I6dh8IvniSraZnVW1yomJLXj6KuqZ+fGdcGgBPH7LWruDiwnGc7Ao6JXMIgogfSFxk4vFc/COKd/48sCBFGqNei/PP4JBq1OG5309AvtmAeUtrkJulFxY5evPUWyfQZXEOqjyTChMROc99lwvQBSls29+gwejx/ja6OCAOnEOCIBRC4iIT9q0/ltViYhS794F54LoxyMlIwm3/d8jvmK1VfGuY5o4BnOockCkUg3MrUqrEnBwHXZDMzmuf6fDWl/55IJeLi2kbHYIgwoPERSbDM/m8xqi8ZHz6bVeMrQnO9JLQiy2X/bMWRxv7cdHULMnnZVrCfkoRl1Ddma0BVvfTshmCSGyoWkwmZWekY+l1xfjp94fH2pSwOereAVNsgaUvvh2TpZQgb/yiHT++9xtZFWUDDpdoHocgiPiGxEUmOp0G55Vkhr21cLS488qRuHGWeLv+7j7p4uIUxGXwazFWbWoAxwFdlsEbmIm16f/n1iZc+9heWlhJEAkKhcUU4l0t9mplCfQBEvzxwCXfNQMAXt7cFPSY/Sf6JJ+PeSpCWEyGN+L0EROxHS8/39sNAOjoccCcrpd8DYIg4gMSF4WwMtmzx6cJeZjTCU9YTELOheVpXL7vB58rdxdNgiDii/j8up0grL6rBPdfK7+cNxaML+SryCZL6EMmBZeCLZGdTg49Vj40tuXLdrT3OIIey07LSWr8TxBEvEGeSxjkZSeOx3LbvBHYf7wPXx8OvTOlFFwyEvqMrw714Nl367H4ZyPw9FsnMc4teAHP71EXgiASEPJcThMmjEjFZecPU9KjMiCcgpzL7gN8HmXPEb6LgW+C3xvSFoJIbMIWl3feeQeXXnopzjzzTLz22muDxqxWKxYvXozy8nJUVFRg+/btYY8R4cF20jxnYnpY55FTLcbw3YxMbI0k81zEkv4EQcQvYYtLSUkJnnrqKcyZM8dvbNWqVTCZTNi6dSuef/55LFmyBBaLJawxIjzOHM3nXMa4V/JnpIbecCwQLCx2qD709smeOfxPKXvAeIoAyHchiEQkbHGZMGECxo8fD22AjeM3bdqE+fPnAwCKi4tRWlqKHTt2hDVGhMePp+fgiZvGo2Q0v+FYskHZr4CSZ77Lb41M8GOFajSRijKCIOKXiOZcGhoaUFRUJLwuKChAU1NTWGNEeCQbtJhcbEKqkfdYioYZFZ1HyTOfeSpCMYBIvsazjgbYtb9bdMElQRDxR8hqsXnz5qGhoSHg2GeffQZdsHa3cUBNTY3iuVVVVSpaEh3k2MxxwGVna3BmUSe+Piy/aLCvzwq524z19FgAaNDW1g5AiwG7I+g5XC4XAA3+uekQvq3X4opznPjO6PgRmKH++xFPJKLdZLMEcVm3bp3ikxcWFqK+vh5mM79KvLGxEdOnTw9rTA6lpaUwGuV/M6+qqkJZWZnsebFEic3TpvE/p061Y8MXbXjtg2bJc43JyUBPgC2TRUhJTQXarcjMygZOdkGj1QEIFhvjRcelSwPQB0N6IcrKxNvYRIvT5fcjHkhEuxPZZpvNFtaXcm8iGharqKjA2rVrAQB1dXWorq7GjBkzwhoj1CcrTY/Lzh/m977YbpsnT8kTFsB/4aVYQp/lY7TukjKHStsoEwQRHcIWl/Xr12PmzJl4//338cwzz2DmzJk4fPgwAGDRokXo7u5GeXk5brrpJixbtgxpaWlhjRGRIdDeKWrvp8JyLXIqwViuxXul/nuft2LfMaoeJIh4JuwV+nPmzAlYhgwAqampWLFihapjRGTQBlhdqXYvTua5cILnIn2udz5/5bv1AIBNj05VyzSCIFSGVugTAIAAleSiYTElOH08F0lz3AJkd3D444uHse84eSwEkQiQuBAAAofANGr1inHjEtr1S1cXFhZraLOhptaCJ988oapNBEFEBhIXAkDgViwqa4uQY1ESDjPo+V9Vu8N/8n8+OYXZ93wjqxUNQRCRhcSFABDYSwlHW4x6/18tJS1dOJ85gaa+urURAPDR/zow+55v0N5Nu1cSRKwhcSFCckZR8Nb4wcg3+29H0NwxAEBezsW3tX+giBoL6f3nv6cAAEcapfc7IwgiMtB+LkRItDJKkpN0GjicnGgxQDiei3cT/nX/PYV+u0uwzyUiQARBRBfyXIjguPVBTkky8yLExUX6+Y619APwLLj0Fo4XNzTg1S1NQvhOqEZzcTjR0i+rHxnHcdj+vw4MBMjpEAQhHxIXIijsoS3Hc2ElzaLiIuOhL1SYCZVm/sewdBE75ttjFvz6qQPYtLs95Pltdhc+39uF/x3pxeNrj+Pl9xsl26YWS1+pxQ2P74v6dQkikpC4ECGRs1JfiufS3hN8B8pgCB2VxfaAcY/VNfHezoETfSHP+8ZHLVj2zzrs2s/vknmiRX5bG6V88FU7dh/oxq793UI+iiCGCpRzIfzIydCjrdsueARywmLMcxGb06agmku0HxnzXNxDLLQlxe5TXfxDnQkeF8WNlWnNDjGUIc+F8OOF2yfi1coS4XWg1jDB0LmPVb8vGf9TbI0M81xYk0ux6FtNbS9WvnPS8wYXek6k2by7DXe9eDh2BhCEipC4EH6YknUYnmkAcwl0MtrAsPyMnDyNFFwingvnFpxTXXb3sQh6LOOljY14b2cbuvuc/DnYuTjg2XdOoqa2V5JdNXW9sPQ7JR0biqf/cxLVtRbaGI0YEpC4EAJ/v2MiXv7jJOE1c1jkyIROCItFxnMJlHPxFRFWMGB3cLh0yR6sc69/8abJnePot7nc5+Xfd7g4rN/Zhj++eCSkTT19DvzxhSN4fO1xyZ9DCnbaXoAYApC4EAIjhicj3xx8c7XxhaEXU7IQmsraIto6ps82+E3mwVj6nXA4Oby4oQFt3XZY+p3oH3Cis9cu5IaYMLEHutjWy770WHmP5XhzP5595yR27uuS9ZmCIccGgohXKKFPBIXpA3vUSfFGwvFcSkalYt/xwBVeclb1d7iT83aHZ9I1j+7FsEw9jHot6lttMKcnuc/L8jQu90/5ZdIaDbB+ZxvW72yTtA3Ah1+1Y3KxKei4nN5rBBGvkLgQQUlN1gFddsELCdSW3xdPzoV/nWLUwmqT9rQUK18+1twv6Rze+IaXWrs8VWqsl5p3S3/vn2K4OKC2ySp4aXL8DI7j8MSbJ5BpCv6nRw04iaEAhcWIoDx4/RjcWFGAYZl6AB7h0CcFFwGW/GeeixwHRu39YwbswUXNs/CSf5BX1/L7xEjJd+yr1+CWZw7ivzWdg87B/i2WkGfeTpcl+Fqf/gHXoJ02XS4OS/5xFJ/vVSfsRhDRgMSFCEpetgE/vzAXGlY15v5tERMMNsbERc6eMHodmyPf1kCIeSHsEr5OgrfXUHWwGwdP+ofpTrTzs483s9Y0nrHFKw+JrraX4pX8c2sT7nj+MA7V89e22V2oOtSDZf+sCzmXIOIFEhdCMlLyKOybuRzPhXkszOth+ZBwqRMJpTEBO94y+BjvnMuSl2vx+2cPDRp3ujhhLQw71O6lLofqrWjpDL5IVIq41DbxXZ3rT9mw9qNmoXCAIBIJyrkQkmE5Bt4bCfKQ9BETKZ6LxsfbkbNoUynMLl/vxjsv48uaD5rw2ofN+N54/jWr6nJIyNMwpCTrmW1Vh3rwwVcdONJAWwgQiQeJCxES9qzXioTF7rxyJD79tgsajQZ1Tf3Q6YIf64vWLVZarbphMTHkLFTctb8bTe0D2LS7DQDQ524DZrPLrzCT4rmwj88WeHb0yu/FRhCxhsSFkIzY6vtLvmvGJd81w9LvxAWTM9E/wD94pQiFJ0/DXkdeXeSU+y59pRYAkJvFFzYwXWJez0AAz4XjONgdnLA9s+e6UqrR+GN8iw4IIpEIO+fy4IMPoqKiApdddhnmz5+P6upqYcxqtWLx4sUoLy9HRUUFtm/fHvYYEX3Yw43lRsS8EVOyDhefnS28lhIW0/qIipSS53BR0jxT41N6LLb3y8p363H5/dV+70sSF7Z+xv1azDPqH3ChsT16nZwJQiphey4zZ87EvffeC71ej+3bt+P222/HBx98AABYtWoVTCYTtm7dirq6OixcuBBbtmyByWRSPEZEH1aem+z+Fi6lbxj79i2lvNhXVJT0JUtL1qFXpR5fwWA6yTyXhrbgbfLX72xzH8sNElgpG6U5fUJ2gbysrw71oMviwCfVnfh8bzfee3iK6qXcBBEOYX9H/MEPfgC9ng8XfOc730FTUxNc7r+gTZs2Yf78+QCA4uJilJaWYseOHWGNEdGH5RaMBv7XRUrUij0fk2QsvNSFkXMRW3ujNkwgeiVUcTFhaO2y4zdPH5DkZbicg8Ni3u1gDpzow5EGK+77x1E8vva4sD7nvc9bMfueb0Rtqm2y4n133oggIo2qOZc1a9bgoosugtb9FbShoQFFRUXCeEFBAZqamsIaI6LP5ecPR9WhHkyflIGNX7RJyokIoTS3uhj1GtjsgcM7vh6LkpxLNL61u4Q+ZNLn1DZZsfGLNuSbDTjW3I//fOLfRNOXLgt/AU8XAc99W7xycGm03R2ae3FDAwDg1S2NaOmy46Y5hSjw6RN3yzMHAQDfL81CbZMVZ41Jk/5BCEImIcVl3rx5aGhoCDj22WefQecuC9qwYQPee+89rFmzRl0Lw6Cmpkbx3KqqKhUtiQ6RtPmu2UDDyUMAkuCwD8C35tj32sN0wKQCLaaOsuBYs87dFz+wADgddgAatLWeAqBFf7816LG+5GdyaOrSwOHwt0ltrP38NexO6df525sHcKhZi7NHuwBo0XSqG6HsZOG9jo4OAFr0WfuDzmGCrddxsDs1qD15CjUnteju6sTkERyGp3PY+D8drHadMGfJi3twoEmLyjkOpCVL/igxg/4Wo4PaNocUl3Xr1oU8ydatW/HUU09h9erVGDZsmPB+YWEh6uvrYTabAQCNjY2YPn16WGNyKC0thdEYvMtvMKqqqlBWViZ7XiyJhs2ZDX3Ah4eQnGxEl3VwviHQtS84j9/vZO0XR6DXJ8HmCPyVP9loQLfVjtzc4cCRNphSU4Auab3EMtNNaOrqgyk1GR2WyCa2tTo9AAfqWqWLS89AMoABFOQNx9fH2qA3pgCQ9tnS0jMB9CBJbwQgvg0yE7zMrGzgZBdMaZn4966egMce70gC4IIhezz+tr4BT/5mPHIy9JI/UzShv8XowGy22WxhfSn3Juycy/bt2/Hoo49i1apVGDFixKCxiooKrF27FgBQV1eH6upqzJgxI6wxIvZ459uLhhlFW/FL2ZlSWJwJ+WExlmuJRlhMSUNJh2tw/kTOmhhW6iznunZ3fkxsjs1dJr52ewuaOwawY0+n5PMThFTCzrncc8890Ov1+N3vfie8t3r1amRnZ2PRokWorKxEeXk5tFotli1bhrQ0Ps6rdIyIPd468fc7JoqWG7OhJF3QQ/z2upeTcmEtY5LU3kAmAFIS+L44fbZcliUuTvnisusA763YRJp2stOxtjViJdVq84u/7MMPv2vGgkvyonZNIjaELS47d+4MOpaamooVK1aoOkbEHo3XgzzUOhbfSrCAx/h4HXJKkdmR8VqGywSCeTByFm9abbyYKdk8bO+xwPvieHO0kQ/PyWlfEy6N7QP45wdNmDQqFRoNcPb49Khdm4gu1LiSkIwnbCV9DqsEExMXFjrLdjesHDlcep6MradJNrA1ONJtSzFE/tefeSqsqkuO58Ie/qwNTCTZ/GWbIs9MKff94yjuXXU0atcjog+JCyEbOZ4Fy5/oRDyLS8/ni0B++F0z/vyLsbi2PF/y+bPcm26dMSIFU8emYem1Y6TbFoXffiYmA3b5Ia5QsHY04XKspR9Pv3UST7x5XJXzEQRAvcUIGeRm8w+zS88bhr+9fVLSHCYpYp7LD8/ORsU0M7RaDYZl6tHRw7dm0Wg8uYpgTBmXhgunZmP6pAxRAQtENHqYMXFhOZBOFZtQji9KRUtn+BuIfVLNn6O2kbovE+pB4kJIJj0lSdgj3pSixajc0IskmDaIbjCm1QzyhlKMvEshpXGxTqvB+ZMzQx8YaK6IGKndTkYswS6X9BQdeqxOpKeIVEkowMXxHQDSU3UozJFfwh8Mh5PDv7Y1Y94Fw0IfTAwZKCxGKOLCKdkYkx+8BJkhRSB8hcfo7mF2w6zg4bEMk849N7T34VuNNr0kI+B1vUlNVvdPQ01xYV2Y2fbTk0amSp5bnBf8C4EGfAeARU/sD8s+X/Yc7cXr25qx8t16Vc9LxDckLkRESXN/u55xVlbQY3xzOBqNBpsenYqfzcgNPkdYP+M/dscVI1F59SjhNaskyzcbAHgacIpVuvm2ypdCfrYh6NjBk+qFnNhnLjAb8NTN43HPgtEAgExT8EDESDMvSJlpwY+JVM0YW68TzYIBIvaQuBARJS/bgFcrSzD/B8GFIhhinoVnAzP/g8rLzLhwiqftPzvG4F5wKVSWiZzfoKARpl6vPIdTMkq693HDrAL8dMZwzJyShUmjTBieqcdVF+XijitGBp2Tnsw/4Me5F7wGKgYQ24UzHIR9aQK4sd/WWfD2p6H7rRGJB4kLEXGGZxok7evii9gUrSAuoc/DjmUeDBMO7/M/9stxeGTRWOG1UYHnolew1oa1XZHjKWWlJeFXPy6EPsnjgd0wqwDnTsrwO9bsLu++YIILv5s3Atdckofs9CTcOKtAtq1K8XR39h+784XDeGF94N6FRGJD4kLELWKCJOwBI0O0mLh4L7gcnZeM2+aNwNRxaYMW9LGHfapR+p8IO+/YAundIJnwyREzsco73/PeNncEkg1a5GYAs8/NQYpRh3/dOxkXfSdbdP4fXziM2fd8I2s76GAwa9UswybiH6oWI2LGqj9Mwqe7v1U019OeP/gxmaYkdFkcQlEB+6af5PZcOA54fvHEQXOK85JR19wv5DXMGXr0nfI0xBSrImPikixjcSbTRjlhODFvbfVdJbA7XGhoG0B1bS/OOzMT6x48K2DH28qrR8GcrsddLx7xG6up4/eJcTg56JM04DgOrd12DM8MnlcKRqCtA5Sw9qNm6LQaXPa9YejucwoFDUR8QuJCxIzCYUaMzVX2wJHisDy/eAIGHBxufvoAAE/Yim32FcgDePLm8WjvduDFDfUBj9GKVP/q3YokR1yYSMppXyPmueS5iwpGDE8OGCbzxjsvFYzn32vAxl1t+M2lhXj+vQY8fcsZ2H2gG5d+b5hoAYE3TAzldCcIxOrN/J5Ou/Z3o7rWIpTFE/EJhcWIqJKRqs7aDCnP4qw0PXKzPN+02QPclMzbwEqSvUk16jBiuNFvV8wx+XyoK1iDzExTkvAQ1UvZftMN+xxSQl2MaCz+ZGzcxe9cyTonv/FxC9Z82IzVmxtln0utsBjbfZOIb0hciKjxyt0leOnOSaqcS04LGvZIY+KSYdLh1btL8KsfF4Y8P7tKqNb+y24Y49naWYYXosRzUbtJ5/OLJ2LFrWeIHsPKiS3ukGD/gAtz76/Gmx+3hDw/8xTFmnaqkdsh4gsKixFRg3kRv728CFZbeIsKhS/vEr7F52UbUNfULyTNNdBgeJZ47sBXuwIVAwTLv8ipGlMWFpN8qCRGiyysZByq59fpMO/jo286AQD/eL8RV14oXmbOSpDFPBcXJ80bHTTHxcn6kkFEF/JciKgz57xhIR9IoZDzULn/2mLceeVIoa2MlN2Q2flZniBQqOtf950prHjXaDyLGJmIiYUAWQjNt0xaSsRLbg81NalREJJiTonY1gFiwrPly3acaPHfvZOKz+Ib8lyIuCdJp/FLBsv5wlpgNqLAbBRi9VKm9rF97Hv4RpOBvBF9knbQg/63c4swpiAZWaYkbP6yXdQb0eu0sLlcnv1uvMQlVIRITn5GDlqtJ4QlB5eLg0YTvHTc5ZLguYhc96m3TgQ/b5zu40OQ50LEOc/89gy8cleJ3/tKHrDs0SbFOxjB9pRxH6v3Kl/+/gQXHrx+jPs1JxyWnpKEqy7KE+YwcdEHKDPW+XguntdSeqVF5oF629wRSEvW4dyJ8jbwumJZDW77v0NBx5mmiImLkmR/oBX/RPxA4kLENRNGpMKc4b+e4eKz+TLaEcPkd+9V8mj29kIqprj8y3wDnFRYVyMiBgXufmfZafxnFO0e7SNaalNxTg7eXFoqu8eY1ebCkYbgvdM8OZfg59i9vxtL/nFUVmLfZufQ1G4LfSAREygsRiQkF07JwiXfzUZ6ivRfYSUVScxLYNeRtHBPWLTJmmv6iwEL891QwbdtGeXO3RiStBhwBF6kqdVq4HJyqif0fWG36cIpWfjYXYKsxvnE9rJZvpbfqMxm55Bs8Nwvsf+yv29owIdfd+D1+85EVhotqIw3yHMhEhKtViNLWAYh44v/T6bnoOIcMxZekoeKc8z4RUXonlws9zDM7XEFinSVjDIBALLTknDJd82Cd5OVHvwzCaGzCFdIFbvX9LCO1jkZ0u5zZ68dbd3+zS/lhLystsHCKjbzq0M9AID2HukbsJ3qGsDSV2rRZ6MOzZGGPBciIVGyRTH7FqyRoS5pKTph2+Xf/5TvOlwVoonv98/KxL7jFlwwORNfHuwJ+IB84PpiHG6wItnAP8CL85OxaHYBys5Ixy0rDgY8L5+P4SIWFmNcV56PMfkpSEvRYcMXbW4bQz/Ar/7zXgDwWzkvx2F0eAmRy8WJzmWiJUVrHU4OLZ0DWPtRC3bt78Yn1Z2YNS1HumGEbMhzIRISJavUBXGRMFXqAzGQYKUadfj9T0d6dnMMcK5kgw6lxWnCa41Ggytm5gpdkgPBBDVSCX2GPkmLi8/OFh7acpp3BsIlw3NhVWPdFgcu/dMeVNUF/6xMXKSc/aHX6rDoif0eW6gWIOKQ50IkJPGydq50jAm1Tf0B17SwBpneVU2vVpagyxLcCxBbw8LyNHJ6l4UDywPNPjcHh9adVHweOYVgDieHhlYbevudcLmAzw8F/6zsfkgJu+3a3z1oDmlL5An7t/S5557DpZdeirlz5+Lyyy/Hxo0bhTGr1YrFixejvLwcFRUV2L59e9hjBAEoLEWW4blIdYx+/ZMivHj7xIAr/ln4yvtBNjzTgPGFwTcGE0vW37ugGFPHpYXtSUglN8uA9x6egtnn8uGjH00z46qLcrFotnjeyWZ34Wijp3pMTsnw+7vbsOjJ/fjW3ZVZTDdY9ZmctTn2MJtnEtIJ23O55pprcPPNNwMAmpubMXv2bFxwwQXIzMzEqlWrYDKZsHXrVtTV1WHhwoXYsmULTCaT4jHi9ESnHVzKGmjtSCg4eNakhDxW4jMoSafByNzA7VPYSv1JI1Oh12kwQcJe92LhvnMnZYTsdKw2TCDfe3gKtBpPMcGGnW1o6hhAXrYBzR0Dg+bc94+j+LbOgtV3lWDVpgZMlPC5Gayk+bC73YwUr0dOwQDrkUZLZCJP2OKSnu5ZcNXX1weNRgOX+6vEpk2b8NhjjwEAiouLUVpaih07dmD27NmKx4jTk9fuORN9Nhe6LQ7sO96naGdLj+cSnZiaKVmHh28ci4kjU4XKq1Aw0Zw2IR1fHuyJpHmy8C0ieG7xRPQPuHD7Sv/Fk8zreGljA/5b04Vjzf6tW4Lh8vEuxXRDSs+yX/xl36Bxl4w8DREequRcXn/9dbzyyitoamrCI488guxsfoFbQ0MDioqKhOMKCgrQ1NQU1pgcampqFH0eAAE3V4p3EtFmQL7do1KAqqrjsq/T3q4FoEVtXS2qXEdFj21p4Y89efJEwGvJsfnAXnl23j0HMCZ14MuDg/88w/3/jcTvx8CADsF8wcaWDgBa9Pf3Bz3Gl66uHgAatLa1AdCKhrw4FwdAg/37D8DWFviYxnb+HmrAgYMGnV1dALQ4duwYqnR1kmxSQiL+Laptc0hxmTdvHhoaAu9x/dlnn0Gn0+Hqq6/G1VdfjQMHDuDOO+/E9773PUFgYklpaSmMRvkruKuqqlBWVhYBiyJHItoMRNfuDw4dA050YuyYMSgLsc3v7oZ64HArRowYibKy4YPGombz298MehnONSNl88S9tfh8bzfOK8nAzn3dg8YsjmQAAzAYkwFIW0mflpYGtFqQmWUGjneKhq84t2ANyx+L/3zVjj/+fJT/Ysp/u++hRgNwQHp6BtDcC1NWIZ7a0oZHFo3DqCBhTaUk4t8is9lms4X1pdybkOKybt06ySebOHEicnNzsWvXLsyaNQuFhYWor6+H2WwGADQ2NmL69OkAoHiMIJRCcXb1uf1nI/HD71qEUJg3Te18LkbODpQs1MXCV1LSKe9+3op9x/vw7metuO5HgYsN+BVCnt+BXfu70dbtwFufnMLtPxsp2T5COmGXnRw54tl/+8SJE9i3bx/Gjx8PAKioqMDatWsBAHV1daiursaMGTPCGiMIpQiNK2NqhXweumEMFv90RKzNCEh6ahLOn5wpuqiViYwUWBiMiYoUcWH5oAGHyMFCDmdwObdtILx9hYjghJ1zWbFiBQ4fPoykpCTodDosWbIE48aNAwAsWrQIlZWVKC8vh1arxbJly3i3N4wxglCMZ8VjQjFtYnQrxJSg1sJO3yS9FHFh/63eh760sQFFXk1NmXVMvJJ89ush1CdscXnmmWeCjqWmpmLFihWqjhGEUuS03I8H3ry/NGFsZSXKRcOMqG9V3qmYlZuzjcWkrGFhgsR5KdFbnwzu0aNxt85hx+rcxXsOBa3+CWnQCn3itEFJb7FYPtylli/HA6yE2u4IL8zEOlcLuRcJz34mQByA/gEnDAF2DfU7lhv8k1Af6i1GnDbI8VxYlCdBHIeY89PvD8ePpplx86VFoQ8WwbOxGP9TysOfHeJ0cZi3tAYvrA9c3Sr1fIQ6kOdCnDakuJO4gbYs9mXBJfmwOThUnEudc6Vg1Gtx+89G4uSpMDfvYol8YbGjlC6j/I+ePr6N/ravO4IeSrtXRg8SF+K04aY5RSjMMUpqoZKWosNtc+OzQiueMSXzAl46xoSaWv/y5FDICYcxWFsfVi2mDRBN1ArVYu457PwkNhGDwmLEaUNaig5XX5wX8c22Tmey0/VY+bsJuOuqUYrmcz6ei5w5UnqMsd5lpCmRh8SFIAhVGVOQgsxUd9sVmTp+wh1Wk9OMUhAXVlYcYKrv6dj57U4Os+/5Bu9+1irPUCIkJC4EQaiOPkmDBRfn4YmbxgcNQ04YkRJ0vpw2+lLWxvh6Qqw7ss3OX2jVpuBFAIQySFwIglAdjUaDa8vzceZoEx68fgx+Mt1TGMGikmLbNZ9UsFZG8HYkeC52dzka83povYv6kLgQBBFxbvUqjmChMr3IepR+GW1ZjjbyLf09LWNCC4XvHMrBqA+JC0EQUYUVVEgpCZeDUL4sp9IswMGN7TbsOdqrllmnLVSKTBBEVNG6W7Eo2U1UDE/TSxldmL0Wa+7a343hmXrcsuIgAGDTo1NVte90g8SFIIiownIuapeEe0qSpc/xFqKlr9Sqas/pDoXFCIKIKiznonJUTNK2x35zJApRU3uYnQdOQ0hcCIKIKsxjUdtzUZKclxJC+3hPB278y358fbgH9a02atMvERIXgiCiCttYTMkeMBNHpgYdk7OqnyElhLbnKN/GZvf+bvzyyf14c0eL7OucjpC4EAQRVbTuuJjOHRdLS5a+tYBYhZmSpSqBqsWE87k49PQ5hBxRR68DAPDlgW75FzoNIXEhCCKqCDkX7eDXUmALL7PT/WuRjjX3y7ZFLOfy8uZG/Pyhb2G18QcxO+0ODr1WZ9h71wx1SFwIgogqbIth5sHIyb0wcVErWyOWc9lS1Q4AGHCLiPceM1cuq8Hja4/Lu5aLE/WUhhokLgRBRBUWDmO5F62MpxCbqyRfE4iWTnvQMZu7SwALtzFPhQnSf2u6Qp7f6eJw76oj+PpwD35y3x786eXTp9yZ1rkQBBFV2OJJtt20FJ3QaHiPQRehSrNA2N1VYaxQgFWJyakW6+lz4OvDvTjSyLf6rzrUo7KV8Qt5LgRBRJV7rh6N8rJs5GbrAUjzQoRtpzWDX0cSdgkmJqyTslOCuHAch44euxBKU8vTSiRIXAiCiCpj8lNwxxWjZHkhwtoYVgyg9grMAGjcSsYWZbLOyVLKl3fu68aCR/Zi5z4+dGYQadI5VFHtE3/xxRcoKSnBa6+9JrxntVqxePFilJeXo6KiAtu3bw97jCCIxGTWNDNGDjf6vS/Nc/EpAoiiI+B08j/Zts1S2vMfONEHANi5ly9bTjF6HrVt3XZ09ATP9QwVVMm59Pb24oknnsDMmTMHvb9q1SqYTCZs3boVdXV1WLhwIbZs2QKTyaR4jCCIxGTxz0YOes30QSfhK65vWCwaYSah9NjHVfEOi73xUTMy05Iwaxq/X43TxaHd4rGXeT3ec65bvhcul7TGmP0DLmg1QJfFAaeLQ77ZX5zjFVU8l8ceewyLFi1Cdnb2oPc3bdqE+fPnAwCKi4tRWlqKHTt2hDVGEMTQQopQ6MIoX2akGJQ97vYe6xv0mi2mBICXNzfh6bdOCq/XfNCEv25Kwqku3jNhRQF2L3GRs8vmvKXV+NVf9+Oel47gxr/sV2J+zAhbXD7++GN0d3ejoqLCb6yhoQFFRUXC64KCAjQ1NYU1RhDE0EJsR0qG1mfBpRLHRcp1vGEJfCl09NjR0GbDlwf5arAeKx9LczjkV5gBQH2rDf0D/DlaOu2obxuQNT8eCBkWmzdvHhoaAu8v/f777+PJJ5/Eyy+/rLphalBTU6N4blVVlYqWRIdEtBlITLvJ5vAZ6NYA0GFYigWHQnzPdTkdADRoa28DoEW/tQ9yl1K6XA7Zc6Ry01+/RU+/BvmZHAANers7AWjR3WMBoIG1f8Dv2sH+P1wccP9bSRif54Lv9/8vdlVhwAmkGtT/DGr/foQUl3Xr1gUd+/LLL3Hq1ClceeWVAICOjg5s374dnZ2duPXWW1FYWIj6+nqYzWYAQGNjI6ZPnw4AisfkUFpaCqNRfoyyqqoKZWVlsufFkkS0GUhMu8lmdSgDMGO6DZZ+Fz7/20HRY41GPXptDpizc4BjHUhPNwHtfaJzfDEY9LDYHKEPVEBPv3t3TWMyABuG5ZiB+k4YjCkA+mGx+Yva2AlT0NHrwNiClEHvW/qdwFs1OHrKX3A37s3GrgM9qm9kxn4/bDZbWF/KvQkrLDZt2jR8/vnn2LZtG7Zt24ZZs2bhtttuw6233goAqKiowNq1awEAdXV1qK6uxowZM8IaIwhi6JBvNkpK6Gt8GpDJSegX5ycD8ORrIglL3LM2L3aRcNijrx/Db1f4i+qAnU/KBPqMuw7wYbdEaPsf0eLrRYsWobu7G+Xl5bjpppuwbNkypKWlhTVGEMTQQkouhD1oOfAPVVn9yLSsZYwC42Qi7Ibp/lnfGnyTsWp3aXNdkxW3/u0geqwOvPFRM7r7nIPOFQibPf6bZqra/uWxxx4b9Do1NRUrVqwIeKzSMYIghhZSFkQKHZQhXyg8vcwi77mwJTByPIuX32/EkQYrXlzfgA++6sBXh3pDnuO091wIgiBCISXExTYJm3NeDi4ozcTPL8yTfH7mGcmtFlMC60PGml5Kwe7Ts6yrL3ReSE4lW6ygxpUEQcSUJK/WLsEWv/9mThEqzslByWgTlow2oaVTemmuJywWhZyL+wPIsY+JC7NPinDYnS7Ut9pQNCx+F1WS50IQREzxtN4P/vBPStJgylhP3lVObt6zrbIS6+TBxLFBxroU5uWwbtFSGmNu2NmGXz65HzW1vfKNjBIkLgRBxJjQeRTfSi+dDHXxLLyMXrWYHCz9fAKfbeHs224mEGyxZnOHR8QOnuyD1eaUff1IQeJCEERMYd/YC3KCh3h8NxTz7ZIsBhOVaITF2Mp8ObB2MKwhplj5MoOVK7P7MGB34ffPHsJDr9XJvn6kIHEhCCKmmJJ1uG/haPz5F2ODHuPrqbAuw2INitNSdAA8D2CdLkxDIwTb2VLYjExCzsW3WowJ0teH4ydMRuJCEETM+X5pFszp+qDjvhEtQ5IGxXnJuPPKkYEnwD/XoqQUOdMU+ZonJpA2u/Q+ZA6vXTJXb25ES0f89R6jajGCIOIeX2HQaDR4bvFEAMATb54IOId5O5owwmLRWXjJC0W/O7E/IKlajD9m/4k+rN/ZJuRg4gkSF4Ig4pYrznGidSBH0VyhCi2KnZSV0NbNr2uRszbG47nwr3sV5HoiDYkLQRBxy3dGcygrG61oLvNUwtlgTCyUlmrUos+mXhuWfhniwo5l+RqXhN0xow3lXAiCGJL4CgOrSisvyw50eEDEHBd9kvzHZ3pK8KqCAyfldXkGgAF3nsYl1ogsRpDnQhDEkIR5Kix0pE/S4l/3nomM1CRsreqQdA6tiLooCZmZUnSKypWDYXVvKCZhaUzUIc+FIIghCUvGG/X8P0blGpGdrpfUKJPBjjSn+38P1ysQF2aLWgjhsTgMi5HnQhBEXDFtQjqqay1ht5Vnnsu5Jek4f3IGzivJlH0OjgOWXluMghwjPt/bBavNhTc+bgHAt6SRi0HBnNwsPVo67QHHrDYmLp73WjoHkJsVga0qZULiQhBE3MB2WOzosUOr1eDw/m8Un8s7gX9BaZboscGaZnIAzjuTF6XRefymY0xc2OmnjDVhz1GLJJv0CsRFLPy2/wSfp3F65VzuX12L591l2rGEwmIEQcQd2en6sBcwChuMSYgY+a+j4X9yARTHN7QlJ9Sld8fqpo6TvvkhC+OJtUazelWtHWvul3zuSELiQhDEkEToRyZBXHzLlM8enxZ06srfT8CShaOFxZlyEvusBY1RhgfDcjtTxybWbrwkLgRBJDQrfz8B//jjJL/32UM/ULhr+a/G4cnfjBde+67EZ2XGgSSgMMc4KMzGjh1fmOKe43/BWdPM7uu4BUlGGbMwJwoLOtWEci4EQSQ0Y/JTAr6fYuTdhECFAVN8vAD2AM83G9DUPiA8yKXUYOmFbZT511oNwNqD/aKiAHYHh6svzsVvLi3En9ccGzRHCtHcSVNNSFwIghiSXFeeD5eLw8wpWSGPZXkNFh2T8vBnR3gaZHr2pXG6l7JceWGucHyyQSfkf5hQGJI0IXuJMdvklFDHAxQWIwhiSJKbpcdDN46FKVl6r30mEFKqunyP0XqJSzCmuBP5wzL5DtCpEmxj2zSzn0rKmWMBeS4EQQxJ5LTY93ghbnFxK4RYpdk9C0Zj7fYWjMpNBtAheD1iVV1XzhyO6ZMycLyl332d0DYyL4cJH29j/C2a9IU8F4IghiRy+lSyRzXzOqRsLFZgNmLxz0YKD39hd0yfzgDeaDQaYb0MIJ5HYSLFvBuHu8dLFDbUVIWwxaWyshIzZ87E5ZdfjssvvxzPPfecMGa1WrF48WKUl5ejoqIC27dvD3uMIAhCjNnn8pVZShpLsi2RtWLuh98c/idrwZKcBDyyaCxevCP0QkaxVf7Mq7muPB8XTc1CeRn/uZRsehYLVAmL/frXv8Y111zj9/6qVatgMpmwdetW1NXVYeHChdiyZQtMJpPiMYIgiEA8eP0YNLTZcNn3huEXFYWKqqt894CRgsY9h5U8D0vncPb4dNE5bD+WVENwF+mu+aPxyuZG5GcbcPf80dh3jO8CIGXrgCMNVowrDFxFFy0iGhbbtGkT5s+fDwAoLi5GaWkpduzYEdYYQRBEIM6dlIG5FwyHVqtBmkhrezHkeCwM1klg6tg03HHFSFxxbuieaMPdCf2zxgb/wnzB5Ey8eMckvxX6WglP7R6rI/RBEUYVz+Xll1/G2rVrMXLkSPzhD3/AuHHjAAANDQ0oKioSjisoKEBTU1NYY3KoqalR9HkAoKqqSvHcWJGINgOJaTfZHD0iazf/CLTb7QA06OuzANCguaUFgBY2my3k9ZM54IpzNJiQVc+fTS/N5lsuAdKTGxHsMex7jpPtvL1OB2+rGEcPH4SzM6QJotcLl5DiMm/ePDQ0NAQc++yzz3D77bdj+PDh0Gq1ePvtt/HLX/4SH3zwAXRSMmIRprS0FEajUfa8qqoqlJWVRcCiyJGINgOJaTfZHD0ibfcjmT3o6nXghQ0NgM2BjIw0oM2C/Lw84NApGIxGlJWVhDzPtGnybS4DYOl3AhsCfwn2PUd+qw3Yth8TR2Vg94GegHOSDVr0D7gw+cwSTByZGtIGX5ttNltYX8q9CSku69atEx3Py8sT/j137lw8+uijaGpqQlFREQoLC1FfXw+zmU9ENTY2Yvr06QCgeIwgCEItWG7khQ38F2idT8VXqjGyBbWBSpFfubsEHT3+Ya2iYUY8smgsRucmY+GjewOej1W7xcNamLDvXHNzs/DvTz75BFqtVhCciooKrF27FgBQV1eH6upqzJgxI6wxgiCISMFy5XlZBiyaXYCl142J6PUCFR3kZhmCeh1nj09HanLwx3Y89SELO+dy9913o62tDRqNBmlpaXjuueeQlMSfdtGiRaisrER5eTm0Wi2WLVuGtLS0sMYIgiAihXcl1hUzc0WOVAdWVpybpcf0SRnuBZniJIm0ANAKjTGHgLisXr066FhqaipWrFih6hhBEESkiMUaEhbqMmfoJR2fpNNg9rlmTJuQgYdeqxs0lmLQohOeDgOxhNq/EARx2qOgAlk1Qq2JCcTv5o0M+P4ffz4KW79qR05G7B/tsbeAIAgixmSmJg1KoqdEOJEfKUpGm1AyOj4Wm5O4EARx2rP0umJs/rId8y4YjnWfnsKFU7NjbVLCk5jyTBAEoSL5ZiOu/1EBMkxJuP5HBXFRbSUH72aY8QJ5LgRBEAmKUa9Fkg546ubx6LE6Y23OIEhcCIIgEpQ3/jQZ0ACGJK2wrXO8QOJCEASRoBgC7BkTL8SvZQRBEETCQuJCEARBqA6JC0EQBKE6JC4EQRCE6pC4EARBEKpD4kIQBEGozpAsReY4DgAwMDCg+Bw2m00tc6JGItoMJKbdZHP0SES7E9Vm9sxkz9Bw0HBqnCXO6OnpwcGDB2NtBkEQREIyYcIEpKfL79bszZAUF5fLBYvFAr1eD00se2kTBEEkEBzHwW63w2QyQasNL2syJMWFIAiCiC2U0CcIgiBUh8SFIAiCUB0SF4IgCEJ1SFwIgiAI1SFxIQiCIFSHxIUgCIJQHRIXgiAIQnWGZPsXpdTW1qKyshKdnZ3IysrC8uXLUVxcHGuzAAAXX3wxDAYDjEYjAODOO+/EjBkzRG2O9udZvnw5Nm/ejPr6erz33nuYMGFCSDtibX8wm4Pd73iwuaOjA3fddReOHz8Og8GA0aNHY9myZTCbzXF9r8Xsjuf7fcstt+DkyZPQarVITU3Fn/70J5SUlMT1vQ5mc1TvM0cIXHvttdzbb7/NcRzHvf3229y1114bY4s8/OAHP+AOHDjg976YzdH+PLt37+YaGhr8bFVqYzTsD2ZzsPsdDzZ3dHRwO3fuFF4/9thj3D333BOWbbG2O57vd3d3t/DvrVu3cnPnzg3LrljaHM37TOLiprW1lSsrK+McDgfHcRzncDi4srIyrq2tLcaW8QT6pRCzOZafx9tWpTZG236p4hJPNjPef/997vrrr0+Ye+1rN8clzv1et24dN2/evIS618xmjovufaawmJvGxkbk5eVBp9MBAHQ6HXJzc9HY2Aiz2Rxj63juvPNOcByHsrIy3HHHHaI2cxwXF59HqY3xYL/v/c7IyIi7e+5yufD666/j4osvTqh77W03I57v93333YdPP/0UHMfhpZdeSoh77WszI1r3mRL6CcKaNWvw7rvv4q233gLHcVi2bFmsTRrSJMr9fuihh5Camoprrrkm1qbIwtfueL/ff/7zn/HRRx/h9ttvx+OPPx5rcyQRyOZo3mcSFzcFBQVobm6G0+kEADidTrS0tKCgoCDGlvEwOwwGAxYsWICvvvpK1OZ4+TxKbYy1/YHudzifJxIsX74cx44dw9NPPw2tVpsw99rXbiAx7jcAzJ07F1988QXy8/MT4l5729zR0RHV+0zi4iYnJwclJSVYv349AGD9+vUoKSmJi5BYX18fenp6APAtsTdu3IiSkhJRm+Pl8yi1MZb2B7vf4XwetXnqqadQU1ODZ599FgaDISzbYm13PN9vi8WCxsZG4fW2bduQmZkZ1/c6mM1GozGq95la7ntx5MgRVFZWoru7GxkZGVi+fDnGjh0ba7Nw4sQJ3HbbbXA6nXC5XBg3bhyWLFmC3NxcUZuj/XkefvhhbNmyBa2trcjOzkZWVhY2bNig2MZo2B/I5ueffz7o/Y4Hmw8dOoQ5c+aguLgYycnJAIARI0bg2Wefjet7HczuysrKuL3fra2tuOWWW2C1WqHVapGZmYm7774bkydPjtt7HczmjIyMqN5nEheCIAhCdSgsRhAEQagOiQtBEAShOiQuBEEQhOqQuBAEQRCqQ+JCEARBqA6JC0EQBKE6JC4EQRCE6pC4EARBEKrz/wG3jT5R2mmV8gAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(len(df_train) + len(df_eval)), df_all)" + ] + }, + { + "cell_type": "markdown", + "id": "9fef8177", + "metadata": {}, + "source": [ + "# Cost Evaluation " + ] + }, + { + "cell_type": "code", + "execution_count": 413, + "id": "5fecde25", + "metadata": {}, + "outputs": [], + "source": [ + "from sktime.performance_metrics.forecasting import mean_squared_scaled_error\n", + "def get_loss_per_key(key: int, path, oracle_filename):\n", + "\n", + " oracle_residual = pd.read_csv(oracle_filename)[\n", + " \"pred_residual\"\n", + " ]\n", + "\n", + " df = pd.read_csv(path)\n", + " residual = df[\"pred_residual\"]\n", + " mask = ~np.isnan(residual)\n", + " loss = mean_squared_scaled_error(\n", + " y_true=oracle_residual[mask], y_pred=residual[mask], y_train=df[\"value\"]\n", + " )\n", + " loss = {\n", + " \"loss\": loss,\n", + " \"n_fits\": df[\"model_version\"].dropna().nunique(),\n", + " }\n", + " return loss" + ] + }, + { + "cell_type": "code", + "execution_count": 414, + "id": "21099224", + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "get_loss_per_key() missing 1 required positional argument: 'oracle_filename'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mbaseline_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m101\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_loss_per_key\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"{artifact_dir}/plan_eval\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0mbaseline_results\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlosses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: get_loss_per_key() missing 1 required positional argument: 'oracle_filename'" + ] + } + ], + "source": [ + "replica = 1\n", + "baseline_results = {}\n", + "for key in range(1, 101, 1):\n", + " losses = get_loss_per_key(key, f\"{artifact_dir}/plan_eval\")\n", + " baseline_results[key] = losses" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d82f9ca7", + "metadata": {}, + "outputs": [], + "source": [ + "slide_size = 12\n", + "baseline_total_cost = 0\n", + "baseline_total_loss = 0\n", + "for key in baseline_results.keys(): \n", + " for loss in baseline_results[key]:\n", + " if loss['slide_size'] == slide_size:\n", + " baseline_total_cost += loss['n_fits']\n", + " baseline_total_loss += loss['loss']\n", + "print(baseline_total_cost, baseline_total_loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da85dcf1", + "metadata": {}, + "outputs": [], + "source": [ + "lp_results = {}\n", + "\n", + "for key in range(1, 101, 1):\n", + " oracle_filename = f\"{artifact_dir}/plan_eval/oracle_key_A4Benchmark-TS{key}.csv\"\n", + " filename = f\"{artifact_dir}/lp_plan_eval/{plan}/{key}.csv\"\n", + " lp_results[key] = get_loss_per_key(key, filename, oracle_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "303fcfcf", + "metadata": {}, + "outputs": [], + "source": [ + "lp_total_cost = 0\n", + "lp_total_loss = 0\n", + "for key in lp_results.keys(): \n", + " lp_total_cost += lp_results[key]['n_fits']\n", + " lp_total_loss += lp_results[key]['loss']\n", + "print(lp_total_cost, lp_total_loss)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "656758ed", + "metadata": {}, + "outputs": [], + "source": [ + "experiments = [(\"max_fits_1100\", 96), (\"max_fits_2100\", 48), (\"max_fits_4200\", 24), (\"max_fits_8400\", 12)]\n", + "\n", + "graph_results = {\"baseline\": [], \"optimized\": [], \"cost\": []}\n", + "\n", + "replica = 1\n", + "for plan, slide_size in experiments:\n", + " print(plan)\n", + " \n", + " baseline_total_cost = 0\n", + " baseline_total_loss = 0\n", + " for key in baseline_results.keys(): \n", + " for loss in baseline_results[key]:\n", + " if loss['slide_size'] == slide_size:\n", + " baseline_total_cost += loss['n_fits']\n", + " baseline_total_loss += loss['loss']\n", + " print(baseline_total_cost, baseline_total_loss)\n", + " \n", + " for key in range(1, 101, 1):\n", + " oracle_filename = f\"{artifact_dir}/plan_eval/oracle_key_A4Benchmark-TS{key}.csv\"\n", + " filename = f\"{artifact_dir}/lp_plan_eval/{plan}/{key}.csv\"\n", + " lp_results[key] = get_loss_per_key(key, filename, oracle_filename)\n", + " \n", + " lp_total_cost = 0\n", + " lp_total_loss = 0\n", + " for key in lp_results.keys(): \n", + " lp_total_cost += lp_results[key]['n_fits']\n", + " lp_total_loss += lp_results[key]['loss']\n", + " print(lp_total_cost, lp_total_loss)\n", + " \n", + " assert lp_total_cost <= baseline_total_cost\n", + " \n", + " graph_results[\"baseline\"].append(baseline_total_loss)\n", + " graph_results[\"optimized\"].append(lp_total_loss)\n", + " graph_results[\"cost\"].append(baseline_total_cost)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 163, + "id": "959d7dc6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Text(0.5, 0, 'Cost Budget'),\n", + " Text(0, 0.5, 'MASE Loss'),\n", + " Text(0.5, 1.0, 'Residual Estimate Loss for Time-Series Decomposition')]" + ] + }, + "execution_count": 163, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmsAAAFSCAYAAACt0ZgcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA8xUlEQVR4nO3dfXyP9f////vsFMOabE3OQtayj8xmS4YaEc1WTpr2JuQd5ayETHJONXpHZMhb3u/elaIwJuktpynLROxtRXK+MzbkdCevHb8//Ly+LTYb216H7Xa9XFwuvY7jdTyPx3G8ntvuPZ/HcbzsDMMwBAAAAFOqZOsCAAAAUDDCGgAAgIkR1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENFdrq1av1/PPPF7i+T58+Wr58+W3vJz4+Xm3btr3tdm7mZsdTERmGobFjx6ply5bq0aNHqe4rOTlZfn5+slgspbqf0kDfKXl+fn46fvx4geuffPJJxcfHl2FFuFPZ8Zw13ClCQkJ0+vRp2dvbq0qVKmrTpo3Gjx+vqlWrlto++/Tpo7CwMPXs2fO22omPj9fo0aO1devWG6739vZW5cqVZWdnZ102ePBgvfDCCwW2eeLECbVv317/+9//5ODgcFv1FcXtnouSOpfFlZCQoFdffVVff/21qlSpctttXftMDMPQ5cuX87W5du1a1a5d+7b2URwLFizQsmXLdObMGVWrVk0tWrTQ7Nmzy2z/N7NixQqNGzdOLi4ukqS77rpLQUFBGjhwoO677z4bV1e2oqKi5OnpqREjRti6FNyBSv83PFCCFixYoEceeUSnTp3SgAED9MEHH5SbX36xsbGqX7++rcsod06ePKl77733loJabm5uviAcEBCg3bt3S/p/YXnnzp1lEpb/auXKlYqNjdW//vUv1atXT6dOndLGjRtvqa2/HmdJat68uZYuXSqLxaKTJ0/qww8/VLdu3fT555+rSZMmpbJPoLxhGhR3pFq1aik4OFhJSUnWZXv27FGvXr0UEBCgsLCwfNMLK1asUPv27eXn56eQkBCtXr3auvzZZ5+1vm/79u164okn5O/vrylTpujPA89z587VqFGjrK9PnDghb29v5ebmSpK+/PJLde7cWX5+fmrfvr0+++yzEjnWvXv3qlu3bmrRooUeeeQRvfXWW5Kk3r17S5JatmwpPz8/7d69+7rj8fb21ieffKKOHTvKz89Ps2fP1rFjxxQREaEWLVro5ZdfVnZ2tiTp3LlzGjRokB5++GG1bNlSgwYNUmpqqiRp1qxZSkhI0JQpU+Tn56cpU6ZIkg4dOqT+/fsrMDBQnTp10ldffVXs48vLy1NMTIwee+wxtWrVSq+99prOnz8vScrKytKoUaMUFBSkgIAAde/eXadPn5ZU8Gf6Z8uXL9cbb7yhPXv2yM/PT3PmzJEkLVu2TI8//rgCAwP14osvKi0t7YbnrGPHjsU6lr/2iT59+mjWrFnq1auX/Pz89OKLL+rMmTMaOXKkWrRooe7du+vEiRPW7YtzPvft26fg4GDVq1dP0tWfiYiICOv68+fP6/XXX1dwcLDatGmjWbNmWadnV6xYoV69eunNN99UYGCg5s6de13fKayWLVu2qEuXLvLz81ObNm20ePHim54be3t71atXT5MmTVJgYKDef/9967rCfnbPnj2rsWPHKjg4WC1bttTgwYOt64r6Od6s71+7TGHBggUKCgq6rj+dP39er732mh5++GE99thjiomJUV5eniTp6NGj6t27t/z9/RUUFKRXXnklXw1Hjx7V559/rjVr1mjx4sXWfiBdnS34/vvvJUnZ2dmaPn26goODFRwcrOnTp19X34cffqhWrVopODhYX3755U3POcoRA7hDPPbYY8b27dsNwzCMlJQUIzQ01Jg6daphGIaRmppqBAYGGps3bzYsFovx3XffGYGBgUZGRoZx8eJFw8/Pzzh06JBhGIaRlpZmHDhwwDAMw/jyyy+NXr16GYZhGBkZGYafn5+xbt06Izs721iyZInh4+NjLFu2zDAMw5gzZ44xcuRIaz3Hjx83mjRpYuTk5BiGYRibNm0yjh49auTl5Rnx8fFGs2bNjMTERMMwDGPHjh1GmzZtCjy2Jk2aGEeOHLnhumeeecZYuXKlYRiGceHCBWP37t033P9fj+dau4MGDTLOnz9vHDhwwGjatKnx3HPPGceOHTP++OMPo3PnzsaKFSsMwzCMzMxM4+uvvzYuXbpknD9/3hg2bJjx0ksvWdvq3bu39VwYhmFcvHjRaNu2rfHFF18YOTk5RmJiohEYGGg9t3/11+2vWb58udGhQwfj2LFjxoULF4whQ4YYo0aNMgzDMJYuXWoMGjTIuHTpkpGbm2vs27fPOH/+fKGf6V/99Zx8//33RmBgoJGYmGhkZWUZU6ZMMSIjI/Ods379+hlnzpwxLl++fMM2Czr/f13Wu3dvo0OHDsbRo0et57tjx47G9u3bjZycHGP06NFGVFTULZ3PVatWGS1btjQWLVpk7N2718jNzc23/qWXXjLGjx9vXLx40Th9+rTRvXt3Y+nSpdZz4uPjY3z00UdGTk6Ocfny5Xzn6Wa1tG7d2ti5c6dhGIZx9uxZaz+/2bm/Zvny5UarVq0Mwyj8Z9cwDOOFF14wXn75ZePs2bNGdna2ER8fX+TPsah9f8eOHYaPj4/x5ptvGllZWUZ8fLzx0EMPWfvX6NGjjRdffNE4f/68cfz4caNjx47WvjxixAgjJibGsFgsxpUrV6zn5VoN136ux4wZY7z77rv5zsOff6fNnj3b6Nmzp3H69GkjIyPDiIiIMGbNmpWvvtmzZxvZ2dnG5s2bjWbNmhlnz5694XlH+cPIGu4oQ4YMkZ+fn9q1ayd3d3cNHz5c0tUpxLZt26pdu3aqVKmSWrduLV9fX23ZskWSVKlSJR08eFBXrlyRh4eH7r///uva3rp1qxo3bqwnnnhCjo6O6tu3r+6+++4i1/boo4+qXr16srOzU2BgoFq3bq2EhIQib//0008rICDA+m/btm2SJAcHBx07dkyZmZmqWrWqmjdvXuQ2JemFF16Qq6ur7r//fjVp0kStW7dW3bp1Va1aNbVt21b79++XdPV6ok6dOqly5cpydXXVSy+9pJ07dxbY7ubNm3Xvvfeqe/fucnBwUNOmTdWpUyetX7++WPWtWbNG/fr1U926dVW1alW9+uqr+uqrr6xTc2fPntXRo0dlb28vX19fubq6SiraZ1rQ/rp3766mTZvKyclJr776qvbs2ZNvhGvgwIFyc3OzXmt1O7p166Z69epZz3fdunX1yCOPyMHBQU888YT1/Bf3fIaHh+uNN97Qd999pz59+uiRRx7RBx98IEk6ffq0tm7dqtdff11VqlRRzZo11a9fP61du9a6vYeHh/r06SMHB4frjvNmtTg4OOi3337ThQsXVKNGDTVt2rRY58TDw0Pnzp2TVPjPbnp6urZu3arJkyerRo0acnR0VGBgoKSifY5F7fvXvPzyy3JyclJgYKDatWundevWyWKx6KuvvtLIkSPl6uqqOnXqqH///taRNwcHByUnJys9PV3Ozs4KCAgo1rm4Zs2aNRoyZIhq1qwpd3d3DRkyJN/onoODg4YMGSJHR0e1a9dOVapU0eHDh29pX7jzcM0a7ijz5s3TI488oh9//FEjR47UmTNnVL16dSUnJ+vrr7/Wpk2brO/Nzc1VUFCQqlSpolmzZunDDz/UuHHj1KJFC40ZM0aNGjXK13Z6erruuece62s7Ozt5eXkVubYtW7Zo3rx5OnLkiPLy8nTlypViXZOzcuXKG16zNn36dM2ZM0edO3dWnTp1NHToUD322GNFbvfPgdPZ2fm619emFS9fvqy33npL27Zts/4hvXjxoiwWi+zt7a9r9+TJk9q7d2++P04Wi0VhYWFFrk26et7vvfde6+t7771Xubm5ysjIUHh4uFJTU/Xqq6/qjz/+UFhYmEaMGFHkz7Sg/f05XFStWlVubm5KS0tTnTp1JKlYn/vNFHb+XVxcdOnSJUmFn8/k5GQ9+eST1uXXrpsLCwtTWFiYcnJytGHDBo0ePVo+Pj6qUaOGcnNzFRwcbN0mLy8v33H9ua//1c0+2zlz5mj+/Pn6xz/+IW9vb40cOVJ+fn5FPidpaWmqUaOGJBX6s5uamqoaNWpY3/tnRfkci9r3Jal69er5rmusXbu20tPTdebMGeXk5OS7caR27drWKdfRo0frvffeU48ePVSjRg3179//lu46Tk9Pv24f6enp1tdubm75riusXLmyte+g/COs4Y4UGBiobt26KTo6WjExMfLy8lJ4eLimTZt2w/e3adNGbdq00ZUrVzR79myNHz9en376ab731KpVy3qNlnT1br+UlBTr68qVK+vKlSvW13/+RZ+dna3hw4crOjpa7du3l6OjowYPHpzvmrdb1aBBA7377rvKy8vTN998o+HDhys+Pj7fnaMl4cMPP9Thw4e1bNky1apVS0lJSXrqqacKPAYvLy+1bNlSS5Ysua39enh46OTJk9bXycnJcnBwUM2aNeXg4KChQ4dq6NChOnHihPUuwp49exbpMy3K/i5duqSzZ8/K09PTuqykz21R3Ox8XgtoN+Lo6KjOnTtr0aJFOnjwoEJDQ+Xk5KQdO3YUeONAYcd4s1qaNWum+fPnKycnR5988oleeeUV6yh2UWzYsMEaBAv72U1PT9e5c+f0xx9/qHr16vnWFeVzLI4//vhDly5dsga2lJQU3X///brrrrvk6Oio5ORkNW7c2Lru2n5q1aplrT0hIUH9+/dXy5Ytr/sfr5v1KQ8PDyUnJ1tHiFNSUuTh4XFLx4Lyh2lQ3LH69u2r77//XklJSQoLC9OmTZu0bds2WSwWZWVlKT4+XqmpqTp9+rS+/fZbXbp0SU5OTqpSpcoNR4ratWungwcP6ptvvlFubq4++uijfIHMx8dHO3fuVHJyss6fP6+FCxda12VnZys7O1vu7u5ycHDQli1btH379hI5ztjYWGVmZqpSpUrWP1j29vZyd3dXpUqVCn2OU3FcvHhRzs7Oql69us6ePZvvAnDp6ijFn/f16KOP6siRI1q1apVycnKUk5OjvXv36tChQwXuIzc3V1lZWdZ/OTk5Cg0N1b///W8dP35cFy9e1KxZs9S5c2c5ODhox44d+vXXX2WxWOTq6ioHBwfZ29sX+TO9ka5du2rFihVKSkpSdna23n33XTVr1sw6GmMrxT2fK1as0ObNm3XhwgXl5eVpy5Yt+u2339SsWTN5eHiodevWevvtt63rjx07ph9//PG2a8nOztbq1at1/vx5OTo6qmrVqkU69xaLRcePH9fUqVP1448/asiQIZJU6M+uh4eH2rZtq8mTJ+vcuXPKycmxTs2Xxuc4d+5cZWdnKyEhQZs3b9YTTzwhe3t7PfHEE5o1a5YuXLigkydPasmSJdZRxnXr1ln/J69GjRqys7NTpUrX/2mtWbNmvinav3ryySc1f/58ZWZmKjMzU/PmzVPXrl1v+VhQvhDWcMdyd3dXeHi4dWQtJiZGCxcuVKtWrdSuXTstXrxYeXl5ysvL05IlS9SmTRsFBgZq586dmjhx4g3be++99/SPf/xDQUFBOnr0qFq0aGFd37p1a3Xp0kVhYWHq1q1bvqlIV1dXvfHGG3rllVfUsmVLxcXFKSQkpFjHEx4eLj8/P+u/6dOnS5K2bdumJ5980rps1qxZcnZ2VuXKlfXiiy/q2WefVUBAgPbs2XNrJ/L/17dvX2VlZenhhx9WRESE2rRpk2/9c889p/Xr16tly5aaNm2aXF1dtXjxYn311Vdq06aNgoOD9c4771jvYLuRSZMmqVmzZtZ/Y8eOVffu3RUWFqbevXurffv2cnJy0vjx4yVdHb0cPny4/P391aVLFwUGBiosLKzIn+mNtGrVSi+//LKGDRum4OBgHT9+XLNmzbr1E1dCins+XV1dtWDBAj322GMKCAjQO++8o0mTJllHrGbMmKGcnBx16dJFLVu21PDhw3Xq1KkSqSU2NlYhISFq0aKFPvvsM82YMaPAtq7dievv76/nnntOFy5c0BdffCFvb29JKvRn99pxODg4qHPnznrkkUf073//W1LJf4533323qlevrjZt2mjUqFGaNGmSdVp9/Pjxqly5sjp06KDIyEiFhoaqe/fukq7elduzZ0/5+fnppZde0rhx41S3bt3r2u/Ro4d+++03BQQE5Luj9ZrBgwfL19fXOrXdtGnTG74PFRMPxQUAVGg3e2g1YGuMrAEAAJgYYQ0AAMDEmAYFAAAwMUbWAAAATKzchjXDMJSVlVUiz7kCAACwlXIb1rKzs5WYmFjoYwQAAADMrtyGNQAAgPKAsAYAAGBihDUAAAATI6wBAACYmIOtCwAAALaRk5OjEydO6MqVK7YupcJwcXFRnTp15OjoWORtCGsAAFRQJ06cULVq1dSgQQPZ2dnZupxyzzAMZWRk6MSJE7rvvvuKvB3ToAAAVFBXrlxRzZo1CWplxM7OTjVr1iz2SCZhDQCACoygVrZu5XwT1gAAAEyMsAYAAEzPz89Px48fv+n7Tpw4IW9vb+Xm5t5w/dy5czVq1KiSLq9UEdYAAECJGzBggN57773rlm/YsEGtW7cuMEwVZPfu3apbt25JlXdHIazdQHaOxdYllJryfGwAAPN4+umnFRsbK8Mw8i1fvXq1unbtKgeHoj2Qorihrjzi0R034ORor8gJm21dRqn4dMqjti4BAFABdOjQQRMnTlRCQoJatmwpSTp37pw2bdqkxYsXKyIiQocOHZKLi4s6duyoqKgoOTk5SZK8vb01YcIE/fvf/1Zubq42btwob29vffPNN6pfv742b96s2bNn69ixY6pWrZp69OihYcOG5dv/l19+qblz50qSnn/+eT3//PM3rHPPnj16++239dtvv6l27doaN26cgoKCSvHMFB8jawAAoMS5uLioc+fOWrVqlXXZunXr1LBhQ1WpUkVjx47Vjh079Nlnn+mHH37Qp59+mm/7DRs2aNmyZfrqq6+ua7ty5cqKjo5WQkKCFi5cqKVLl2rDhg353hMfH69vvvlGixcv1gcffKDvv//+unbS0tI0aNAgvfTSS/rxxx81ZswYDR8+XJmZmSVzEkoIYQ0AAJSKp556Sl9//bX1uWKrVq3S008/LV9fXzVv3lwODg6qU6eOIiIitHPnznzbDhw4UG5ubnJxcbmu3aCgIHl7e6tSpUp64IEH9OSTT+rHH3/M954hQ4aoSpUq8vb2Vrdu3RQXF3ddO7GxsWrbtq3atWunSpUqqXXr1vL19dWWLVtK8CzcPqZBAQBAqQgICJC7u7u+/fZbNWvWTImJiXr//fd1+PBhvf3220pMTNTly5dlsVjUtGnTfNt6eXkV2O7PP/+sd955RwcPHlROTo6ys7P1xBNPFLj9vffeqwMHDlzXTnJysr7++mtt2rTJuiw3N9d006CENQAAUGrCw8O1atUqHT58WK1bt9bdd9+tkSNH6sEHH9Q//vEPubq66l//+pfWr1+fb7vCHh47cuRI9e7dW//85z/l7Oys6dOn68yZM/nek5KSokaNGkm6Gso8PDyua8fLy0vh4eGaNm1aCRxp6WEaFAAAlJqnnnpKP/zwg5YtW6annnpKknTx4kVVrVpVVatW1aFDh7R06dJitXnx4kXVqFFDzs7O2rt37w2nOGNiYnT58mUdPHhQK1asUJcuXa57T1hYmDZt2qRt27bJYrEoKytL8fHxSk1NvaVjLS2ENQAAUGrq1KkjPz8/Xb58We3bt5ckjRkzRnFxcWrRooXGjx9/wyBVmIkTJ2rOnDny8/PTvHnz1Llz5+veExgYqMcff1z9+vXT888/r+Dg4Ove4+XlpZiYGC1cuFCtWrVSu3bttHjxYuXl5d3awZYSO+OvD0ApJ7KyspSYmChfX185OzsXe3se3QEAKO+SkpLk4+Nj6zIqnOKed0bWAAAATIywBgAAYGKENQAAABMjrAEAAJgYYQ0AAMDECGsAAAAmRlgDAAAwMcIaAACQJGXnWO6odisKvhsUAABIkpwc7UvlofBFfSC7t7e3fvrpJ1WtWrXEa/izqKgo+fr6qnfv3lq6dKmysrLUr1+/Ut3n7SCsAQCACuvZZ5+1dQk3RVgDAACm8eGHH2r79u06c+aMXn31VXXq1EmSNHLkSB0+fFg5OTmqV6+e3nzzTdWoUUO///67xo4dq8uXLysvL09PP/20BgwYoOzsbM2aNUs7d+5UTk6OmjRpokmTJl03ajd37lxdunRJY8aM0YoVKxQXF6fq1avr4MGDqlatmubOnatatWpJkhYtWqT169fLYrHI09NTU6dOta4rTVyzBgAATMPOzk6fffaZ5s+frwkTJigjI0OSNG7cOK1YsUJr1qxR48aNtWjRIknSp59+qrZt22r16tWKi4tTjx49JEn//Oc/Va1aNX3xxReKjY2Vh4eHPvjgg5vuf9++fRozZozWrl2rxo0b6+OPP5YkxcbG6tixY1q2bJlWrlyptm3b6u233y6ls5AfI2sAAMA0evbsKUlq2LChHnzwQe3Zs0ft27dXbGys1qxZo5ycHF26dEkNGjSQJLVs2VLR0dHKyclRUFCQHn74YUnSxo0bdeHCBa1fv16SlJ2drQceeOCm+2/RooW8vLwkSQ899JC+//57a3uJiYl6+umnJUkWi0Wurq4leuwFIawBAABTMgxDdnZ2SkhI0NKlS/XZZ5/J3d1da9as0bJlyyRJnTp1UvPmzbV9+3YtWrRIX375pd555x0ZhqGJEyeqVatWxdqns7Oz9b/t7e1lsVistbz00kvWkbuyxDQoAAAwjS+//FKSdOTIESUlJemhhx7SH3/8IVdXV7m5uSk7O9v6Hkk6evSoatWqpW7dumnIkCHat2+fJCkkJET/+te/dOXKFUnShQsXdOjQoVuuKyQkRJ9++qnOnTsn6epI3S+//HLL7RUHI2sAAEDS1eehFfUxG8Vt18nRvkjvdXJyUq9evXTmzBlNmTJFNWvWtF6T1rlzZ3l6esrX19caytatW6c1a9bI0dFRdnZ2ev311yVJAwcO1Pvvv68ePXrIzs5OdnZ2Gjp0qBo1anRLx/DUU0/p7Nmz6t27t6SrI23PPvtskaZWb5edYRhGqe/FBrKyspSYmChfX998Q5pFVRrPmTGD0vghBADcmZKSkuTj42PrMiqc4p53pkEBAABMjLAGAABgYmUS1qKjoxUSEiJvb28dOHDAuvzw4cOKiIhQp06dFBERoSNHjhRpHQAAQEVRJmGtffv2+uSTT3TvvffmWz5x4kRFRkZq/fr1ioyM1IQJE4q0DgAAoKIok7AWEBBgfcDcNRkZGdq/f79CQ0MlSaGhodq/f78yMzMLXQcAAFCR2OzRHSkpKfL09JS9/dVbee3t7eXh4aGUlBQZhlHgOnd392LtJzExsdi1+fv7F3ubO8muXbtsXQIAwAQcHBx08eJFW5dR4WRnZ9/wb3FB+aPcP2ftVh/dUZ6V9zAKACiapKSkfF9snpebrUoOTiW+n9JqV5JOnDih7du3KyIiwrrshRde0Pjx41WvXr1itRUeHq7PP/9cLi4ut11XfHy8oqOjtWLFiuvWOTk56aGHHipyWzYLa15eXkpLS5PFYrF+nUN6erq8vLxkGEaB6wAAQOmo5OCkA+/0K/F2m4z6V4m3ec3Jkyf1+eef5wtr177kvbhiY2NLqqwSZbOwVrNmTfn4+CguLk7h4eGKi4uTj4+PdZqzsHUAAKB82rp1q959911ZLBa5u7trypQpSk1N1fTp09W0aVP98ssvsre319tvv63GjRtrypQpOnHihMLDw1W/fn3NmTNHISEhWrBggZo0aaI+ffqoadOm2rt3r06ePKnnnntOnp6e+vjjj5Wenq7Ro0erc+fOkiRvb2/99NNPOnbsmKKioqw1HTt2TK+88or69u2rLVu2aP78+crOzpajo6PGjh2r5s2bS5JmzZqlr776Sp6envq///u/EjsnZRLWpk2bpm+++UanT59W//795ebmprVr12rSpEmKiopSTEyMqlevrujoaOs2ha0DAADlT0ZGhl577TV9/PHHaty4sZYvX65Ro0Zp1KhR+vXXX/XGG28oMDBQK1eu1GuvvaYVK1ZowoQJBU43XpOamqqPP/5Yp06dUseOHdWvXz999tln2rt3r4YOHWoNa9f4+PhYR9m2bt2qt956S127dtWxY8cUExOjxYsXy9XVVQcPHtQLL7ygzZs3a+PGjdq4caNWrVolFxcXDRkypMTOS5mEtTfeeENvvPHGdcsbNWqk5cuX33CbwtYBAIDy5+eff9YDDzygxo0bS5K6d++uyZMn6+LFi6pfv74CAwMlXb22bPz48bpw4UKR2n3iiSdUqVIleXp6ys3NTR06dJAkNW3aVGlpacrKyrrh9e1JSUmaNGmSPvzwQ7m7u2vdunU6duyY/va3v1nfk5ubq9OnTys+Pl5dunSxXgPYo0cPxcTE3Nb5uKbc32AAAADuDIZhyM7OrsTb/XMQs7e3t76+9tSJ3Nzc68Jaamqqhg8frpkzZ6pBgwbW5W3atNGMGTNuWHtp4eumAACAKfj5+SkpKUmHDh2SJK1cuVIPPvigqlatqqNHjyohIUGStGbNGjVp0kSurq5ydXUt8ghbUV24cEGDBg3SiBEj8j1BoXXr1tq2bZsOHjxoXbZ3715JUqtWrbRu3TpdunRJFotFX375ZYnVw8gaAACQdPURG6Vx52ZRH93h7u6uGTNmaNSoUcrNzZW7u7tmzpyp1NRU642Hb775pipVqmQd3fL29tZ9992n0NBQNWzYUHPmzLntev/73//q8OHDWrhwoRYuXChJGjBggMLCwjRz5kyNGzdOV65cUU5Ojlq0aKFmzZrpscce0549e/TUU0/Jw8NDQUFBSktLu+1aJMnOKM1xOxvKyspSYmLiLT9nLXLC5pIvygQ+nfKorUsAAJhEUlKSfHx8bF3GTRX2zLI7UXHPO9OgAAAAJkZYq2DycrNtXUKpKu/HBwAVUVBQULkZVbsVXLNWwZTW06nNojSfkg0AgC0wsgYAQAVWTi9dN61bOd+ENQAAKigXFxdlZGQQ2MqIYRjKyMgo9hfFMw0KAEAFVadOHZ04cUKnTp2ydSkVhouLi+rUqVOsbQhrAABUUI6OjrrvvvtsXQZugmlQAAAAEyOsAQAAmBhhDQAAwMQIawAAACZGWAMAADAxwhoAAICJEdYAAABMjLAG3IbsHIutSyhV5f34AOBOwENxgdvg5GivyAmbbV1Gqfl0yqO2LgEAKjxG1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENAADAxAhrAAAAJkZYAwAAMDHCGgAAgIkR1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENAADAxAhrAAAAJkZYAwAAMDHCGgAAgIkR1gAUKC8329YllKryfnwAygcHWxcgSZs2bdJ7770nwzCUl5enYcOGqWPHjjp8+LCioqJ09uxZubm5KTo6Wg0aNLB1uUCFUcnBSQfe6WfrMkpNk1H/KtP9ZedY5ORoX6b7LCvl+dgAW7N5WDMMQ6+99po++eQTNWnSRL/88oueffZZdejQQRMnTlRkZKTCw8MVGxurCRMm6KOPPrJ1yQBwS5wc7RU5YbOtyygVn0551NYlAOWWKaZBK1WqpPPnz0uSzp8/Lw8PD505c0b79+9XaGioJCk0NFT79+9XZmamLUsFAAAoUzYfWbOzs9Ps2bM1ePBgValSRRcvXtTChQuVkpIiT09P2dtfHVa3t7eXh4eHUlJS5O7uXuT2ExMTi12Tv79/sbeBeezatavM9kVfufPRX0pOWZ5LoDwq6HeEzcNabm6uFi5cqJiYGPn7+2vXrl0aMWKEZsyYUSLt+/r6ytnZuUTawp2hvP9BRMmiv5QcziVQOmw+DZqUlKT09HTrD7m/v78qV64sZ2dnpaWlyWKxSJIsFovS09Pl5eVly3IBAADKlM3D2j333KPU1FT9/vvvkqRDhw7p9OnTql+/vnx8fBQXFydJiouLk4+PT7GmQAEAuBNl51hsXUKpKu/HV9JsPg1aq1YtTZo0SS+//LLs7OwkSW+99Zbc3Nw0adIkRUVFKSYmRtWrV1d0dLSNqwUAoPSV5zuHJe4eLi6bhzVJCgsLU1hY2HXLGzVqpOXLl9ugIgAAAHOw+TQoAAAACkZYAwAAMDHCGgDgtpX371kt78cHczPFNWsAgDsb3yMLlB5G1gAAAEyMsAYAAGBihDUAAFCmyvs1gCV9fFyzBgAAyhTXOBYPI2sAAAAmRlgDAAAwsVsKa1euXFF2dvmebwYAADCDIoW16Oho7d27V5K0efNmBQYGqmXLltq4cWOpFgcAAFDRFSmsrVmzRvfff78kad68eZo5c6bmz5+vWbNmlWpxAAAAFV2R7ga9fPmyKleurDNnzuj48ePq1KmTJOnkyZOlWhwAAEBFV6Sw1qBBA61evVrHjh1T69atJUmZmZlycXEp1eIAAAAquiKFtYkTJ+rNN9+Uo6Ojpk+fLkn67rvvrMENAAAApaNIYa1Zs2b67LPP8i0LCwtTWFhYqRQFAACAq4p0g8GOHTt0/PhxSVJ6errGjBmjsWPH6tSpU6VaHAAAQEVXpLA2efJk2dvbS7r6GI/c3FzZ2dlp/PjxpVocAABARVekadC0tDTVrl1bubm5+u6777Rx40Y5OjqqTZs2pV0fAABAhVaksObq6qrTp0/r4MGDatSokapWrars7Gzl5uaWdn0AAAAVWpHCWu/evdWjRw/l5OTo9ddflyT99NNPatiwYakWBwAAUNEVKawNHDhQjz/+uOzt7VWvXj1Jkqenp6ZNm1aqxQEAAFR0RQprklS3bl3t3r1be/fulaenp/z8/OTgUOTNAQAAcAuKlLYOHTqkl156SVeuXJGXl5dSUlLk7OysBQsWqFGjRqVdIwAAQIVVpLA2efJkPfPMMxowYIDs7OwkSYsXL9akSZP0n//8p1QLBAAAqMiK9Jy1X375Rf3797cGNUnq27evfvnll1IrDAAAAEUMax4eHvrxxx/zLUtISJCHh0epFAUAAICrijQNOmLECA0ePFiPPvqoateureTkZG3evFkzZ84s7foAAAAqtCKNrLVv314rVqzQ/fffr4sXL+r+++/XihUr1KFDh9KuDwAAoEIr8rM37rvvPg0ePNj6Ojs7W48++qg2b95cGnUBAABARRxZK0hqampJ1QEAAIAbuK2w9ue7QwEAAFDybiusAQAAoHQVes3a6NGjCxw9s1gspVIQAAAA/p9Cw1r9+vUL3XjIkCElWgwAAADyKzSsDR06tEyKyMrK0ptvvqkffvhBzs7Oat68uaZOnarDhw8rKipKZ8+elZubm6Kjo9WgQYMyqQkAAMAMivzojtI0c+ZMOTs7a/369bKzs9Pp06clSRMnTlRkZKTCw8MVGxurCRMm6KOPPrJxtQAAAGXH5jcYXLx4UatWrdLLL79svT7u7rvvVkZGhvbv36/Q0FBJUmhoqPbv36/MzExblgsAAFCmbD6ydvz4cbm5uen9999XfHy8qlatqpdfflkuLi7y9PSUvb29JMne3l4eHh5KSUmRu7t7kdtPTEwsdk3+/v7F3gbmsWvXrjLbF33lzkd/QVHRV1Act9JfCvrcbR7WcnNzdfz4cT344IMaM2aMfv75Z7344ot67733SqR9X19fOTs7l0hbuDPwSw7FQX9BUdFXUBwl2V8KnQadNm1avtfLly/P93rYsGG3XUDt2rXl4OBgne586KGHdNddd8nFxUVpaWnWR4RYLBalp6fLy8vrtvcJAABwpyg0rK1YsSLf65kzZ+Z7vX379tsuwN3dXUFBQda2Dh8+rIyMDDVo0EA+Pj6Ki4uTJMXFxcnHx6dYU6AAAAB3ukKnQQ3DKPR1SZk8ebJef/11RUdHy8HBQTNmzFD16tU1adIkRUVFKSYmRtWrV1d0dHSp7B8AAMCsCg1rf/32gtL6LtC6devqP//5z3XLGzVqdN3UKwAAQEVSaFizWCzasWOHdUQtNzc33+u8vLzSrxAAAKACKzSs1axZU6+//rr1tZubW77XXD8GAABQugoNaxs3biyrOgAAAHADxf4Gg99//13//e9/dfLkydKoBwAAAH9SaFh7++23FRsba329atUqhYaGavz48ercubO2bNlS6gUCAABUZIWGtQ0bNqhly5bW1++++67GjRunHTt2aPLkyZo3b16pFwgAAFCRFRrWMjMzVbt2bUnSgQMHdPbsWfXs2VOSFBYWpiNHjpR6gQAAABVZoWGtWrVqOn36tCQpISFBvr6+cnJyknT1MR6l9ZBcAAAAXFXo3aCdO3fWiBEj9Pjjj2vJkiV64YUXrOt+/vln1a1bt9QLBAAAqMgKHVkbOXKkgoKC9P333+uZZ57Rs88+a12XlJSkiIiIUi8QAACgIit0ZM3R0VFDhw694bq+ffvqt99+K5WiAAAAcFWhYe1Gzpw5o7Vr12rlypX69ddflZiYWBp1AQAAQEUMa7m5udq8ebNWrlypLVu2yGKx6O9//7vmz59f2vUBAABUaIWGtX379mnVqlWKi4uTJHXq1ElLlizRK6+8on79+qlmzZplUiQAAEBFVWhY69mzp9zc3PTGG2+oc+fOcnC4+nY7O7syKQ4AAKCiK/Ru0CFDhqhatWoaP368Ro8erY0bNyo3N7esagMAAKjwCg1rw4YN03//+18tWrRIVapU0ejRo9W6dWudO3dOBw4cKKsaAQAAKqxCw9o1LVu21PTp07V9+3aNGzdOgYGBGjBggHr06FHa9QEAAFRoxXp0h4uLi8LCwhQWFqa0tDTFxsaWVl0AAADQTcJacnJyoRuHhoaWaDEAAADIr9CwFhISYr3z80Zf2m5nZ6ekpKTSqQwAAACFhzVvb29lZWXp6aefVlhYmDw8PMqqLgAAAOgmYS02NlYHDhzQypUrFRkZqYYNGyo8PFwdO3aUi4tLWdUIAABQYd30btAmTZpozJgx+vbbb9WvXz9t3rxZwcHB+t///lcW9QEAAFRoRXp0hyQdOXJEO3fu1J49e+Tj46Pq1auXZl0AAADQTaZBz549q7Vr12rlypW6ePGiwsPD9fHHH6t27dplVR8AAECFVmhYa9OmjerUqaPw8HA99NBDkqSjR4/q6NGj1ve0atWqdCsEAACowAoNa7Vq1VJWVpaWLVumZcuWXbfezs5O3377bakVBwAAUNEVGtY2btxYVnUAAADgBop8gwEAAADKHmENAADAxAhrAAAAJkZYAwAAMDHCGgAAgIkR1gAAAEyMsAYAAGBihDUAAAATM1VYe//99+Xt7a0DBw5Ikg4fPqyIiAh16tRJEREROnLkiG0LBAAAKGOmCWv/+9//tGfPnnxfEj9x4kRFRkZq/fr1ioyM1IQJE2xYIQAAQNkzRVjLzs7WlClTNHHiRNnZ2UmSMjIytH//foWGhkqSQkNDtX//fmVmZtqyVAAAgDJV6HeDlpX33ntPYWFhqlu3rnVZSkqKPD09ZW9vL0myt7eXh4eHUlJS5O7uXuS2ExMTi12Pv79/sbeBeezatavM9kVfufPRX1BU9BUUx630l4I+d5uHtd27d2vfvn0aNWpUqbTv6+srZ2fnUmkb5sQvORQH/QVFRV9BcZRkf7H5NOjOnTv1+++/q3379goJCVFqaqoGDBigY8eOKS0tTRaLRZJksViUnp4uLy8vG1cMAABQdmwe1gYOHKjvvvtOGzdu1MaNG3XPPfdo8eLF6tKli3x8fBQXFydJiouLk4+PT7GmQAEAAO50Np8GLcykSZMUFRWlmJgYVa9eXdHR0bYuCQAAoEyZLqxt3LjR+t+NGjXS8uXLbVgNAACAbdl8GhQAAAAFI6wBAACYGGENAADAxAhrAAAAJkZYAwAAMDHCGgAAgIkR1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENAADAxAhrAAAAJkZYAwAAMDHCGgAAgIkR1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENAADAxAhrAAAAJkZYAwAAMDHCGgAAgIkR1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENAADAxAhrAAAAJkZYAwAAMDHCGgAAgIkR1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENAADAxAhrAAAAJuZg6wLOnDmj1157TceOHZOTk5Pq16+vKVOmyN3dXYcPH1ZUVJTOnj0rNzc3RUdHq0GDBrYuGQAAoMzYfGTNzs5Of//737V+/XqtWbNGdevW1TvvvCNJmjhxoiIjI7V+/XpFRkZqwoQJNq4WAACgbNk8rLm5uSkoKMj6unnz5kpOTlZGRob279+v0NBQSVJoaKj279+vzMxMW5UKAABQ5mwe1v4sLy9PS5cuVUhIiFJSUuTp6Sl7e3tJkr29vTw8PJSSkmLjKgEAAMqOza9Z+7OpU6eqSpUq6t27t/bv318ibSYmJhZ7G39//xLZN2xj165dZbYv+sqdj/6CoqKvoDhupb8U9LmbJqxFR0fr6NGjWrBggSpVqiQvLy+lpaXJYrHI3t5eFotF6enp8vLyKla7vr6+cnZ2LqWqYUb8kkNx0F9QVPQVFEdJ9hdTTIPOmjVLiYmJmjdvnpycnCRJNWvWlI+Pj+Li4iRJcXFx8vHxkbu7uy1LBQAAKFM2H1k7ePCgFixYoAYNGqhXr16SpDp16mjevHmaNGmSoqKiFBMTo+rVqys6OtrG1QIAAJQtm4e1+++/X7/++usN1zVq1EjLly8v44oAAADMwxTToAAAALgxwhoAAICJEdYAAABMjLAGAABgYoQ1AAAAEyOsAQAAmBhhDQAAwMQIawAAACZGWAMAADAxwhoAAICJEdYAAABMjLAGAABgYoQ1AAAAEyOsAQAAmBhhDQAAwMQIawAAACZGWAMAADAxwhoAAICJEdYAAABMjLAGAABgYoQ1AAAAEyOsAQAAmBhhDQAAwMQIawAAACZGWAMAADAxwhoAAICJEdYAAABMjLAGAABgYoQ1AAAAEyOsAQAAmBhhDQAAwMQIawAAACZGWAMAADAxwhoAAICJEdYAAABMjLAGAABgYoQ1AAAAEzN9WDt8+LAiIiLUqVMnRURE6MiRI7YuCQAAoMyYPqxNnDhRkZGRWr9+vSIjIzVhwgRblwQAAFBmHGxdQGEyMjK0f/9+LVmyRJIUGhqqqVOnKjMzU+7u7oVuaxiGJCk7O/uW9l29st0tbWd2WVlZynOpZusySk1WVlaZ77O89hWJ/lIaymt/oa+UvPLaVyT6S2GcnJxkZ5f/s7czrqUaE0pMTNSYMWO0du1a67IuXbpo5syZatq0aaHbnj9/XgcOHCjtEgEAAEqMr6+vnJ2d8y0z9cja7ahataqaNGkiR0fH6xIqAACAGTk5OV23zNRhzcvLS2lpabJYLLK3t5fFYlF6erq8vLxuum2lSpVUrVr5HWIFAAAVg6lvMKhZs6Z8fHwUFxcnSYqLi5OPj89Nr1cDAAAoL0x9zZokHTp0SFFRUfrjjz9UvXp1RUdHq2HDhrYuCwAAoEyYPqwBAABUZKaeBgUAAKjoCGsAAAAmRlgDAAAwMcIaAACAiRHW7nDR0dEKCQmRt7d3vm9sKGi5JB0+fFgRERHq1KmTIiIidOTIkSKtw53tzJkzeuGFF9SpUyd17dpVQ4cOVWZmpiT6Cwr2/vvvW/tFYX1Ioq9UZJs2bdJTTz2l8PBwde3aVd98802+9X/uR9fQX4rBwB1t586dRnJysvHYY48Zv/76602XG4Zh9OnTx1i1apVhGIaxatUqo0+fPkVahzvbmTNnjB07dlhfv/3228bYsWMNw6C/4MYSExONAQMGGI8++qjx66+/FtqHDIO+UlHl5eUZAQEB1t8dSUlJRvPmzQ2LxWIYxvX96Br6S9ER1sqJG/2RvdHy06dPG/7+/kZubq5hGIaRm5tr+Pv7GxkZGYWuQ/nz9ddfG3379s23jP6Ca7KysoxnnnnGOHbsWIG/X/7ch+grFVdeXp4RGBhoJCQkGIZhGD/++KPRsWNHwzAK7kf0l+Ix9ddNoeSlpKTI09NT9vb2kiR7e3t5eHgoJSVFhmEUuI5vjShf8vLytHTpUoWEhBT6PvpLxfXee+8pLCxMdevWveH6v/Yh+krFZWdnp9mzZ2vw4MGqUqWKLl68qIULF0oquB/RX4qHa9aACmjq1KmqUqWKevfubetSYEK7d+/Wvn37FBkZWeB76EO4Jjc3VwsXLlRMTIw2bdqk+fPna8SIEUpISLhpP0LRMLJWwXh5eSktLU0Wi0X29vayWCxKT0+Xl5eXDMMocB3Kj+joaB09elQLFixQpUqF//8a/aVi2rlzp37//Xe1b99ekpSamqoBAwborbfeUnBw8A37EH2l4kpKSlJ6err8/f0lSf7+/qpcubLi4+ML7Ec+Pj70l2JgZK2CqVmzpnx8fBQXFydJiouLk4+Pj9zd3Qtdh/Jh1qxZSkxM1Lx58+Tk5HTT99NfKqaBAwfqu+++08aNG7Vx40bdc889Wrx4sYKDgwvsQ/SViuuee+5Ramqqfv/9d0lXv9P79OnT+tvf/lZgP6K/FA/fDXqHmzZtmr755hudPn1ad911l9zc3LR27doCl0tXf5CioqL0xx9/qHr16oqOjlbDhg1vug53toMHDyo0NFQNGjSQi4uLJKlOnTqaN28e/QWFCgkJ0YIFC2RnZ1dgH5LoKxXZ6tWrtWjRItnZ2UmShg8frg4dOuR7z7V+1KRJE0n0l+IgrAEAAJgY06AAAAAmRlgDAAAwMcIaAACAiRHWAAAATIywBgAAYGKENQAoIydOnJC3t7dyc3NtXQqAOwhhDcAdYc2aNerWrZv8/PwUHBysv//970pISLitNkNCQvT9998XuD4+Pl4PPPCA/Pz85OfnpzZt2mjOnDm3tc+S0qdPHy1fvtzWZQAoA3zdFADTW7JkiT744ANNnjxZwcHBcnR01LZt2/Ttt98qICCgVPft4eGhrVu3SpKOHz+uv/3tb3rwwQeve+AnAJQWRtYAmNr58+c1Z84cTZgwQR07dlSVKlXk6OiokJAQjRkzRpKUnZ2t6dOnKzg4WMHBwZo+fbqys7MlSZmZmRo0aJACAgIUGBioyMhI5eXlafTo0UpOTtaLL74oPz8/LVq06Ka11K1bV35+fvrtt98k3Xha888jXhaLRdHR0QoKClL79u21ZcuWfO1dC39+fn7q16+fJk+erFGjRlnX79mzR7169VJAQIDCwsIUHx8v6erXhiUkJGjKlCny8/PTlClTbuMMAzA7RtYAmNru3buVlZWlxx9/vMD3zJ8/Xz///LNiY2NlZ2enwYMHKyYmRq+88oqWLFkiT09P/fDDD5Kkn3/+WXZ2dpo5c6Z27dqladOm6ZFHHilSLUeOHNFPP/2kXr16Fen9y5Yt06ZNm7Rq1SpVrlxZw4YNy7d+1KhRatGihZYsWaK9e/dq4MCBCgkJkSSlpaVp0KBBmjFjhtq0aaMffvhBw4cP17p16zRixAj99NNPCgsLU8+ePYtUC4A7FyNrAEzt7Nmzuuuuu+TgUPD/W65Zs0ZDhgxRzZo15e7uriFDhmj16tWSJAcHB506dUrJyclydHRUQECA9fsLiyI9PV0BAQFq0aKFOnXqpIceekj+/v5F2nbdunXq27evvLy85ObmpkGDBlnXJScna9++fRo+fLicnJwUEBBgDWqSFBsbq7Zt26pdu3aqVKmSWrduLV9f3+tG5wCUf4Q1AKbm5uamM2fOFHoHZXp6umrXrm19Xbt2baWnp0uSBgwYoPr16+v5559X+/bt9cEHHxRr/x4eHkpISNBPP/2khIQEOTs7Kyoqqkjbpqeny8vLK19df15Xo0YNVa5c2brsz+9NTk7W119/rYCAAOu/Xbt26dSpU8WqH8Cdj7AGwNT8/Pzk7OysDRs2FPgeDw8PJScnW1+npKTIw8NDkuTq6qqoqCh9++23WrBggZYsWWKdEi2uatWqqWvXrtq0aZMkqUqVKpKkK1euWN/z5zBVq1YtpaSk5Kvrz+vOnTuny5cv33C9l5eXwsPDlZCQYP23Z88eDRw48JZqB3DnIqwBMLVq1app+PDhmjJlijZs2KDLly8rJydHW7Zs0YwZMyRJTz75pObPn6/MzExlZmZq3rx56tq1qyRp06ZNOnr0qAzDkKurq+zt7VWp0tVffXfffbeOHz9e5FouXryotWvXqnHjxpIkd3d3eXp6KjY2VhaLRV988UW+9jp37qz//Oc/Sk1N1blz5/KN6t17773y9fXV3LlzlZ2drd27d1tDoCSFhYVp06ZN2rZtmywWi7KyshQfH6/U1NRbqh3AnYuwBsD0+vfvr6ioKMXExKhVq1Z69NFH9cknn1gfnzF48GD5+voqLCxMYWFhatq0qQYPHixJOnr0qPr37y8/Pz9FRETo2WefVVBQkCRp4MCBmj9/vgICArR48eIb7js9Pd36nLWQkBCdO3dO77zzjnX91KlTtXjxYgUFBem3336Tn5+fdd0zzzyj4OBghYeH6+mnn1bHjh3ztf3OO+9oz549CgoK0uzZs9WlSxc5OTlJujqyFhMTo4ULF6pVq1Zq166dFi9erLy8PEnSc889p/Xr16tly5aaNm1aCZ1pAGZkZxiGYesiAADSK6+8ooYNG2r48OG2LgWAiTCyBgA2snfvXh07dkx5eXnaunWrvv32Wx62C+A6PGcNAGzk9OnTGjZsmM6ePat77rlHkyZN0oMPPmjrsgCYDNOgAAAAJsY0KAAAgIkR1gAAAEyMsAYAAGBihDUAAAATI6wBAACYGGENAADAxP4/pKnTvw7Eb9EAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Network error resolved after 0:00:38.544998, resuming normal operation.\n" + ] + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "\n", + "x = 'Factor'\n", + "\n", + "df = pd.DataFrame({\n", + " x: graph_results[\"cost\"], \n", + " 'baseline': graph_results[\"baseline\"], \n", + " \"optimized\": graph_results[\"optimized\"],\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars=x).rename(columns=str.title)\n", + "seaborn.barplot(x=x, y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "seaborn.despine(fig)\n", + "\n", + "ax1.set(xlabel=\"Cost Budget\", ylabel=f'MASE Loss', title='Residual Estimate Loss for Time-Series Decomposition')\n", + "#ax1.legend_.remove()\n", + "#plt.legend(loc='lower center')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb5185be", + "metadata": {}, + "outputs": [], + "source": [ + "baseline_results[2]" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "id": "526e9b31", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'baseline': [113.47347746568275,\n", + " 89.57722060605525,\n", + " 81.12720697469311,\n", + " 78.42078584418643],\n", + " 'optimized': [95.26955983050661,\n", + " 81.53832641325205,\n", + " 76.2934822322845,\n", + " 74.68576266515468],\n", + " 'cost': [1100, 2099, 4197, 8375]}" + ] + }, + "execution_count": 144, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "{**graph_results}" + ] + }, + { + "cell_type": "markdown", + "id": "a1e429ec", + "metadata": {}, + "source": [ + "# Plot different numbers of replicas" + ] + }, + { + "cell_type": "code", + "execution_count": 269, + "id": "38d0548e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "plan_baseline_1_lifo 172.98378216386888\n", + "plan_baseline_1_lifo 112.67935494700063\n", + "plan_baseline_6_lifo 180.18115087379635\n", + "plan_baseline_6_lifo 115.77955410145175\n", + "plan_baseline_12_lifo 184.2670899767375\n", + "plan_baseline_12_lifo 120.07497015444703\n", + "plan_baseline_18_lifo 190.24309735984917\n", + "plan_baseline_18_lifo 122.9708625384093\n", + "plan_baseline_24_lifo 196.18648185699215\n", + "plan_baseline_24_lifo 128.18398403462876\n", + "plan_baseline_48_lifo 225.58043199493562\n", + "plan_baseline_48_lifo 144.9251630795228\n", + "plan_baseline_96_lifo 320.8503011206009\n", + "plan_baseline_96_lifo 189.32209967308512\n", + "plan_baseline_168_lifo 431.41793348258363\n", + "plan_baseline_168_lifo 293.96569319942313\n", + "plan_baseline_192_lifo 544.0725854282231\n", + "plan_baseline_192_lifo 339.90741345037276\n", + "plan_baseline_336_lifo 949.2024557323098\n", + "plan_baseline_336_lifo 705.3804094682863\n", + "plan_baseline_672_lifo 1917.3666698011584\n", + "plan_baseline_672_lifo 1555.133039236265\n" + ] + }, + { + "data": { + "text/plain": [ + "{1: [172.98378216386888, 112.67935494700063],\n", + " 6: [180.18115087379635, 115.77955410145175],\n", + " 12: [184.2670899767375, 120.07497015444703],\n", + " 18: [190.24309735984917, 122.9708625384093],\n", + " 24: [196.18648185699215, 128.18398403462876],\n", + " 48: [225.58043199493562, 144.9251630795228],\n", + " 96: [320.8503011206009, 189.32209967308512],\n", + " 168: [431.41793348258363, 293.96569319942313],\n", + " 192: [544.0725854282231, 339.90741345037276],\n", + " 336: [949.2024557323098, 705.3804094682863],\n", + " 672: [1917.3666698011584, 1555.133039236265]}" + ] + }, + "execution_count": 269, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiments = [(\"max_fits_1100\", 96), (\"max_fits_2100\", 48), (\"max_fits_4200\", 24), (\"max_fits_8400\", 12)]\n", + "replicas = [1, 2]\n", + "slides = [1, 6, 12, 18, 24, 48, 96, 168, 192, 336, 672]\n", + "graph_results = {\"baseline\": [], \"optimized\": [], \"cost\": []}\n", + "prio = \"lifo\"\n", + "replica_results = {}\n", + "\n", + "for slide in slides: \n", + " replica_results[slide] = []\n", + " for replica in replicas: \n", + " baseline_plan = f\"plan_baseline_{slide}_{prio}\"\n", + " \n", + " total_loss = 0\n", + " for key in range(1, 101, 1):\n", + " oracle_filename = f\"{oracle_dir}/{key}.csv\"\n", + " \n", + " lp_filename = f\"{results_dir}/replica_{replica}/{baseline_plan}/{key}.csv\"\n", + " \n", + " baseline_filename = f\"{results_dir}/replica_{replica}/{baseline_plan}/{key}.csv\"\n", + " results = get_loss_per_key(key, baseline_filename, oracle_filename)\n", + " #print(results)\n", + " total_loss += results[\"loss\"]\n", + " \n", + " replica_results[slide].append(total_loss)\n", + " print(baseline_plan, total_loss)\n", + " \n", + "replica_results" + ] + }, + { + "cell_type": "code", + "execution_count": 206, + "id": "2b9c7c38", + "metadata": {}, + "outputs": [], + "source": [ + "del replica_results[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 270, + "id": "8678b39b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Text(0.5, 0, 'Num Replicas'),\n", + " Text(0, 0.5, 'MASE Loss'),\n", + " Text(0.5, 1.0, 'Residual Estimate Loss for Time-Series Decomposition')]" + ] + }, + "execution_count": 270, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnIAAAFSCAYAAAB2ajI+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABKUElEQVR4nO3de1wU9f4/8Neyy6KgiGAQCl7yKIfkqMgtL2SBZRqCaYURWpZd1NRUVFIUBdFWPYEXCDuHPKfi2NHkImhh5qUyw0t6jLA0QkRBkIuKoCy7O78//Dq/UC4L7rK78Ho+Hj4ezWdm3vOeWVjefT4zn5EIgiCAiIiIiEyOmaETICIiIqLWYSFHREREZKJYyBERERGZKBZyRERERCaKhRwRERGRiWIhR0RERGSiWMgRNWL37t147bXXGl0/depU7Ny584GPk52djccff/yB4zSnufPpiARBwHvvvQcvLy88//zzej1WUVER3N3doVar9XocfeDPju65u7ujsLCw0fXPPvsssrOz2zAjMlUSziNH7YGfnx/KysoglUphaWkJX19fLF++HFZWVno75tSpUxEYGIgXXnjhgeJkZ2dj0aJF+Pbbbxtc7+Ligs6dO0MikYhts2bNwhtvvNFozEuXLsHf3x+//PILZDLZA+WnjQe9Frq6li114sQJLFiwAF999RUsLS0fONbdz0QQBNy6datezD179qBnz54PdIyWSExMxI4dO1BZWYmuXbti2LBhiIuLa7PjNyclJQXLli1Dp06dAADdu3eHj48P3nzzTfTr18/A2bWt8PBwODg4YP78+YZOhUyQ/r/hidpIYmIiRowYgatXr+L111/HRx991G6+GNPT09GnTx9Dp9HuXL58Gb169WpVEadSqeoVyZ6enjh16hSA/19IHz9+vE0K6XulpqYiPT0d//rXv9C7d29cvXoVBw4caFWse89Tl4YOHYrt27dDrVbj8uXL+PjjjzFp0iT897//xcCBA/VyTKL2hkOr1O489NBDGDVqFM6ePSu2nT59GlOmTIGnpycCAwPrDVmkpKTA398f7u7u8PPzw+7du8X2l156SdzuyJEjeOaZZ+Dh4YGoqCj8uTN78+bNCAsLE5cvXboEFxcXqFQqAMCuXbswbtw4uLu7w9/fH59//rlOzvXMmTOYNGkShg0bhhEjRmDt2rUAgNDQUACAl5cX3N3dcerUqfvOx8XFBcnJyXj66afh7u6OuLg4XLx4EcHBwRg2bBjmzZsHpVIJALh+/TreeustPPbYY/Dy8sJbb72FK1euAABiY2Nx4sQJREVFwd3dHVFRUQCAvLw8TJ8+Hd7e3hg7diz27t3b4vPTaDRISEjAk08+ieHDh2Px4sWoqqoCANTW1iIsLAw+Pj7w9PTE5MmTUVZWBqDxz/TPdu7ciYiICJw+fRru7u7YtGkTAGDHjh146qmn4O3tjbfffhslJSUNXrOnn366Redy78/E1KlTERsbiylTpsDd3R1vv/02KisrsXDhQgwbNgyTJ0/GpUuXxP1bcj1//vlnjBo1Cr179wZw53ciODhYXF9VVYWlS5di1KhR8PX1RWxsrDjkm5KSgilTpmDNmjXw9vbG5s2b7/vZaSqXw4cPY/z48XB3d4evry+SkpKavTZSqRS9e/fGypUr4e3tjS1btojrmvrdvXbtGt577z2MGjUKXl5emDVrlrhO28+xuZ/9u7c+JCYmwsfH576fp6qqKixevBiPPfYYnnzySSQkJECj0QAACgoKEBoaCg8PD/j4+ODdd9+tl0NBQQH++9//IiMjA0lJSeLPAXBnlOGHH34AACiVSsTExGDUqFEYNWoUYmJi7svv448/xvDhwzFq1Cjs2rWr2WtO7YhA1A48+eSTwpEjRwRBEITi4mIhICBAiI6OFgRBEK5cuSJ4e3sLhw4dEtRqtfD9998L3t7eQnl5uVBdXS24u7sLeXl5giAIQklJiXDu3DlBEARh165dwpQpUwRBEITy8nLB3d1d+PLLLwWlUils27ZNcHV1FXbs2CEIgiBs2rRJWLhwoZhPYWGhMHDgQKGurk4QBEE4ePCgUFBQIGg0GiE7O1sYPHiwkJOTIwiCIPz444+Cr69vo+c2cOBA4cKFCw2ue/HFF4XU1FRBEATh5s2bwqlTpxo8/r3nczfuW2+9JVRVVQnnzp0TBg0aJEybNk24ePGicOPGDWHcuHFCSkqKIAiCUFFRIXz11VdCTU2NUFVVJcyZM0eYOXOmGCs0NFS8FoIgCNXV1cLjjz8ufPHFF0JdXZ2Qk5MjeHt7i9f2Xvfuf9fOnTuFMWPGCBcvXhRu3rwpzJ49WwgLCxMEQRC2b98uvPXWW0JNTY2gUqmEn3/+WaiqqmryM73Xvdfkhx9+ELy9vYWcnByhtrZWiIqKEkJCQupds1dffVWorKwUbt261WDMxq7/vW2hoaHCmDFjhIKCAvF6P/3008KRI0eEuro6YdGiRUJ4eHirrmdaWprg5eUl/OMf/xDOnDkjqFSqeutnzpwpLF++XKiurhbKysqEyZMnC9u3bxeviaurq/DJJ58IdXV1wq1bt+pdp+ZyGTlypHD8+HFBEATh2rVr4s95c9f+rp07dwrDhw8XBKHp311BEIQ33nhDmDdvnnDt2jVBqVQK2dnZWn+O2v7s//jjj4Krq6uwZs0aoba2VsjOzhaGDBki/nwtWrRIePvtt4WqqiqhsLBQePrpp8Wf5fnz5wsJCQmCWq0Wbt++LV6Xuznc/b1esmSJ8MEHH9S7Dn/+TouLixNeeOEFoaysTCgvLxeCg4OF2NjYevnFxcUJSqVSOHTokDB48GDh2rVrDV53an/YI0ftxuzZs+Hu7o7Ro0fD1tYWc+fOBXBnWPLxxx/H6NGjYWZmhpEjR8LNzQ2HDx8GAJiZmeH8+fO4ffs27O3tMWDAgPtif/vtt/jLX/6CZ555Bubm5njllVfQo0cPrXN74okn0Lt3b0gkEnh7e2PkyJE4ceKE1vs/99xz8PT0FP999913AACZTIaLFy+ioqICVlZWGDp0qNYxAeCNN95Aly5dMGDAAAwcOBAjR46Es7Mzunbtiscffxy5ubkA7ty/NHbsWHTu3BldunTBzJkzcfz48UbjHjp0CL169cLkyZMhk8kwaNAgjB07FllZWS3KLyMjA6+++iqcnZ1hZWWFBQsWYO/eveJw37Vr11BQUACpVAo3Nzd06dIFgHafaWPHmzx5MgYNGgS5XI4FCxbg9OnT9XrG3nzzTdjY2Ij3dj2ISZMmoXfv3uL1dnZ2xogRIyCTyfDMM8+I17+l1zMoKAgRERH4/vvvMXXqVIwYMQIfffQRAKCsrAzffvstli5dCktLS9jZ2eHVV1/Fnj17xP3t7e0xdepUyGSy+86zuVxkMhl+//133Lx5E926dcOgQYNadE3s7e1x/fp1AE3/7paWluLbb7/FqlWr0K1bN5ibm8Pb2xuAdp+jtj/7d82bNw9yuRze3t4YPXo0vvzyS6jVauzduxcLFy5Ely5d4OTkhOnTp4s9djKZDEVFRSgtLYWFhQU8PT1bdC3uysjIwOzZs2FnZwdbW1vMnj27Xq+gTCbD7NmzYW5ujtGjR8PS0hL5+fmtOhaZHt4jR+1GfHw8RowYgWPHjmHhwoWorKyEtbU1ioqK8NVXX+HgwYPitiqVCj4+PrC0tERsbCw+/vhjLFu2DMOGDcOSJUvQv3//erFLS0vx8MMPi8sSiQSOjo5a53b48GHEx8fjwoUL0Gg0uH37dovuAUpNTW3wHrmYmBhs2rQJ48aNg5OTE9555x08+eSTWsf9czFqYWFx3/Ldocpbt25h7dq1+O6778Q/stXV1VCr1ZBKpffFvXz5Ms6cOVPvD5darUZgYKDWuQF3rnuvXr3E5V69ekGlUqG8vBxBQUG4cuUKFixYgBs3biAwMBDz58/X+jNt7Hh/LjysrKxgY2ODkpISODk5AUCLPvfmNHX9O3XqhJqaGgBNX8+ioiI8++yzYvvd+/QCAwMRGBiIuro67N+/H4sWLYKrqyu6desGlUqFUaNGiftoNJp65/Xnn/V7NffZbtq0CR9++CH+/ve/w8XFBQsXLoS7u7vW16SkpATdunUDgCZ/d69cuYJu3bqJ2/6ZNp+jtj/7AGBtbV3vPsqePXuitLQUlZWVqKurq/cQS8+ePcVh3EWLFmHjxo14/vnn0a1bN0yfPr1VT0eXlpbed4zS0lJx2cbGpt59jJ07dxZ/dqj9YyFH7Y63tzcmTZoEhUKBhIQEODo6IigoCKtXr25we19fX/j6+uL27duIi4vD8uXL8Z///KfeNg899JB4Txhw56nE4uJicblz5864ffu2uPznPwJKpRJz586FQqGAv78/zM3NMWvWrHr32LVW37598cEHH0Cj0WDfvn2YO3cusrOz6z3hqgsff/wx8vPzsWPHDjz00EM4e/YsJk6c2Og5ODo6wsvLC9u2bXug49rb2+Py5cviclFREWQyGezs7CCTyfDOO+/gnXfewaVLl8SnHV944QWtPlNtjldTU4Nr167BwcFBbNP1tdVGc9fzbvHWEHNzc4wbNw7/+Mc/cP78eQQEBEAul+PHH39s9CGGps6xuVwGDx6MDz/8EHV1dUhOTsa7774r9n5rY//+/WKR2NTvbmlpKa5fv44bN27A2tq63jptPseWuHHjBmpqasRirri4GAMGDED37t1hbm6OoqIi/OUvfxHX3T3OQw89JOZ+4sQJTJ8+HV5eXvf9T1lzP1P29vYoKioSe5aLi4thb2/fqnOh9odDq9QuvfLKK/jhhx9w9uxZBAYG4uDBg/juu++gVqtRW1uL7OxsXLlyBWVlZfjmm29QU1MDuVwOS0vLBnuYRo8ejfPnz2Pfvn1QqVT45JNP6hVrrq6uOH78OIqKilBVVYWtW7eK65RKJZRKJWxtbSGTyXD48GEcOXJEJ+eZnp6OiooKmJmZiX/MpFIpbG1tYWZm1uQ8VS1RXV0NCwsLWFtb49q1a/VuRgfu9G78+VhPPPEELly4gLS0NNTV1aGurg5nzpxBXl5eo8dQqVSora0V/9XV1SEgIAD//ve/UVhYiOrqasTGxmLcuHGQyWT48ccf8dtvv0GtVqNLly6QyWSQSqVaf6YNmTBhAlJSUnD27FkolUp88MEHGDx4sNiLYygtvZ4pKSk4dOgQbt68CY1Gg8OHD+P333/H4MGDYW9vj5EjR+L9998X11+8eBHHjh174FyUSiV2796NqqoqmJubw8rKSqtrr1arUVhYiOjoaBw7dgyzZ88GgCZ/d+3t7fH4449j1apVuH79Ourq6sThfn18jps3b4ZSqcSJEydw6NAhPPPMM5BKpXjmmWcQGxuLmzdv4vLly9i2bZvYO/nll1+K/wPYrVs3SCQSmJnd/2fXzs6u3rDvvZ599ll8+OGHqKioQEVFBeLj4zFhwoRWnwu1LyzkqF2ytbVFUFCQ2COXkJCArVu3Yvjw4Rg9ejSSkpKg0Wig0Wiwbds2+Pr6wtvbG8ePH0dkZGSD8TZu3Ii///3v8PHxQUFBAYYNGyauHzlyJMaPH4/AwEBMmjSp3vBmly5dEBERgXfffRdeXl7IzMyEn59fi84nKCgI7u7u4r+YmBgAwHfffYdnn31WbIuNjYWFhQU6d+6Mt99+Gy+99BI8PT1x+vTp1l3I//PKK6+gtrYWjz32GIKDg+Hr61tv/bRp05CVlQUvLy+sXr0aXbp0QVJSEvbu3QtfX1+MGjUKGzZsEJ+0a8jKlSsxePBg8d97772HyZMnIzAwEKGhofD394dcLsfy5csB3On1nDt3Ljw8PDB+/Hh4e3sjMDBQ68+0IcOHD8e8efMwZ84cjBo1CoWFhYiNjW39hdORll7PLl26IDExEU8++SQ8PT2xYcMGrFy5UuzpWrduHerq6jB+/Hh4eXlh7ty5uHr1qk5ySU9Ph5+fH4YNG4bPP/8c69atazTW3SeGPTw8MG3aNNy8eRNffPEFXFxcAKDJ39275yGTyTBu3DiMGDEC//73vwHo/nPs0aMHrK2t4evri7CwMKxcuVIcql++fDk6d+6MMWPGICQkBAEBAZg8eTKAO08Pv/DCC3B3d8fMmTOxbNkyODs73xf/+eefx++//w5PT896T97eNWvWLLi5uYnD5YMGDWpwO+qYOCEwERFRI5qbsJvI0NgjR0RERGSiWMgRERERmSgOrRIRERGZqDbpkausrMQbb7yBsWPHYsKECXjnnXdQUVEBAMjPz0dwcDDGjh2L4OBgXLhwQdyvteuIiIiIOoI2KeQkEglmzJiBrKwsZGRkwNnZGRs2bAAAREZGIiQkBFlZWQgJCcGKFSvE/Vq7rjmCIKC2tlYn83gRERERGUqbFHI2Njbw8fERl4cOHYqioiKUl5cjNzcXAQEBAICAgADk5uaioqKi1eu0oVQqkZOT0+RUCERERETGrs3f7KDRaLB9+3b4+fmJM2DfnTBSKpXC3t4excXFEAShVetsbW21ziUnJ0f3J0hERESkYx4eHg22t3khFx0dDUtLS4SGht73UuK25ubmBgsLC4PmQERERNRabVrIKRQKFBQUIDExEWZmZnB0dERJSYn44m21Wo3S0lI4OjpCEIRWrSMiIiLqKNpsHrnY2Fjk5OQgPj4ecrkcwJ33y7m6uiIzMxMAkJmZCVdXV9ja2rZ6HREREVFH0SbzyJ0/fx4BAQHo27cvOnXqBABwcnJCfHw88vLyEB4ejhs3bsDa2hoKhQKPPPIIALR6XXNqa2uRk5Nz39BqXV0dLl26hNu3b+v4ChinTp06wcnJCebm5oZOhYiIiFqhQ04I3Fghl5+fj65du8LOzg4SicSAGeqfIAgoLy9HVVUV+vXrZ+h0iIiIqBX4iq4/uX37doco4oA7c/vZ2dl1mN5HIiKi9oiF3D06QhF3V0c6VyIiovaIhRwRERGRiWIhZwDu7u4oLCxsdrtLly7BxcUFKpWqwfWbN29GWFiYrtMjIiIiE8FCTguvv/46Nm7ceF/7/v37MXLkyEYLrcacOnUKzs7OukqPiIiIOigWclp47rnnkJ6ejnsf8N29ezcmTJgAmUy7eZVbWvARdUSCSmPU8YiIjEmbv6LLFI0ZMwaRkZE4ceIEvLy8AADXr1/HwYMHkZSUhODgYOTl5aFTp054+umnER4eLk567OLighUrVuDf//43VCoVDhw4ABcXF+zbtw99+vTBoUOHEBcXh4sXL6Jr1654/vnnMWfOnHrH37VrFzZv3gwAeO211/Daa681mOfp06fx/vvv4/fff0fPnj2xbNky+Pj46PHKEOmeRGaGkrhjOovn8K63zmIRERkb9shpoVOnThg3bhzS0tLEti+//BKPPPIILC0t8d577+HHH3/E559/jqNHj+I///lPvf3379+PHTt2YO/evffF7ty5MxQKBU6cOIGtW7di+/bt2L9/f71tsrOzsW/fPiQlJeGjjz7CDz/8cF+ckpISvPXWW5g5cyaOHTuGJUuWYO7cuaioqNDNRSAiIiKjw0JOSxMnTsRXX30lzruWlpaG5557Dm5ubhg6dChkMhmcnJwQHByM48eP19v3zTffhI2NjfhWiz/z8fGBi4sLzMzM8Ne//hXPPvssjh2r3xsxe/ZsWFpawsXFBZMmTRJfTfZn6enpePzxxzF69GiYmZlh5MiRcHNzw+HDh3V4FYiIiMiYcGhVS56enrC1tcU333yDwYMHIycnB1u2bEF+fj7ef/995OTk4NatW1Cr1Rg0aFC9fR0dHRuN+7///Q8bNmzA+fPnUVdXB6VSiWeeeabR/Xv16oVz587dF6eoqAhfffUVDh48KLapVCoOrRIREbVjLORaICgoCGlpacjPz8fIkSPRo0cPLFy4EI8++ij+/ve/o0uXLvjXv/6FrKysevs1NfHuwoULERoain/+85+wsLBATEwMKisr621TXFyM/v37A7hTsNnb298Xx9HREUFBQVi9erUOzpSIiIhMAYdWW2DixIk4evQoduzYgYkTJwIAqqurYWVlBSsrK+Tl5WH79u0tilldXY1u3brBwsICZ86caXDYNCEhAbdu3cL58+eRkpKC8ePH37dNYGAgDh48iO+++w5qtRq1tbXIzs7GlStXWnWuREREZPxYyLWAk5MT3N3dcevWLfj7+wMAlixZgszMTAwbNgzLly9vsMhqSmRkJDZt2gR3d3fEx8dj3Lhx923j7e2Np556Cq+++ipee+01jBo16r5tHB0dkZCQgK1bt2L48OEYPXo0kpKSoNFw6gUiIqL2SiLcOzlaB1BbW4ucnBy4ubnBwsJCbD979ixcXV0NmFnb64jnTMaP048QEWmHPXJEREREJoqFHBEREZGJYiFHREREZKJYyBERERGZKBZyRERERCaKhRwRERGRiWIhR0RERGSiWMg1Q1CpTSouERERdRx812ozJDIprn74mc7jPjQzVKvtFAoFsrKycPnyZWRkZGDgwIE6z4WIiIhME3vkjJy/vz+Sk5PRq1cvQ6dCRERERoY9ckbO09PT0CkQERGRkWqTQq6h4cFLly5h9uzZ4jZVVVW4efMmjh27845FPz8/yOVy8V2oYWFh8PX1BQDk5+cjPDwc165dg42NDRQKBfr27dsWp0JERERkNNqkkPP398e0adPw8ssvi21OTk5IT08Xl2NiYqBW138AYNOmTQ3eExYZGYmQkBAEBQUhPT0dK1aswCeffKK/EyAiIiIyQm1yj5ynpyccHR0bXa9UKpGRkYHJkyc3G6u8vBy5ubkICAgAAAQEBCA3NxcVFRU6y5eIiIjIFBjFPXIHDhyAg4MDBg0aVK89LCwMgiDAw8MDCxYsgLW1NYqLi+Hg4ACpVAoAkEqlsLe3R3FxMWxtbVt03JycnHrLMpkM1dXV9dqsrKxacUbaufdYTdFoNLh161aL9tGGUqnEyZMndRqT6EF4eHjoPCZ/xonI1DX23WgUhdyuXbvu641LTk6Go6MjlEolYmJiEBUVhQ0bNuj0uG5ubuI9eABw9uzZ+wo3QaXWeqqQlhBUaq2KxNWrV2Pfvn0oKyvDrFmzYGNjgz179ugsD7lcjiFDhugsHpEx0kdxSERkDAxeyJWUlOD48eNYt25dvfa7Q7FyuRwhISGYOXOm2F5SUgK1Wg2pVAq1Wo3S0tImh24fhEQmNWjciIgIRERE6CUHIiIiMm0Gn0cuNTUVo0ePRvfu3cW2mpoaVFVVAQAEQcDevXvh6uoKALCzs4OrqysyMzMBAJmZmXB1dW3xsCoRERGRqWuTHrk/Dw9Onz693vBgamoqli1bVm/78vJyzJkzB2q1GhqNBv3790dkZKS4fuXKlQgPD0dCQgKsra2hUCja4jSIiIiIjIpEEATB0Em0tdraWuTk5DR4j9zdnr+OoiOeMxm/krhjOovl8K63zmIRERkbgw+tEhEREVHrsJAjIiIiMlEs5IiIiIhMlMGnHzF2GpUSZjK5weLW1tZizZo1OHr0KCwsLDB06FBER0frPB8iIiIyPSzkmmEmkyNvc5DO4/afk978RgDWr18PCwsLZGVlQSKRoKysTOe5EBERkWliIWfEqqurkZaWhsOHD0MikQAAevToYeCsiIiIyFjwHjkjVlhYCBsbG2zZsgWTJk3C1KlTceLECUOnRUREREaChZwRU6lUKCwsxKOPPoqUlBSEhYVhzpw5uHnzpqFTIyIiIiPAQs6I9ezZEzKZDAEBAQCAIUOGoHv37sjPzzdwZkRERGQMWMgZMVtbW/j4+ODIkSMAgPz8fJSXl6NPnz4GzoyIiIiMAR92aIZGpdT6CdOWxtVm+pFVq1Zh6dKlUCgUkMlkWLduHaytrXWeDxEREZkeFnLN0Mccci2J6+zsjE8//VQvORAREZFp49AqERERkYliIUdERERkoljIEREREZkoFnJEREREJoqFHBEREZGJYiFHREREZKJYyDVDrVKaVFwiIiLqODiPXDOkMjkyPx6n87gBr32p1XYKhQJZWVm4fPkyMjIyMHDgQFRWVmLx4sW4ePEi5HI5+vTpg6ioKNja2uo8TyIiIjJe7JEzcv7+/khOTkavXr3ENolEghkzZiArKwsZGRlwdnbGhg0bDJglERGRcRNUGqOO11rskTNynp6e97XZ2NjAx8dHXB46dCi2b9/elmkRERGZFInMDCVxx3QWz+Fdb53FehDskTNxGo0G27dvh5+fn6FTISIiojbGQs7ERUdHw9LSEqGhoYZOhYiIiNpYmxRyCoUCfn5+cHFxwblz58R2Pz8/PPPMMwgKCkJQUBC+++47cV1+fj6Cg4MxduxYBAcH48KFC1qt60gUCgUKCgoQFxcHMzPW5ERERB1Nm/z1b+iG/bs2bdqE9PR0pKenw9fXV2yPjIxESEgIsrKyEBISghUrVmi1rqOIjY1FTk4O4uPjIZfLDZ0OERERGUCbPOzQ0A37TSkvL0dubi62bdsGAAgICEB0dDQqKiogCEKj6/Qx/YZapdR6qpCWxpXKmi/AVq9ejX379qGsrAzTp0+HjY0N4uLikJiYiL59+2LKlCkAACcnJ8THx+s8TyIiIjJeBn9qNSwsDIIgwMPDAwsWLIC1tTWKi4vh4OAAqVQKAJBKpbC3t0dxcTEEQWh0nT4KOW2KLX3GjYiIQERExH3tv/32m65TIiIiIhNj0EIuOTkZjo6OUCqViImJQVRUVJvOh5aTk1NvWSaTobq6us2ObwyUSiVOnjxp6DSIRB4eHjqPyZ9xIjL175bG8jdoIefo6AgAkMvlCAkJwcyZM8X2kpISqNVqSKVSqNVqlJaWwtHREYIgNLqupdzc3GBhYSEunz17FlZWVro5ORMhl8sxZMgQQ6dBpFf6+AInIjKG7xaDPepYU1ODqqoqAIAgCNi7dy9cXV0BAHZ2dnB1dUVmZiYAIDMzE66urrC1tW1yHREREVFH0iY9cg3dsJ+YmIg5c+ZArVZDo9Ggf//+iIyMFPdZuXIlwsPDkZCQAGtraygUCq3WEREREXUUEkEQBEMn0dZqa2uRk5PT4NDq3V7BjqIjnjMZv/b4Gh0iMrz2+N3CWWSJiIiITBQLuWao1EqDxm3srRgHDx7ExIkTERQUhAkTJmDfvn16yZOIiIiMl8HnkTN2Mqkcsf8Zq/O480OytNrO398f06ZNw8svvyy2CYKAxYsXIzk5GQMHDsSvv/6Kl156CWPGjOGruoiIiDoQFnJGrrG3YpiZmYlP/VZVVcHe3p5FHBERUQfDQs4ESSQSxMXFYdasWbC0tER1dTW2bt1q6LSIiIiojbELxwSpVCps3boVCQkJOHjwID788EPMnz+/w72VgoiIqKNjIWeCzp49i9LSUnFGaQ8PD3Tu3Bl5eXkGzoyIiIjaEgs5E/Twww/jypUr+OOPPwAAeXl5KCsrQ+/evQ2cGREREbUl3iPXDJVaqfUTpi2NK5PKm92uobdi7NmzBytXrsS8efMgkUgAAGvXroWNjY3O8yQiIiLjxUKuGdoUW/qMGxERgYiIiPvaAwMDERgYqOu0iIiIyIRwaJWIiIjIRLGQIyIiIjJRLOSIiIiITBQLOSIiIiITxUKOiIiIyESxkCMiIiIyUSzkmqFU15lUXCIiIuo4OI9cM+RSc4xLf1vncb8MSmx2m8rKSixevBgXL16EXC5Hnz59EBUVBVtbW3GbLVu2YPPmzcjIyMDAgQN1nicREREZL/bIGTGJRIIZM2YgKysLGRkZcHZ2xoYNG8T1v/zyC06fPo2ePXsaMEsiIiIyFBZyRszGxgY+Pj7i8tChQ1FUVAQAUCqViIqKQmRkpPiaLiIiIupYWMiZCI1Gg+3bt8PPzw8AsHHjRgQGBsLZ2dnAmREREZGhsJAzEdHR0bC0tERoaChOnTqFn3/+GSEhIYZOi4iIiAyIhZwJUCgUKCgoQFxcHMzMzHD8+HH88ccf8Pf3h5+fH65cuYLXX38d33//vaFTJSIiojbEp1aNXGxsLHJycvDRRx9BLpcDAN588028+eab4jZ+fn5ITEzkU6tEREQdDAu5ZijVdVpNFdKauHKpeZPbnD9/HomJiejbty+mTJkCAHByckJ8fLzO8yEiIiLT0yaFnEKhQFZWFi5fvizOd9bcHGl+fn6Qy+WwsLAAAISFhcHX1xcAkJ+fj/DwcFy7dg02NjZQKBTo27evXnJvrtjSZ9wBAwbgt99+a3a7AwcO6CIlIiIiMjFtco+cv78/kpOT0atXL7GtuTnSAGDTpk1IT09Henq6WMQBQGRkJEJCQpCVlYWQkBCsWLGiLU6DiIiIyKi0SSHn6ekJR0fHem1NzZHWlPLycuTm5iIgIAAAEBAQgNzcXFRUVOg2aSIiIiIjZxT3yN07R9pdYWFhEAQBHh4eWLBgAaytrVFcXAwHBwdIpVIAgFQqhb29PYqLi+u9uoqIiIiovTOKQu7Pc6TdlZycDEdHRyiVSsTExCAqKuq+odcHlZOTU29ZJpOhurpap8cwdkqlEidPnjR0GkQiDw8PncfkzzgRmfp3S2P5G7yQuztHWmJiIszM/v9I792hWLlcjpCQEMycOVNsLykpgVqthlQqhVqtRmlp6X1Dt9pwc3MTH6YAgLNnz8LKyuoBz8i0yOVyDBkyxNBpEOmVPr7AiYiM4bulVffI3b59G0ql8oEPfneOtPj4eHGONACoqalBVVUVAEAQBOzduxeurq4AADs7O7i6uiIzMxMAkJmZCVdXVw6rEhERUYejVSGnUChw5swZAMChQ4fg7e0NLy8vrae9WL16NR5//HFcuXIF06dPx7PPPivOkVZaWoopU6YgKCgIs2fPBnDngYapU6diwoQJCAgIQH5+PiIjI8V4K1euxGeffYaxY8fis88+w6pVq1p63lpTqlVGEXfLli1wcXHBuXPnAAAHDx7ExIkTERQUhAkTJmDfvn36SJOIiIiMmFZDqxkZGZg7dy4AID4+HuvXr0fXrl2xdu3a+x5QaEhERAQiIiLua29sjjRnZ2ekpaU1Gq9///7YuXOnNqk/MLlUhmd3bdV53D2T39J6219++QWnT59Gz549AdzppVy8eDGSk5MxcOBA/Prrr3jppZcwZsyYesPTRERE1L5p9Vf/1q1b6Ny5MyorK1FYWIixY8dixIgRuHz5sr7z6/CUSiWioqIQGRkJiUQitpuZmYnDz1VVVbC3t2cRR0RE1MFo1SPXt29f7N69GxcvXsTIkSMBABUVFejUqZNekyNg48aNCAwMhLOzs9gmkUgQFxeHWbNmwdLSEtXV1di6Vfe9hkRERGTctOrCiYyMxH/+8x9kZ2dj3rx5AIDvv/9eLOpIP06dOoWff/4ZISEh9dpVKhW2bt2KhIQEHDx4EB9++CHmz5/f4aZOISIi6ui06pEbPHgwPv/883ptgYGBCAwM1EtSdMfx48fxxx9/wN/fHwBw5coVvP7665g2bRpKS0vFx549PDzQuXNn5OXlYfDgwYZMmYiIiNqQVj1yP/74IwoLCwEApaWlWLJkCd577z1cvXpVr8l1dG+++Sa+//57HDhwAAcOHMDDDz+MpKQkTJw4EVeuXMEff/wBAMjLy0NZWRl69+5t4IyJiIioLWnVI7dq1SokJSUBuDMVCQBYWFhg+fLlSExM1F92RkCpVrXoCdOWxJVLWzcf80MPPYSVK1di3rx54gMQa9euhY2NjQ4zJCIiImOnVSVRUlKCnj17QqVSiT1E5ubm8PX11Xd+BtfaYksfcf88bx+HtomIiEiraqJLly4oKyvD+fPn0b9/f1hZWUGpVEKl0s9kuURERETUPK0KudDQUDz//POoq6vD0qVLAQA//fQTHnnkEb0mR0RERESN06qQe/PNN/HUU09BKpWKN9Q7ODhg9erVek2OiIiIiBqn9Y1azs7OOHXqFM6cOQMHBwe4u7tDJtPP/WNERERE1DytKrG8vDzMnDkTt2/fhqOjI4qLi2FhYYHExET0799f3zkSERERUQO0nn7kxRdfxOuvvy5Od5GUlISVK1fi008/1WuCRERERNQwrSYE/vXXXzF9+vR6L21/5ZVX8Ouvv+otMWOhVKtNKi4RERF1HFr1yNnb2+PYsWMYPny42HbixAnY29vrLTFjIZdKEfhFus7j7n4+SKvtDh06hI0bN0KlUqFbt25Yu3YtnJ2dUVtbizVr1uDo0aOwsLDA0KFDER0drfM8iYiIyHhpVcjNnz8fs2bNwhNPPIGePXuiqKgIhw4dwvr16/WdX4d2/fp1LFmyBJ9//jn69euH9PR0rFy5EklJSVi/fj0sLCyQlZUFiUSCsrIyQ6dLREREbUyroVV/f3+kpKRgwIABqK6uxoABA5CSkoIxY8boO78OraCgAD169EC/fv0AAKNHj8b333+PK1euIC0trd4runr06GHIVImIiMgAtJ4/pF+/fpg1a5a4rFQq8cQTT+DQoUP6yItw55qXlZXhzJkzGDx4MDIyMgAAFy9ehI2NDbZs2YLs7GxYWVlh3rx58PT0NHDGRERE1Ja06pFrzJUrV3SVBzWga9euiI2Nxdq1azFp0iSUl5fD2toaAFBYWIhHH30UKSkpCAsLw5w5c3Dz5k0DZ0xERERt6YFm9P3zU6ykHyNGjMCIESMAAGVlZUhKSkKvXr0gk8kQEBAAABgyZAi6d++O/Px8/O1vfzNkukRERNSGHqhHjvTv6tWrAACNRoMPPvgAU6ZMQa9eveDj44MjR44AAPLz81FeXo4+ffoYMlUiIiJqY032yC1atKjRXjd1B5kHTalWaz1VSEvjyqXSZreLi4vDTz/9hLq6OowcORJhYWEA7kzSvHTpUigUCshkMqxbt04cdiUiIqKOoclCrrkentmzZ+s0GWOkTbGlz7gxMTENtjs7O/OtGkRERB1ck4XcO++801Z5EBEREVEL8R45IiIiIhPVJoWcQqGAn58fXFxccO7cObE9Pz8fwcHBGDt2LIKDg3HhwoUHXkdERETUUbRJIefv74/k5GT06tWrXntkZCRCQkKQlZWFkJAQrFix4oHXEREREXUUbVLIeXp6wtHRsV5beXk5cnNzxbnQAgICkJubi4qKilavIyIiIupImnzYYfXq1YiIiBCXd+7ciRdeeEFcnjNnDjZv3tyqAxcXF8PBwQHS/3t6UyqVwt7eHsXFxRAEoVXrbG1tW5RDTk5OvWWZTIbq6upWnY+pUiqVOHnypKHTIBJ5eHjoPCZ/xonI1L9bGsu/yUIuJSWlXiG3fv36eoXc3QlpTZWbmxssLCzE5bNnz8LKyqreNkq1BnKp7jsutY2rUCiQlZWFy5cvIyMjAwMHDgQA1NbWYs2aNTh69CgsLCwwdOhQREdHAwAOHjyIjRs3QhAEaDQazJkzB08//XSD8eVyOYYMGaK7EyMyQvr4AiciMobvliYLOUEQmlx+EI6OjigpKYFarYZUKoVarUZpaSkcHR0hCEKr1umDXGqGF3blNL9hC+2c7KbVdv7+/pg2bRpefvnleu3r16+HhYUFsrKyIJFIUFZWBuDOZ7R48WIkJydj4MCB+PXXX/HSSy9hzJgxMDPjQ8pERETtSZN/2e99q4Mu361qZ2cHV1dXZGZmAgAyMzPh6uoKW1vbVq9rjxq6v7C6uhppaWmYN2+e+Jn06NFDXG9mZoaqqioAQFVVFezt7VnEERERtUNN9sip1Wr8+OOPYk+cSqWqt6zRaLQ6yOrVq7Fv3z6UlZVh+vTpsLGxwZ49e7By5UqEh4cjISEB1tbWUCgU4j6tXdcRFBYWwsbGBlu2bEF2djasrKwwb948eHp6QiKRIC4uDrNmzYKlpSWqq6uxdetWQ6dMREREetBkIWdnZ4elS5eKyzY2NvWWte0Fi4iIqHev3V39+/fHzp07G9yntes6ApVKhcLCQjz66KNYsmQJ/ve//+Htt9/G119/jU6dOmHr1q1ISEiAh4cHTp48ifnz52PPnj333f9HREREpq3JQu7AgQNtlQe1QM+ePSGTycQpWIYMGYLu3bsjPz8fAFBaWiregOnh4YHOnTsjLy8PgwcPNljOREREpHstvnHqjz/+wNdff43Lly/rIx/Sgq2tLXx8fMSnhvPz81FeXo4+ffrg4YcfxpUrV/DHH38AAPLy8lBWVobevXsbMmVqpzQq3T0ARURELddkj9z7778PV1dXBAUFAQDS0tKwdOlSWFtbo6amBps3b8bo0aPbJFFDUao1Wj9h2tK42kw/0tj9hatWrcLSpUuhUCggk8mwbt06WFtbA7hzD+GfH4RYu3YtbGxsdH4ORGYyCX5NKNFpzL/OctBpPCKi9qzJQm7//v2YNm2auPzBBx9g2bJlePnll5Gamor4+Ph2X8jpYw65lsRt7P5CZ2dnfPrppw3uExgYiMDAwAfKj4iIiIxfk9VERUUFevbsCQA4d+4crl27Jk4IHBgYyJfVExERERlQk4Vc165dxYlmT5w4ATc3N8jlcgB3npzU5QTBRERERNQyTQ6tjhs3DvPnz8dTTz2Fbdu24Y033hDX/e9//4Ozs7PeEyQiIiKihjXZI7dw4UL4+Pjghx9+wIsvvoiXXnpJXHf27FkEBwfrPUEiIiIialiTPXLm5uZ45513Glz3yiuv4Pfff9dLUkRERETUvCYLuYZUVlZiz549SE1NxW+//YacHN2/UJ6IiIiImqdVIadSqXDo0CGkpqbi8OHDUKvVmDFjBj788EN952dwarUAqVRiMnGJiIio42iykPv555+RlpaGzMxMAMDYsWOxbds2vPvuu3j11VdhZ2fXJkkaklQqwY5dZTqP++LkHlptp1AokJWVhcuXLyMjIwMDBw4EABw6dAgbN26ESqVCt27dsHbtWjg7O6OyshKLFy/GxYsXIZfL0adPH0RFRWn9XlwiIiIyHU0Wci+88AJsbGwQERGBcePGQSa7s/ndNwaQ/vn7+2PatGl4+eWXxbbr169jyZIl+Pzzz9GvXz+kp6dj5cqVSEpKgkQiwYwZM+Dj4wPgTiG4YcMGrFmzxlCnQERERHrS5FOrs2fPRteuXbF8+XIsWrQIBw4cgEqlaqvcCICnpyccHR3rtRUUFKBHjx7o168fAGD06NH4/vvvUVFRARsbG7GIA4ChQ4eiqKioTXMmIiKittFkITdnzhx8/fXX+Mc//gFLS0ssWrQII0eOxPXr13Hu3Lm2ypHu0a9fP5SVleHMmTMAgIyMDABAcXFxve00Gg22b98OPz+/Ns+RiIiI9E+rhx28vLzg5eWF5cuXY9++fUhPT8frr7+ORx99FF988YW+c6R7dO3aFbGxsVi7di1qa2vx+OOPw9raWhz6vis6OhqWlpYIDQ01UKZERESkTy2afqRTp07iC9lLSkqQnp6ur7yoGSNGjMCIESMAAGVlZUhKSqr3pg2FQoGCggIkJibCzKzJjlciIiIyUU0Wcs3dWxUQEKDTZEh7V69exUMPPQSNRoMPPvgAU6ZMgaWlJQAgNjYWOTk5+Oijj8R34xIREVH702Qh5+fnJz6hKgjCfeslEgnOnj2rn8yMhFotaD1VSEvjajOP3OrVq7Fv3z6UlZVh+vTpsLGxwZ49exAXF4effvoJdXV1GDlyJMLCwgAA58+fR2JiIvr27YspU6YAAJycnBAfH6/zcyAiIiLDarKQc3FxQW1tLZ577jkEBgbC3t6+rfIyGvqatFfbuBEREYiIiLivPSYmpsHtBwwYgN9+++2BciMiIiLT0GQhl56ejnPnziE1NRUhISF45JFHEBQUhKeffhqdOnVqqxyJiIiIqAHN3gU/cOBALFmyBN988w1effVVHDp0CKNGjcIvv/zSFvkRERERUSO0fpzxwoULOH78OE6fPg1XV1dYW1vrMy8iIiIiakaTQ6vXrl3Dnj17kJqaiurqagQFBeGzzz5Dz5492yo/IiIiImpEk4Wcr68vnJycEBQUhCFDhgC483qogoICcZvhw4frN0MiIiIialCThdxDDz2E2tpa7NixAzt27LhvvUQiwTfffPNACVy6dAmzZ88Wl6uqqnDz5k0cO3YMfn5+kMvlsLCwAACEhYXB19cXAJCfn4/w8HBcu3YNNjY2UCgU6Nu37wPlQkRERGRKmizkDhw4oPcEnJyc6r0hIiYmBmq1WlzetGkTBg4ceN9+kZGRCAkJQVBQENLT07FixQp88sknOs9PoxJgJtP9FCTaxp01axYuXboEMzMzWFpaYvny5XB1dW20HQBqa2uxZs0aHD16FBYWFhg6dCiio6N1fg5ERERkWC16RZe+KZVKZGRkICkpqcntysvLkZubi23btgG484aJ6OhoVFRUwNbWVqc5mckk+DWhRKcxAeCvsxy02k6hUKBr164AgP3792Pp0qVITU1ttB0A1q9fDwsLC2RlZUEikaCsrEzn+RMREZHhGVUhd+DAATg4OGDQoEFiW1hYGARBgIeHBxYsWABra2sUFxfDwcEBUqkUACCVSmFvb4/i4uIWFXI5OTn1lmUyGaqrq+u1WVlZPcAZNe3eYzXEzMxM3K68vByCIKC6urrR9pqaGqSmpuKrr75CTU0NAKBz586NHkupVOLkyZM6OiPqaDw8PAydglb4M05E+vi+asvvlsbyN6pCbteuXZg8ebK4nJycDEdHRyiVSsTExCAqKgobNmzQ2fHc3NzE++8A4OzZs3ot3O6l7bGWLVuGI0eOQBAE/POf/xT3a6i9sLAQ3bt3x8cff4zs7GxYWVlh3rx58PT0bDC2XC4XH2Qhaq9MpeAkItNiDN8tWs8jp28lJSU4fvw4JkyYILY5OjoCuFNshISE4KeffhLbS0pKxHvp1Go1SktLxe3bm5iYGBw6dAjz58/HunXrmmxXqVQoLCzEo48+ipSUFISFhWHOnDm4efOmodInIiIiPTGaQi41NRWjR49G9+7dAQA1NTWoqqoCAAiCgL1794o389vZ2cHV1RWZmZkAgMzMTLi6uur8/jhjM3HiRGRnZ6OysrLR9p49e0ImkyEgIAAAMGTIEHTv3h35+fmGSJmIiIj0yGiGVlNTU7Fs2TJxuby8HHPmzIFarYZGo0H//v0RGRkprl+5ciXCw8ORkJAAa2trKBQKQ6StV9XV1bhx44bY03jgwAF069YN5ubmKC4uvq/dxsYGEokEPj4+OHLkCEaNGoX8/HyUl5ejT58+hjwVIiIi0gOjKeSysrLqLTs7OyMtLa3R7fv374+dO3fqOas704Ro+4RpS+M2N/3IrVu3MG/ePNy6dQtmZmbo1q0bEhMTcfv27QbbJZI78VatWoWlS5dCoVBAJpNh3bp1fKUaERFRO2Q0hZyx0sccctrG7dGjR4MTMQNotB24UwR/+umnrc6NiIjIkPQ1h2t7xEKOiIiIjIo+5nDVx+iaMTCahx2IiIiIqGVYyBERERGZKBZyRERERCaKhRwRERGRiWIhR0RERGSiWMg1Q1BpTCouERERdRycfqQZEpkZSuKO6Tyuw7veWm1XW1uLNWvW4OjRo7CwsMDQoUPx1ltvYfbs2eI2VVVVuHnzJo4dO4bKykosXrwYFy9ehFwuR58+fRAVFdXuX19GRETUEbGQM3Lr16+HhYUFsrKyIJFIUFZWhh49eiA9PV3cJiYmBmq1GgAgkUgwY8YM+Pj4AAAUCgU2bNiANWvWGCR/IiIi0h8OrRqx6upqpKWlYd68eeLrt3r06FFvG6VSiYyMDEyePBkAYGNjIxZxADB06FAUFRW1XdJERETUZljIGbHCwkLY2Nhgy5YtmDRpEqZOnYoTJ07U2+bAgQNwcHDAoEGD7ttfo9Fg+/bt8PPza6uUiYiIqA2xkDNiKpUKhYWFePTRR5GSkoKwsDDMmTMHN2/eFLfZtWuX2Bt3r+joaFhaWiI0NLStUiYiIqI2xELOiPXs2RMymQwBAQEAgCFDhqB79+7Iz88HAJSUlOD48eOYMGHCffsqFAoUFBQgLi4OZmb8mImIiNoj/oU3Yra2tvDx8cGRI0cAAPn5+SgvL0efPn0AAKmpqRg9ejS6d+9eb7/Y2Fjk5OQgPj4ecrm8zfMmIiKitsGnVpshqDRaTxXS0rgSWfN19KpVq7B06VIoFArIZDKsW7cO1tbWAO4UcsuWLau3/fnz55GYmIi+fftiypQpAAAnJyfEx8fr/ByIiIjIsFjINUObYkufcZ2dnfHpp582uC4rK+u+tgEDBuC33357oNyIiIjINHBolYiIiMhEsZAjIiIiMlEs5O4hCIKhU2gzHelciYiI2iMWcn/SqVMnlJeXd4gCRxAElJeXo1OnToZOhYiIiFqJDzv8iZOTEy5duoSrV68aOpU20alTJzg5ORk6DSIiImolFnJ/Ym5ujn79+hk6DSK9UasFSKUSQ6dBREQ6wkKOqAORSiXYsatMZ/FenNxDZ7GIiKjljKKQ8/Pzg1wuh4WFBQAgLCwMvr6+yM/PR3h4OK5duwYbGxsoFAr07dsXAJpcR0RERNQRGM3DDps2bUJ6ejrS09Ph6+sLAIiMjERISAiysrIQEhKCFStWiNs3tY6IiIioIzCaQu5e5eXlyM3NFV8YHxAQgNzcXFRUVDS5joiIiKijMIqhVeDOcKogCPDw8MCCBQtQXFwMBwcHSKVSAIBUKoW9vT2Ki4shCEKj62xtbQ15GkRERERtxigKueTkZDg6OkKpVCImJgZRUVF49dVX9X7cnJwcvR+DyJh4eHgYOgWDOHnypKFTIKIWMJXvqrb8bmnsmhhFIefo6AgAkMvlCAkJwcyZM/Hee++hpKQEarUaUqkUarUapaWlcHR0hCAIja5rCTc3N/EBCyJqv0zljwIRmRZj+G4x+D1yNTU1qKqqAnDnbQN79+6Fq6sr7Ozs4OrqiszMTABAZmYmXF1dYWtr2+Q6IiIioo7C4D1y5eXlmDNnDtRqNTQaDfr374/IyEgAwMqVKxEeHo6EhARYW1tDoVCI+zW1joiIiKgjMHgh5+zsjLS0tAbX9e/fHzt37mzxOiIiIqKOwOBDq0RERETUOizkiIiIiEwUCzkiIiIiE8VCjoiIiMhEsZAjMmJKtcbQKRARkREz+FOrRNQ4udQML+zS3RtIdk5201ksIiIyPPbIEREREZkoFnJEREREJoqFHBEREZGJYiFHREREZKJYyBERERGZKBZyRERERCaKhRwRERGRiWIhR0RERGSiWMgRERERmSgWckREREQmioUcERERPRC1WjB0Ch0W37VKRERED0QqlWDHrjKdxXtxcg+dxWrv2CNHREREZKJYyBHpiFKtNnQKRETUwXBolUhH5FIpAr9I12nM3c8H6TQeERG1L+yRIyIiIjJRLOSIiIiITBQLOSIiog5GqdYYOgXSEd4jR0RE1MHIpWZ4YVeOzuLtnOyms1jUMgYv5CorK7F48WJcvHgRcrkcffr0QVRUFGxtbeHn5we5XA4LCwsAQFhYGHx9fQEA+fn5CA8Px7Vr12BjYwOFQoG+ffsa8EyIiIiI2pbBh1YlEglmzJiBrKwsZGRkwNnZGRs2bBDXb9q0Cenp6UhPTxeLOACIjIxESEgIsrKyEBISghUrVhgifSIiIiKDMXghZ2NjAx8fH3F56NChKCoqanKf8vJy5ObmIiAgAAAQEBCA3NxcVFRU6DVXIiIiImNi8KHVP9NoNNi+fTv8/PzEtrCwMAiCAA8PDyxYsADW1tYoLi6Gg4MDpFIpAEAqlcLe3h7FxcWwtbU1VPpEREREbcqoCrno6GhYWloiNDQUAJCcnAxHR0colUrExMQgKiqq3rDrg8rJ0d2NnkQeHh6GToEacfLkSUOnQGRU+H2lG2353dLYZ2Y0hZxCoUBBQQESExNhZnZnxNfR0REAIJfLERISgpkzZ4rtJSUlUKvVkEqlUKvVKC0tFbfXlpubm/ggBRG1X/yjRUT6YAzfLQa/Rw4AYmNjkZOTg/j4eMjlcgBATU0NqqqqAACCIGDv3r1wdXUFANjZ2cHV1RWZmZkAgMzMTLi6unJYlYiIiDoUg/fInT9/HomJiejbty+mTJkCAHByckJ4eDjmzJkDtVoNjUaD/v37IzIyUtxv5cqVCA8PR0JCAqytraFQKAx1CmSilGoV5FKD/woQERG1msH/ig0YMAC//fZbg+vS0tIa3a9///7YuXOnnrKijkAuleHZXVt1Fm/P5Ld0FouIiEgbRjG0SkREREQtx0KOiIiIyESxkCOToVTXGToFIiIio2Lwe+TIOKhVSkhlcp3FU6lqIZPpdmoXudQc49Lf1lm8L4MSdRaLiEhflGo15P83AT7RvVjImSiNSgkzHRZeUpkcmR+P01m8gNe+ROx/xuosHgDMD8nSaTwiIlMgl0oR+EW6TmPufj5Ip/HIcFjItRFBpYZEprv/ozKTyZG3WXe/iP3n6PZLgoiIiPSPhVwbkcikuPrhZzqL99DMUJ3FIiIiItPEhx2IiIiITBQLOSIiIiITxUKOiIhIh5RqlaFToA6E98gRERHpEF//R22JPXJERNShcbJxMmXskSMiIr3hZONE+sVCjojIhOl6jkpONk5kWljIERGZMH3MUcnJxolMB++RIyIiIjJRLOSIiIiITBQLOSIiIiITxUKOiIiIyESxkCMiIiIyUSzkiIiIiEwUCzkiIiIiE8VCjoiIiMhEsZAjIiIiMlEs5IiIiIhMlEkXcvn5+QgODsbYsWMRHByMCxcuGDolIiIiojZj0oVcZGQkQkJCkJWVhZCQEKxYscLQKRERERG1GZmhE2it8vJy5ObmYtu2bQCAgIAAREdHo6KiAra2tk3uKwgCAECpVOo9zz9Tyc11Fqu2thZqi246jSc1t9FpPAuZ7uLdjWkj7aLbeDK5TuN1k0l1Fu//xxR0Gk8mU+k0nkauu3h3Y6osdBuvveN3y4PH5HfLg8fjd4v+yeVySCSSem0S4W5VY2JycnKwZMkS7NmzR2wbP3481q9fj0GDBjW5b1VVFc6dO6fvFImIiIh0xs3NDRYW9atRk+2RexBWVlYYOHAgzM3N76tsiYiIiIyRXH5/T6/JFnKOjo4oKSmBWq2GVCqFWq1GaWkpHB0dm93XzMwMXbt2bYMsiYiIiPTHZB92sLOzg6urKzIzMwEAmZmZcHV1bfb+OCIiIqL2wmTvkQOAvLw8hIeH48aNG7C2toZCocAjjzxi6LSIiIiI2oRJF3JEREREHZnJDq0SERERdXQs5IiIiIhMFAs5IiIiIhPFQo6IiIjIRLGQI2qEQqGAn58fXFxc+CYQItKZyspKvPHGGxg7diwmTJiAd955BxUVFYZOi0wUCzmiRvj7+yM5ORm9evUydCpE1I5IJBLMmDEDWVlZyMjIgLOzMzZs2GDotMhEsZAjaoSnp6dWbwohImoJGxsb+Pj4iMtDhw5FUVGRATMiU8ZCjoiIyEA0Gg22b98OPz8/Q6dCJoqFHBERkYFER0fD0tISoaGhhk6FTJTM0AkQERF1RAqFAgUFBUhMTISZGftVqHVYyBEREbWx2NhY5OTk4KOPPoJcLjd0OmTC+K5VokasXr0a+/btQ1lZGbp37w4bGxvs2bPH0GkRkYk7f/48AgIC0LdvX3Tq1AkA4OTkhPj4eANnRqaIhRwRERGRieKgPBEREZGJYiFHREREZKJYyBERERGZKBZyRERERCaKhRwRERGRiWIhR0RkJFxcXFBQUAAAWLFiBaejIKJmcUJgIjIpfn5+uH37Nvbv3w9LS0sAwM6dO7F79258+umnej/+1KlTcfr0achkMsjlcnh5eWHFihWwt7fX6XGioqJ0Go+I2if2yBGRyVGr1fjkk08MdvwVK1bg1KlT+Prrr1FTUwOFQmGwXIioY2MhR0Qm5/XXX8fHH3+MGzdu3Lfu0qVLcHFxgUqlEtumTp2KnTt3AgBSUlIwZcoUrFmzBp6envD398dPP/2ElJQUjB49GsOHD0dqaqpWeVhbW8Pf3x+//vqr2JaXl4fp06fD29sbY8eOxd69e8V14eHhWLFiBaZPnw53d3eEhobi8uXLDcYODw9HbGysuLx//34EBQVh2LBhGDNmDL799lsAwK5duzBu3Di4u7vD398fn3/+ubhPRUUF3nrrLXh6esLb2xshISHQaDRanRsRmQYWckRkctzc3ODt7Y2kpKRW7X/mzBm4uLggOzsbAQEBWLBgAX7++Wd8/fXXWL9+PaKiolBdXd1snMrKSnz99dfo3bs3AKCmpgavvfYaAgIC8MMPP+CDDz7AqlWrcP78eXGfjIwMzJo1C9nZ2fjrX/+KsLAwrfJdsmQJFi9ejBMnTiA5ORm9evUCANjZ2WHr1q346aefsHbtWqxduxa//PILAGDbtm1wcHDA0aNHceTIESxYsAASiaQ1l4yIjBQLOSIySXPnzsVnn32GioqKFu/r5OSEyZMnQyqVYvz48SguLsbs2bMhl8sxatQoyOVyXLx4sdH9V69eDQ8PDzz22GOorKzE8uXLAQCHDh1Cr169MHnyZMhkMgwaNAhjx45FVlaWuO8TTzwBLy8vyOVyzJ8/H6dPn0ZxcXGT+X7xxReYPHkyRo4cCTMzMzg4OKB///5ivN69e0MikcDb2xsjR47EiRMnAAAymQxXr15FUVERzM3N4enpyUKOqJ1hIUdEJmngwIF44okn8NFHH7V4Xzs7O/G/7760vEePHmKbhYVFkz1yEREROHnyJHbv3o0bN27gypUrAIDLly/jzJkz8PT0FP9lZGTg6tWr4r4PP/yw+N9WVlbo1q0bSktLm8y3uLhY7PW71+HDh/Hiiy/C29sbnp6e+Pbbb1FZWQngzhB0nz598Nprr8Hf379V14qIjBufWiUikzV37lw899xzeO2118S2u0+y3r59G126dAGAeoWULrm4uGDmzJmIiopCamoqHB0d4eXlhW3btjW6z92iDwCqq6tx/fr1Zp94dXR0bLCHUKlUYu7cuVAoFPD394e5uTlmzZoFQRAAAF26dEF4eDjCw8Nx/vx5TJs2DX/7298wfPjwVp4xERkb9sgRkcnq06cPxo8fX2/aEVtbWzg4OCA9PR1qtRpffPEFCgsL9ZbDxIkTUV5ejm+++QZPPPEELly4gLS0NNTV1aGurg5nzpxBXl6euP3hw4dx4sQJKJVKbNy4EUOGDIGjo2OTx3j++eeRkpKCo0ePQqPRoKSkBHl5eVAqlVAqlbC1tYVMJsPhw4dx5MgRcb+DBw+ioKAAgiCgS5cukEqlMDPj1z5Re8LfaCIyabNnz0ZNTU29tujoaCQlJcHHxwe///473N3d9XZ8uVyOqVOnIiEhAV26dEFSUhL27t0LX19fjBo1Chs2bIBSqRS3DwgIQHx8PHx8fPDLL79g/fr1zR5j8ODBWLt2LdasWQMPDw+EhoaiqKgIXbp0QUREBN599114eXkhMzMTfn5+4n4FBQXiE7LBwcF46aWX4OPjo5frQESGIRHu9sETEZFehYeHw8HBAfPnzzd0KkTUTrBHjoiIiMhEsZAjIiIiMlEcWiUiIiIyUeyRIyIiIjJRLOSIiIiITBQLOSIiIiITxUKOiIiIyESxkCMiIiIyUSzkiIiIiEzU/wMu94CpINRt5wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "\n", + "x = 'Factor'\n", + "\n", + "df = pd.DataFrame({\n", + " x: replicas, \n", + " **replica_results,\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars=x).rename(columns=str.title)\n", + "seaborn.barplot(x=x, y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "seaborn.despine(fig)\n", + "\n", + "ax1.set(xlabel=\"Num Replicas\", ylabel=f'MASE Loss', title='Residual Estimate Loss for Time-Series Decomposition')" + ] + }, + { + "cell_type": "code", + "execution_count": 272, + "id": "dd593565", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "max_fits_1100 1893.9663657607266\n", + "max_fits_1100 133.45026146609104\n", + "max_fits_1100 117.59550187337948\n", + "max_fits_2100 2875.0872626231935\n", + "max_fits_2100 1447.6250864288265\n", + "max_fits_2100 99.89538248542472\n", + "max_fits_4200 3430.718917641645\n", + "max_fits_4200 2591.967671743391\n", + "max_fits_4200 95.01342291496671\n", + "max_fits_8400 3586.4346343021443\n", + "max_fits_8400 3006.052341175111\n", + "max_fits_8400 93.57666588051953\n" + ] + }, + { + "data": { + "text/plain": [ + "{'max_fits_1100': [1893.9663657607266, 133.45026146609104, 117.59550187337948],\n", + " 'max_fits_2100': [2875.0872626231935, 1447.6250864288265, 99.89538248542472],\n", + " 'max_fits_4200': [3430.718917641645, 2591.967671743391, 95.01342291496671],\n", + " 'max_fits_8400': [3586.4346343021443, 3006.052341175111, 93.57666588051953]}" + ] + }, + "execution_count": 272, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "experiments = [(\"max_fits_1100\", 96), (\"max_fits_2100\", 48), (\"max_fits_4200\", 24), (\"max_fits_8400\", 12)]\n", + "replicas = [1, 2, 4]\n", + "slides = [1, 6, 12, 18, 24, 48, 96, 168, 192, 336, 672]\n", + "graph_results = {\"baseline\": [], \"optimized\": [], \"cost\": []}\n", + "\n", + "replica_results = {}\n", + "\n", + "\n", + " \n", + "for plan, slide in experiments: \n", + " replica_results[plan] = []\n", + " \n", + " for replica in replicas:\n", + " total_loss = 0\n", + " for key in range(1, 101, 1):\n", + " oracle_filename = f\"{oracle_dir}/{key}.csv\"\n", + " lp_filename = f\"{results_dir}/replica_{replica}/{plan}/{key}.csv\"\n", + " results = get_loss_per_key(key, lp_filename, oracle_filename)\n", + " total_loss += results[\"loss\"]\n", + " \n", + " replica_results[plan].append(total_loss)\n", + " print(plan, total_loss)\n", + " \n", + "replica_results" + ] + }, + { + "cell_type": "code", + "execution_count": 245, + "id": "fa644e00", + "metadata": {}, + "outputs": [], + "source": [ + "static_results = {1: [213.2200178532724, 150.7256273025718, 133.3349003094353],\n", + " 6: [218.75801939773078, 156.0948132080653, 135.81574629721845],\n", + " 12: [220.90611494937247, 159.05690147515068, 138.24448802117672],\n", + " 18: [229.63341620779627, 163.1813146667572, 140.3471142793597],\n", + " 24: [233.91725185369373, 164.71255155633278, 144.22561887879107],\n", + " 48: [268.3977607679711, 184.14616983478135, 158.3310894771346],\n", + " 96: [348.50466291276604, 229.46322062727472, 189.37872271737845],\n", + " 168: [474.50432909190295, 319.6199513026192, 281.5650612571233],\n", + " 192: [609.2301332698065, 399.7143084337964, 333.28670707646245],\n", + " 336: [908.5841349053487, 728.1723528503993, 650.7431132687982],\n", + " 672: [1848.5207568587812, 1612.2188363608043, 1489.8733845259228]}" + ] + }, + { + "cell_type": "code", + "execution_count": 246, + "id": "2a2bdeb1", + "metadata": {}, + "outputs": [], + "source": [ + "policy_results = {'max_fits_1100': [1893.9663657607266, 133.45026146609104, 117.59550187337948],\n", + " 'max_fits_2100': [2875.0872626231935, 1447.6250864288265, 99.89538248542472],\n", + " 'max_fits_4200': [3430.718917641645, 2591.967671743391, 95.01342291496671],\n", + " 'max_fits_8400': [3586.4346343021443, 3006.052341175111, 93.57666588051953]}" + ] + }, + { + "cell_type": "code", + "execution_count": 273, + "id": "239a5274", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'baseline': [213.2200178532724, 150.7256273025718],\n", + " 'policy': [1893.9663657607266, 133.45026146609104]}" + ] + }, + "execution_count": 273, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "replicas = [1, 2]\n", + "results = {\"baseline\": [], \"policy\": []}\n", + "for i in range(len(replicas)): \n", + " \n", + " best_baseline = None\n", + " for key in static_results.keys(): \n", + " if best_baseline is None or static_results[key][i] <= best_baseline: \n", + " best_baseline = static_results[key][i]\n", + " results[\"baseline\"].append(best_baseline)\n", + " \n", + " best_baseline = None\n", + " for key in policy_results.keys(): \n", + " if best_baseline is None or policy_results[key][i] <= best_baseline: \n", + " best_baseline = policy_results[key][i]\n", + " results[\"policy\"].append(best_baseline)\n", + " \n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 274, + "id": "ee33ec46", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Text(0.5, 0, 'Num Replicas'),\n", + " Text(0, 0.5, 'MASE Loss'),\n", + " Text(0.5, 1.0, 'Residual Estimate Loss for Time-Series Decomposition')]" + ] + }, + "execution_count": 274, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnIAAAFSCAYAAAB2ajI+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA+CElEQVR4nO3de1xUdR7/8RcCgwohQcGSlyxLIvlZyC1TsqAyjbDU0lgt7W5ethSTSsVQKtJN0zSrNatds9W8IGhhZVpZ3nONtNVY7xIoF0VRBobz+8Of84sUBATGg+/n4+FjO+d7Lp9zZoZ57/d7zhknwzAMRERERMR0mji6ABERERGpHQU5EREREZNSkBMRERExKQU5EREREZNSkBMRERExKQU5EREREZNSkBOpxLJly3jssccqbR84cCALFy684P2sX7+e22677YK3cz7nO55LkWEYvPjii4SFhdG3b9963dehQ4cIDg7GZrPV637qg947dS84OJj9+/dX2n7vvfeyfv36BqxIzMpJz5GTxiAqKoojR47g7OxM8+bNiYyMZNy4cbi7u9fbPgcOHEhsbCwPPvjgBW1n/fr1jB49mm+//fac7QEBATRr1gwnJyf7vGeffZYnn3yy0m0eOHCA6OhofvnlF1xcXC6ovuq40HNRV+eypjZt2sTIkSP54osvaN68+QVv68xrYhgGJ0+erLDN5cuXc9VVV13QPmpi9uzZLFiwgIKCAi677DI6derEtGnTGmz/57N48WJefvllmjZtCsDll19OREQETz31FNdcc42Dq2tYCQkJ+Pn58fzzzzu6FDGh+v8LL9JAZs+eza233srhw4d5/PHHee+99xrNH8bU1FSuvvpqR5fR6Bw8eJCWLVvWKsSVlZVVCMmhoaH89NNPwP8P0hs3bmyQIP1nS5YsITU1lQ8//JA2bdpw+PBhVq1aVatt/fk469LNN9/M/PnzsdlsHDx4kA8++IDevXvz73//m/bt29fLPkUaGw2tSqNz5ZVX0rVrV3bs2GGft3XrVvr3709oaCixsbEVhiwWL15MdHQ0wcHBREVFsWzZMvv8hx9+2L7c2rVrueeeewgJCSEpKYk/dmbPmDGD+Ph4+/SBAwcICAigrKwMgEWLFtGjRw+Cg4OJjo7m008/rZNj3bZtG71796ZTp07ceuutvPbaawAMGDAAgLCwMIKDg/npp5/OOp6AgADmzZvH3XffTXBwMNOmTWPfvn3069ePTp068be//Q2r1QrA0aNHefrpp7nlllsICwvj6aef5vfffwdg6tSpbNq0iaSkJIKDg0lKSgIgKyuLwYMHEx4eTvfu3VmxYkWNj6+8vJxZs2Zxxx130LlzZ1544QWKiooAKCkpIT4+noiICEJDQ+nTpw9HjhwBKn9N/2jhwoWMHTuWrVu3EhwczPTp0wFYsGABd911F+Hh4TzzzDPk5OSc85zdfffdNTqWP78nBg4cyNSpU+nfvz/BwcE888wzFBQUMGrUKDp16kSfPn04cOCAff2anM+ff/6Zrl270qZNG+D0Z6Jfv3729qKiIl566SW6du1KZGQkU6dOtQ/5Ll68mP79+/Pqq68SHh7OjBkzznrvVFXLmjVr6NmzJ8HBwURGRjJnzpzznhtnZ2fatGnDhAkTCA8P5+2337a3VfXZLSws5MUXX6Rr166EhYXx7LPP2tuq+zqe771/5tKH2bNnExERcdb7qaioiBdeeIFbbrmFO+64g1mzZlFeXg7A3r17GTBgACEhIURERPDcc89VqGHv3r38+9//Ji0tjTlz5tjfB3B6lOGHH34AwGq1kpycTNeuXenatSvJycln1ffBBx/QuXNnunbtyqJFi857zqURMUQagTvuuMNYu3atYRiGkZ2dbcTExBgTJ040DMMwfv/9dyM8PNxYvXq1YbPZjO+//94IDw838vLyjBMnThjBwcFGVlaWYRiGkZOTY+zcudMwDMNYtGiR0b9/f8MwDCMvL88IDg42Pv/8c8NqtRpz5841AgMDjQULFhiGYRjTp083Ro0aZa9n//79Rvv27Y3S0lLDMAzjm2++Mfbu3WuUl5cb69evNzp27GhkZmYahmEY69atMyIjIys9tvbt2xt79uw5Z9tDDz1kLFmyxDAMwzh+/Ljx008/nXP/fz6eM9t9+umnjaKiImPnzp1Ghw4djEceecTYt2+fcezYMaNHjx7G4sWLDcMwjPz8fOOLL74wiouLjaKiImP48OHGkCFD7NsaMGCA/VwYhmGcOHHCuO2224zPPvvMKC0tNTIzM43w8HD7uf2zP69/xsKFC40777zT2Ldvn3H8+HFj6NChRnx8vGEYhjF//nzj6aefNoqLi42ysjLj559/NoqKiqp8Tf/sz+fkhx9+MMLDw43MzEyjpKTESEpKMuLi4iqcs0GDBhkFBQXGyZMnz7nNys7/n+cNGDDAuPPOO429e/faz/fdd99trF271igtLTVGjx5tJCQk1Op8Ll261AgLCzPef/99Y9u2bUZZWVmF9iFDhhjjxo0zTpw4YRw5csTo06ePMX/+fPs5CQwMND7++GOjtLTUOHnyZIXzdL5aunTpYmzcuNEwDMMoLCy0v8/Pd+7PWLhwodG5c2fDMKr+7BqGYTz55JPG3/72N6OwsNCwWq3G+vXrq/06Vve9v27dOiMwMNB49dVXjZKSEmP9+vXGTTfdZH9/jR492njmmWeMoqIiY//+/cbdd99tfy8///zzxqxZswybzWacOnXKfl7O1HDmcz1mzBjjzTffrHAe/vg3bdq0acaDDz5oHDlyxMjLyzP69etnTJ06tUJ906ZNM6xWq7F69WqjY8eORmFh4TnPuzQ+6pGTRmPo0KEEBwfTrVs3vL29GTFiBHB6WPK2226jW7duNGnShC5duhAUFMSaNWsAaNKkCbt27eLUqVP4+vpy/fXXn7Xtb7/9luuuu4577rkHV1dXHn30Ua644opq13b77bfTpk0bnJycCA8Pp0uXLmzatKna6z/wwAOEhoba/3333XcAuLi4sG/fPvLz83F3d+fmm2+u9jYBnnzySTw8PLj++utp3749Xbp0oXXr1lx22WXcdtttbN++HTh9/VL37t1p1qwZHh4eDBkyhI0bN1a63dWrV9OyZUv69OmDi4sLHTp0oHv37mRkZNSovrS0NAYNGkTr1q1xd3dn5MiRrFixwj7cV1hYyN69e3F2diYoKAgPDw+geq9pZfvr06cPHTp0wGKxMHLkSLZu3VqhZ+ypp57Cy8vLfm3Xhejduzdt2rSxn+/WrVtz66234uLiwj333GM//zU9n7169WLs2LF8//33DBw4kFtvvZX33nsPgCNHjvDtt9/y0ksv0bx5c3x8fBg0aBDLly+3r+/r68vAgQNxcXE56zjPV4uLiwu//fYbx48fp0WLFnTo0KFG58TX15ejR48CVX92c3Nz+fbbb3nllVdo0aIFrq6uhIeHA9V7Hav73j/jb3/7GxaLhfDwcLp168bnn3+OzWZjxYoVjBo1Cg8PD1q1asXgwYPtPXYuLi4cOnSI3Nxc3NzcCA0NrdG5OCMtLY2hQ4fi4+ODt7c3Q4cOrdAr6OLiwtChQ3F1daVbt240b96c3bt312pfYj66Rk4ajZkzZ3LrrbeyYcMGRo0aRUFBAZ6enhw6dIgvvviCb775xr5sWVkZERERNG/enKlTp/LBBx/w8ssv06lTJ8aMGUO7du0qbDs3N5e//OUv9mknJyf8/f2rXduaNWuYOXMme/bsoby8nFOnTtXoGqAlS5ac8xq55ORkpk+fTo8ePWjVqhXDhg3jjjvuqPZ2/xhG3dzczpo+M1R58uRJXnvtNb777jv7l+yJEyew2Ww4Ozuftd2DBw+ybdu2Cl9cNpuN2NjYatcGp897y5Yt7dMtW7akrKyMvLw8evXqxe+//87IkSM5duwYsbGxPP/889V+TSvb3x+Dh7u7O15eXuTk5NCqVSuAGr3u51PV+W/atCnFxcVA1efz0KFD3Hvvvfb5Z67Ti42NJTY2ltLSUr766itGjx5NYGAgLVq0oKysjK5du9rXKS8vr3Bcf3yv/9n5Xtvp06fzzjvv8Pe//52AgABGjRpFcHBwtc9JTk4OLVq0AKjys/v777/TokUL+7J/VJ3XsbrvfQBPT88K11FeddVV5ObmUlBQQGlpaYWbWK666ir7MO7o0aN566236Nu3Ly1atGDw4MG1ujs6Nzf3rH3k5ubap728vCpcx9isWTP7e0caPwU5aXTCw8Pp3bs3KSkpzJo1C39/f3r16sWkSZPOuXxkZCSRkZGcOnWKadOmMW7cOD755JMKy1x55ZX2a8Lg9F2J2dnZ9ulmzZpx6tQp+/QfvwSsVisjRowgJSWF6OhoXF1defbZZytcY1dbbdu25c0336S8vJyVK1cyYsQI1q9fX+EO17rwwQcfsHv3bhYsWMCVV17Jjh07uP/++ys9Bn9/f8LCwpg7d+4F7dfX15eDBw/apw8dOoSLiws+Pj64uLgwbNgwhg0bxoEDB+x3Oz744IPVek2rs7/i4mIKCwvx8/Ozz6vrc1sd5zufZ8Lbubi6utKjRw/ef/99du3aRUxMDBaLhXXr1lV6E0NVx3i+Wjp27Mg777xDaWkp8+bN47nnnrP3flfHV199ZQ+JVX12c3NzOXr0KMeOHcPT07NCW3Vex5o4duwYxcXF9jCXnZ3N9ddfz+WXX46rqyuHDh3iuuuus7ed2c+VV15pr33Tpk0MHjyYsLCws/5P2fneU76+vhw6dMjes5ydnY2vr2+tjkUaHw2tSqP06KOP8sMPP7Bjxw5iY2P55ptv+O6777DZbJSUlLB+/Xp+//13jhw5wtdff01xcTEWi4XmzZufs4epW7du7Nq1i5UrV1JWVsbHH39cIawFBgayceNGDh06RFFREe+++669zWq1YrVa8fb2xsXFhTVr1rB27do6Oc7U1FTy8/Np0qSJ/cvM2dkZb29vmjRpUuVzqmrixIkTuLm54enpSWFhYYWL0eF078Yf93X77bezZ88eli5dSmlpKaWlpWzbto2srKxK91FWVkZJSYn9X2lpKTExMXz00Ufs37+fEydOMHXqVHr06IGLiwvr1q3jv//9LzabDQ8PD1xcXHB2dq72a3ou9913H4sXL2bHjh1YrVbefPNNOnbsaO/FcZSans/FixezevVqjh8/Tnl5OWvWrOG3336jY8eO+Pr60qVLF15//XV7+759+9iwYcMF12K1Wlm2bBlFRUW4urri7u5erXNvs9nYv38/EydOZMOGDQwdOhSgys+ur68vt912G6+88gpHjx6ltLTUPtxfH6/jjBkzsFqtbNq0idWrV3PPPffg7OzMPffcw9SpUzl+/DgHDx5k7ty59t7Jzz//3P5/AFu0aIGTkxNNmpz9tevj41Nh2PfP7r33Xt555x3y8/PJz89n5syZ3HfffbU+FmlcFOSkUfL29qZXr172HrlZs2bx7rvv0rlzZ7p168acOXMoLy+nvLycuXPnEhkZSXh4OBs3biQxMfGc23vrrbf4+9//TkREBHv37qVTp0729i5dutCzZ09iY2Pp3bt3heFNDw8Pxo4dy3PPPUdYWBjp6elERUXV6Hh69epFcHCw/V9ycjIA3333Hffee6993tSpU3Fzc6NZs2Y888wzPPzww4SGhrJ169bancj/59FHH6WkpIRbbrmFfv36ERkZWaH9kUceISMjg7CwMCZNmoSHhwdz5sxhxYoVREZG0rVrV6ZMmWK/0+5cJkyYQMeOHe3/XnzxRfr06UNsbCwDBgwgOjoai8XCuHHjgNO9niNGjCAkJISePXsSHh5ObGxstV/Tc+ncuTN/+9vfGD58OF27dmX//v1MnTq19ieujtT0fHp4eDB79mzuuOMOQkNDmTJlChMmTLD3dL3xxhuUlpbSs2dPwsLCGDFiBIcPH66TWlJTU4mKiqJTp058+umnvPHGG5Vu68wdwyEhITzyyCMcP36czz77jICAAIAqP7tnjsPFxYUePXpw66238tFHHwF1/zpeccUVeHp6EhkZSXx8PBMmTLAP1Y8bN45mzZpx5513EhcXR0xMDH369AFO3z384IMPEhwczJAhQ3j55Zdp3br1Wdvv27cvv/32G6GhoRXuvD3j2WefJSgoyD5c3qFDh3MuJ5cmPRBYRESkEud7YLeIo6lHTkRERMSkFORERERETEpDqyIiIiImpR45EREREZO6JIOcYRiUlJTUyXO8RERERBzlkgxyVquVzMzMKh+FICIiInKxuySDnIiIiEhjoCAnIiIiYlIKciIiIiImpSAnIiIiYlIuji5AREREzKe0tJQDBw5w6tQpR5fSaDRt2pRWrVrh6upa7XUU5ERERKTGDhw4wGWXXUbbtm1xcnJydDmmZxgGeXl5HDhwgGuuuaba62loVURERGrs1KlT+Pj4KMTVEScnJ3x8fGrcw6kgJyIiIrWiEFe3anM+FeRERERETEpBTkRERBqV4OBg9u/ff97lDhw4QEBAAGVlZedsnzFjBvHx8XVdXp1SkBMRERGHevzxx3nrrbfOmv/VV1/RpUuXSoNWZX766Sdat25dV+Vd1BTkRM6jvEy/yWtmev1ELn4PPPAAqampGIZRYf6yZcu47777cHGp3kM2ahr4GgM9fkTkPJq4WNg5ZZCjy5Baah//oaNLEJHzuPPOO0lMTGTTpk2EhYUBcPToUb755hvmzJlDv379yMrKomnTptx9990kJCRgsVgACAgIYPz48Xz00UeUlZWxatUqAgICWLlyJVdffTWrV69m2rRp7Nu3j8suu4y+ffsyfPjwCvtftGgRM2bMAOCxxx7jscceO2edW7du5fXXX+e3337jqquu4uWXXyYiIqIez8z5qUdOREREHKpp06b06NGDpUuX2ud9/vnnXHvttTRv3pwXX3yRdevW8emnn/Ljjz/yySefVFj/q6++YsGCBaxYseKsbTdr1oyUlBQ2bdrEu+++y/z58/nqq68qLLN+/XpWrlzJnDlzeO+99/jhhx/O2k5OTg5PP/00Q4YMYcOGDYwZM4YRI0aQn59fNyehlhTkRERExOHuv/9+vvjiC/tz1JYuXcoDDzxAUFAQN998My4uLrRq1Yp+/fqxcePGCus+9dRTeHl50bRp07O2GxERQUBAAE2aNOGGG27g3nvvZcOGDRWWGTp0KM2bNycgIIDevXuTnp5+1nZSU1O57bbb6NatG02aNKFLly4EBQWxZs2aOjwLNaehVREREXG40NBQvL29+frrr+nYsSOZmZm8/fbb7N69m9dff53MzExOnjyJzWajQ4cOFdb19/evdLv/+c9/mDJlCrt27aK0tBSr1co999xT6fotW7Zk586dZ23n0KFDfPHFF3zzzTf2eWVlZQ4fWlWQExERkYtCr169WLp0Kbt376ZLly5cccUVjBo1ihtvvJG///3veHh48OGHH5KRkVFhvaoepDtq1CgGDBjAP/7xD9zc3EhOTqagoKDCMtnZ2bRr1w44Hdh8fX3P2o6/vz+9evVi0qRJdXCkdUdDqyIiInJRuP/++/nxxx9ZsGAB999/PwAnTpzA3d0dd3d3srKymD9/fo22eeLECVq0aIGbmxvbtm0757DprFmzOHnyJLt27WLx4sX07NnzrGViY2P55ptv+O6777DZbJSUlLB+/Xp+//33Wh1rXVGQExERkYtCq1atCA4O5uTJk0RHRwMwZswY0tPT6dSpE+PGjTtnyKpKYmIi06dPJzg4mJkzZ9KjR4+zlgkPD+euu+5i0KBBPPbYY3Tt2vWsZfz9/Zk1axbvvvsunTt3plu3bsyZM4fy8vLaHWwdcTL+/NCWS0BJSQmZmZkEBQXh5ubm6HLEBPT4EfPS40dE6seOHTsIDAx0dBmNTk3Pq3rkRERERExKQU5ERETEpBTkRERERExKQU5ERETEpBTkRERERExKQU5ERETEpBTkRERERExKQU5ERETqhLXUZqrtNgYN8lurKSkpZGRkcPDgQdLS0mjfvj0HDhxg6NCh9mWKioo4fvw4GzZsACAqKgqLxWJ/YG98fDyRkZEA7N69m4SEBAoLC/Hy8iIlJYW2bds2xKGIiIhIJSyuzsSNX13n2/0k6fbzLhMQEMCWLVtwd3ev8/3/UUJCAkFBQQwYMID58+dTUlLCoEGD6nWfVWmQIBcdHc0jjzzCX//6V/u8Vq1akZqaap9OTk7GZquYuKdPn0779u3P2l5iYiJxcXH06tWL1NRUxo8fz8cff1x/ByAiIiLyJw8//LCjS2iYIBcaGlplu9VqJS0tjTlz5px3W3l5eWzfvp25c+cCEBMTw8SJE8nPz8fb27tO6hURERHz+eCDD1i7di0FBQWMHDmS7t27AzBq1Ch2795NaWkpbdq04dVXX6VFixb873//48UXX+TkyZOUl5fzwAMP8Pjjj2O1Wpk6dSobN26ktLSU9u3bM2HChLN6+2bMmEFxcTFjxoxh8eLFpKen4+npya5du7jsssuYMWMGV155JQDvv/8+GRkZ2Gw2/Pz8mDhxor3tQjRIkDufVatW4efnR4cOHSrMj4+PxzAMQkJCGDlyJJ6enmRnZ+Pn54ezszMAzs7O+Pr6kp2dXeMgl5mZWWfHII1XSEiIo0uQC7R582ZHlyDS6Li4uHDixIkK8+pzWPPP+zqXsrIy5syZw549exg8eDA33ngj3t7ePP/881x++eUAzJw5k1mzZjFixAg+/vhjOnfuzJNPPgnAsWPHOHHiBP/4xz9o2rQpH330EQBvvfUWb7/9NsOGDaOsrIySkhJOnDiB1WqltLSUEydOUFJSwrZt2/j3v//NX/7yFyZOnMgHH3zAsGHDWL58OVlZWcydO5cmTZqwcOFCkpOTSU5OPusYrFbrOf9mVfZddFEEuUWLFtGnT58K8+bNm4e/vz9Wq5Xk5GSSkpKYMmVKne43KCjIfg2eiDReCuMidW/Hjh31fj3aH1VnX3Fxcbi7u9OhQwc6dOjAzp07iY6OZsGCBaSlpVFaWkpxcTFt27bF3d2dzp07k5KSAkBERAS33HILTk5OfPfddxw/fpxVq1YBp8PVDTfcgLu7Oy4uLri5ueHu7o7FYqGsrAx3d3fc3NwICQmhXbt2wOm/Oz/88APu7u6sXbuWzMxMBgwYAIDNZsPDw+Ocx2SxWLjpppuqfV4cHuRycnLYuHEjb7zxRoX5/v7+wOkDiouLY8iQIfb5OTk52Gw2nJ2dsdls5Obm2pcXERERMQwDJycnNm3axPz58/n000/x9vYmLS2NBQsWANC9e3duvvlm1q5dy/vvv8+iRYuYMmUKhmGQmJhI586da7TPP3YOnckoZ2oZMmQIffv2rbsD/H8c/viRJUuW0K1bN3uXJ0BxcTFFRUXA6YNfsWIFgYGBAPj4+BAYGEh6ejoA6enpBAYG6vo4ERGRS9yiRYsA2LNnDzt27OCmm27i2LFjeHh44OXlhdVqtS8DsHfvXq688kp69+7N0KFD+fnnn4HTT8748MMPOXXqFADHjx8nKyur1nVFRUXxySefcPToUeB0D9+vv/5a6+39UYP0yE2aNImVK1dy5MgRBg8ejJeXF8uXLwdOB7mXX365wvJ5eXkMHz4cm81GeXk57dq1IzEx0d4+YcIEEhISmDVrFp6envZuUREREXEca6mtWo8Kqc12La7O513OYrHQv39/CgoKSEpKwsfHh9tuu41ly5bRo0cP/Pz8CAoKsge2zz//nLS0NFxdXXFycuKll14C4KmnnuLtt9+mb9++ODk54eTkxLBhw+zDpjV1//33U1hYaB9aNQyDhx9+mBtuuKFW2/sjJ8MwjAveismUlJSQmZmpa+Sk2nZOGeToEqSW2sd/6OgSRBqlHTt22EfLpO7U9Lw6fGhVRERERGpHQU5ERETEpBTkRERERExKQU5ERETEpBTkRERERExKQU5ERETEpBTkREREpE6Ul1lNtd3169fTu3dv4PQvTQ0cOLBe9lOfHP4TXSIiItI4NHGx1MtzNxvieZB+fn7885//rPf91DX1yImIiIjpBQQEMGPGDPr370/37t3JyMiwt3377bfcf//93HfffTz66KPs3bv3rPUPHDhARESEffqnn37i4YcfJjY2ltjYWL7//ntWrFjB008/bV/GarXStWtXsrOz6/fgqqAeOREREWkUnJyc+PTTT/nf//7Hww8/TGhoKAAvvPAC//rXv7juuutYuHAh8fHxLFy4sNLtFBYWMmzYMGbMmEGnTp2w2WwcP34cd3d3Jk+ezP79+2ndujUrVqzgpptuwt/fv6EO8SzqkRMREZFG4cEHHwTg2muv5cYbb2Tr1q385z//4YYbbuC6664DoE+fPuzYsYPjx49Xup2tW7fSrl07OnXqBICzszMtWrTAxcWFfv368emnnwLwySef8Ne//rWej6pqCnIiIiLS6BiGgZOTk/1/a7puZR566CHS09PZvHkzx44do3Pnzhda6gVRkBMREZFGYdGiRQDs2bOHHTt2cNNNNxEcHMyOHTvIysoCYMmSJdx44414eHhUup3g4GCysrL46aefALDZbBw9ehQAb29vbr31VkaOHElcXFyNQ2Jd0zVyIiIiUifKy6z1codpeZmVJi6W8y5nsVjo378/BQUFJCUl4ePjA8Abb7xBfHw8ZWVleHt7M3ny5Cq34+XlxYwZM3j99dcpLi6mSZMmjBkzhltvvRWAvn378sUXX/DAAw9c+MFdICejqv7DRqqkpITMzEyCgoJwc3NzdDliAvVxO700jIZ4bIHIpWjHjh0EBgY6ugy7gIAAtmzZgru7e73va9asWRw+fJjExMQ633ZNz6t65ERERESq6d5778XZ2Zk5c+Y4uhRAQU5EREQagf/+978Nsp/ly5c3yH6qSzc7iIiIiJiUgpyIiIjUyiV4mX29qs35VJATERGRGmvatCl5eXkKc3XEMAzy8vJo2rRpjdbTNXIiIiJSY61ateLAgQMcPnzY0aU0Gk2bNqVVq1Y1WkdBTkRERGrM1dWVa665xtFlXPI0tCoiIiJiUgpyIiIiIialICciIiJiUg0S5FJSUoiKiiIgIICdO3fa50dFRXHPPffQq1cvevXqxXfffWdv2717N/369aN79+7069ePPXv2VKtNRERE5FLRIEEuOjqaefPm0bJly7Papk+fTmpqKqmpqURGRtrnJyYmEhcXR0ZGBnFxcYwfP75abSIiIiKXigYJcqGhofj7+1d7+by8PLZv305MTAwAMTExbN++nfz8/CrbRERERC4lDn/8SHx8PIZhEBISwsiRI/H09CQ7Oxs/Pz+cnZ0BcHZ2xtfXl+zsbAzDqLTN29u7RvvOzMys8+ORxickJMTRJcgF2rx5s6NLEBG5IJV9Fzk0yM2bNw9/f3+sVivJyckkJSUxZcqUBtt/UFAQbm5uDbY/EXEMhXERaawcetfqmeFWi8VCXFwcW7Zssc/PycnBZrMBYLPZyM3Nxd/fv8o2ERERkUuJw4JccXExRUVFwOnfF1uxYgWBgYEA+Pj4EBgYSHp6OgDp6ekEBgbi7e1dZZuIiIjIpcTJaIBfu500aRIrV67kyJEjXH755Xh5eTF79myGDx+OzWajvLycdu3aMXbsWHx9fQHIysoiISGBY8eO4enpSUpKCtdee+1526qjpKSEzMxMDa1Kte2cMsjRJUgttY//0NEliIjUmwYJchcbBTmpKQU581KQE5HGTL/sICIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSCnIiIiIiJqUgJyIiImJSLg2xk5SUFDIyMjh48CBpaWm0b9+egoICXnjhBfbt24fFYuHqq68mKSkJb29vAKKiorBYLLi5uQEQHx9PZGQkALt37yYhIYHCwkK8vLxISUmhbdu2DXEoIiIiIheNBumRi46OZt68ebRs2dI+z8nJiSeeeIKMjAzS0tJo3bo1U6ZMqbDe9OnTSU1NJTU11R7iABITE4mLiyMjI4O4uDjGjx/fEIchIiIiclFpkCAXGhqKv79/hXleXl5ERETYp2+++WYOHTp03m3l5eWxfft2YmJiAIiJiWH79u3k5+fXbdEiIiIiF7kGGVo9n/LycubPn09UVFSF+fHx8RiGQUhICCNHjsTT05Ps7Gz8/PxwdnYGwNnZGV9fX7Kzs+3DstWVmZlZZ8cgjVdISIijS5ALtHnzZkeXICJyQSr7LroogtzEiRNp3rw5AwYMsM+bN28e/v7+WK1WkpOTSUpKOmvo9UIFBQXZr8ETkcZLYVxEGiuH37WakpLC3r17mTZtGk2a/P9yzgzFWiwW4uLi2LJli31+Tk4ONpsNAJvNRm5u7llDtyIiIiKNnUOD3NSpU8nMzGTmzJlYLBb7/OLiYoqKigAwDIMVK1YQGBgIgI+PD4GBgaSnpwOQnp5OYGBgjYdVRURERMzOyTAMo753MmnSJFauXMmRI0e4/PLL8fLyYtq0acTExNC2bVuaNm0KQKtWrZg5cyb79+9n+PDh2Gw2ysvLadeuHWPHjsXX1xeArKwsEhISOHbsGJ6enqSkpHDttddWu56SkhIyMzM1tCrVtnPKIEeXILXUPv5DR5cgIlJvahXkTp06RZMmTSr0opmJgpzUlIKceSnIiUhjVq2h1ZSUFLZt2wbA6tWrCQ8PJywsjFWrVtVrcSIiIiJSuWoFubS0NK6//noAZs6cyeTJk3nnnXeYOnVqvRYnIiIiIpWr1uNHTp48SbNmzSgoKGD//v10794dgIMHD9ZrcSIiIiJSuWoFubZt27Js2TL27dtHly5dAMjPz7ffpCAiIiIiDa9aQS4xMZFXX30VV1dXkpOTAfj+++/toU5EREREGl6DPH7kYqO7VqWmdNeqeemuVRFpzKp1s8O6devYv38/ALm5uYwZM4YXX3yRw4cP12txIiIiIlK5agW5V155xf4j9SkpKZSVleHk5MS4cePqtTgRERERqVy1rpHLycnhqquuoqysjO+//55Vq1bh6upKZGRkfdcnIiIiIpWoVpDz8PDgyJEj7Nq1i3bt2uHu7o7VaqWsrKy+6xMRERGRSlQryA0YMIC+fftSWlrKSy+9BMCWLVtq9PumIiIiIlK3qhXknnrqKe666y6cnZ1p06YNAH5+fkyaNKleixMRERGRylUryAG0bt2an376iW3btuHn50dwcDAuLtVeXURERETqWLWSWFZWFkOGDOHUqVP4+/uTnZ2Nm5sbs2fPpl27dvVdo4iIiIicQ7WC3CuvvMJDDz3E448/jpOTEwBz5sxhwoQJ/POf/6zXAkVERETk3Kr1HLlff/2VwYMH20McwKOPPsqvv/5ab4WJiIiISNWqFeR8fX3ZsGFDhXmbNm3C19e3XooSERERkfOr1tDq888/z7PPPsvtt9/OVVddxaFDh1i9ejWTJ0+u7/pEREREpBLV6pGLjo5m8eLFXH/99Zw4cYLrr7+exYsXc+edd9Z3fSIiIiJSiWo/P+Saa67h2WeftU9brVZuv/12Vq9eXR91iYiIiMh5VKtHrjK///57XdUhIiIiIjV0QUHuj3exioiIiEjDuqAgJyIiIiKOU+U1cqNHj660181ms9VLQSIiIiJSPVUGuauvvrrKlYcOHVqnxYiIiIhI9VUZ5IYNG1YnO0lJSSEjI4ODBw+SlpZG+/btAdi9ezcJCQkUFhbi5eVFSkoKbdu2vaA2ERERkUtFg1wjFx0dzbx582jZsmWF+YmJicTFxZGRkUFcXBzjx4+/4DYRERGRS0WDBLnQ0FD8/f0rzMvLy2P79u3ExMQAEBMTw/bt28nPz691m4iIiMilpNoPBK5r2dnZ+Pn54ezsDICzszO+vr5kZ2djGEat2ry9vR11OCIiIiINzmFB7mKQmZnp6BLEBEJCQhxdglygzZs3O7oEEZELUtl3UZVBbtKkSYwdO9Y+vXDhQh588EH79PDhw5kxY0atCvL39ycnJwebzYazszM2m43c3Fz8/f0xDKNWbTUVFBSEm5tbreoXEfNQGBeRxqrKa+QWL15cYXry5MkVpteuXVvrHfv4+BAYGEh6ejoA6enpBAYG4u3tXes2ERERkUtJlT1yhmFUOV1dkyZNYuXKlRw5coTBgwfj5eXF8uXLmTBhAgkJCcyaNQtPT09SUlLs69S2TURERORS4WRUkc46derEli1b7NPh4eFs2LCh0nazKCkpITMzU0OrUm07pwxydAlSS+3jP3R0CSIi9abKHjmbzca6devsPXFlZWUVpsvLy+u/QhERERE5pyqDnI+PDy+99JJ92svLq8K0rksTERERcZwqg9yqVasaqg4RERERqaEa/7LD//73P7788ksOHjxYH/WIiIiISDVVGeRef/11UlNT7dNLly4lJiaGcePG0aNHD9asWVPvBYqIiIjIuVUZ5L766ivCwsLs02+++SYvv/wy69at45VXXmHmzJn1XqCIiIiInFuVQS4/P5+rrroKgJ07d1JYWGj/ZYfY2Fj27NlT7wWKiIiIyLlVGeQuu+wyjhw5AsCmTZsICgrCYrEApx9FUtsHBIuIiIjIhavyrtUePXrw/PPPc9dddzF37lyefPJJe9t//vMfWrduXe8FioiIiMi5VdkjN2rUKCIiIvjhhx946KGHePjhh+1tO3bsoF+/fvVeoIiIiIicW5U9cq6urgwbNuycbY8++ii//fZbvRQlIiIiIudXZZA7l4KCApYvX86SJUv473//S2ZmZn3UJSIiIiLnUa0gV1ZWxurVq1myZAlr1qzBZrPxxBNP8M4779R3fSIiIiJSiSqD3M8//8zSpUtJT08HoHv37sydO5fnnnuOQYMG4ePj0yBFioiIiMjZqgxyDz74IF5eXowdO5YePXrg4nJ6cScnpwYpTkREREQqV+Vdq0OHDuWyyy5j3LhxjB49mlWrVlFWVtZQtYmIiIhIFaoMcsOHD+fLL7/k/fffp3nz5owePZouXbpw9OhRdu7c2VA1ioiIiMg5VBnkzggLCyM5OZm1a9fy8ssvEx4ezuOPP07fvn3ruz4RERERqUSNHj/StGlTYmNjiY2NJScnh9TU1PqqS0RERETOo8ogd+jQoSpXjomJqdNiRERERKT6qgxyUVFR9jtUDcM4q93JyYkdO3bUT2UiIiIiUqUqg1xAQAAlJSU88MADxMbG4uvr21B1iYiIiMh5VBnkUlNT2blzJ0uWLCEuLo5rr72WXr16cffdd9O0adOGqlFEREREzuG8d622b9+eMWPG8PXXXzNo0CBWr15N165d+eWXXxqiPhERERGpRLUePwKwZ88eNm7cyNatWwkMDMTT07M+6xIRERGR86hyaLWwsJDly5ezZMkSTpw4Qa9evfjXv/7FVVddVWcFHDhwgKFDh9qni4qKOH78OBs2bCAqKgqLxYKbmxsA8fHxREZGArB7924SEhIoLCzEy8uLlJQU2rZtW2d1iYiIiFzsqgxykZGRtGrVil69enHTTTcBsHfvXvbu3WtfpnPnzhdUQKtWrSo8jy45ORmbzWafnj59Ou3btz9rvcTEROLi4ujVqxepqamMHz+ejz/++IJqERERETGTKoPclVdeSUlJCQsWLGDBggVntTs5OfH111/XWTFWq5W0tDTmzJlT5XJ5eXls376duXPnAqefZzdx4kTy8/Px9vaus3pERERELmZVBrlVq1Y1VB32/fn5+dGhQwf7vPj4eAzDICQkhJEjR+Lp6Ul2djZ+fn44OzsD4OzsjK+vL9nZ2QpyIiIicsmo0U901bdFixbRp08f+/S8efPw9/fHarWSnJxMUlISU6ZMqbP9ZWZm1tm2pPEKCQlxdAlygTZv3uzoEkRELkhl30UXTZDLyclh48aNvPHGG/Z5/v7+AFgsFuLi4hgyZIh9fk5ODjabDWdnZ2w2G7m5ufblqysoKMh+I4WINF4K4yLSWFX78SP1bcmSJXTr1o3LL78cgOLiYoqKioDTPw+2YsUKAgMDAfDx8SEwMJD09HQA0tPTCQwM1LCqiIiIXFIumh65JUuW8PLLL9un8/LyGD58ODabjfLyctq1a0diYqK9fcKECSQkJDBr1iw8PT1JSUlxRNkiIiIiDuNkGIbh6CIaWklJCZmZmRpalWrbOWWQo0uQWmof/6GjSxARqTcXzdCqiIiIiNSMgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISbk4ugCAqKgoLBYLbm5uAMTHxxMZGcnu3btJSEigsLAQLy8vUlJSaNu2LUCVbSIiIiKXgoumR2769OmkpqaSmppKZGQkAImJicTFxZGRkUFcXBzjx4+3L19Vm4iIiMil4KIJcn+Wl5fH9u3biYmJASAmJobt27eTn59fZZuIiIjIpeKiGFqF08OphmEQEhLCyJEjyc7Oxs/PD2dnZwCcnZ3x9fUlOzsbwzAqbfP29q72PjMzM+vlWKRxCQkJcXQJcoE2b97s6BJERC5IZd9FF0WQmzdvHv7+/litVpKTk0lKSmLQoEH1vt+goCD7dXki0ngpjItIY3VRDK36+/sDYLFYiIuLY8uWLfj7+5OTk4PNZgPAZrORm5uLv79/lW0iIiIilwqHB7ni4mKKiooAMAyDFStWEBgYiI+PD4GBgaSnpwOQnp5OYGAg3t7eVbaJiIiIXCqcDMMwHFnA/v37GT58ODabjfLyctq1a8fYsWPx9fUlKyuLhIQEjh07hqenJykpKVx77bUAVbadT0lJCZmZmRpalWrbOWWQo0uQWmof/6GjSxARqTcOD3KOoCAnNaUgZ14KciLSmDl8aFVEREREakdBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETMrF0QUUFBTwwgsvsG/fPiwWC1dffTVJSUl4e3sTFRWFxWLBzc0NgPj4eCIjIwHYvXs3CQkJFBYW4uXlRUpKCm3btnXgkYiIiIg0LIf3yDk5OfHEE0+QkZFBWloarVu3ZsqUKfb26dOnk5qaSmpqqj3EASQmJhIXF0dGRgZxcXGMHz/eEeWLiIiIOIzDg5yXlxcRERH26ZtvvplDhw5VuU5eXh7bt28nJiYGgJiYGLZv305+fn691ioiIiJyMXH40OoflZeXM3/+fKKiouzz4uPjMQyDkJAQRo4ciaenJ9nZ2fj5+eHs7AyAs7Mzvr6+ZGdn4+3tXe39ZWZm1vkxSOMTEhLi6BLkAm3evNnRJYiIXJDKvosuqiA3ceJEmjdvzoABAwCYN28e/v7+WK1WkpOTSUpKqjDseqGCgoLs19+JSOOlMC4ijZXDh1bPSElJYe/evUybNo0mTU6X5e/vD4DFYiEuLo4tW7bY5+fk5GCz2QCw2Wzk5ubalxcRERG5FFwUQW7q1KlkZmYyc+ZMLBYLAMXFxRQVFQFgGAYrVqwgMDAQAB8fHwIDA0lPTwcgPT2dwMDAGg2rioiIiJidw4dWd+3axezZs2nbti39+/cHoFWrViQkJDB8+HBsNhvl5eW0a9eOxMRE+3oTJkwgISGBWbNm4enpSUpKiqMOQURERMQhnAzDMBxdREMrKSkhMzOzQa+Rs5basLg6N8i+pO7tnDLI0SVILbWP/9DRJYiI1BuH98hdKiyuzsSNX+3oMqQWPkm63dEliIiInNNFcY2ciIiIiNScgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIiISSnIiYiIiJiUgpyIiIlZS22OLkFqSa+d1AU9EFhExMT0sHHz0sPGpS6oR05ERMQBysusji5BLsDF8vqpR05ERMQBmrhY9DvOJnax/I6zeuRERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETEpBTkRERMSkFORERERETMrUQW737t3069eP7t27069fP/bs2ePokkREREQajKmDXGJiInFxcWRkZBAXF8f48eMdXZKIiIhIg3FxdAG1lZeXx/bt25k7dy4AMTExTJw4kfz8fLy9vatc1zAMAKxWa73X+UeezZwadH9SN0pKSihvepmjy5BaKikpcXQJ9U5/W8xJf1vMzRF/WywWC05OFT/vTsaZVGMymZmZjBkzhuXLl9vn9ezZk8mTJ9OhQ4cq1y0qKmLnzp31XaKIiIhInQkKCsLNza3CPNP2yF0Id3d32rdvj6ur61nJVkRERORiZLFYzppn2iDn7+9PTk4ONpsNZ2dnbDYbubm5+Pv7n3fdJk2acNll6s4WERERczPtzQ4+Pj4EBgaSnp4OQHp6OoGBgee9Pk5ERESksTDtNXIAWVlZJCQkcOzYMTw9PUlJSeHaa691dFkiIiIiDcLUQU5ERETkUmbaoVURERGRS52CnIiIiIhJKciJiIiImJSCnIiIiIhJKciJVCIlJYWoqCgCAgL0SyAiUmcKCgp48skn6d69O/fddx/Dhg0jPz/f0WWJSSnIiVQiOjqaefPm0bJlS0eXIiKNiJOTE0888QQZGRmkpaXRunVrpkyZ4uiyxKQU5EQqERoaWq1fChERqQkvLy8iIiLs0zfffDOHDh1yYEViZgpyIiIiDlJeXs78+fOJiopydCliUgpyIiIiDjJx4kSaN2/OgAEDHF2KmJSLowsQERG5FKWkpLB3715mz55NkybqV5HaUZATERFpYFOnTiUzM5P33nsPi8Xi6HLExPRbqyKVmDRpEitXruTIkSNcfvnleHl5sXz5ckeXJSImt2vXLmJiYmjbti1NmzYFoFWrVsycOdPBlYkZKciJiIiImJQG5UVERERMSkFORERExKQU5ERERERMSkFORERExKQU5ERERERMSkFOROQiERAQwN69ewEYP368HkchIuelBwKLiKlERUVx6tQpvvrqK5o3bw7AwoULWbZsGf/85z/rff8DBw5k69atuLi4YLFYCAsLY/z48fj6+tbpfpKSkup0eyLSOKlHTkRMx2az8fHHHzts/+PHj+enn37iyy+/pLi4mJSUFIfVIiKXNgU5ETGdxx9/nA8++IBjx46d1XbgwAECAgIoKyuzzxs4cCALFy4EYPHixfTv359XX32V0NBQoqOj2bJlC4sXL6Zbt2507tyZJUuWVKsOT09PoqOj+fXXX+3zsrKyGDx4MOHh4XTv3p0VK1bY2xISEhg/fjyDBw8mODiYAQMGcPDgwXNuOyEhgalTp9qnv/rqK3r16kWnTp248847+fbbbwFYtGgRPXr0IDg4mOjoaD799FP7Ovn5+Tz99NOEhoYSHh5OXFwc5eXl1To2ETEHBTkRMZ2goCDCw8OZM2dOrdbftm0bAQEBrF+/npiYGEaOHMnPP//Ml19+yeTJk0lKSuLEiRPn3U5BQQFffvklbdq0AaC4uJjHHnuMmJgYfvjhB958801eeeUVdu3aZV8nLS2NZ599lvXr13PDDTcQHx9frXrHjBnDCy+8wKZNm5g3bx4tW7YEwMfHh3fffZctW7bw2muv8dprr/HLL78AMHfuXPz8/Pjxxx9Zu3YtI0eOxMnJqTanTEQuUgpyImJKI0aM4F//+hf5+fk1XrdVq1b06dMHZ2dnevbsSXZ2NkOHDsVisdC1a1csFgv79u2rdP1JkyYREhLCLbfcQkFBAePGjQNg9erVtGzZkj59+uDi4kKHDh3o3r07GRkZ9nVvv/12wsLCsFgsPP/882zdupXs7Owq6/3ss8/o06cPXbp0oUmTJvj5+dGuXTv79tq0aYOTkxPh4eF06dKFTZs2AeDi4sLhw4c5dOgQrq6uhIaGKsiJNDIKciJiSu3bt+f222/nvffeq/G6Pj4+9v8+86PlV1xxhX2em5tblT1yY8eOZfPmzSxbtoxjx47x+++/A3Dw4EG2bdtGaGio/V9aWhqHDx+2r/uXv/zF/t/u7u60aNGC3NzcKuvNzs629/r92Zo1a3jooYcIDw8nNDSUb7/9loKCAuD0EPTVV1/NY489RnR0dK3OlYhc3HTXqoiY1ogRI3jggQd47LHH7PPO3Ml66tQpPDw8ACoEqboUEBDAkCFDSEpKYsmSJfj7+xMWFsbcuXMrXedM6AM4ceIER48ePe8dr/7+/ufsIbRarYwYMYKUlBSio6NxdXXl2WefxTAMADw8PEhISCAhIYFdu3bxyCOP8H/+z/+hc+fOtTxiEbnYqEdOREzr6quvpmfPnhUeO+Lt7Y2fnx+pqanYbDY+++wz9u/fX2813H///eTl5fH1119z++23s2fPHpYuXUppaSmlpaVs27aNrKws+/Jr1qxh06ZNWK1W3nrrLW666Sb8/f2r3Effvn1ZvHgxP/74I+Xl5eTk5JCVlYXVasVqteLt7Y2Liwtr1qxh7dq19vW++eYb9u7di2EYeHh44OzsTJMm+rMv0pjoEy0ipjZ06FCKi4srzJs4cSJz5swhIiKC3377jeDg4Hrbv8ViYeDAgcyaNQsPDw/mzJnDihUriIyMpGvXrkyZMgWr1WpfPiYmhpkzZxIREcEvv/zC5MmTz7uPjh078tprr/Hqq68SEhLCgAEDOHToEB4eHowdO5bnnnuOsLAw0tPTiYqKsq+3d+9e+x2y/fr14+GHHyYiIqJezoOIOIaTcaYPXkRE6lVCQgJ+fn48//zzji5FRBoJ9ciJiIiImJSCnIiIiIhJaWhVRERExKTUIyciIiJiUgpyIiIiIialICciIiJiUgpyIiIiIialICciIiJiUgpyIiIiIib1fwEBTjqtgdR2AwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "\n", + "x = 'Factor'\n", + "\n", + "df = pd.DataFrame({\n", + " x: replicas, \n", + " **results,\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars=x).rename(columns=str.title)\n", + "seaborn.barplot(x=x, y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "seaborn.despine(fig)\n", + "\n", + "ax1.set(xlabel=\"Num Replicas\", ylabel=f'MASE Loss', title='Residual Estimate Loss for Time-Series Decomposition')" + ] + }, + { + "cell_type": "code", + "execution_count": 277, + "id": "9a4dba29", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "min() arg is an empty sequence", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubplots\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mtidy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmelt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid_vars\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mseaborn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbarplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Value'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Variable'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtidy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0max1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mseaborn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdespine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/seaborn/_decorators.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 44\u001b[0m )\n\u001b[1;32m 45\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 47\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/seaborn/categorical.py\u001b[0m in \u001b[0;36mbarplot\u001b[0;34m(x, y, hue, data, order, hue_order, estimator, ci, n_boot, units, seed, orient, color, palette, saturation, errcolor, errwidth, capsize, dodge, ax, **kwargs)\u001b[0m\n\u001b[1;32m 3177\u001b[0m ):\n\u001b[1;32m 3178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3179\u001b[0;31m plotter = _BarPlotter(x, y, hue, data, order, hue_order,\n\u001b[0m\u001b[1;32m 3180\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mci\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_boot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3181\u001b[0m \u001b[0morient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpalette\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaturation\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/seaborn/categorical.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, x, y, hue, data, order, hue_order, estimator, ci, n_boot, units, seed, orient, color, palette, saturation, errcolor, errwidth, capsize, dodge)\u001b[0m\n\u001b[1;32m 1584\u001b[0m self.establish_variables(x, y, hue, data, orient,\n\u001b[1;32m 1585\u001b[0m order, hue_order, units)\n\u001b[0;32m-> 1586\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestablish_colors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpalette\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaturation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1587\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimate_statistic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mci\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_boot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/seaborn/categorical.py\u001b[0m in \u001b[0;36mestablish_colors\u001b[0;34m(self, color, palette, saturation)\u001b[0m\n\u001b[1;32m 317\u001b[0m \u001b[0;31m# Determine the gray color to use for the lines framing the plot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 318\u001b[0m \u001b[0mlight_vals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcolorsys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrgb_to_hls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrgb_colors\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m \u001b[0mlum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlight_vals\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m.6\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 320\u001b[0m \u001b[0mgray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrgb2hex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlum\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 321\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: min() arg is an empty sequence" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl8AAAE1CAYAAADZOIW8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAU7ElEQVR4nO3dUWyV9f3H8Q8tgkZZHESwjBkiU2xkeoHJdiGLQ7RsFnGbSlI1c8SazMUlLjHqpkCnydYlu1DGskgy1NUL1yzDUAkS4wVjUdwaE2CdmCgGnRWkhLiB0no4/4v9JTLUHmr51YOvV2LSNr/Wb/MN8e1zHs4zrlqtVgMAQBENYz0AAMDnifgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgoaNr87OzsyfPz+zZ8/Oyy+//JFnKpVKOjo6smDBglxxxRXp7u4e9UEBAE4Gw8bX5Zdfnscffzxf+tKXPvbMunXrsmvXrmzcuDFPPPFEVq5cmTfeeGNUBwUAOBkMG1+XXHJJmpqaPvHM+vXrc91116WhoSGTJ0/OggULsmHDhlEbEgDgZDF+NH5If39/pk+ffuTzpqamvPXWWzV//+HDh3PgwIGccsopGTdu3GiMBABwQlSr1QwNDeX0009PQ8Px3z4/KvH1aR04cOBj7ycDAPgsOv/88zNp0qTj/r5Ria+mpqa8+eabueiii5IceyVsOKecckqS//4SEyZMGI2RKGz79u2ZM2fOWI/BCNhdfbO/+mV39WtwcDAvv/zykX45XqMSXwsXLkx3d3euvPLK7N+/P88880wef/zxmr//g5caJ0yYkIkTJ47GSIwBu6tfdlff7K9+2V19G+mtUsO+UPnAAw/kG9/4Rt5666384Ac/yFVXXZUkaW9vz7Zt25IkixcvzowZM3LllVfm+uuvz49+9KN8+ctfHtFAAAAns2GvfN1777259957j/n66tWrj3zc2NiYjo6O0Z0MAOAk5B3uAQAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQ0PhaDu3cuTN333139u/fnzPPPDOdnZ2ZOXPmUWcGBgZyzz33pL+/P0NDQ/n617+ee++9N+PH1/SvAAD4XKjpytfy5cvT1taWp59+Om1tbVm2bNkxZ373u99l1qxZWbduXdatW5d//OMf2bhx46gPDABQz4aNr4GBgfT19aW1tTVJ0tramr6+vuzbt++oc+PGjcuBAwdy+PDhDA4OZmhoKNOmTTsxUwMA1KlhXxPs7+/PtGnT0tjYmCRpbGzM1KlT09/fn8mTJx85d9ttt+X222/PpZdemnfffTc33HBD5s6de1zDbN++/TjH57Okt7d3rEdghOyuvtlf/bK7z6dRuyFrw4YNmT17dh599NEcOHAg7e3t2bBhQxYuXFjzz5gzZ04mTpw4WiNRUG9v73HHNp8Ndlff7K9+2V39OnTo0Ke6YDTsy45NTU3ZvXt3KpVKkqRSqWTPnj1pamo66lxXV1euvvrqNDQ0ZNKkSZk/f362bNky4sEAAE5Gw8bXlClT0tzcnJ6eniRJT09Pmpubj3rJMUlmzJiRTZs2JUkGBwfz3HPP5bzzzjsBIwMA1K+a/rbjihUr0tXVlZaWlnR1daWjoyNJ0t7enm3btiVJfvrTn6a3tzeLFi3KNddck5kzZ+b6668/cZMDANShmu75mjVrVrq7u4/5+urVq498fM4552TNmjWjNxkAwEnIO9wDABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQTXF186dO7NkyZK0tLRkyZIlee211z7y3Pr167No0aK0trZm0aJF2bt372jOCgBQ98bXcmj58uVpa2vL4sWL8+STT2bZsmV57LHHjjqzbdu2/OY3v8mjjz6as846K//+978zYcKEEzI0AEC9GvbK18DAQPr6+tLa2pokaW1tTV9fX/bt23fUuUceeSRLly7NWWedlSSZNGlSJk6ceAJGBgCoX8Ne+erv78+0adPS2NiYJGlsbMzUqVPT39+fyZMnHzn3yiuvZMaMGbnhhhty8ODBXHHFFfnhD3+YcePG1TzM9u3bR/Ar8FnR29s71iMwQnZX3+yvftnd51NNLzvWolKpZMeOHVmzZk0GBwdzyy23ZPr06bnmmmtq/hlz5sxxtaxO9fb2Zu7cuWM9BiNgd/XN/uqX3dWvQ4cOfaoLRsO+7NjU1JTdu3enUqkk+W9k7dmzJ01NTUedmz59ehYuXJgJEybkjDPOyOWXX56tW7eOeDAAgJPRsPE1ZcqUNDc3p6enJ0nS09OT5ubmo15yTP57L9jmzZtTrVYzNDSU559/PhdccMGJmRoAoE7V9FYTK1asSFdXV1paWtLV1ZWOjo4kSXt7e7Zt25YkueqqqzJlypR8+9vfzjXXXJOvfOUrufbaa0/c5AAAdaime75mzZqV7u7uY76+evXqIx83NDTknnvuyT333DN60wEAnGS8wz0AQEHiCwCgIPEFAFCQ+AIAKEh8AQAUJL4AAAoSXwAABYkvAICCxBcAQEHiCwCgIPEFAFCQ+AIAKEh8AQAUJL4AAAoSXwAABYkvAICCxBcAQEHiCwCgIPEFAFCQ+AIAKEh8AQAUJL4AAAoSXwAABYkvAICCxBcAQEHiCwCgIPEFAFCQ+AIAKEh8AQAUJL4AAAoSXwAABYkvAICCxBcAQEHiCwCgIPEFAFCQ+AIAKEh8AQAUJL4AAAoSXwAABYkvAICCxBcAQEHiCwCgIPEFAFCQ+AIAKEh8AQAUVFN87dy5M0uWLElLS0uWLFmS11577WPPvvrqq7n44ovT2dk5WjMCAJw0aoqv5cuXp62tLU8//XTa2tqybNmyjzxXqVSyfPnyLFiwYFSHBAA4WQwbXwMDA+nr60tra2uSpLW1NX19fdm3b98xZx9++OFcdtllmTlz5qgPCgBwMhg2vvr7+zNt2rQ0NjYmSRobGzN16tT09/cfde6ll17K5s2bc/PNN5+QQQEATgbjR+OHDA0N5b777ssvfvGLI5E2Etu3bx+NcRgjvb29Yz0CI2R39c3+6pfdfT4NG19NTU3ZvXt3KpVKGhsbU6lUsmfPnjQ1NR058/bbb2fXrl259dZbkyTvvPNOqtVq/vOf/+T++++veZg5c+Zk4sSJI/g1GGu9vb2ZO3fuWI/BCNhdfbO/+mV39evQoUOf6oLRsPE1ZcqUNDc3p6enJ4sXL05PT0+am5szefLkI2emT5+eLVu2HPl85cqVOXjwYO66664RDwYAcDKq6W87rlixIl1dXWlpaUlXV1c6OjqSJO3t7dm2bdsJHRAA4GRS0z1fs2bNSnd39zFfX7169Ueev/322z/dVAAAJynvcA8AUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILEFwBAQeILAKAg8QUAUJD4AgAoSHwBABQkvgAAChJfAAAFiS8AgILG13Jo586dufvuu7N///6ceeaZ6ezszMyZM486s2rVqqxfvz6NjY0ZP3587rjjjsybN+9EzAwAULdqiq/ly5enra0tixcvzpNPPplly5blscceO+rMRRddlKVLl+a0007LSy+9lBtvvDGbN2/OqaeeekIGBwCoR8O+7DgwMJC+vr60trYmSVpbW9PX15d9+/YddW7evHk57bTTkiSzZ89OtVrN/v37R39iAIA6NuyVr/7+/kybNi2NjY1JksbGxkydOjX9/f2ZPHnyR37P2rVrc8455+Tss88+rmG2b99+XOf5bOnt7R3rERghu6tv9le/7O7zqaaXHY/HCy+8kAcffDC///3vj/t758yZk4kTJ472SBTQ29ubuXPnjvUYjIDd1Tf7q192V78OHTr0qS4YDfuyY1NTU3bv3p1KpZIkqVQq2bNnT5qamo45++KLL+bOO+/MqlWrcu655454KACAk9Ww8TVlypQ0Nzenp6cnSdLT05Pm5uZjXnLcunVr7rjjjjz00EO58MILT8y0AAB1rqb3+VqxYkW6urrS0tKSrq6udHR0JEna29uzbdu2JElHR0fee++9LFu2LIsXL87ixYuzY8eOEzc5AEAdqumer1mzZqW7u/uYr69evfrIx3/6059GbyoAgJOUd7gHAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgsQXAEBB4gsAoCDxBQBQkPgCAChIfAEAFCS+AAAKEl8AAAWJLwCAgmqKr507d2bJkiVpaWnJkiVL8tprrx1zplKppKOjIwsWLMgVV1yR7u7u0Z4VAKDu1RRfy5cvT1tbW55++um0tbVl2bJlx5xZt25ddu3alY0bN+aJJ57IypUr88Ybb4z6wAAA9Wz8cAcGBgbS19eXNWvWJElaW1tz//33Z9++fZk8efKRc+vXr891112XhoaGTJ48OQsWLMiGDRtyyy23DDtEtVpNkgwODo709+Az4NChQ2M9AiNkd/XN/uqX3dWnD3rlg345XsPGV39/f6ZNm5bGxsYkSWNjY6ZOnZr+/v6j4qu/vz/Tp08/8nlTU1PeeuutmoYYGhpKkrz88svHNTyfLdu3bx/rERghu6tv9le/7K6+DQ0N5dRTTz3u7xs2vko4/fTTc/755+eUU07JuHHjxnocAICPVa1WMzQ0lNNPP31E3z9sfDU1NWX37t2pVCppbGxMpVLJnj170tTUdMy5N998MxdddFGSY6+EfZKGhoZMmjRpBOMDAJQ3kiteHxj2hvspU6akubk5PT09SZKenp40Nzcf9ZJjkixcuDDd3d05fPhw9u3bl2eeeSYtLS0jHgwA4GQ0rlrD3WKvvPJK7r777rzzzjv5whe+kM7Ozpx77rlpb2/Pj3/843z1q19NpVLJz3/+8/z1r39NkrS3t2fJkiUn/BcAAKgnNcUXAACjwzvcAwAUJL4AAAoSXwAABYkvAICCisaXB3TXt1r2t2rVqlx11VW5+uqr893vfjd/+ctfyg/KMWrZ3QdeffXVXHzxxens7Cw3IJ+o1v2tX78+ixYtSmtraxYtWpS9e/eWHZSPVMv+BgYGcuutt2bRokVZuHBhVqxYkffff7/8sByls7Mz8+fPz+zZsz/2KTwj6pZqQTfddFN17dq11Wq1Wl27dm31pptuOubMn//85+rSpUurlUqlOjAwUJ03b1719ddfLzkmH6OW/W3atKl68ODBarVarf7zn/+szp07t/ruu+8WnZNj1bK7arVaff/996s33nhj9Sc/+Un1l7/8ZckR+QS17G/r1q3Vb33rW9U9e/ZUq9Vq9Z133qm+9957Refko9WyvwceeODIn7nBwcHqtddeW33qqaeKzsmx/va3v1XffPPN6je/+c3qjh07PvLMSLql2JWvDx7Q3dramuS/D+ju6+vLvn37jjr3cQ/oZmzVur958+bltNNOS5LMnj071Wo1+/fvLz0uH1Lr7pLk4YcfzmWXXZaZM2cWnpKPU+v+HnnkkSxdujRnnXVWkmTSpEmZOHFi8Xk5Wq37GzduXA4cOJDDhw9ncHAwQ0NDmTZt2liMzIdccsklxzzR53+NpFuKxdcnPaD7f8+N9AHdnDi17u/D1q5dm3POOSdnn312qTH5CLXu7qWXXsrmzZtz8803j8GUfJxa9/fKK6/k9ddfzw033JDvfOc7+e1vf5uqt3Ecc7Xu77bbbsvOnTtz6aWXHvln7ty5YzEyx2kk3eKGe06IF154IQ8++GB+/etfj/Uo1GBoaCj33XdfOjo6jvxHgvpSqVSyY8eOrFmzJn/4wx+yadOmPPnkk2M9FjXasGFDZs+enc2bN2fTpk35+9//7lWfk1ix+PrwA7qTDPuA7g/09/e7cvIZUOv+kuTFF1/MnXfemVWrVuXcc88tPSr/o5bdvf3229m1a1duvfXWzJ8/P48++mj++Mc/5r777hursfl/tf7Zmz59ehYuXJgJEybkjDPOyOWXX56tW7eOxch8SK376+rqytVXX52GhoZMmjQp8+fPz5YtW8ZiZI7TSLqlWHx5QHd9q3V/W7duzR133JGHHnooF1544ViMyv+oZXfTp0/Pli1b8uyzz+bZZ5/N97///Vx//fW5//77x2ps/l+tf/ZaW1uzefPmVKvVDA0N5fnnn88FF1wwFiPzIbXub8aMGdm0aVOSZHBwMM8991zOO++84vNy/EbSLUWf7egB3fWtlv1973vfy7/+9a+jbhT91a9+ldmzZ4/h5NSyuw9buXJlDh48mLvuumuMJubDatnf4cOH09nZmU2bNqWhoSGXXnpp7rrrrjQ0uLtkrNWyv127dmX58uXZu3dvKpVKvva1r+VnP/tZxo8fP9bjf6498MAD2bhxY/bu3ZsvfvGLOfPMM/PUU0996m7xYG0AgIL8LxEAQEHiCwCgIPEFAFCQ+AIAKEh8AQAUJL4AAAoSXwAABYkvAICC/g95qFmz3s7qbQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "\n", + "x = 'Factor'\n", + "\n", + "df = pd.DataFrame({\n", + " x: graph_results[\"cost\"], \n", + " 'baseline': graph_results[\"baseline\"], \n", + " \"optimized\": graph_results[\"optimized\"],\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars=x).rename(columns=str.title)\n", + "seaborn.barplot(x=x, y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "seaborn.despine(fig)\n", + "\n", + "ax1.set(xlabel=\"Cost Budget\", ylabel=f'MASE Loss', title='Residual Estimate Loss for Time-Series Decomposition')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9bccc31", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/stl/offline/config_gen.py b/stl/offline/config_gen.py index 6e52b11..91f9bf4 100644 --- a/stl/offline/config_gen.py +++ b/stl/offline/config_gen.py @@ -6,7 +6,6 @@ import numpy as np import pandas as pd from absl import app, flags -from ortools.linear_solver import pywraplp from sktime.performance_metrics.forecasting import mean_squared_scaled_error FLAGS = flags.FLAGS @@ -23,8 +22,30 @@ required=True, ) +# TODO(simon): add flags for lp solver constraint +flags.DEFINE_integer( + "max_n_fits", + default=None, + help="Max fits for LP", + required=False, +) + +flags.DEFINE_integer( + "max_loss", + default=None, + help="Max loss for LP", + required=False, +) + +flags.DEFINE_string( + "objective", + default="min_loss", + help="LP optimization goal", + required=False, +) -def run_lp(df: pd.DataFrame, max_n_fits=None, max_loss=None, objective="min_loss"): +def run_lp(df: pd.DataFrame, objective="min_loss"): + from ortools.linear_solver import pywraplp """Run through mixed integer program to generate the best plan. Input: @@ -35,6 +56,8 @@ def run_lp(df: pd.DataFrame, max_n_fits=None, max_loss=None, objective="min_loss Output: plan(Dict[str, int]): a dictionary mapping key -> optimal n_fits such that loss is minimal. """ + max_n_fits = FLAGS.max_n_fits + max_loss = FLAGS.max_loss assert all(df.columns == ["key", "n_fits", "loss"]) assert objective in {"min_loss", "min_fits"} @@ -96,17 +119,17 @@ def run_lp(df: pd.DataFrame, max_n_fits=None, max_loss=None, objective="min_loss def get_loss_per_key(key: int, csv_dir): - key_one = glob(f"{csv_dir}/slide_*_key_A4Benchmark-TS{key}.csv") + key_one = glob(f"{csv_dir}/fifo_slide_*_key_{key}.csv") assert len(key_one) > 0 - oracle_residual = pd.read_csv(f"{csv_dir}/oracle_key_A4Benchmark-TS{key}.csv")[ + oracle_residual = pd.read_csv(f"./oracle/{key}.csv")[ "pred_residual" ] losses = [] for path in key_one: slide_size = int( - os.path.basename(path).split("_key_A4")[0].replace("slide_", "") + os.path.basename(path).split("_key_")[0].replace("fifo_slide_", "") ) df = pd.read_csv(path) residual = df["pred_residual"] diff --git a/stl/offline/default_plans.py b/stl/offline/default_plans.py new file mode 100644 index 0000000..8bcd66f --- /dev/null +++ b/stl/offline/default_plans.py @@ -0,0 +1,8 @@ +import json + +plan_dir = "/data/wooders/stl/results" +slides = [1, 6, 12, 18, 24, 48, 96, 168, 192, 336, 672] + +for slide in slides: + weights = {i: slide for i in range(1, 101, 1)} + open(f"{plan_dir}/plan_baseline_{slide}.json", "w").write(json.dumps(weights)) diff --git a/stl/offline/evaluate_loss.py b/stl/offline/evaluate_loss.py new file mode 100644 index 0000000..29d0242 --- /dev/null +++ b/stl/offline/evaluate_loss.py @@ -0,0 +1,48 @@ +from sktime.performance_metrics.forecasting import mean_squared_scaled_error +import numpy as np +import pandas as pd +from tqdm import tqdm +import argparse + +def get_loss_per_key(key: int, csv_dir, oracle_dir): + path = f"{csv_dir}/{key}.csv" + + oracle_residual = pd.read_csv(f"{oracle_dir}/oracle_key_A4Benchmark-TS{key}.csv")[ + "pred_residual" + ] + + df = pd.read_csv(path) + print(path) + residual = df["pred_residual"] + print("residual", len(residual.tolist())) + mask = ~np.isnan(residual) + print("residual", len(residual[mask].tolist())) + loss = mean_squared_scaled_error( + y_true=oracle_residual[mask], y_pred=residual[mask], y_train=df["value"] + ) + loss = { + "loss": loss, + "n_fits": df["model_version"].dropna().nunique(), + } + return loss + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Specify experiment config") + parser.add_argument("--csv-path", type=str) + parser.add_argument("--oracle-path", type=str) + args = parser.parse_args() + + raw_data = [] + for key in tqdm(range(1, 101)): + entry = get_loss_per_key(key, csv_dir=args.csv_path, oracle_dir=args.oracle_path) + raw_data.append({"key": key, **entry}) + + df = pd.DataFrame(raw_data) + print("loss per n_fits") + print(df.groupby("n_fits")["loss"].describe()) + print(f"loss per key (sample of 10 out of {len(df)})") + print(df.groupby("key")["loss"].describe().sample(10)) + df.to_csv("final_results.csv") + diff --git a/stl/offline/evaluation.py b/stl/offline/evaluation.py index 92ff30b..f3a47ac 100644 --- a/stl/offline/evaluation.py +++ b/stl/offline/evaluation.py @@ -1,16 +1,22 @@ import argparse +import time +from multiprocessing import Pool +import json import os import bisect - +from tqdm import tqdm import numpy as np import pandas as pd +import time from statsmodels.tsa.seasonal import STL def train(data, window_size, seasonality): window = data[-window_size:] values = [r["value"] for r in window] + st = time.time() stl_result = STL(values, period=seasonality, robust=True).fit() + print(time.time() - st) timestamp = data[-1]["timestamp"] return { "timestamp": timestamp, @@ -37,27 +43,43 @@ def predict(event, model): SEASONALITY = 24 * 7 -def offline_eval(yahoo_csv_path, plan_json_path): - df = pd.read_csv(yahoo_csv_path) - df["timestamp"] = list(range(len(df))) +def offline_eval(yahoo_csv_path, plan_json_path, key, output_path): - # Headers - # processing_time window_start_seq_id window_end_seq_id key + print(output_path) + + # get plan DF for key plan_df = pd.read_json(plan_json_path) + if key is not None: + plan_df_key = plan_df[plan_df["key"] == int(key)] + else: + plan_df_key = plan_df + plan_df_key.index = pd.RangeIndex(start=0, stop=len(plan_df_key.index)) + + # get original data + df = pd.read_csv(yahoo_csv_path) + df["timestamp"] = list(range(len(df))) # Given our model versions from offline plan, run training on corresponding # events. offline_stl = {} - for _, row in plan_df.iterrows(): - records = df.iloc[row.window_start_seq_id : row.window_end_seq_id + 1].to_dict( + print(plan_df_key) + for _, row in tqdm(plan_df_key.iterrows()): # note: doesn't preserve types + st = time.time() + records = df.iloc[int(row.window_start_seq_id) : int(row.window_end_seq_id) + 1].to_dict( orient="records" ) + #print("find time", time.time() - st) # The yahoo dataset seasonaly can be 12hr, daily, and weekly. # Each record is an hourly record. Here we chose weekly seasonality. + st = time.time() trained = train(records, window_size=len(records), seasonality=SEASONALITY) + #print("fit time", time.time() - st) offline_stl[row.processing_time] = trained + print(offline_stl.keys()) + + # Assign the trained model with every events in the source file. def find_freshest_model_version(event_time, model_versions): model_loc = bisect.bisect_left(model_versions, event_time) - 1 @@ -66,12 +88,13 @@ def find_freshest_model_version(event_time, model_versions): return model_versions[model_loc] df["model_version"] = [ - find_freshest_model_version(et, plan_df["processing_time"]) + find_freshest_model_version(et, plan_df_key["processing_time"]) for et in df["timestamp"] ] # Run prediction! predicted = [] + print("running prediction") for _, row in df.iterrows(): model_version = row["model_version"] if np.isnan(model_version): @@ -96,10 +119,27 @@ def find_freshest_model_version(event_time, model_versions): add_df = pd.DataFrame(predicted) for new_col in add_df.columns: df[new_col] = add_df[new_col] - return df + print("writing", output_path) + df.to_csv(output_path, index=None) + +def offline_eval_all(yahoo_path, plan_json_path, output_path, param_path): + + policy_params = json.load(open(param_path)) + + # loop through each key + inputs = [] + for key in policy_params.keys(): + key_output_path = f"{output_path}/{key}.csv" + inputs.append((f"{yahoo_path}/{key}.csv", plan_json_path, key, key_output_path)) + p = Pool(100) + p.starmap(offline_eval, inputs) + p.close() + return -def offline_oracle(yahoo_csv_path): + + +def offline_oracle(yahoo_csv_path, output_path): df = pd.read_csv(yahoo_csv_path) df["timestamp"] = list(range(len(df))) df["model_version"] = "oracle" @@ -111,15 +151,20 @@ def offline_oracle(yahoo_csv_path): df["pred_seasonality"] = oracle_model["stl_result"].seasonal df["pred_staleness"] = 0 - return df + df.to_csv(output_path) -def run_exp(csv_path, plan_path, output_path, run_oracle=False): +def run_exp(csv_path, plan_path, output_path, run_policy=False, run_oracle=False, param_path=None): if run_oracle: - df = offline_oracle(csv_path) + df = offline_oracle(csv_path, output_path) + elif run_policy: + offline_eval_all(csv_path, plan_path, output_path, param_path) else: - df = offline_eval(csv_path, plan_path) - df.to_csv(output_path, index=None) + + # Headers + # processing_time window_start_seq_id window_end_seq_id key + #plan_df = pd.read_json(plan_path) + offline_eval(csv_path, plan_path, None, output_path) def _ensure_dir(path): @@ -132,19 +177,23 @@ def main(): parser.add_argument("--offline-yahoo-csv-path", type=str) parser.add_argument("--offline-plan-path", type=str) parser.add_argument("--output-path", type=str) - parser.add_argument("--offline-run-oracle", type=bool, default=False) + parser.add_argument("--offline-run-oracle", default=False, action='store_true') + parser.add_argument("--run-policy", default=False, action='store_true') + parser.add_argument("--param-path", type=str, default=None) args = parser.parse_args() assert args.offline_yahoo_csv_path if not args.offline_run_oracle: assert args.offline_plan_path - _ensure_dir(args.output_path) + #_ensure_dir(args.output_path) run_exp( csv_path=args.offline_yahoo_csv_path, plan_path=args.offline_plan_path, output_path=args.output_path, run_oracle=args.offline_run_oracle, + run_policy=args.run_policy, + param_path=args.param_path, ) diff --git a/stl/offline/extend_data.py b/stl/offline/extend_data.py new file mode 100644 index 0000000..98ce1bc --- /dev/null +++ b/stl/offline/extend_data.py @@ -0,0 +1,57 @@ +import numpy as np +import pandas as pd +import random +import statistics +import glob +import os + +max_length = 1680 # double length +noise = 2 +max_seasonality = 24*7 +# over_sampling_rate = 1 +path = "yahoo_train_data/" +output_path = "yahoo_eval_data/" +input_path = "yahoo_train_data/*" +files = glob.glob(input_path) +print(files) +for filename in files: + df = pd.read_csv(filename) + + max_outlier_value, min_outlier_value = max(df['noise']), min(df['noise']) + mean, stddev = statistics.mean(df['noise']), statistics.stdev(df['noise']) + + initial_trend = df['trend'][0] + last_trend = df['trend'].iloc[-1] + trend_subtracted_series = df['trend'] - initial_trend + # trend_subtracted_series = np.repeat(trend_subtracted_series, over_sampling_rate) + + seasonality = df['seasonality1'] + df['seasonality2'] + df['seasonality3'] + # seasonality = np.repeat(seasonality, over_sampling_rate) + + repeat_length = (len(trend_subtracted_series) // max_seasonality) * max_seasonality + + count = 0 + generated_trend = [last_trend] * max_length + generated_noise = [0] * max_length + generated_outlier = [0] * max_length + generated_seasonality = [0] * max_length + + for i in range(max_length): + if count >= repeat_length: + count = 0 + last_trend = generated_trend[i-1] + generated_trend[i] = last_trend + trend_subtracted_series[count] + generated_seasonality[i] = seasonality[count] + generated_noise[i] = random.gauss(mean, stddev) + generated_outlier[i] = 0 + if random.randint(0, 100) > 100 - noise: + if random.randint(0, 100) > 50: + generated_outlier[i] = max_outlier_value * random.randint(70,100) // 100 + else: + generated_outlier[i] = min_outlier_value * random.randint(70,100) // 100 + count += 1 + + new_df = pd.DataFrame({"trend": generated_trend, "noise": generated_noise, "outlier": generated_outlier, "seasonality": generated_seasonality }) + new_df['value'] = new_df['trend'] + new_df['noise'] + new_df['outlier'] + new_df['seasonality'] + print(os.path.basename(filename)) + new_df.to_csv(os.path.join(output_path, os.path.basename(filename))) diff --git a/stl/offline/log_data.py b/stl/offline/log_data.py new file mode 100644 index 0000000..b8eb566 --- /dev/null +++ b/stl/offline/log_data.py @@ -0,0 +1,45 @@ +import wandb +import configparser +import os + + +def log_experiment(run, config): + # log experiment output + artifact = wandb.Artifact("results", type='dataset') + artifact.add_dir("/data/wooders/stl/results") + run.log_artifact(artifact) + +def log_train(run, config): + # log experiment output + artifact = wandb.Artifact("yahoo_train_data", type='dataset') + artifact.add_dir("yahoo_train_data") + run.log_artifact(artifact) + +def log_eval(run, config): + # log experiment output + artifact = wandb.Artifact("yahoo_eval_data", type='dataset') + artifact.add_dir("yahoo_eval_data") + run.log_artifact(artifact) + +def log_oracle(run, config): + # log experiment output + artifact = wandb.Artifact("oracle", type='dataset') + artifact.add_dir("oracle") + run.log_artifact(artifact) + + + +if __name__ == "__main__": + + print("Running wandb logging on data") + run = wandb.init(job_type="dataset-creation", project="stl") + + # configuration file + config = configparser.ConfigParser() + config.read("config.yml") + + log_experiment(run, config) + log_train(run, config) + log_eval(run, config) + log_oracle(run, config) + diff --git a/stl/offline/run_1_simulate_windows.sh b/stl/offline/run_1_simulate_windows.sh index 70dc721..a25823e 100644 --- a/stl/offline/run_1_simulate_windows.sh +++ b/stl/offline/run_1_simulate_windows.sh @@ -1,7 +1,28 @@ -set -xe +set -ex -for slide in 1 6 12 18 24 48 96 168 192 336 672 +data_dir="./yahoo_train_data" +result_dir="/data/wooders/stl/results" +tmp_script=`mktemp` + +for key_prio in "lifo" "fifo" +do +for data in `ls $data_dir/*` do - python simulation.py --model_runtime_s 0 --total_runtime_s 2000 --per_key_records_per_second 1 \ - --window_size 672 --slide_size ${slide} --output_path result/offline_1_slide/plan/slide_${slide}_plan.json + key=`basename $data` + for slide in 6 12 18 24 48 96 168 192 336 672 + do + echo \" python simulation.py --num_keys 100 --model_runtime_s 1.5 --total_runtime_s 2000 --per_key_records_per_second 1 --key_prio_policy ${key_prio} --window_size 672 --slide_size ${slide} --output_path ${result_dir}/plan/${key_prio}_slide_${slide}_plan.json --num_mapper_replicas 1\" >> $tmp_script + done +done done + +cat $tmp_script | xargs -n 1 -P 36 bash -l -c + +#set -xe +# +#for replicas in +#for slide in 1 6 12 18 24 48 96 168 192 336 672 +#do +# python simulation.py --model_runtime_s 0 --total_runtime_s 2000 --per_key_records_per_second 1 \ +# --window_size 672 --slide_size ${slide} --output_path result/offline_1_slide/plan/slide_${slide}_plan.json +#done diff --git a/stl/offline/run_2_eval_yahoo_keys.sh b/stl/offline/run_2_eval_yahoo_keys.sh index 1d9865e..a3cb4bc 100644 --- a/stl/offline/run_2_eval_yahoo_keys.sh +++ b/stl/offline/run_2_eval_yahoo_keys.sh @@ -1,17 +1,37 @@ set -ex -data_dir="/home/ubuntu/ydata-labeled-time-series-anomalies-v1_0/A4Benchmark/" +data_dir="./yahoo_train_data" +results_dir="/data/wooders/stl/results" tmp_script=`mktemp` -for data in `ls $data_dir/A4Benchmark-TS*` +for key_prio in "lifo" "fifo" +do +for data in `ls $data_dir/*` do key=`basename $data` for slide in 6 12 18 24 48 96 168 192 336 672 do - echo python evaluation.py --offline-yahoo-csv-path $data \ - --offline-plan-path ./result/offline_1_slide/plan/slide_${slide}_plan.json \ - --output-path ./result/offline_1_slide/plan_eval/slide_${slide}_key_${key} >> $tmp_script + echo \" python evaluation.py --offline-yahoo-csv-path $data \ + --offline-plan-path ${results_dir}/plan/${key_prio}_slide_${slide}_plan.json \ + --output-path ${results_dir}/single_key/${key_prio}_slide_${slide}_key_${key} \" >> $tmp_script done done +done + +cat $tmp_script | xargs -n 1 -P 144 bash -l -c + -cat $tmp_script | parallel --bar bash -l -c \ No newline at end of file +#set -ex +# +#data_dir="/data/wooders/stl/yahoo" +# +#for data in `ls $data_dir/A4/*` +#do +# key=`basename $data` +# for slide in 6 12 18 24 48 96 168 192 336 672 +# do +# python evaluation.py --offline-yahoo-csv-path $data \ +# --offline-plan-path ./result/offline_1_slide/plan/slide_${slide}_plan.json \ +# --output-path ./result/offline_1_slide/plan_eval/slide_${slide}_key_${key} +# done +#done diff --git a/stl/offline/run_3_eval_oracle.sh b/stl/offline/run_3_eval_oracle.sh index 9262e2a..5fdbef5 100644 --- a/stl/offline/run_3_eval_oracle.sh +++ b/stl/offline/run_3_eval_oracle.sh @@ -1,14 +1,18 @@ set -ex -data_dir="/home/ubuntu/ydata-labeled-time-series-anomalies-v1_0/A4Benchmark/" +#data_dir="/home/ubuntu/ydata-labeled-time-series-anomalies-v1_0/A4Benchmark/" +data_dir="./yahoo_eval_data" +output_path="./oracle" tmp_script=`mktemp` -for data in `ls $data_dir/A4Benchmark-TS*` +#for data in `ls $data_dir/A4Benchmark-TS*` +for data in `ls $data_dir/*` do key=`basename $data` - echo python evaluation.py --offline-yahoo-csv-path $data \ - --offline-run-oracle true \ - --output-path ./result/offline_1_slide/plan_eval/oracle_key_${key} >> $tmp_script + echo \" python evaluation.py --offline-yahoo-csv-path $data \ + --offline-run-oracle \ + --output-path ${output_path}/${key} \" >> $tmp_script done -cat $tmp_script | parallel --bar bash -l -c \ No newline at end of file +cat $tmp_script | xargs -n 1 -P 36 bash -l -c +#cat $tmp_script | parallel --bar bash -l -c diff --git a/stl/offline/run_4_generate_plan.sh b/stl/offline/run_4_generate_plan.sh index 0d06e8d..ef514b5 100644 --- a/stl/offline/run_4_generate_plan.sh +++ b/stl/offline/run_4_generate_plan.sh @@ -3,6 +3,13 @@ set -ex # TODO(simon): use a workflow engine for step tracking # e.g. https://dagster.io/ +#python config_gen.py \ +# --csv_dir "./result/offline_1_slide/plan_eval" \ +# --output_path "./result/offline_1_slide/min_loss_plan.json" + +MAX_FITS=500 python config_gen.py \ - --csv_dir "./result/offline_1_slide/plan_eval" \ - --output_path "./result/offline_1_slide/min_loss_plan.json" \ No newline at end of file + --csv_dir "/data/wooders/stl/results/single_key" \ + --output_path "/data/wooders/stl/results/max_fits_${MAX_FITS}.json" \ + --max_n_fits ${MAX_FITS} + diff --git a/stl/offline/run_5_simulate_lp_plan.sh b/stl/offline/run_5_simulate_lp_plan.sh index 7e353f0..bf9daeb 100644 --- a/stl/offline/run_5_simulate_lp_plan.sh +++ b/stl/offline/run_5_simulate_lp_plan.sh @@ -1,8 +1,36 @@ set -ex +PARAM_DIR="offline_1_slide" +PLAN_DIR="offline_1_slide" +OUTPUT_CSV_PATH="offline_1_slide/lp_plan_eval" +TRAIN_PATH="./yahoo_train_data" +EVAL_PATH="./yahoo_eval_data" -python simulation.py --model_runtime_s 0.02 --total_runtime_s 150 \ - --per_key_records_per_second 100 \ - --num_mapper_replicas 2 --num_keys 100 \ - --window_size 672 --slide_size 0 \ - --per_key_slide_size_plan result/offline_1_slide/min_loss_plan.json \ - --output_path result/offline_1_slide/lp_eval/varying_slide_size_trace.json \ No newline at end of file + +for replicas in 8 +do +for plan in "max_fits_1100" "max_fits_2100" "max_fits_4200" "max_fits_8400" +do + mkdir -p ${PLAN_DIR}/replica_${replicas} + + # re-run simulation with lp-generated weights + python simulation.py --model_runtime_s 1.5 --total_runtime_s 2000 \ + --per_key_records_per_second 1 \ + --num_mapper_replicas ${replicas} \ + --window_size 672 --slide_size 0 \ + --per_key_slide_size_plan ${PARAM_DIR}/${plan}.json \ + --output_path ${PLAN_DIR}/replica_${replicas}/plan_${plan}.json \ + --source_data_path ${TRAIN_PATH} + + mkdir -p ${PLAN_DIR}/replica_${replicas}/${plan} + + # run evaluation with simulation results + python evaluation.py --offline-yahoo-csv-path $EVAL_PATH \ + --offline-plan-path ${PLAN_DIR}/replica_${replicas}/plan_${plan}.json \ + --output-path ${PLAN_DIR}/replica_${replicas}/${plan} \ + --param-path ${PARAM_DIR}/${plan}.json \ + --run-policy + + # get final results + #python evaluate_loss.py --offline-yahoo-csv-path $SOURCE_PATH --predicted-csv-path $OUTPUT_CSV_PATH --output-path +done +done diff --git a/stl/offline/run_6_simulate_baseline.sh b/stl/offline/run_6_simulate_baseline.sh new file mode 100644 index 0000000..c0cb4ed --- /dev/null +++ b/stl/offline/run_6_simulate_baseline.sh @@ -0,0 +1,37 @@ +set -xe +PARAM_DIR="/data/wooders/stl/results" +PLAN_DIR="/data/wooders/stl/results" +TRAIN_PATH="./yahoo_train_data" +EVAL_PATH="./yahoo_eval_data" + +for key_policy in "fifo" +do +for replicas in 1 2 4 8 +do + for slide in 672 1 6 12 18 24 48 96 168 192 336 + do + plan="plan_baseline_${slide}_${key_policy}" + param="plan_baseline_${slide}" + mkdir -p ${PLAN_DIR}/replica_${replicas} + python simulation.py \ + --model_runtime_s 1.5 \ + --total_runtime_s 2000 \ + --per_key_records_per_second 1 \ + --window_size 672 \ + --slide_size ${slide} \ + --per_key_slide_size_plan ${PARAM_DIR}/${param}.json \ + --output_path ${PLAN_DIR}/replica_${replicas}/${plan}.json \ + --source_data_path $TRAIN_PATH \ + --num_mapper_replicas ${replicas} \ + --key_prio_policy ${key_policy} + + mkdir -p ${PLAN_DIR}/replica_${replicas}/${plan} + python evaluation.py --offline-yahoo-csv-path $EVAL_PATH \ + --offline-plan-path ${PLAN_DIR}/replica_${replicas}/${plan}.json \ + --output-path ${PLAN_DIR}/replica_${replicas}/${plan} \ + --param-path ${PARAM_DIR}/${param}.json \ + --run-policy + + done +done +done diff --git a/stl/offline/simulation.py b/stl/offline/simulation.py index 41461bb..6431293 100644 --- a/stl/offline/simulation.py +++ b/stl/offline/simulation.py @@ -26,7 +26,7 @@ flags.DEFINE_enum( "key_prio_policy", - "fifo", + "lifo", list(prio_policies.keys()), "The prioritization policy for a given key.", ) @@ -45,7 +45,7 @@ flags.DEFINE_float("total_runtime_s", 14, "When to end the simulation.") flags.DEFINE_float( "model_runtime_s", - 0.2, + 0.01, "The latency for the map function (when processing a single record).", ) flags.DEFINE_integer("window_size", 24 * 7, "The sliding window size.") @@ -63,7 +63,7 @@ None, "path to generated per key's window slide size config.", ) -flags.DEFINE_integer("num_mapper_replicas", None, "number of replicas for mapper") +flags.DEFINE_integer("num_mapper_replicas", 1, "number of replicas for mapper") def _get_config() -> Dict: @@ -74,14 +74,22 @@ def _get_config() -> Dict: def main(argv): env = simpy.Environment() # source --source_to_window_queue--> window --windows_to_mapper_queue--> mapper + + if FLAGS.per_key_slide_size_plan is not None: + policy_params = json.load(open(FLAGS.per_key_slide_size_plan)) + keys = policy_params.keys() + else: + keys = [i+1 for i in range(FLAGS.num_keys)] + + print(FLAGS.key_prio_policy) source_to_window_queue = simpy.Store(env) windows_to_mapper_queue = { - i: PerKeyPriorityQueue( + key: PerKeyPriorityQueue( env, processing_policy=prio_policies[FLAGS.key_prio_policy], load_shedding_policy=load_shed_policies[FLAGS.key_load_shed_policy], ) - for i in range(FLAGS.num_keys) + for key in keys } Source( env, @@ -89,7 +97,8 @@ def main(argv): num_keys=FLAGS.num_keys, next_queue=source_to_window_queue, total_run_time=FLAGS.total_runtime_s, - data_file=FLAGS.source_data_path, + keys=keys, + data_dir=FLAGS.source_data_path, ) WindowOperator( env, @@ -104,7 +113,7 @@ def main(argv): source_queues=windows_to_mapper_queue, model_run_time_s=FLAGS.model_runtime_s, # TODO(simon): customize this once we want different key selection policy - key_selection_policy_cls=RoundRobinLoadBalancer, + key_selection_policy_cls=RoundRobinLoadBalancer(FLAGS.num_mapper_replicas), num_replicas=FLAGS.num_mapper_replicas, ) env.run(until=FLAGS.total_runtime_s) @@ -112,7 +121,7 @@ def main(argv): plan = m.plan config = _get_config() if FLAGS.output_path: - os.makedirs(os.path.split(FLAGS.output_path)[0], exist_ok=True) + #os.makedirs(os.path.split(FLAGS.output_path)[0], exist_ok=True) with open(FLAGS.output_path, "w") as f: json.dump(plan, f, indent=2) with open(FLAGS.output_path + ".config.json", "w") as f: diff --git a/stl/scratch.ipynb b/stl/scratch.ipynb index 962bf16..2334fd9 100644 --- a/stl/scratch.ipynb +++ b/stl/scratch.ipynb @@ -3,33 +3,33 @@ { "cell_type": "code", "execution_count": null, - "source": [], + "metadata": {}, "outputs": [], - "metadata": {} + "source": [] } ], "metadata": { - "orig_nbformat": 4, + "interpreter": { + "hash": "a10b01f403a1542ddbe951c0fc128eb6a019580013b1191ba1a82a0d150f03e0" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python", - "version": "3.7.10", - "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, - "pygments_lexer": "ipython3", + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", "nbconvert_exporter": "python", - "file_extension": ".py" - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3.7.10 64-bit ('ralf': conda)" - }, - "interpreter": { - "hash": "a10b01f403a1542ddbe951c0fc128eb6a019580013b1191ba1a82a0d150f03e0" + "pygments_lexer": "ipython3", + "version": "3.7.10" } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/wikipedia/README.md b/wikipedia/README.md new file mode 100644 index 0000000..983b625 --- /dev/null +++ b/wikipedia/README.md @@ -0,0 +1,60 @@ +# Wikipedia Experiment Pipeline + +### Configuration +Update `config.yml` + +### Generating simulation data +Run parts of the pipeline using flags: +``` +python generate_data.py \ + --run_query_recentchanges # query wikipedia recentchanges api + --run_query_doc_versions # query wikipedia docs api + --run_recent_changes # process raw changes data into changes.csv file + --run_parse_docs # process raw doc data with wikiparser + --run_get_questions # process raw questions into questions.csv + --run_get_pageviews # process raw pageview data into pageviews.csv + --run_generate_diffs # compute diffs between different version + --run_generate_simulation_data # generate simulation data + --run_check_dataset # check dataset + --run_generate_embeddings # embed documents +``` +To update simulation data, make sure you have the embeddings and diffs already download, and run: +``` +python generate_data.py --run_generate_simulation_data --run_get_questions --run_check_dataset +``` + + +## Offline Simulation Pipeline +Download the data with `./download_data.sh` (warning: 100s of GBs) and update `config.yml`. + +Run the simulation in stages to go from raw Wikipedia API data to simulation results: + +``` +./run_0_generate_data.sh # generate simulation data from questions.csv file +./run_1_generate_plan.sh # run simulations to generate plan +./run_2_prepare_data.sh # use plan to determine questions / embedding versions at each timestep +./run_3_run_predictions.sh # run DPR model on embeddings +./run_4_run_optimal_predictons.sh # generate optimal predictions +``` + +### Logging Data +To save the current data, run +``` +python log_data.py +``` + +### Logging Experiments +TODO + +## Online Pipeline (ralf) +(NOTE: incomplete) +Run the server +``` +python wiki_server.py +``` +Run the client +``` +python wiki_client.py +``` + + diff --git a/wikipedia/config.yml b/wikipedia/config.yml deleted file mode 100644 index 74128ff..0000000 --- a/wikipedia/config.yml +++ /dev/null @@ -1,28 +0,0 @@ -[directory] -data_dir = /data/wooders/wikipedia -revisions_dir = %(data_dir)s/recentchanges -raw_doc_dir = %(data_dir)s/doc_xml/ -parsed_doc_dir = %(data_dir)s/doc_pkl/ -parsed_tmp_dir = %(data_dir)s/parsed_tmp/ -diff_dir = %(data_dir)s/diffs/ -embedding_dir = %(data_dir)s/embeddings/ - -[files] -data_dir = /data/wooders/wikipedia -raw_questions_file = %(data_dir)s/10052021_questions_revid.csv -model_file = %(data_dir)s/bert-base-encoder.cp -changes_file = %(data_dir)s/changes.csv -titles_file = %(data_dir)s/top_titles.csv -revisions_file = %(data_dir)s/title_revisions_timestamps.json -edits_file = %(data_dir)s/edits.csv -questions_file = %(data_dir)s/questions.csv -pageview_file = %(data_dir)s/top_title_views.csv - -[simulation] -data_dir = /data/wooders/wikipedia -plan_dir = /data/wooders/wiki-plans -init_data_file = %(data_dir)s/init_data.json -stream_edits_file = %(data_dir)s/edit_stream.json -stream_questions_file = %(data_dir)s/question_stream.json - - diff --git a/wikipedia/notebooks/Wikipedia Plots.ipynb b/wikipedia/notebooks/Wikipedia Plots.ipynb new file mode 100644 index 0000000..6b56acc --- /dev/null +++ b/wikipedia/notebooks/Wikipedia Plots.ipynb @@ -0,0 +1,2502 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 266, + "id": "e0030940", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "import wandb\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "id": "594d6d4e", + "metadata": {}, + "source": [ + "# Plot Wikipedia Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 267, + "id": "016e13bb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: wandb version 0.12.4 is available! To upgrade, please run:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: $ pip install wandb --upgrade\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " Syncing run toasty-plasma-547 to Weights & Biases (docs).
\n", + "\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact questions:latest, 84.36MB. 4 files... Done. 0:0:0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact prediction_results:latest, 6400.51MB. 413 files... Done. 0:0:0\n" + ] + } + ], + "source": [ + "run = wandb.init(job_type=\"evaluation\", project=\"wiki-workload\")\n", + "pageview_dir = run.use_artifact('pageviews:latest').download()\n", + "questions_dir = run.use_artifact('questions:latest').download()\n", + "predictions_dir = run.use_artifact('prediction_results:latest').download()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7690f6d7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0titleedit_count2021080500202108060020210807002021080800202108090020210810002021081100...20210828002021082900202108300020210831002021090100202109020020210903002021090400weightsdoc_id
00Deaths in 20211877383536313496656...69506368505239460.02851165984422
112021 Atlantic hurricane season14381151689714...820285121150.00380557798785
22Neeraj Chopra11563732434...560492130.00217051150040
33Fall of Kabul (2021)10091891212161012...11169920155100.00487668481047
44Great Britain at the 2020 Summer Paralympics989135641689...3868107470.00339760043578
..................................................................
211211List of fungi of South Africa203897132149...10761135560.00346768354495
212212Mister Supranational 2021203897132149...10761135560.00346767918135
2132132021–22 FC Barcelona season20219292927282723...21262916272043180.01269867089631
214214Hamid Karzai International Airport20114261517261417...1910251326142270.007258487602
215215Characters of the Marvel Cinematic Universe20114261517261417...1910251326142270.00725862372638
\n", + "

216 rows × 36 columns

\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 title edit_count \\\n", + "0 0 Deaths in 2021 1877 \n", + "1 1 2021 Atlantic hurricane season 1438 \n", + "2 2 Neeraj Chopra 1156 \n", + "3 3 Fall of Kabul (2021) 1009 \n", + "4 4 Great Britain at the 2020 Summer Paralympics 989 \n", + ".. ... ... ... \n", + "211 211 List of fungi of South Africa 203 \n", + "212 212 Mister Supranational 2021 203 \n", + "213 213 2021–22 FC Barcelona season 202 \n", + "214 214 Hamid Karzai International Airport 201 \n", + "215 215 Characters of the Marvel Cinematic Universe 201 \n", + "\n", + " 2021080500 2021080600 2021080700 2021080800 2021080900 2021081000 \\\n", + "0 38 35 36 31 349 66 \n", + "1 11 5 16 8 9 7 \n", + "2 3 7 3 2 4 3 \n", + "3 18 9 12 12 16 10 \n", + "4 13 5 6 4 16 8 \n", + ".. ... ... ... ... ... ... \n", + "211 8 9 7 13 21 4 \n", + "212 8 9 7 13 21 4 \n", + "213 19 29 29 27 28 27 \n", + "214 14 26 15 17 26 14 \n", + "215 14 26 15 17 26 14 \n", + "\n", + " 2021081100 ... 2021082800 2021082900 2021083000 2021083100 \\\n", + "0 56 ... 69 50 63 68 \n", + "1 14 ... 8 20 2 8 \n", + "2 4 ... 5 6 0 4 \n", + "3 12 ... 11 16 9 9 \n", + "4 9 ... 3 8 6 8 \n", + ".. ... ... ... ... ... ... \n", + "211 9 ... 10 7 6 1 \n", + "212 9 ... 10 7 6 1 \n", + "213 23 ... 21 26 29 16 \n", + "214 17 ... 19 10 25 13 \n", + "215 17 ... 19 10 25 13 \n", + "\n", + " 2021090100 2021090200 2021090300 2021090400 weights doc_id \n", + "0 50 52 39 46 0.028511 65984422 \n", + "1 5 12 11 5 0.003805 57798785 \n", + "2 9 2 1 3 0.002170 51150040 \n", + "3 20 15 5 10 0.004876 68481047 \n", + "4 10 7 4 7 0.003397 60043578 \n", + ".. ... ... ... ... ... ... \n", + "211 13 5 5 6 0.003467 68354495 \n", + "212 13 5 5 6 0.003467 67918135 \n", + "213 27 20 43 18 0.012698 67089631 \n", + "214 26 14 22 7 0.007258 487602 \n", + "215 26 14 22 7 0.007258 62372638 \n", + "\n", + "[216 rows x 36 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pageview_df = pd.read_csv(f\"{pageview_dir}/pageviews.csv\")\n", + "pageview_df" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5b5d1edc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA4nklEQVR4nO3dd3xUVd7H8c9vZlKA0BOQJgSlGDpEsCyoYAELiIJlBRVFREFRl11xXR/xeXTX3XXXtSAsCrKsuKgoKyIqNYpSQ5EqRWqkhd7S5zx/nEkyCRNyA2k3/N6vV5iZW88MM9977rnn3ivGGJRSSlVcnrIugFJKqZKlQa+UUhWcBr1SSlVwGvRKKVXBadArpVQF5yvrAoQSHR1tmjRpUtbFUEop11ixYsVBY0xMqHHlMuibNGlCYmJiWRdDKaVcQ0R2FjROm26UUqqC06BXSqkKToNeKaUquHLZRq+UOj8ZGRkkJSWRmppa1kVRxSwyMpKGDRsSFhbmeB4NeqUqoKSkJKpWrUqTJk0QkbIujiomxhgOHTpEUlISsbGxjufTphulKqDU1FRq166tIV/BiAi1a9cu8p6aBr1SFZSGfMV0Lv+vGvRFsS0BDv1c1qVQSqki0aAvis+Hw6I3y7oUSilVJBr0RZGVDlmZZV0KpSqMSZMmMXz4cADGjRvH5MmTc4bv2bPnrPMuXLiQVq1a0b59e1JSUkq8rG6mvW6Kwvjtn1Kq2A0dOjTn+aRJk2jdujX169cvcPopU6YwcuRIBg0alGd4VlYWXq+3xMrpRhr0RaFBr1zopS/Ws2HP8WJdZlz9arx4W6tCp/vggw948803SU9Pp0uXLrzzzjtMnjyZP/3pT9SrV4/mzZsTEREBwOjRo4mKisq51tV9991HpUqVWLx4MZUqVcqz3Pfee4+PP/6Yb775hrlz5/LII4/w0ksvUa9ePVavXs3atWsZNWoUCQkJpKWlMWzYMB599FGMMTzxxBPMnz+f2NhYjDE89NBD9OvXL2e90dHRJCYmMnLkSBISEjh16hRPPPEEa9euJTMzk9GjR9OnTx8mTZrEjBkzOH36ND///DN9+/blL3/5CwBff/01v//978nKyiI6Opo5c+bQokULFi1aRExMDH6/n+bNm7NkyRKio6OL9f8mFA36otCgV8qxjRs38tFHH/HDDz8QFhbG448/zgcffMCLL77IihUrqF69Otdddx0dOnTIM1+/fv14++23ee2114iPjw+57MGDB/P9999z66230q9fPxISEli2bBnr1q0jNjaW8ePHU716dZYvX05aWhpXX301N954I6tWrWLTpk2sXbuW/fv3ExcXx0MPPXTW9/HKK6/QvXt3Jk6cyNGjR+ncuTPXX389AKtXr2bVqlVERETQokULnnjiCSIjI3nkkUf47rvviI2N5fDhw3g8HgYMGMCUKVN46qmnmDt3Lu3atSuVkAcN+qLRoFcu5KTmXRLmzZvHihUruPzyywFISUlh0aJFXHvttcTE2Kvp3n333WzevLlY1te5c+eck4hmz57NmjVrmDZtGgDHjh1jy5YtfPfdd9x77714vV7q169P9+7dC13u7NmzmTFjBq+99hpgz1HYtWsXAD169KB69eoAxMXFsXPnTo4cOUK3bt1yylKrVi0AHnroIfr06cNTTz3FxIkTz2hyKkka9EVhcv5RShXCGMMDDzzAn/70p5xh//3vf5k+fXqJrK9KlSp51v3WW29x00035Zlm1qxZBfZD9/l8+P22Ihd8QpIxhk8//ZQWLVrkmX7p0qU5zU4AXq+XzMxMjDEh19GoUSPq1q3L/PnzWbp0KVOmTCn6mzxH2uumKLRGr5RjPXr0YNq0aRw4cACAw4cP06FDBxISEjh06BAZGRl88sknIeetWrUqJ06cOOd133TTTYwdO5aMjAwANm/ezKlTp+jWrRtTp04lKyuLvXv3smDBgpx5mjRpwooVKwD49NNP8yzrrbfewhhbyVu1atVZ133llVfy7bffsn379pz3nW3w4MEMGDCAu+66q1QPGGvQF4UGvVKOxcXF8fLLL3PjjTfStm1bbrjhBvbu3cvo0aO58soruf766+nYsWPIeR988EGGDh16zl0nBw8eTFxcHB07dqR169Y8+uijZGZm0rdvX5o1a0abNm147LHHuOaaa3LmefHFFxkxYgRdu3bNE8IvvPACGRkZtG3bltatW/PCCy+cdd0xMTGMHz+eO+64g3bt2nH33XfnjOvduzcnT54s1WYbAMneSpUn8fHxplzeYerli6DZ9XD3B2VdEqXOauPGjVx22WVlXYxy78EHH8w5oFsaEhMTefrpp1m4cOF5LSfU/6+IrDDGhDx6rW30RWH8UA43jEqp8u/VV19l7Nixpdo2n02Dvii06UapUte3b9+c9u5sf/7zn8840HouJk2adN7LcGrUqFGMGjWq1NYXTIO+KDTolSp1JdVL50Li6GCsiPQUkU0islVEztgkiUhLEVksImkiMjLEeK+IrBKRmcVR6DKjQa+UcqFCg15EvMAYoBcQB9wrInH5JjsMPAm8VsBiRgAbz6Oc5YTRoFdKuY6TGn1nYKsxZpsxJh2YCvQJnsAYc8AYsxzIyD+ziDQEbgHeK4bylp3sg7B6MFYp5TJOgr4BsDvodVJgmFP/AH4HnLUqLCJDRCRRRBKTk5OLsPhSkl2T1xq9UsplnAR9qPOFHVVrReRW4IAxZkVh0xpjxhtj4o0x8dnXwShXNOiVcqXf/va3tGrVit/+9rdlXZQy46TXTRLQKOh1Q+DsdwTIdTXQW0RuBiKBaiLygTFmQNGKWQ5o0CtV5s7lWvP//Oc/SU5OznNdGoDMzEx8vguj46GTd7kcaCYiscAvwD3Ar50s3BjzHPAcgIhcC4x0ZchDUNBrG71yma9Gwb61xbvMi9pAr1cLneyVV15h8uTJNGrUiJiYGDp16sTMmTNzLkF88OBB4uPj2bFjB1lZWSGvIZ+QkJDnWvN33nkn0dHRjBgxAoDnn3+eunXr8uSTT56x/t69e3Pq1Cm6dOnCc889x1dffUWtWrVYtWoVHTt25PHHH2fYsGEkJydTuXJl3n33XVq2bMn27dv59a9/TWZmJj179uT111/n5MmTJCQk8NprrzFzpu1AOHz4cOLj43nwwQdZsWIFzzzzDCdPniQ6OppJkyZRr149rr32Wrp06cKCBQs4evQoEyZMoGvXrmRlZfHss8/yzTffICI88sgjxMXF8fbbb+d0KZ0zZw5jx47ls88+O6//rkKD3hiTKSLDgW8ALzDRGLNeRIYGxo8TkYuARKAa4BeRp4A4Y0zx3u2gLGmNXqkiWbFiBVOnTmXVqlVkZmbSsWNHOnXqVOD0EyZMCHkNeSDPteZ37NjBHXfcwYgRI/D7/UydOpVly5aFXOaMGTOIiopi9erVAHz11Vds3ryZuXPn4vV66dGjB+PGjaNZs2YsXbqUxx9/nPnz5zNixAgee+wx7r//fsaMGVPoe83IyOCJJ57g888/JyYmho8++ojnn3+eiRMnAnbvYdmyZcyaNYuXXnqJuXPnMn78eLZv386qVavw+XwcPnyYmjVr5mx4YmJieP/994vlujiO9luMMbOAWfmGjQt6vg/bpHO2ZSQACUUuYXmhQa/cykHNuyQsXLiQvn37UrlyZcDWrs+moGvIh4eH57nWfJMmTahduzarVq1i//79dOjQgdq1azsuV//+/fF6vZw8eZJFixbRv3//nHFpaWkA/PDDDzlXsBw4cCDPPvvsWZe5adMm1q1bxw033ADYJqZ69erljL/jjjsA6NSpEzt27ABg7ty5DB06NKf5KPu69QMHDuSDDz5g0KBBLF68OOc+uufjwmigKg4a9EoVWajrsp/tuu+hriGfkJCQ51rzYK9OOWnSJPbt21foHaLyy16W3++nRo0aObX9opY9uPzGGFq1asXixYtDLiv7+ED2Neuz5wm1jkGDBnHbbbcRGRlJ//79i+U4gl6m2CkNeqWKpFu3bkyfPp2UlBROnDjBF198AeS97nt27R0KvoZ8KH379uXrr79m+fLl53zNm2rVqhEbG5tzTXxjDD/++CMAV199NVOnTgXIcxGyxo0bs2HDBtLS0jh27Bjz5s0DoEWLFiQnJ+cEfUZGBuvXrz/r+m+88UbGjRuXE/zZ162vX78+9evX5+WXX+bBBx88p/eWnwa9UzknTGnQK+VEx44dufvuu2nfvj133nknXbt2BWDkyJGMHTuWq666ioMHD+ZMX9A15EMJDw/nuuuuO+8beEyZMoUJEybQrl07WrVqxeeffw7AG2+8wZgxY7j88ss5duxYzvSNGjXirrvuom3bttx3330597sNDw9n2rRpPPvss7Rr14727duzaNGis6578ODBXHzxxbRt25Z27drx4Ycf5oy77777aNSoEXFx+S9CcG70evROnToEf20K9TvAkISyLo1SZ1Uer0c/evRooqKiGDnyjMthFZnf76djx4588sknNGvWrBhKd3ZRUVGcPHmyxNeTbfjw4XTo0IGHH3445PiiXo9ea/ROadONUuXChg0buPTSS+nRo0ephHxp69SpE2vWrGHAgOLria4HY53SoFfqvIwePbpYlhMXF8e2bdvyDFu7di0DBw7MMywiIoKlS5cWyzpLszafffyiOGnQO6UnTCmXKahXR0XUpk2bAnvPVDTn0tyuTTdOaY1euUhkZCSHDh06p1BQ5ZcxhkOHDhEZGVmk+bRG75QGvXKRhg0bkpSURLm8Eqw6L5GRkTRseNbzU8+gQe+UBr1ykbCwsJwzSZXSphunNOiVUi6lQe+YnjCllHInDXqn9FaCSimX0qB3SptulFIupUHvlPajV0q5lAa9U1qjV0q5lAa9Uxr0SimXchT0ItJTRDaJyFYRGRVifEsRWSwiaSIyMmh4IxFZICIbRWS9iIwozsKXKg16pZRLFXrClIh4gTHADUASsFxEZhhjNgRNdhh4Erg93+yZwG+MMStFpCqwQkTm5JvXHTTolVIu5aRG3xnYaozZZoxJB6YCfYInMMYcMMYsBzLyDd9rjFkZeH4C2Ag0KJaSlzYNeqWUSzkJ+gbA7qDXSZxDWItIE6ADEPK6oSIyREQSRSSxXF6fQ+8wpZRyKSdBH+o6p0XqYygiUcCnwFPGmOOhpjHGjDfGxBtj4mNiYoqy+NKhQa+UciknQZ8ENAp63RDY43QFIhKGDfkpxpjPila8ciQn4LUfvVLKXZwE/XKgmYjEikg4cA8ww8nCxd71YAKw0Rjz93MvZjmgJ0wppVyq0F43xphMERkOfAN4gYnGmPUiMjQwfpyIXAQkAtUAv4g8BcQBbYGBwFoRWR1Y5O+NMbOK/Z2UND0Yq5RyKUfXow8E86x8w8YFPd+HbdLJ73tCt/G7jwa9Usql9MxYpzTolVIupUHvlAa9UsqlNOid0qBXSrmUBr1j2o9eKeVOGvROaY1eKeVSGvROBfef1770SikX0aB3Krgmr0GvlHIRDXqn8gS9Nt8opdxDg94pDXqllEtp0DulQa+UcikNeqc06JVSLqVB75QGvVLKpTToncrTvVKDXinlHhr0TmmNXinlUhr0TmmNXinlUhr0Tmm4K6VcSoPeKW26UUq5lKOgF5GeIrJJRLaKyKgQ41uKyGIRSRORkUWZ1zU06JVSLlVo0IuIFxgD9MLeB/ZeEYnLN9lh4EngtXOY1x006JVSLuWkRt8Z2GqM2WaMSQemAn2CJzDGHDDGLAcyijqva2jQK6VcyknQNwB2B71OCgxzwvG8IjJERBJFJDE5Odnh4kuRBr1SyqWcBL2EGOb0Or2O5zXGjDfGxBtj4mNiYhwuvjRp90qllDs5CfokoFHQ64bAHofLP595yxet0SulXMpJ0C8HmolIrIiEA/cAMxwu/3zmLV/0hCmllEv5CpvAGJMpIsOBbwAvMNEYs15EhgbGjxORi4BEoBrgF5GngDhjzPFQ85bQeylZeocppZRLFRr0AMaYWcCsfMPGBT3fh22WcTSvK2nQK6VcSs+MdUrb6JVSLqVB75QGvVLKpTTondKgV0q5lAa9Uxr0SimX0qB3SrtXKqVcSoPeKa3RK6VcSoPeKa3RK6VcSoPeKe1Hr5RyKQ16p/LU4jXolVLuoUHvlLbRK6VcSoPeKQ16pZRLadA7pUGvlHIpDXqnNOiVUi6lQe+Ydq9USrmTBr1TWqNXSrmUBr1TesKUUsqlNOid0hq9UsqlHAW9iPQUkU0islVERoUYLyLyZmD8GhHpGDTuaRFZLyLrROQ/IhJZnG+g1OQJ+rIrhlJKFVWhQS8iXmAM0AuIA+4Vkbh8k/UCmgX+hgBjA/M2AJ4E4o0xrbH3jb2n2EpfmrRGr5RyKSc1+s7AVmPMNmNMOjAV6JNvmj7AZGMtAWqISL3AOB9QSUR8QGVgTzGVvXRp0CulXMpJ0DcAdge9TgoMK3QaY8wvwGvALmAvcMwYMzvUSkRkiIgkikhicnKy0/KXHg16pZRLOQl6CTEsfyt1yGlEpCa2th8L1AeqiMiAUCsxxow3xsQbY+JjYmIcFKuUadArpVzKSdAnAY2CXjfkzOaXgqa5HthujEk2xmQAnwFXnXtxy5AGvVLKpZwE/XKgmYjEikg49mDqjHzTzADuD/S+uQLbRLMX22RzhYhUFhEBegAbi7H8pUf70SulXMpX2ATGmEwRGQ58g+01M9EYs15EhgbGjwNmATcDW4HTwKDAuKUiMg1YCWQCq4DxJfFGSpwGvVLKpQoNegBjzCxsmAcPGxf03ADDCpj3ReDF8yhj+aBNN0opl9IzY53SWwkqpVxKg94pvZWgUsqlNOidMn4QT+5zpZRyCQ16p4wfPL7c50op5RIa9E5p0CulXEqD3ikNeqWUS2nQO2a0jV4p5Uoa9E4ZAx5v4LkGvVLKPTTondKmG6WUS2nQO5Un6LUfvVLKPTTonTL+oKYbDXqllHto0DulTTdKKZfSoHdKg14p5VIa9E5p0CulXEqD3injB9HulUop99Ggd0r70SulXEqD3ikNeqWUSzkKehHpKSKbRGSriIwKMV5E5M3A+DUi0jFoXA0RmSYiP4nIRhG5sjjfQKnRNnqllEsVGvQi4gXGAL2AOOBeEYnLN1kvoFngbwgwNmjcG8DXxpiWQDtce3NwPWFKKeVOTmr0nYGtxphtxph0YCrQJ980fYDJxloC1BCReiJSDegGTAAwxqQbY44WX/FLUZ4TprRGr5RyDydB3wDYHfQ6KTDMyTRNgWTgfRFZJSLviUiVUCsRkSEikigiicnJyY7fQKkJrtHrrQSVUi7iJOglxLD8SVfQND6gIzDWGNMBOAWc0cYPYIwZb4yJN8bEx8TEOChWKdPulUopl3IS9ElAo6DXDYE9DqdJApKMMUsDw6dhg999su8ZKx4NeqWUqzgJ+uVAMxGJFZFw4B5gRr5pZgD3B3rfXAEcM8bsNcbsA3aLSIvAdD2ADcVV+FJl/CCiQa+Uch1fYRMYYzJFZDjwDeAFJhpj1ovI0MD4ccAs4GZgK3AaGBS0iCeAKYGNxLZ841zEaI1eKeVKhQY9gDFmFjbMg4eNC3pugGEFzLsaiD/3IpYTJvtWgqJBr5RyFT0z1iltulFKuZQGvVN5DsZq90qllHto0DulvW6UUi6lQe+U1uiVUi6lQe9UTtDrwVillLto0Dtl/IAejFVKuY8GvVPaRq+UcikNeqcMGvRKKVfSoHdKa/RKKZfSoHcq54QpPRirlHIXDXqntHulUsqlNOid0qYbpZRLadA7pf3olVIupUHvVHCNXm8lqJRyEQ16p/TqlUopl9Kgd8rojUeUUu6kQe+YBr1Syp0cBb2I9BSRTSKyVURGhRgvIvJmYPwaEemYb7xXRFaJyMziKnip0143SimXKjToRcQLjAF6AXHAvSISl2+yXkCzwN8QYGy+8SOAjedd2rKU3UavtxJUSrmMkxp9Z2CrMWabMSYdmAr0yTdNH2CysZYANUSkHoCINARuAd4rxnKXPj1hSinlUk6CvgGwO+h1UmCY02n+AfwOOGs1WESGiEiiiCQmJyc7KFYp06YbpZRLOQl6CTEsf5U25DQicitwwBizorCVGGPGG2PijTHxMTExDopVyvSEKaWUSzkJ+iSgUdDrhsAeh9NcDfQWkR3YJp/uIvLBOZe2LOW58Yg23Sil3MNJ0C8HmolIrIiEA/cAM/JNMwO4P9D75grgmDFmrzHmOWNMQ2NMk8B8840xA4rzDZQabbpRSrmUr7AJjDGZIjIc+AbwAhONMetFZGhg/DhgFnAzsBU4DQwquSKXgewavAa9UsqFCg16AGPMLGyYBw8bF/TcAMMKWUYCkFDkEpYHGvRKKRfTM2OdyA52DXqllAtp0DuRE/R6hymllPto0DtxRo1ee90opdxDg94JbbpRSrmYBr0TeYJem26UUu6iQe9EnjZ6rdErpdxFg96J/E03eitBpZSLaNA7oW30SikX06AvCg16pZQLadA7oTV6pZSLadA7oQdjlVIuVqGC3u83pGZkFf+Cg2v0iJ4wpZRylQoT9BlZftr/72zeWbC1+Beu/ejdY8tcyEgt61IoVa5UmKAP83qoUy2SDXtPFP/Cc4Jdm27KtaO7YMqdsPGLsi6JUuVKhQl6gMvqVeOnfceLf8F6MNYdUo7Yx9SjZVoMpcqbChb0VUk6ksLx1IziXbBe1Mwd0k8HHk+WbTmUKmcqVtBfVA2An4q7+UZr9O6QfirweLpsy6FUOeMo6EWkp4hsEpGtIjIqxHgRkTcD49eISMfA8EYiskBENorIehEZUdxvINhl9WzQb9xbzM03eocpd8iuyWdcwEGfcgROHbR/mellXRpVThR6K0ER8QJjgBuAJGC5iMwwxmwImqwX0Czw1wUYG3jMBH5jjFkpIlWBFSIyJ9+8xaZutQhqVg7ToL9QZVzgTTfrp8MnD+a+rtcOHv2uzIqjyg8nNfrOwFZjzDZjTDowFeiTb5o+wGRjLQFqiEg9Y8xeY8xKAGPMCWAj0KAYy5+HiNDyomolEPT5T5jSNvpy6UJvujkY6Frc6y8Qe03ua3XBcxL0DYDdQa+TODOsC51GRJoAHYCloVYiIkNEJFFEEpOTkx0UK7RW9avx074TxXvilPajd4cLvekm9SiEVYEuj0LjqyDjFPhL4ARC5TpOgl5CDMtfpT3rNCISBXwKPGWMCVndNsaMN8bEG2PiY2JiHBQrtKsurU1app/lOw6f8zLOLJwG/Tnz++Fft8HrbeC/j5fsunJq9Bdo003qMYisbp9H2ONVpJVAd2PlOk6CPgloFPS6IbDH6TQiEoYN+SnGmM/OvajOXNG0NuE+Dwmbzn2v4Ax6rZtzl3oUtn8Hx3+BjTNLdl053Ssv1Bp9UNBHBoI+VYNeOQv65UAzEYkVkXDgHmBGvmlmAPcHet9cARwzxuwVEQEmABuNMX8v1pIXoHK4jy6xtUjYdKD4FqrdK89d9slLtWIh7RhkppXcurTpJqhGX9U+ao1e4SDojTGZwHDgG+zB1I+NMetFZKiIDA1MNgvYBmwF3gWy99GvBgYC3UVkdeDv5uJ+E/ld0zyGn5NPsftwMf3gNejPXcpR+1j7Uvt4+lDJrSun6eZUya2jPAvZdFMClwRRrlNo90oAY8wsbJgHDxsX9NwAw0LM9z2h2+9L1HUt6/DylxuZvuoXnuzR7PwXqLcSPHc5NfpL7OOpg1CtfsmsK6d75QUc9DEt7XNtulFBKs6ZscbA/Jdh82wuiYnixri6jP9uGwdPFkNTgfajP3epx+xj7UDQnz5Ycuu64Jtugmv0gUdtulFUpKAXgaXjYetcAH7XsyUpGVk8/dFqZq3dy7bkk5hz7v+uQX/O8jfdnCrJoA9quinpcx38flg7rfx0XzQmX9BrG73K5ajpxjWiYuCU7W1zaZ0onrmhOW/P38rCLTZcrm0Rw7gBnYgM8xZtuXrC1LnLqdGXRtBn1+QNZKRAeOWSW9eOhfDpw1CpJlzao+TW41TaCfs91V43KoSKU6MHqJIb9ADDrruUH1+8kc+HXc3IG5uTsCmZYVNWFr1mf8YdpspxjX7Oi5D4flmXIlfqUfCEQdV6IN4SbroJapsv6eabIzvs44l9Jbsep7I3qNlB74u0n3tFqtEnrah4layMFPhlRYmvpkIHPUC4z0O7RjUY3r0Zz998GfN+OpBTw3csz41HynnQ/zgV1n1a1qXIlXoMKtUAjwcq1y7hGv1J8FUKPC/hA7LHAieCnyrG8zXOR/6gF7G1+opSo9+1BN7rDj/PK+uSFK+V/4Z3u8OKSSW6mooX9CcL7j//wFVNiKkawbsLtxVtueWxe2VWJuxbB/s32PZisO3Fp5LhWFLZli1YytHc8KkSXbLdKzNOQ1Sd3Ocl6Wh5DfoaucMiqlac7pW7A1dO2b++bMtR3A5tsY8zn4HJt8Mng0pkNRUr6KPqQMphyAp945Fwn4cHrmzMwi0HWbnriPPlFhb0ZXGP0sVvw7irYeyVsGaqHXb6EJgsexaqvxxsjCBwgLCGfV4luuSCMSsTMlNzg/5Cr9GD7UtfUZpu9qyyjwc3l205ituRHRDdHFrfYfdIS+jyHRUr6KsErpFzllrjfV0aU7NyGP3GLmLYhyv5cs1eTqZlnn25Zwv6AxvhTw1h74/nWfgi2r8eouqCNxySf7LDTu63j1npuc/LWvDZmpWjS67pJiMQ7NnfgdIK+rPsQZaq7PMVgoM+snrFabrJCfotZVuO4nZkhz334c73YPBcuO+TEllNxep1k/0jP3kAql4UcpKaVcKZ/fQ1/PPbn/l0ZRJfrtlLuNdD6wbVaBoTRdOYKjSNrkLLi6rRuHZlJLhNPtStBHf+AP4M2LPaXv+7tBzeZr8gR3flNiMEh/ux3VCtXumVpyApR6FmE/u8SnTJHYzN7nFTGk03/iw4HrjcU3mv0R/dVTblKU6nD9tA9PgqVo3e74cjO6F5zxJfVcUK+uwf+amz17Jiqkbwh1vjGNWrJSt2HmHOhv2s/eUY321OZtqK3Pbti2tV5toWMdwQfoCuwNo9x6lzMoM6xs/BE2lUjfQRsXetPfU3uxdGaTmyHS67DTCha5dHd0GjzqVbplCC+3ZXjravszLAG1a868muwVcphaabE3vBn2l7tZS3oM++9AEE2uiPlU15itPe1fbx0uth89ew+kOY+XRuE+1FrWHIt/YAtJuc3AdZabkVoRJUsYI+u0bvsHnA5/XQpWltujStnTPsRGoGOw6eZvXuIyRsSuaTxCR2Zm2jazj8z4yNXOPdzVM+w+WvzAGEz8MX0s4D8xYv42/rF1I7Kpw2DarTtHIK0f6DZNZpS1Skj6gI+1clwkfVSF/R+/IHSzlqm6dqNbVf9p/nBwof1NXv2O6Qs5YqYwJNNzXs6yqBz/n0oQL3uM5ZdttmaTTdZO9BXdQa9q6xNTNPAa2g+zfAfx+DAZ/aPZqSknoMwquCN+gnXVF63fyy0j626W+Dfs7/2O9UhwFw+Gd7Z619ayCssu2dk5/HBy1vyT23oLzIrhxq0BdRcNPNOaoaGUabhtVp07A6A69sQmpGFsfWpMMX8L+3t6XyrkOwHv6vdxwnUtOJW5gEBi4NO0j9GpHsO57KP7/bxv95xtPJu4R2ae/iD3EoJMLnoVqlMKqEe6kU7iMqwkuT2lWoUy2CSJ+XyDAvkWEeIsICz32ewDAvNY+tpxmQHNaAypHHqXxiH+lpKYSf3I+EVwWPNzeMiupkMnz5DBwI3O1RPHDTH6HZDUVfVsZpW/PN6XWTvSFOLoGgDwR7UZpuDm+zQVwlGi6+wvm6sjei9TvatuOUI7kbsfw2f21rpD8vgLb9na+jqIL3nLJFVAucSGXcV9sNtmuJPeGu4eX29alk6PY76P68/a2vnw6bvoa1H8OhAu6q1XUk9Hjh3MuwZ7XtzVY3zlawioMG/TmKqAreiGLdnY4M8xIZFQ5Am4Y1ILUarIeBDfbZH9J3aRBZncZygPcesF/E1IwsfGNfxHc4hS8HNORIZCNOpGVyKi2Tk2mZnEjN5HhqBsdTMjiVlkXE6X30O/Amow8OYvPpKLL8Zz8p5BbPEsaEw8DpybTxHOevYYYbRv+H3/pW08YTRapEcGjlKv7wUwIRORsI+xgTFUGT6CpUCvMS5vMQ4fUQ5hMqZZ3kkl3TaLzlX/jSj3Gy8Q2Ix0vl3d+S+sN4kmteRdjJPfir1sfj9RDmEapVCit4z+T0YfsHth892KYbKJnmjox8bfSF9V4wBj7oZ2uEAMOWQ0xz+3zfOrsnEt3Cnm2dX3a7d4OOkDjBNhVWqgHTh9omtXrt4ZbXAstaax93/lDyQZ/9OWeLrGZ7YWWchvAqJbdusG3Nnw+zPZ/AhlfvtyCs0vktN/20PQu50yCo3sieCJaZCu1/bcdH1YEGnWDRW5B+Am79h23iCfbfx+zGoPsfCt7gpR6DaQ/b//dLesC1o3Kn3ToPPrjDPg+rAoNmQf325/e+wAa9eOz7KmEVK+hF7H98cQdJZkru8rOvIfJ+L6gWuFtii5vhx//k1KoiM4/DYXvQ6DLPbri0w5nLzMqAuaOhc19I/AJ2L+Kr7j2g20gysvykZmSRmmEf0zJzn6dm+Lnox+WwDh69vQdRB2vD8vE8c3kk7Xek4zd1MVKVS9KSuCbmFPFHZtHoxCZSTASTK91Hwp46JAcdh8j2dtgbNPMuZZm/Bf+T8Qw/bbgYgP/1nabf9m954e9v8u/wV5mQ2Yv/yxyYM1+Ez0O9iDRGmA9I8P2KteHtuZSd/P3Ucxz0xNAE+PvCA6xctZSanOQNhJlfzmBOTE3CvEKYx25owryewJ/g83gI93nweYRwj6F26k58Hg8p1Zvi8/kI93rwBabNni96z36aAttPhdHE4+PE8WOcOpZixwetw+cRe4D94BYb8p0fhWX/tDXvmOa299Q/u9k31/By2xMiv2O77clfNRrb1ycP2Jr92o9twC1/F7qNtHst+9bYaXYuOvP/P/jAYq2m5xeKwecr5PznBL6rqcdLPuiXv2vfY9Nr7MHqtZ/Y93/jy6GnTzliD2gn/wQbZoRuaqvVFGK72mBvfqNtHruojW2iqRWbO12zm+zZpdUa2Oac/Md/Wt9h2/T3r7Pzh7L2E9g6x26kv33VdsBodbv9f/pihO0C2ftt+HQwTOkPQxKg+nne/vrITqjWEHzh57ccBypW0EPuSVPGwKp/2wNm7e+1/azFE7otNSvDtuOF2tqf2Aez/8ce5KvVFOq0sl+WHd9Dwh/tHsSl19ug3/6d/cI1CmoG2L8e4nqfudwNn9u+8Cv/bWsi2cO6jcwJr6qRBbzH9ckQdRF9uzSHwz5YDrfH+uGX47ZsVepA4mJG73zA1ujqtoKjm7nC/wo8MYeUyLqkZWaRnuUnI8uQnpHFxROGcbhRPzKv+DO/ScsiI8uP3xii956g8uI5jK82EZMqPOz7ip71T3MisgELGj/BiZQM7tw4gktOraJP+lzWhP+KJikbiDKniMqyP95TniqcTs/kuL8Km73NaHb0B/6e1oeMLEOm35YhI8tPRpafzCxDZtAezSPemQwK+xCAlzPu472sW874OMLI5Hbvj/w1DAb+ez1fhofz2ZLNvPT9/JAfX5hXGOz9kmc9cHNie97gYo7NncoT37ViZOa73Eo483xd6Zk0n35/m8lpbzV8gQ2QzyO8engxmVKPybP38Udg0lc/0PvIJE5GNufTGiN5+sgQPv5kCpuqd+X5Qz+T6q1G5YOb+Pv0H0iNqIXg576fhnPx8ZU5ZdpSuzuzW/8Vr0fwiuD1CD6v4BHB5xE8HvvoFfCZdCSsEh4Rqp7aSeymCcTsXcfxOvFs2noQEcEjEH3cyyXAhh1JZNSshDcrjTprxhBxfBcZ1Ztw5PJnEI8Hj4DXY9clAh6xzyN2f0ul9R8hvghSuz0PUXUD0+VO4xHwkoX8+JHtPXKv/b/ii6dg8RhocQs0vjLvf8DR3fYckOwDyFXrn9mUZ7Js8K6bBuFR0PhqO/ye/5wZ5C162d9il0dDH+RveRt8+RtY/R/oWj90M9vqD+1ve0gCTH8UvvuL/QNA4OHZtnPDgGkw/lob/vd9UrQmMWNsTvy8wL7e/q3dgJSCihn0h7fBx/fDxhk23MMrwzd/sAdker2ad3pjYHIfW5v69Sd5NwTGwGeP2IOHg77MrTE1udr+YezeQ/YFu2Y+bV/XamrXG3URHAhxJp8x9kdQo7G91oXxQ5chsPBv8NMsu6tq/NDpQahz2ZnzH9meW6Op1gAQ++M5ecD2ra/V1LaNN+8Ft/zN1jz2rrF7IdMeptKgWVQK8+R+SQ/9DKlHqNXiV1x1Sb4Dhpf1geVPUSn1AHT7LZw+TIMtc+DAt7S8vIdtyz+1Cm57E8+eVbTflgBVG0G738HXzwLwQr+roH5gr+bb/rDgjyQ81jpvs0hQO7Lfb8M+I8tPxIQ/kmlaYYzhd7KGe29/iWqrx3EiugPHoztQZec8Yhc/R2ZYFJyGF+/sTPjcqlxbJ4rI1m3IzPITcWIXLXZNxRjDyrp3cjC8IXet38j+jEvo1LItv+y5hq4HPqRnEw+9tnzPumpd2VT9dm7ePY+eVbawvHJXsgJl8macIjZjC59V7s+uNFtL7n5oCrX8+3nF9xjL9tTiIaKI3L2Qn5PC8WD4T9Z1PMznJK36hvmmI/2Zx8XelbyZdQcb/RfT07OMWw4m8P43yziI/Y5FkM4A71ymZ/2Kw+QeRHza9wmPe2eQ4G/HflOTq72L8ODnF1ODd7dfzJStS3OmvdaTxKRwWPjx60SQQRfPRup4drPX1KKeTGfk4jAS/O3P/H4F1r8g4hm8pBJFCuNWpfJ6Zr8804STwSPeL4mQdJ70HWDoupbMf/4rPB6oJt2Y5plF1Pv9eUReYp/UBRG8YvhL1l9pa9L4S9jTHPTUZl1WHHLcG7SRsY+jwl7j2tPfsjj8Sl79Z2LQBoacjVn2xunSOm+y86fmsHlZyGlGRLan5ZIxsGQMC2r24+voQRiPDwHqpO/iN7+sYEbd4Xz/6Rp8DKVZk3jCTBoCHIlswP4VlfGsXIeIcMVFj9Jz6+usf6s/KWHVMeJlbZ0+HK5yadB6BZ9JJ5x0Gh9eRL3ja6iemsTFh77nVEQMWZ5IIJyNlbuydelO+z6ASuFe+rQ/zz2FEOTcL91bcuLj401iYuK5zfz5MFj1gb2A1nXP2Qt8Hf/FjgurAiM3w08z7W557UtsLfxft9nxN78GnR+xwffLCntlwin9oNdfbRAXJOUo/DmwG599QlXd1jaM92+AJ1fmnX7nYni/pw3hZjfZsxcjqsE/Wtvx3gi7jPod4MEv7XUwGnS0bZHpJ+EfbWxz0e3v2OlfawGNr4L1n0GPF+GKx2H3Eoi9Jm+NY/kEe6D11tdh8TvQ7m4b3j9OtbWYxxbZ2n9+H95jrzHy9HrbNOb3wxttbdvigfXQ9Fq4a3LeefxZ8LeWtv36yVW5B7D2rIbx18DtY207a8oR+Hy43fN5ZD5UrpW7jEM/w1sd7cFg44fZf4D2A2D1B3nXFVYl94Sp5/fb2mKtpramF3c7TLrVNpOI2L2dPm/b/9ernoDrR9uDfRNvsrXGnT/AgM8gthv8uQm0vct+Xtl+XgD/vt32omnaHV6OsRvV2G5w/wy7jo/vh6RE+NXTMGukff/vXJXbBAhw6Q05NUJz4CfknS5kXv9/ZHQeRqbfT/j80UQse5u0pjdyuPe/yPSDnNhD/clXk1GjKZJ+Ak/6CVJrt2Jn17+SWrkefmM3kn4Dxhiq7E+k3Zy7AcgMiyI9vCYbO/yBg3Wu5Fff3EKmrwrfXvcpVY9spNGu6TT85WuOVWvGknav0OiXr+j402vMvvw9Wm2bQLVTO/joqpkYPGQZg9+fxfUbnqd58mwATvtqMrbTTDLw4TeGLL+h+uldPLz5UapkHj3jK/VZvaf5oWZfjDH4jS2zP/u53z6vnHmMEftG8d/q97O6Uhf8xmAC02X5DQYC85/56M/z2lAnax/tM36kedZWemd+c0Z5MvHSO/xdDlMDQ/a8ALnrNNjPF+PnVd7gKtZigMqkcYJK9Mv4X3aaOnhNJiN9H/OI90u8YvP1pIkkhQgmZd7EO1m9MQWcqxodFUHiH64POa4wIrLCGBMfapyjGr2I9ATeALzAe8aYV/ONl8D4m4HTwIPGmJVO5i12ja+2V7m77R+2J0WDeJjxJHS8Hxa8DJ88aHcJw6NsgGz8wra31m1tr/xYv6MNvUNb7FmnNRrbmvXZVKphu3ulHYfbx8H0IdAw3l6xceNM2/6Y3UZ6Yj98NsTWvNvdm7ftNK6PPfjUZwxs/sruHk64Ibcf8cVX2hp8ytG8ZarRCJKW2+dRdSEs0oZvfh3vtwetZj5tX//wJnQZCruX2a552Xcnyq/nH+0eQ/aBTo/Hlj171/bK4WfO4/HaJqvl79kNZrZ67eyezsp/2zK+f3Pg2jwGvvodtLvHNqVd0sM2ZQFc1tuOn/0HG/LNe9rpjuy0y47tBmO62BPXfBG2DXfLbPs3+wW7cbx3qr2z1fs326D2RdruemA3+tEtbMjXbW3L5fFCk1/Btm/zvq+di+zGvFGXwIXaom1/6B4v5m5Um15ry75ysi1fzVi4+wPbRgy2jO3uyZle6rSEBvH4VkzE5/XZ78vyd6BmLBHbZlNvyf/Zjeq2BWCyiLjvw5w9uiggxKY58FnHw8pLod09+H71G3weD51yRr4Inw2m9xcd7JnU3nC49Hoit31Ln/k32JS7pAc33tIf1gp8+jCDGybZdv/pQ+2BxKx0e4CzbhsqV67Nbxq1zleAODg4F376Mu/gqvW4o01/7iioS2oey3nawVTO2I0eP8+3e7hBfDEtmdWiKCcu9cp9mryZiAk3kOD5DXh8GONH/Bn4295DZnRL/Be1w9u4K5UQHjWGIX7IMubMjVQJ3rmu0Bq9iHiBzcANQBL2ZuH3GmM2BE1zM/AENui7AG8YY7o4mTeU86rRF8QYeDvedr9qeLn9sWZfKKnb7yB+ELzbw55darLg8kdse1qfMfagTGE+vBsq1YK+Y2HNx3Yd+9bCxwPtj6FJV9us9NEAG1APzrS19IJkZcI7XWx5r3nWzjt3tA2tK4fDTa/kTvvlSHswDOC+T6HZWWoEG2favZ6rR8C8l6Dnq7Z9slJNeCD/Pd/P4vB2eLO93csYPC90W+XJA7bHQvt78w5fORlmPGE3LiYLBk63P75v/5w7TXiUDfyL2sAjgSsWvtvd9ql+fPGZTVrzX7Eb7WFLYGIv2LUIWt9pa+uXdLe1eLAbtX1rIK5vwV0isy0ZC1+Pgns+tM1+AO/fYvcehiTY11Pushv6O8bnznd0N7x9ua3Bx90Od/3r7OsB+PEjW0HIFt3ctgt/MsgGfLb8//fnyu+HJWPsOSc1LoZWfe3eVPJmWPNRoOnwAXtwOSMV/tbc7qVlptoKTOs7bBnb3+furpvFZd86ezwhO08bdYGWJX577DzOVqN3EvRXAqONMTcFXj8HYIz5U9A0/wQSjDH/CbzeBFwLNCls3lBKJOjBtosv+KM9i65WU9sWvv1buOpJ+2Pdv97W+Nr0s80qZzsRJr+c2w0GfemPJcEb7eyuvR1p2/nv+lfoGnd+e3+0TQDxD9nlHtxqu4ldNTxvDw1/FmxLsBuuXz1ja/Rnk/2+Jtxo33P6KdtLpPsfnL3XbInv2+alc+lqNv8V+P51W9Nt0RMy0+H7v0OdOFvj3TLH1tDb3m2bpQB2LbU9ZbK71gUzgX1tjwc+e9Q2KQ2eZw/Gi5xbGJ0+DJN72w12dHNbOTi4Ba54LDdsg9cbLCPVhmJENeffobQTuXesCo+yJz9l3zkKAr2+qpVNsK7/r93gRNawlYTgJjZVLpxv0PcDehpjBgdeDwS6GGOGB00zE3g1cDNwRGQe8Cw26M86b9AyhgBDAC6++OJOO3fuLOr7LJwxNtQiogqeJiPF7tYX14/p1CF7GvruZTa4r3jM1qDKg6RE2/NHvHDd73Pv61pagpu0ilNWpt1T8EWc/7IyUuG7v+ZeTtbjs3tYMS3Of9lKFaPzbaMPlXj5tw4FTeNkXjvQmPHAeLA1egflKjqRs4c8nP8JHvlVqW3/ajW17bLlScN46D+p7NZfUn27vT6KrUNZWOT5nVGpVDng5NeQBASfutUQ2ONwmnAH8yqllCpBzg57QzMRiRWRcOAeIP9RuxnA/WJdARwzxux1OK9SSqkSVGiN3hiTKSLDgW+wXSQnGmPWi8jQwPhxwCxsj5ut2O6Vg842b4m8E6WUUiFVvBOmlFLqAnS2g7EV61aCSimlzqBBr5RSFZwGvVJKVXAa9EopVcGVy4OxIpIMnOupsdGAs5vGXnj0swlNP5eC6WdTsPL22TQ2xoS4JVo5DfrzISKJBR15vtDpZxOafi4F08+mYG76bLTpRimlKjgNeqWUquAqYtCPL3ySC5Z+NqHp51Iw/WwK5prPpsK10SullMqrItbolVJKBdGgV0qpCq7CBL2I9BSRTSKyVURGlXV5ypqI7BCRtSKyWkQSA8NqicgcEdkSeKxZ2HIqAhGZKCIHRGRd0LACPwsReS7wPdokIjeVTalLRwGfzWgR+SXw3VkduCd09rgL4rMRkUYiskBENorIehEZERjuyu9NhQj6wE3Ix2BvzR4H3CsicWVbqnLhOmNM+6C+vqOAecaYZsC8wOsLwSSgZ75hIT+LwPfmHqBVYJ53At+vimoSZ342AK8HvjvtjTGz4IL7bDKB3xhjLgOuAIYF3r8rvzcVIuiBzsBWY8w2Y0w6MBXoU8ZlKo/6AP8KPP8XcHvZFaX0GGO+Aw7nG1zQZ9EHmGqMSTPGbMfeY6FzaZSzLBTw2RTkgvlsjDF7jTErA89PABuBBrj0e1NRgr4BsDvodVJg2IXMALNFZEXgxusAdQN3/iLwWKfMSlf2Cvos9LtkDReRNYGmnezmiQvysxGRJkAHYCku/d5UlKB3fBPyC8jVxpiO2OasYSLSrawL5BL6XYKxwCVAe2Av8LfA8AvusxGRKOBT4CljzPGzTRpiWLn5bCpK0Du5gfkFxRizJ/B4AJiO3Y3cLyL1AAKPB8quhGWuoM/igv8uGWP2G2OyjDF+4F1ymyAuqM9GRMKwIT/FGPNZYLArvzcVJej1JuRBRKSKiFTNfg7cCKzDfiYPBCZ7APi8bEpYLhT0WcwA7hGRCBGJBZoBy8qgfGUmO8gC+mK/O3ABfTYiIsAEYKMx5u9Bo1z5vSn05uBuoDchP0NdYLr9ruIDPjTGfC0iy4GPReRhYBfQvwzLWGpE5D/AtUC0iCQBLwKvEuKzCNz4/mNgA7bnxTBjTFaZFLwUFPDZXCsi7bFNDzuAR+GC+2yuBgYCa0VkdWDY73Hp90YvgaCUUhVcRWm6UUopVQANeqWUquA06JVSqoLToFdKqQpOg14ppSo4DXqllKrgNOiVUqqC+389y7ghyc99lAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "df = pd.DataFrame({\n", + " \"edit_frequency\": pageview_df.edit_count / pageview_df.edit_count.sum(),\n", + " \"query_frequency\": pageview_df[\"2021080600\"] / pageview_df[\"2021080600\"].sum()\n", + "})\n", + "\n", + "df.plot()" + ] + }, + { + "cell_type": "markdown", + "id": "1ca13ffa", + "metadata": {}, + "source": [ + "# Plot DPR Model Accuracy Results " + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "39b1975e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Finishing last run (ID:2s3jbe1y) before initializing another..." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
Waiting for W&B process to finish, PID 34365... (success)." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Label(value=' 0.22MB of 0.22MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "
\n", + "Synced 7 W&B file(s), 0 media file(s), 0 artifact file(s) and 1 other file(s)\n", + "
Synced divine-shadow-168: https://wandb.ai/ucb-ralf/wiki-workload%20/runs/2s3jbe1y
\n", + "Find logs at: ./wandb/run-20211012_194624-2s3jbe1y/logs
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Successfully finished last run (ID:2s3jbe1y). Initializing new run:
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: wandb version 0.12.4 is available! To upgrade, please run:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: $ pip install wandb --upgrade\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " Syncing run breezy-cloud-170 to Weights & Biases (docs).
\n", + "\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "run = wandb.init(job_type=\"evaluation\", project=\"wiki-workload\")\n", + "artifact = run.use_artifact('prediction_results:latest')\n", + "artifact_dir = artifact.download()" + ] + }, + { + "cell_type": "code", + "execution_count": 217, + "id": "101571e2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'/home/eecs/wooders/DPR'" + ] + }, + "execution_count": 217, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "artifact_dir" + ] + }, + { + "cell_type": "code", + "execution_count": 218, + "id": "03e14929", + "metadata": {}, + "outputs": [], + "source": [ + "artifact_dir = \"/home/eecs/wooders/DPR\"" + ] + }, + { + "cell_type": "code", + "execution_count": 219, + "id": "eaf30e01", + "metadata": {}, + "outputs": [], + "source": [ + "constants = [0.01, 0.05, 1.0, 10.0]\n", + "#constants = [0.25]\n", + "policies = [\"lifo\", \"fifo\"]\n", + "#key_policies = [\"random\", \"weighted_random\", \"round_robin\", \"weighted_round_robin\"]\n", + "key_policies = [\"round_robin\"]\n", + "#key_policies = [\"weighted_random\", \"weighted_round_robin\"]\n", + "d = artifact_dir\n", + "metric = 'top10'" + ] + }, + { + "cell_type": "code", + "execution_count": 231, + "id": "96209574", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/eecs/wooders/DPR/plan-round_robin_fifo-always_process-0.01-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_fifo-always_process-0.05-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_fifo-always_process-1.0-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_fifo-always_process-10.0-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-0.01-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-0.05-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-1.0-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-10.0-100.json\n" + ] + }, + { + "data": { + "text/plain": [ + "{'plan-round_robin_fifo-always_process': [0.39603393208873827,\n", + " 0.6827773461716538,\n", + " 0.8776121979738054,\n", + " 0.8791895221727837],\n", + " 'plan-round_robin_lifo-always_process': [0.39388374885232,\n", + " 0.46513799624895036,\n", + " 0.8024342585399157,\n", + " 0.8759956368544546]}" + ] + }, + "execution_count": 231, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "constants = [0.01, 0.05, 1.0, 10.0]\n", + "#constants = [0.25]\n", + "policies = [\"fifo\", \"lifo\"]\n", + "#key_policies = [\"random\", \"weighted_random\", \"round_robin\", \"weighted_round_robin\"]\n", + "key_policies = [\"round_robin\"]\n", + "#key_policies = [\"weighted_random\", \"weighted_round_robin\"]\n", + "d = artifact_dir\n", + "metric = 'top10'\n", + "\n", + "event_results = {}\n", + "for policy in policies: \n", + " for key_policy in key_policies: \n", + " scores = []\n", + " name = f\"plan-{key_policy}_{policy}-always_process\"\n", + " for constant in constants: \n", + " print(f'{d}/{name}-{constant}-100.json')\n", + " with open(f'{d}/{name}-{constant}-100.json') as results_file:\n", + " results = json.load(results_file)\n", + " scores.append(1-results[metric])\n", + " event_results[name] = scores\n", + "event_results" + ] + }, + { + "cell_type": "code", + "execution_count": 232, + "id": "332c0ff6", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmgAAAFCCAYAAABFMCGEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA3oUlEQVR4nO3deVxU9f7H8dcwgIJLConiirskLrhboqZezYRwySXTUpNSKsrKXXHL3HPLJc2dzLQ0Fc0sy27d3MXU0NxTZDFBNBe2YX5/WNwfF0RcGI7wfj4ePh4z53znnM/MfIG337N8TVar1YqIiIiIGIZdbhcgIiIiIukpoImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMHkmYBmtVpJTExEF6WKiIjIoy7PBLSkpCSOHj1KUlJSbpciIiIi8kDyTEATERERySsU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0ICkZEtul5Cj8vr7ExERyWvsc7sAI3B0MNMzeGdul5FjQoKfBMy5XUaOSU1Jws7e0Wb7S0q24OiQNz/PvPzeREQeJQpo+YCdvSMnpvfJ7TJyTLX3ltt0f3k50K8e3zK3SxAREXSIU0RERMRwFNBEREREDEYBTURERMRgFNBEREREDEYBTURERMRgFNBEREREDEYBTURERMRgFNBEREREDEYBTURERMRgFNBEREREDEYBTURERMRgFNBEREREDEYBTURERMRgFNBEREREDEYBTURERMRgFNBEREREDEYBTURERMRg7G21o7NnzzJs2DDi4+MpVqwYU6ZMwcPDI12b2NhYhg8fTlRUFMnJyTRp0oRRo0Zhb2+zMkVERERync1G0MaMGUPPnj355ptv6NmzJ8HBwRnaLFy4kMqVK7N582Y2b97Mb7/9xvbt221VooiIiIgh2CSgxcbGEh4ejq+vLwC+vr6Eh4cTFxeXrp3JZOLGjRukpqaSlJREcnIyJUuWtEWJIiIiIoZhk4AWFRVFyZIlMZvNAJjNZtzc3IiKikrXLjAwkLNnz9KsWbO0f/Xr17dFiSICpKYk5XYJOSqvvz8RyTsMdXLXtm3bqF69OitWrODGjRsEBASwbds2nnnmmWxv4+jRo/e8X4XAR9+BAwdstq+83F/s7B05Mb1PbpeRY6q9t9ymfUVEJCtZ/T2xSUBzd3cnJiYGi8WC2WzGYrFw6dIl3N3d07ULCQnhgw8+wM7OjiJFitCqVSv27NlzTwHNy8uLAgUKPOy3IAaXl0OTPFzqKyLyKLDJIU5XV1c8PT0JDQ0FIDQ0FE9PT1xcXNK1K1u2LP/+978BSEpKYteuXVStWtUWJYqIiIgYhs2u4hw7diwhISG0a9eOkJAQxo0bB0BAQABHjhwBYMSIERw4cAA/Pz86duyIh4cH3bp1s1WJIiIiIoZgs3PQKleuzLp16zIsX7x4cdrj8uXLs2zZMluVJCIiImJImklARERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREckhSsiW3S8hRef39ieQm+9wuQEQkr3J0MNMzeGdul5FjVo9vmdsliORZGkETERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRibBbSzZ8/SvXt32rVrR/fu3Tl37lym7bZu3Yqfnx++vr74+flx+fJlW5UoIiIiYgj2ttrRmDFj6NmzJ/7+/mzcuJHg4GBWrlyZrs2RI0f46KOPWLFiBSVKlOCvv/7C0dHRViWKiIiIGIJNRtBiY2MJDw/H19cXAF9fX8LDw4mLi0vXbvny5fTr148SJUoAUKRIEQoUKGCLEkVEREQMwyYBLSoqipIlS2I2mwEwm824ubkRFRWVrt3p06e5cOECL774Ip06dWL+/PlYrVZblCgiIiJiGDY7xJkdFouF33//nWXLlpGUlET//v0pXbo0HTt2zPY2jh49es/7rV+//j2/RozlwIEDNtuX+sujTX3l4bLl5ymS12T1O8ImAc3d3Z2YmBgsFgtmsxmLxcKlS5dwd3dP16506dI888wzODo64ujoSOvWrTl8+PA9BTQvLy8dFs2H8sMfQnk41FceLn2eIjnDJoc4XV1d8fT0JDQ0FIDQ0FA8PT1xcXFJ187X15eff/4Zq9VKcnIyu3fvpkaNGrYoUURERMQwbHabjbFjxxISEkK7du0ICQlh3LhxAAQEBHDkyBEAOnTogKurK88++ywdO3akSpUqPP/887YqUURERMQQbHYOWuXKlVm3bl2G5YsXL057bGdnx/Dhwxk+fLityhIRERExHM0kICIiImIwCmgiIiIiBpOtgJaamprTdYiIiIjI3+4a0CwWC3Xr1iUpKckW9YiIiIjke3cNaGazGQ8PD65cuWKLekRERETyvWxdxenn58eAAQN46aWXKFWqVLp1TZs2zZHCRERERPKrbAW0zz77DIC5c+emW24ymdixY8fDr0pEREQkH8tWQPv+++9zug4RERER+Vu2b1SbkpJCWFgYMTExlCpVirp162Jvb6i51kVERETyhGwlrNOnTzNw4EASEhJwd3cnKiqKAgUKsHDhQipXrpzTNYqIiIjkK9kKaOPGjaNbt2688sormEwmAJYsWcLYsWNZtWpVjhYoIiIikt9k60a1x48fp2/fvmnhDODll1/m+PHjOVaYiIiISH6VrYDm5ubG3r170y3bv38/bm5uOVKUiIiISH6WrUOcgwYNIjAwkJYtW1K6dGkiIyPZuXMn06ZNy+n6RERERPKdbI2gPf3002zYsIGqVaty48YNqlatyvr162nTpk1O1yciIiKS79x1BM1iseDt7c3+/fsJDAy0RU0iIiIi+Zrm4hQRERExGM3FKSIiImIwmotTRERExGDuGtBSU1OZOHEi9evXx9HR0RY1iYiIiORrdz0Hzc7OjsDAQIUzERERERvJ1m02GjZsyKFDh3K4FBERERGBbJ6DVrp0aQICAmjdujWlSpVKN+XTW2+9lWPFiYiIiORH2QpoiYmJaTeljYmJydGCRERERPK7bAW0SZMm5XQdIiIiIvK3LM9B+/rrr9M9P3PmTLrny5cvf+gFiYiIiOR3WQa0kSNHpnveo0ePdM/nzJnz8CsSERERyeeyDGhWq/WenouIiIjIg8syoP3/qzWz81xEREREHtxdLxKwWq1p/zJ7LiIiIiIPV5YB7ebNmzzxxBNpz61Wa9pzq9WqETQRERGRHJBlQNNE6CIiIiK2l2VAK1OmjK3qEBEREZG/ZWsuThERERGxHQU0ERG5L6kpSbldQo7K6+9PjC1bUz2JiIj8Lzt7R05M75PbZeSYau8tz+0SJB/TCJqIiIiIwdx1BO3zzz9nw4YNnDx5kps3b+Ls7EzVqlXp3Lkz3bp1s0WNIiIiIvlKlgFt2rRp7Ny5k759+1KjRg2KFCnC9evXOXbsGMuXL+fChQu8++67tqpVREREJF/IMqB9+eWXbNq0CTc3t3TLa9asiY+PD88995wCmoiIiMhDdk+TpYuIiIhIzstyBO3555/n5Zdfpl+/flSvXj3tEOfx48dZvnw5Xbt2tVWdIiIiIvlGlgFt8ODBlCtXji+//JJTp06lXSRQpUoVevfuTY8ePWxVp4iIiEi+cderOHv06KEgJiIiImJDD3QftMjIyIdVh4iIiIj87b4DWlJSEq1bt36YtYiIiIgIdznEuW/fvjuuS0rSHGUiIiIiOSHLgNa7d29KlCiBnZ1mhBIRERGxlSwDWunSpZk+fTr16tXLsC4xMZG6devmVF0iIiIi+VaWQ2NeXl4cPXo003Umkwl3d/ccKUpEREQkP8syoM2YMYMXXngh03WOjo58//332d7R2bNn6d69O+3ataN79+6cO3fujm3PnDlDnTp1mDJlSra3LyIiIpJXZBnQHBwccHBweCg7GjNmDD179uSbb76hZ8+eBAcHZ9rOYrEwZswY2rRp81D2KyIiIvKoydbZ/0lJScyePZu2bdtSt25d2rZty6xZs0hMTMzWTmJjYwkPD8fX1xcAX19fwsPDiYuLy9B20aJFtGzZEg8Pj+y/CxEREZE8JFsBbezYsezevZuRI0fyxRdfMHLkSPbt28fYsWOztZOoqChKliyJ2WwGwGw24+bmRlRUVLp2x48f5+eff6ZPnz739CZERERE8pK7TvUEsGPHDr799luKFi0KQJUqVahTpw5t27Z9aIUkJyczevRoJk2alBbk7sedLmrISv369e97f2IMBw4csNm+1F8ebeorci9s2V8k/8nqd0S2Atrjjz/OrVu30gIa3L7NRokSJbJVgLu7OzExMVgsFsxmMxaLhUuXLqW7CvTPP//k/PnzvPrqqwBcu3YNq9XK9evXmTBhQrb2A7evPC1QoEC220veoD+Ekl3qK3IvbNlfkpItODrc/wCFkaUmJ2Hn4JjbZeSY1JQk7Owf7vvLVkDz9/enf//+9O7dm5IlSxIdHc2nn36Kv78/u3btSmvXtGnTTF/v6uqKp6cnoaGh+Pv7ExoaiqenJy4uLmltSpcuzZ49e9Kez507l5s3bzJ06ND7fW8iIiKPDEcHMz2Dd+Z2GTli9fiWnJjeJ7fLyDHV3lv+0LeZrYC2Zs0aABYuXJhh+T/rTCYTO3bsuOM2xo4dy7Bhw5g/fz5FixZNu4VGQEAAQUFB1KpV677egIiIiEhek62Adi/3O7uTypUrs27dugzLFy9enGn7N99884H3KSIiIvIoylZAA0hJSSEsLIyYmBhKlSpF3bp1sbfP9stFREREJJuylbBOnz7NwIEDSUhIwN3dnaioKAoUKMDChQupXLlyTtcoIiIikq9kK6CNGzeObt268corr2AymQBYsmQJY8eOZdWqVTlaoIiIiEh+k60b1R4/fpy+ffumhTOAl19+mePHj+dYYSIiIiL5VbYCmpubG3v37k23bP/+/bi5ueVIUSIiIiL5WbYOcQ4aNIjAwEBatmxJ6dKliYyMZOfOnUybNi2n6xMRERHJd7I1gta6dWvWr19P1apVuXHjBlWrVmX9+vW0adMmp+sTERERyXeyNYK2ZMkSXnnlFQIDA9MtX7ZsGX379s2RwkRERETyq2yNoM2bNy/T5QsWLHioxYiIiIjIXUbQ/plnMzU1ld27d2O1WtPWRUREUKhQoZytTkRERCQfyjKgjRw5EoDExERGjBiRttxkMlGiRAlGjRqVs9WJiIiI5ENZBrR/5uAcMmQIU6dOtUlBIiIiIvldts5BUzgTERERsZ1sBTQRERERsR0FNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRh7W+3o7NmzDBs2jPj4eIoVK8aUKVPw8PBI12bevHls3boVs9mMvb09gwYNwsfHx1YlioiIiBiCzQLamDFj6NmzJ/7+/mzcuJHg4GBWrlyZrk3t2rXp168fTk5OHD9+nF69evHzzz9TsGBBW5UpIiIikutscogzNjaW8PBwfH19AfD19SU8PJy4uLh07Xx8fHBycgKgevXqWK1W4uPjbVGiiIiIiGHYZAQtKiqKkiVLYjabATCbzbi5uREVFYWLi0umr/nqq68oX748pUqVuqd9HT169J7rq1+//j2/RozlwIEDNtuX+sujTX1F7oX6i2TX/fSVrL5zmx3ivBd79+5l9uzZLF269J5f6+XlRYECBXKgKjEy/WKT7FJfkXuh/iLZ9bD7ik0Ocbq7uxMTE4PFYgHAYrFw6dIl3N3dM7QNCwtj8ODBzJs3j0qVKtmiPBERERFDsUlAc3V1xdPTk9DQUABCQ0Px9PTMcHjz8OHDDBo0iDlz5lCzZk1blCYiIiJiODa7D9rYsWMJCQmhXbt2hISEMG7cOAACAgI4cuQIAOPGjSMhIYHg4GD8/f3x9/fn999/t1WJIiIiIoZgs3PQKleuzLp16zIsX7x4cdrjL7/80lbliIiIiBiWZhIQERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDsdlk6bkpOTmZiIgIEhIS7thmYPvHbFiRbR07doxkn1dyu4wcc+zYMZvvM6/2l0eir1itmK7H4vDbduySb+V2NSIiOSJfBLSIiAiKFCmCh4cHJpMp0zZnLv5l46psp1KZIiREn83tMnJMwVIVbb7PvNpfHoW+YrVauXLdlVigwKGNuV2OiEiOyBeHOBMSEnB1db1jOBORR4fJZKJ4YSeshV1zuxQRkRyTLwIaoHAmkoeYTCbQz7SI5GH54hDn/0pKtuDoYE63rFKZIg+83VuJKURd1jkxIiIi8mDyZUBzdDDTM3jnQ9/u6vEtH/o2RUREJP/JN4c4HxXtWzXg1q2buV3GA7kYFU2L57o9tO0tWLaKGfMXZ7pu7cYtLF++/L63ffDgQXx9fenYsSO7d+8mICCA8+fPA3Du3Dk6duxIx44d2bRp033v43/lhe9YRERyVr4cQcvPUlIs2Nub797Qhh6kpm7+HR7oKs6NGzfSsWNH+vfvD0CTJk3S1m3fvh1vb2/GjBlz39vPaywWC2azsfqPiEhepICWC9q3asCLLwVwcP8erl2Lp0//12nWvHWGdosXzOLI4YOkJCdT9LFiDBocTMlS7sRERxI0oDft/Tqzb89/SExI4O3BwXjVqpthGzHRkbzQ+SV6dOzA7gNhdPhXa5rU92bCjDlciY/HbDYTFNCXpxo34GJUND1fC+LHTWsB0j3/5/Hzfs/y0+59JCQmMHbIIOrV9gJgzYZNhKzbwOMuLjTwrn3Xz2D0pOk4Ozlz/uJFrsRfZc3ij1i6ei2h23cAULN6NYa/FYizsxMA0TGXeH3IaCJjYqhYvhzjhr5DkcKFWLBsFUl2BRg6dCjr168nNDSUokWLcvLkSYoUKcLcuXMpUaJEpjV88sknfP311xQsWJDNmzfz+eef8+yzz7Jw4UKOHz/OihUrSE1N5eDBg8ydOxer1UpwcDBxcXFYUk306f86DRo9mem2v1wbwo8/bMdiScHRsQBvvD2MylWqp2tzYN8uNq5fw/hJs4m/EscLXdoyIngyPi3bsG7NCm5cv06f/q/fsR98NGsypdzL8Hz33gCcOnmcyRNGsHjFl3y9ZQNffbEaBwdHUq2pjAieTLnyHpnW+u22zfyw42ucnQtx+VIkRZ0LMnHkYEqWeJyNX29n2/c/Uvyxxzjzx3nGDhlEbNwVZi9aRmqqheLFijH63SDKly0NwIYt37D6y68AcLC3Z+7k8bi6FOen3Xv5ZNUaEpOScLC3Z/Abr1G7pifnzl9g9KQZJCQmYklNxf+Zf/Fyj+f54eddfPTJCsxmO1IsFoa/FUhD7zp37VciInmFAlouMdnZ8eFHS4k4f453gl7Bq5Y3xYq7pGvTrWcfAga+DcC2LV+xdPEcho+eBMC1a1fxfKI2fV55ne+/+5pli+YwY+7STPcVHx9PxQrlGdj39h/yFwe8RRe/9nTu8Aynz/1Bv6DBbFi56K41x1+9Ru2anrwZ0Ict337P7I+XsmLeh5w4fYZPVq3h80/m4epSnIkffpStz+Bw+DGWzJ6Gs1NBft69j9DtO1g570MKOTsz6oPpfLxyNYMG3L5p6sHDR1m7ZD6uLsUJnvwhi1au5t3AgAzbPHLkCJs2bcLd3Z1Ro0YREhLCoEGDMt1///79OXXqFF5eXvTq1Svduueee44//viDmzdvMnToUAC6du1Kt27d6Nq1Kz/851eGvB3Ax8u/oFix4hm23bptB7p0u73NsAN7mDtzErPmLU/XpmYtb6ZMHEVKSgqHDu7F84naHArbi0/LNhw6uI/ne7wE3LkfPNepO2NHDqJLt16YTCY2f7UWX/+umEwmlnw8m4VLPqeEWymSkpJITU3N8rv47civzFv0Kc2b1mLW5AlMnbuQGeNH3a7/yG+sWzKfcmVKE3slngHvjmDJnKlU9qjA+i3bGP7+FD5dOJt9Yb+y5NM1LJ87g8ddXbh58xZms5kLFyNZtGI1C6ZPpHChQpw6e47Xh4zmm3Wr+PyrUJo1achrL78IwLW/bt9fbv7SlYwc9Ab16nhhsVi4lcVNpkVE8iIFtFzSrr0/AGXLe1ClanWOhx+hyVMt0rXZv+c/bN64joRbN7FYLOnWOTk507ipDwA1PGvxyYJZd9xXgQIFaPd0cwBu3LzJ76fO0LF9WwAqe1SgepVKHP7tOFUreWRZs7OTEy2ebAxA7SdqpJ0Xti/sMD5NGuHqcjuodPFrz/ad/77rZ9CmRTOcnQoCsPtAGM+0akHhQoXStjF17sK0ts2bNk7bfqcO7Zg8e36m26xXrx7u7u4A1KlTh19++eWudWTH9evXOXbsGF26dAGggkclKlX5+3t7snmG9qdOHOPzT5fx119XMdnZcfHC+QxtChYsSIUKlTh+7ChhB/fS86X+LPl4NsnJyZw8cYyaXrdHjO7UD8pXqEgp9zLs3/sLNZ6oxe5f/s2rA9+5/d7rNuTDqeNp8mRzGjVphnvpslm+v5q16lD27xG2Th2e4fm+A9LWedeqSbkyt0fIjoQfp1qVilT2qABAx/Zt+WDmPG7cvMlPu/fi27YNj7ve/o/GP6Ofv+w9wIXIKPoFDU7bpsViITbuCvXr1GLG/MUkp6TQ0LsOjf4eJWtUry7T5y+ibUsfnmrc8K59U0Qkr1FAMwCrlQz3dIqJjmLR/A+ZvWAlpdzLEH70V6ZMHJW23sHBIe2x2WyHxZICwGchS/j5x9uHCV8NfIdS7qVxcnJKuw+c1WrNtAaTyYS92ZxupCUpKTldG0fH/+7Tzs4uQ2i8V85OTmmPrVZrhnvV3enedZm1/UeBAgXSHpvN5geu8W5MJhNnz5xi+qRgAGrXrU+/V4OYOHYo02Ytpkq1GsRe/pNe3dpn+vq69Rry68G9HA8/yhtvD6dYcVd27thGpUpVcXQscNd+4N+5B6Ebv+D8H2d5yudpChUuDMDo8dM4cfw3fg3bz7B3BvDGoOE0bPxU9t7U/3y+//97Aism7vS93GFzWHmqUQMmjhycYV2bFs2oXdOTXfsOsPTTtXy19RsmjRrK4Dde4+Tps+wN+5XBYybSu1tnuvhl/hmKiORF+TKgJSVbcuSWGLcSU7Ld9tttm3ihd38uRpznzKnfqeHplW79zZs3sHdwoLiLK6mpqWzd/GW2tvtCr1d4odd/51KMiY5Mt75woUJUr1KJTdu+o+OzbTn7xwVOnDpLrSeqU7RwEVJSLJyPiKR82dJs/e6HbO2zoXdtln22jtgr8bgWL8aGLd9k63X/X9MG9Zi5cAk9u/jj7OTEhi3baFzfO239T7v3Ehcfj0uxYmza9q3Nz0cqXLgwnp6ebNiwgS5dunDh/DnOnD5BdU8vihUrzrzFq9Pa3rhxHYvFwuNuJQEI3bjujtutU68R0z4YTbnyHjg4OFC3XkNCVizimQ4dgbv3g4aNn2LxgpmcPnmc8ZNnA2CxpBATHU11Ty+qe3oRFRnB6VO/ZxnQwo/+ysWI81QqU5ONWXy+tWt6MnbKLM7+cYGKFcqxadt31KhamULOzrR4sjFjp86k63PP4upSnJs3b2Fvb6Zpw/osXP4pp86eo0pFDwCOHvsdL8/qnI+IpGzpUvi3b0v5smUInvwhAOfOX6Bq5YpUrVyRm7du8dvxEwpoIpKv5MuA9r83qQXbz63o4ODIu2/24+rVeN58Z0SG888qVqqCT4s2DOjXnRJuJalVpz5HDoc9lH1PGjWUCTPmELJuPWazmYkjB+NSrBgAQ94cwGvvDqd0Kbdsh6BqlSvRv1d3+rz+Dq4uxfFp2uiea2rWpCEnzpyld+Dt88VqVq/Kq71fSFvfqF5dxkyeSURUFB7lyvJu4Kv3vI8HNX36dIKDg1m+fDmWVBODh4/P9PyzQoUK07vva7w18CXc3ErRoHHmFxIA1PD04trVeOp6NwSgrndDln8yL+353fqBnZ0drdt2YP/eX6hUuRoAFksqH04dy43rf2Ey2VHCrSR9A97I8r3VqlOfkOUfM23iubSLBDLjUqwYE0cOZtiEyVgsty8S+GDkEAAa1K1Nvxe78+o7w7GzM+Hg4MDcSeOoULYMH4wawtips0hMTCQ5OYW6tZ7Ay7M623/4N1u++x4HewdMJhj65u1Dq7M+Xsb5ixexN5spUrgQY4dkfh6hiEheZbLe6ZjXIyYxMZGjR4/i5eWV7jAXwLFjx/D09Mzy9bYMaO1bNWD9ln/j5ORsk/09ChNgP4j8Pln6iMGBtO/QGZ+Wbe7r9d9u28ye3T8xauzUR6qvnPgjgoI/Lbmn11R7b3nOFJOFnLgptlGsHt+SE9P75HYZOUb95eFRX7l3ulGtyCPqxO/h9H3RH+dChXmqeavcLkdERB6ifHmIM7d9/f3+3C7BJo6fPE3w5BkZlvfo5EdnX9udT7Ru3TpCQkIyLJ88efJdR1aNrFr1J1j26cZstw8a0DvDRRM1nvDizUEj+Nczfg+7PBEReQAKaJJjalStzNolmd8Ow5a6du1K165dc7uMXDdn4arcLkFERLJJhzhFREREDEYBTURERMRgFNBEREREDCZfnoOWmpKEnb1jumWVyhR54O0mJyZy4XLSA29HRERE8rd8GdDs7B1z5H4st++D8mABzdb3SMsJF6Oi6flaED9uWvtQtrdg2Spu3krIdHL0tRu3kFqgMH369Lmvbbdq1YqFCxdSrVo1Ro4cSadOnWjQoAFXrlxh4MCB3Lp1Cz8/P/r37/+A7+K2l1/wY9wHM/GoWOWhbE9ERPKmfBnQ8rOUFAv29hlnUshND1JTN/8OD+1GtRMnTkx7vGvXLooWLcqaNWseyrbzAovFgtlsrL4jIpJXKaDlgvatGvDiSwEc3L+Ha9fi6dP/dZo1b52h3eIFszhy+CApyckUfawYgwYHU7KUOzHRkQQN6E17v87s2/MfEhMSeHtwMF616mbYRkx0JC90fokeHTuw+0AYHf7Vmib1vZkwYw5X4uMxm80EBfTlqcYNMox8/f/n/zx+3u9Zftq9j4TEBMYOGUS92rfnEF2zYRMh6zbwuIsLDbxr3/UzGD1pOs5Ozpy/eJEr8VdZs/gjlq5eS+j22xO916xejeFvBeLsfHui7uiYS7w+ZDSRMTFULF+OcUPfoUjhQixYtookuwIMHTqU9evXExoaStGiRTl58iRFihRh7ty5lChRIlvfS+/evenXrx9OTk5MnTqV69ev4+/vz+jRo/Hw8GDMmDGcP38eAL/OPWnT1jfT7fywYxsbv/yM5JTbk833H/A23vXST38Vcf4cE8YM4eNla7FYUujWsTUvvPgKz/d4iX/v/JZdP+9k6KiJfLk2hB9/2I7FkoKjYwHeeHsYlatUZ92aFfwZE03gW0MBuBIXS2DACyz7dBNhB3azcukC7OzMWCwpBAYNoXbdBpnWevjQfhZ+NIMqVWtw5vQJnJ0cGffeG1T2qMC+sF+Z9tHHeNfy4rffTxDQ+wVcixdjypwF3EpIwKlgQYYGDcTLszoAP/6yh4XLQ0hJScFksuP9Ee9SrXIlDocfZ/bHS7lx8yYAgf1607xpY2KvxDN8wmRi4+IBaNLAm8FvvMaho+FMmjUPq9VKckoKr/Z+gfZtns7WdygiklcooOUSk50dH360lIjz53gn6BW8anlnmI+zW88+BAx8G4BtW75i6eI5DB89CYBr167i+URt+rzyOt9/9zXLFs1hxtylme4rPj6eihXKM7BvbwBeHPAWXfza07nDM5w+9wf9ggazYeWiu9Ycf/UatWt68mZAH7Z8+z2zP17KinkfcuL0GT5ZtYbPP5mHq0txJn74UbY+g8Phx1gyexrOTgX5efc+QrfvYOW8Dynk7MyoD6bz8crVDBpwe+L3g4ePsnbJfFxdihM8+UMWrVyd6SHPI0eOsGnTJtzd3Rk1ahQhISEMGnRv8zg2adKEoKAgdu7cyZw5cwB4++23qVq1KvPmzePSpUs859+RKlVrZHqosn6DJrRs1Q6TyUTE+XMMey+QkLVb07UpW96DmzdvEBd7mZjoSCpUqMyhsH083+MlDh3cS916t+fibN22A1269QIg7MAe5s6cxKx5y3mmQyde69OVvq++iZOTM19v2UDLVu0oWLAgq5Z9zOtvDcOrtjcWi4WEhFtZvt+zZ04y4M33qF2nPr/u/Y5RH0zns0VzATh55hwjB73B8LcDSU5OxrdnP8YNHUSTBvXYcyCMd4PfJ3T1Ui5GxzBu2iyWzZ1OhbJlSEpKIjklhWt/Xef9GXOZN3U8JVxd+TM2lp6vvcWXy2qy9dvvcS/pxqIPJwNw7a/b02ctW72WXl074deuDVarlb+u37in709EJC/QVZy5pF17f+D2H+oqVatzPPxIhjb79/yHt1/vw4B+3fhy7SrOnDqRts7JyZnGTX0AqOFZi6jIi3fcV4ECBWj3dHMAbty8ye+nztCxfVsAKntUoHqVShz+7fhda3Z2cqLFk40BqP1EDS5ERgGwL+wwPk0a4epye+LwLn7ZmyWgTYtmODsVBGD3gTCeadWCwoUKYTKZ6OLXnj0H/jspePOmjdO236lDO/YePJTpNuvVq4e7uzsAderUSRvxelC7du2iR48eALi5udGoSTN+Dct8RoioyAhGDnmD1/p2Y9KEEVyJiyUu7nKGdnXq1ufQwb2EHdxLe7/O/HkphuTkZMIO7KXO35OlnzpxjMFvBTCgXzcWLZiZ1geKFClK4yebs2P7ViyWFLZt2UCH556/vV3vBixeMJMv1qzkwvmzFCpUOMv3VrpMOWrXqQ+Av78/J8+c4/qN26GofNnS1PF6AoBz5yNwcLCnSYN6ADSu742Dgz3nzkewe/9BmjVuSIWyZQBwdHSkkLMzv/4WTmR0NK8PGU23VwJ5fchoTMCFi5HUfqIGu/Yd5MMFi/nxlz04Od0eLW3oXYeln65l0crVHDn2O0WLZF2/iEhepBE0A7BaAZMp3bKY6CgWzf+Q2QtWUsq9DOFHf2XKxFFp6x0cHNIem812WCwpAHwWsoSff7x9mPDVwHco5V4aJycnTH9v32q1ZlqDyWTC3mwmNTU1bVlSUnK6No6O/92nnZ1dhmmD7pXz33+Q/6nL9D+fwf8+z6rtPwoUKJD22Gw2P3CNWdVjMpm4djWe4e8FAlC2XAWGB09i8vsjCRg4iCebtSQ1NZWO7ZuRnJTx4pE69Rpx6OA+oqMvMnjEBI4ePsjO778BoJR7GZKTk5k4dijTZi2mSrUaxF7+k17d/ht+/Tt3Z8rEURQrXpxy5StStlwFAF57/V3OnjnFr2H7+GDcMDo9/yLtfTvd13tO9x1hxUQmn7vJxB26FVarlaqVKrJs7vRM169dMp9d+w8Sun0HS1d/zoqPPqRX1060eLIxuw+EMXn2fJ5sWI83+ve5r/pFRB5V+TKgpaYk5cjM88mJidlu++22TbzQuz8XI85z5tTv1PD0Srf+5s0b2Ds4UNzFldTUVLZu/jJb232h1yu80OuVtOcx0ZHp1hcuVIjqVSqxadt3dHy2LWf/uMCJU2ep9UR1ihYuQkqKhfMRkZQvW5qt3/2QrX029K7Nss/WEXslHtfixdiw5Ztsve7/a9qgHjMXLqFnF3+cnZzYsGUbjet7p63/afde4uLjcSlWjE3bvqWhd5173seDaNq0KZ9//jlBQUH8+eef7NvzHzp16UnRx4oxb/HqdG1v3LhOKffSAHyzdSPJyZlf2Vu3XkOWf/IRjz1WnBIlSuJdrxHLPplHvfq3RymTkhKxWCw87lYSgNCN69K93qNiFYoWfYyP533I60FD0pZHnD9HxUpVqFipCrdu3eTE7+FZBrTIixc4ejgMr9rebN68maqVPChcqFCGdhXLlyMpOZm9B3+lUb067D34KykpFjzKlcHR0YHFqz7jj4iL6Q5x1vV6gvMRF9NeA3D02O/UrFGNi9ExlCzxOO1bt6RebS/8evYjNTWV8xcj8ShXlnJlSuPs5MSmbd/d7esREclz8mVA+997oAGcufiXTWtwcHDk3Tf7cfVqPG++MyLD+WcVK1XBp0UbBvTrTgm3ktSqU58jh8PusLV7M2nUUCbMmEPIuvWYzWYmjhyMS7FiAAx5cwCvvTuc0qXcsh2CqlWuRP9e3enz+ju4uhTHp2mju7/ofzRr0pATZ87SO/D2+WI1q1fl1d4vpK1vVK8uYybPJCIqCo9yZXk38NV73seDGDVqFMHBwfj53Z5UvG/AG1SoWDnTtq8FvsP40e/h+ngJatWpR9Gij2XarkSJkjg5OVPz74s76ng35M9L0dTxvn1Cf6FChend9zXeGvgSbm6laND4yQzbaPdsR1Z8Mo+GTZqlLVu6+CMiL57HbLanUOHCvP1ecJbvrVKVauz8/hs+njcDp4IOvD/ivUzbOTg4MGP8qHQXCUwfNxIHBwcqlC1D8HtvMWTsB6SmpmJnZ8f7w9+jauWKzP5gLDMXfsK0jxaSnJxC2dKlmDNpHPvDDrNy7ZdpI7ej3n0TOzs7Vn+5kX1hv+Jg74CjowPDggZmWb+ISF5kst7pmNcjJjExkaNHj+Ll5ZXuMBfAsWPH8PT0zPL1tgxotr7XWaUyRUiIPmuTfeWGh3WbjXth60B/J7OmT6Bs2Qo83+Ol+3r94UP7+WTh7LSJ1B+lvnLijwgK/rTknl6TEyPnd9MzeKfN92krq8e3zJF7ShqF+svDo75y73SRgMgjKPbyn/R/qTMXIy7g27FbbpcjIiIPWb48xJnbvv4+86v/8prjJ08TPHlGhuU9OvnR2Td7V3o+DOvWrSMkJCTD8smTJ991ZNWoXB8vwScr12e7/diRg/jzUky6ZSXcSjJ24sy00TMRETEOBTTJMTWqVmbtkvm5XQZdu3ala9euuV1Grho7cWZulyAiIvcg3xzizCOn2okIf/8862daRPKwfBHQChYsSGxsrEKaSB5gtVq5cv0WpuuxuV2KiEiOyReHOMuWLUtERAR//vnnHdtcjk+wYUW2lXitIMnXMt7JPq9wuGL77y6v9pdHoq9YrZiux+Lw2/bcrkREJMfki4Dm4OBAxYpZ34ohr17aDLB6vLcub37I8mp/yet9RUTkUWGzQ5xnz56le/futGvXju7du3Pu3LkMbSwWC+PGjaNNmzb861//Yt26dRk3JCIiIpLH2SygjRkzhp49e/LNN9/Qs2dPgoMz3t188+bNnD9/nu3bt/P5558zd+5cIiIibFWiiIiIiCHY5BBnbGws4eHhLFu2DABfX18mTJhAXFwcLi7/neJo69atdO3aFTs7O1xcXGjTpg3btm2jf//+d93HPxcAJGUyKXV2FHXKfPLtvCAxMZHUgkVyu4wck3gPc6A+LHm1v6ivPHx5ta+A+ktOyKv9RX3lzhwdHTGZMn7vNpnq6ejRowwdOpQtW7akLXv22WeZNm0aNWvWTFvm5+fHxIkTqV27NgCLFy8mJiaGUaNG3XUff/31FydOnHj4xYuIiIjkkMymqIQ8dJFAoUKFqFatGg4ODpkmURERERGjcXR0zHS5TQKau7s7MTExWCwWzGYzFouFS5cu4e7unqFdZGRk2ghaVFQUpUuXztY+7OzsKFIk7w6fioiISP5hk4sEXF1d8fT0JDQ0FIDQ0FA8PT3TnX8G8Mwzz7Bu3TpSU1OJi4vju+++o127drYoUURERMQwbHIOGsDp06cZNmwY165do2jRokyZMoVKlSoREBBAUFAQtWrVwmKxMH78eP7zn/8AEBAQQPfu3W1RnoiIiIhh2CygiYiIiEj25Iu5OEVEREQeJQpoIiIiIgajgCYiIiJiMApoIiIiIgajgJaHTZkyhVatWlG9enXNsiAZ3Kl/nD17lu7du9OuXTu6d+/OuXPncq9IyRVXrlwhICCAdu3a4efnxxtvvEFcXByQdf9Q38k/7uf3h/rOPbJKnrVv3z5rZGSk9emnn7b+/vvvuV2OGMyd+kfv3r2tX331ldVqtVq/+uora+/evXOrRMklV65cse7evTvt+eTJk63Dhw+3Wq1Z9w/1nfzjfn5/qO/cG42g5WENGjTIMFuDyD8y6x+xsbGEh4fj6+sLgK+vL+Hh4WmjJ5I/FCtWjMaNG6c9r1u3LpGRkVn2D/Wd/OVef3+o79y7PDMXp4g8uKioKEqWLInZbAbAbDbj5uZGVFRUhpk/JH9ITU3ls88+o1WrVln2D6vVqr6Tz91v/1DfyZxG0ERE5I4mTJiAs7MzvXr1yu1SRPIVjaCJSBp3d3diYmKwWCyYzWYsFguXLl3SofJ8asqUKfzxxx8sXLgQOzu7LPuH1WpV38nn7rd/qO9kTiNoIpLG1dUVT09PQkNDAQgNDcXT0zNfH2bIr2bOnMnRo0eZN28ejo6OQNb9Q31H7rd/qO9kTnNx5mHvv/8+27dv5/LlyxQvXpxixYqxZcuW3C5LDOJO/eP06dMMGzaMa9euUbRoUaZMmUKlSpVyu1yxoZMnT+Lr64uHhwcFCxYEoGzZssybNy/L/qG+k3/cz+8P9Z17o4AmIiIiYjA6xCkiIiJiMApoIiIiIgajgCYiIiJiMApoIiIiIgajgCYiIiJiMApoIiIiIgajmQRE5JHSqlUrLl++jNlsxtnZGR8fH0aPHk2hQoVyuzQRkYdGI2gi8shZuHAhYWFhfPXVV4SHh7No0aLcLgmAlJSU3C5BRPIIBTQReWSVKFGCZs2acezYMQAOHTpEjx49aNCgAc899xx79uxJa7t+/Xpat26Nt7c3rVq1YtOmTQCkpqYyf/58nn76aZo2bcqQIUP466+/ANizZw/NmzdPt89WrVrxyy+/ADB37lyCgoJ47733qFevHhs2bCA+Pp7hw4fTrFkzGjZsSGBgYNprf/jhB/z9/WnQoAE9evTg+PHjaesWLVqEj48P3t7etGvXjl27duXMhyYijwQd4hSRR1Z0dDQ//fQTjRs3JiYmhtdee42pU6fi4+PDrl27CAoK4uuvv6ZgwYK8//77fPHFF1SqVIlLly5x9epV4HZw27BhAytXrsTFxYWhQ4cyfvx4pk2blq0aduzYwezZs5k6dSpJSUkEBQXh7OzMli1bcHZ2JiwsDIDffvuNESNGsHDhQry8vNi0aROBgYFs27aNiIgIPv30U7744gtKlixJREQEqampOfa5iYjxaQRNRB45r7/+Ot7e3rRo0QIXFxeCgoLYuHEjzZs3p0WLFtjZ2fHUU0/h5eXFjz/+CICdnR0nT54kISEBNzc3qlatCsDmzZvp06cP5cqVo1ChQrzzzjts3bo124cr69atS5s2bbCzs+PatWv8+9//Zty4cTz22GM4ODjQqFEjANauXUv37t2pU6cOZrOZTp064eDgwKFDhzCbzSQlJXH69GmSk5MpW7Ys5cuXz5kPT0QeCQpoIvLImTdvHmFhYaxatYozZ85w5coVIiMj2bZtGw0aNEj7d+DAAf7880+cnZ2ZOXMma9asoVmzZrz66qucPn0agEuXLlGmTJm0bZcpU4aUlBRiY2OzVUupUqXSHkdHR/PYY4/x2GOPZWgXGRnJsmXL0tUXHR3NpUuXqFChAiNGjGDu3Lk8+eSTDBo0iJiYmAf8lETkUaZDnCLyyGrUqBGdO3dmypQp1KlTB39/f95///1M2/r4+ODj40NCQgKzZs1i9OjRrF69Gjc3Ny5evJjWLjIyEnt7e1xdXYmJiSEhISFtncViIS4uLt12TSZT2uNSpUpx9epVrl27RtGiRdO1c3d3Z8CAAQwcODDT+vz8/PDz8+P69esEBwczffr0bB9mFZG8RyNoIvJIe/nll/nll1+oX78+P/zwAz/99BMWi4XExET27NlDdHQ0ly9fZseOHdy8eRNHR0ecnZ0xm80A+Pr6smLFCi5cuMCNGzeYOXMm7du3x97enooVK5KYmMjOnTtJTk5mwYIFJCUl3bEWNzc3mjdvzrhx47h69SrJycns27cPgK5du7JmzRp+/fVXrFYrN2/eZOfOnVy/fp0zZ86wa9cukpKScHR0pECBAmn1iUj+pIAmIo80FxcX/P39WbFiBfPnz+fjjz+madOmtGjRgiVLlpCamkpqairLli3Dx8eHRo0asW/fPsaMGQNAly5deO655+jVqxetW7fG0dGR0aNHA1CkSBHGjBnDqFGjaN68OU5OTukOaWZm6tSp2Nvb0759e5588klWrFgBQK1atZgwYQLjx4+nYcOGtG3blvXr1wOQlJTEjBkzaNy4Mc2aNSMuLo5Bgwbl4KcmIkZnslqt1twuQkRERET+SyNoIiIiIgajgCYiIiJiMApoIiIiIgajgCYiIiJiMApoIiIiIgajgCYiIiJiMApoIiIiIgajgCYiIiJiMApoIiIiIgbzfzhXrAT0Frb9AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "resources = [int(10 / c) for c in constants] \n", + "df = pd.DataFrame({\n", + " 'Model Runtime Const': resources, \n", + " **event_results\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars='Model Runtime Const').rename(columns=str.title)\n", + "seaborn.barplot(x='Model Runtime Const', y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "ax1.set(xlabel='Resources', ylabel=f'{metric} Error')\n", + "ax1.legend_.remove()\n", + "plt.legend(loc='lower left')\n", + "seaborn.despine(fig)" + ] + }, + { + "cell_type": "code", + "execution_count": 229, + "id": "6d536763", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-0.01-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-0.05-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-1.0-100.json\n", + "/home/eecs/wooders/DPR/plan-round_robin_lifo-always_process-10.0-100.json\n", + "/home/eecs/wooders/DPR/plan-weighted_round_robin_lifo-always_process-0.01-100.json\n", + "/home/eecs/wooders/DPR/plan-weighted_round_robin_lifo-always_process-0.05-100.json\n", + "/home/eecs/wooders/DPR/plan-weighted_round_robin_lifo-always_process-1.0-100.json\n", + "/home/eecs/wooders/DPR/plan-weighted_round_robin_lifo-always_process-10.0-100.json\n" + ] + }, + { + "data": { + "text/plain": [ + "{'plan-round_robin_lifo-always_process': [0.39388374885232,\n", + " 0.46513799624895036,\n", + " 0.8024342585399157,\n", + " 0.8759956368544546],\n", + " 'plan-weighted_round_robin_lifo-always_process': [0.39394652792491625,\n", + " 0.44209022922208885,\n", + " 0.6753066365327118,\n", + " 0.7929938554982696]}" + ] + }, + "execution_count": 229, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "constants = [0.01, 0.05, 1.0, 10.0]\n", + "#constants = [0.25]\n", + "policies = [\"lifo\"]\n", + "#key_policies = [\"random\", \"weighted_random\", \"round_robin\", \"weighted_round_robin\"]\n", + "key_policies = [\"round_robin\", \"weighted_round_robin\"]\n", + "#key_policies = [\"weighted_random\", \"weighted_round_robin\"]\n", + "d = artifact_dir\n", + "metric = 'top10'\n", + "\n", + "key_results = {}\n", + "for policy in policies: \n", + " for key_policy in key_policies: \n", + " scores = []\n", + " name = f\"plan-{key_policy}_{policy}-always_process\"\n", + " for constant in constants: \n", + " print(f'{d}/{name}-{constant}-100.json')\n", + " with open(f'{d}/{name}-{constant}-100.json') as results_file:\n", + " results = json.load(results_file)\n", + " scores.append(1-results[metric])\n", + " key_results[name] = scores\n", + "key_results" + ] + }, + { + "cell_type": "code", + "execution_count": 230, + "id": "511f1c65", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmgAAAFCCAYAAABFMCGEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA7jElEQVR4nO3dd1xV9R/H8RdchoCaQixXzhTFvc2VmmZh2BKlLEeUUlmWuffIHDlz5ybnT01F00qz8cstDsJRjkxBVFwpwoXL/f1h0Y9AxMHlCO/n49Hjwb3ne875XPgGb7/fc87Xzmq1WhERERERw7DP6QJEREREJC0FNBERERGDUUATERERMRgFNBERERGDUUATERERMZhcE9CsViuJiYnoplQRERF52OWagGY2m4mMjMRsNud0KSIiIiL3JdcENBEREZHcQgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgFNRERExGAU0EREREQMRgENMCdZcrqEbJXbP5+IiEhu45DTBRiBk6OJ4MHbcrqMbLNkeNOcLkFERETugkbQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBTQRERERAxGAU1ERETEYBxsdaKTJ0/St29frly5QqFChRgzZgwlS5ZM0yYuLo5+/foRExNDUlIS9erVY+DAgTg42KxMERERkRxnsxG0IUOGEBwczObNmwkODmbw4MHp2sycOZMyZcqwfv161q9fzy+//MLXX39tqxJFREREDMEmAS0uLo6oqCgCAgIACAgIICoqikuXLqVpZ2dnx40bN0hJScFsNpOUlIS3t7ctShTJMnOSJadLyDa5+bOJiDxMbDJ3GBMTg7e3NyaTCQCTyYSXlxcxMTG4u7untgsNDeXdd9+lYcOG3Lx5k1deeYWaNWve1bkiIyPvur67PcfDaO/evTldQq5Rs2ZNggdvy+kyssWS4U3VV0REbCSz/GGoi7s2bdpE+fLlWbhwITdu3CAkJIRNmzbx9NNPZ/kY/v7+ODs7Z2OVD6e8EELlwVBfERHJeTaZ4vT19SU2NhaL5db0icVi4fz58/j6+qZpFxYWxnPPPYe9vT0FChSgWbNm7Ny50xYl5mopyeacLiFb5fbPJyIieY9NRtA8PDzw8/MjPDycwMBAwsPD8fPzSzO9CVCsWDF++OEHqlSpgtlsZvv27Tz11FO2KDFXs3dw4tj4TjldRrZ5vNeCnC5BRETkgbLZXZxDhw4lLCyMVq1aERYWxrBhwwAICQnh0KFDAPTv35+9e/fSpk0b2rZtS8mSJWnXrp2tShQRERExBJtdg1amTBlWrlyZ7v05c+akfl2iRAnmz59vq5JEREREDEkrCYiIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiKRKSTbndAnZKrd/PhHJPRxyugARMQ57ByeOje+U02Vkm8d7LcjpEkREskQjaCIiIiIGo4AmIiIiYjAKaCIiIiIGo4AmIiIiYjAKaCIiIiIGo4AmIiIiYjAKaCIiIiIGo4AmIiIiYjAKaCIiIiIGo4AmIiIiYjAKaCIiIiIGo4AmIiIiYjAKaCIiIiIGo4AmIiIiYjAKaCIiIiIGo4AmIiIiYjAKaCIiIiIGY7OAdvLkSYKCgmjVqhVBQUGcOnUqw3YbN26kTZs2BAQE0KZNGy5evGirEkVEREQMwcFWJxoyZAjBwcEEBgaydu1aBg8ezKJFi9K0OXToEJ999hkLFy7E09OTP//8EycnJ1uVKCIiImIINhlBi4uLIyoqioCAAAACAgKIiori0qVLadotWLCALl264OnpCUCBAgVwdna2RYkiIiIihmGTEbSYmBi8vb0xmUwAmEwmvLy8iImJwd3dPbXd8ePHKVasGK+88grx8fE89dRTdO/eHTs7uyyfKzIy8q7rq1mz5l3vI8ayd+9em51L/eXhZsu+IiKSmcz+nthsijMrLBYLR48eZf78+ZjNZt544w2KFClC27Zts3wMf39/jbrlQQpNklXqKyLyMLDJFKevry+xsbFYLBbgVhA7f/48vr6+adoVKVKEp59+GicnJ/Lnz0/z5s05ePCgLUoUERERMQybBDQPDw/8/PwIDw8HIDw8HD8/vzTTm3Dr2rSffvoJq9VKUlISO3bsoEKFCrYoUURERMQwbPaYjaFDhxIWFkarVq0ICwtj2LBhAISEhHDo0CEAnn32WTw8PHjmmWdo27YtZcuW5aWXXrJViSIiIiKGYLNr0MqUKcPKlSvTvT9nzpzUr+3t7enXrx/9+vWzVVkiIiIihqOVBEREREQMRgFNRERExGCyFNBSUlKyuw4RERER+csdA5rFYqFatWqYzWZb1CMiIiKS590xoJlMJkqWLMnly5dtUY+IiIhInpeluzjbtGlDt27deO211/Dx8UmzrX79+tlSmIiIiEhelaWAtnTpUgCmTp2a5n07Ozu2bNny4KsSERERycOyFNC2bt2a3XWIiIiIyF+y/KDa5ORkIiIiiI2NxcfHh2rVquHgYKi11kVERERyhSwlrOPHj9O9e3cSEhLw9fUlJiYGZ2dnZs6cSZkyZbK7RhEREZE8JUsBbdiwYbRr146uXbtiZ2cHwNy5cxk6dCiLFy/O1gJFRERE8posPaj2yJEjdO7cOTWcAbz++uscOXIk2woTERERyauyFNC8vLzYtWtXmvf27NmDl5dXthQlIiIikpdlaYqzZ8+ehIaG0rRpU4oUKUJ0dDTbtm1j3Lhx2V2fiIiISJ6TpRG0J598kjVr1lCuXDlu3LhBuXLlWL16NS1atMju+kRERETynDuOoFksFqpXr86ePXsIDQ21RU0iIiIieZrW4hQRERExGK3FKSIiImIwWotTRERExGDuGNBSUlIYNWoUNWvWxMnJyRY1iYiIiORpd7wGzd7entDQUIUzERERERvJ0mM2ateuzf79+7O5FBERERGBLF6DVqRIEUJCQmjevDk+Pj5plnx67733sq04ERERkbwoSwEtMTEx9aG0sbGx2VqQiIiISF6XpYA2evTo7K5DRERERP6S6TVoX331VZrXJ06cSPN6wYIFD7wgERERkbwu04A2YMCANK/bt2+f5vWUKVMefEUiIiIieVymAc1qtd7VaxERERG5f5kGtP+/WzMrr0VERETk/t3xJgGr1Zr6X0avRUREROTByjSgxcfHU7FixdTXVqs19bXVatUImoiIiEg2yDSgaSF0EREREdvLNKAVLVrUVnWIiIiIyF+ytBaniIiIiNiOApqISDYxJ1lyuoRslds/n0hOytJSTyIicvecHE0ED96W02VkmyXDm+Z0CSK5lkbQRERERAzmjiNoy5cvZ82aNfz666/Ex8fj6upKuXLleOGFF2jXrp0tahQRERHJUzINaOPGjWPbtm107tyZChUqUKBAAa5fv87hw4dZsGABf/zxBx9++KGtahURERHJEzINaKtWrWLdunV4eXmleb9SpUo0atSI5557TgFNRERE5AG7q8XSRURERCT7ZTqC9tJLL/H666/TpUsXypcvnzrFeeTIERYsWMDLL79sqzpFRMRgUpLN2Ds45XQZ2Sa3fz4xtkwD2kcffUTx4sVZtWoVv/32W+pNAmXLlqVjx460b9/eVnWKiIjB2Ds4cWx8p5wuI9s83mtBTpcgedgd7+Js3769gpiIiIiIDd3Xc9Cio6MfVB0iIiIi8pd7Dmhms5nmzZs/yFpEREREhDtMce7evfu228xm8wMvRkRERETuENA6duyIp6cn9vZaEUpERETEVjINaEWKFGH8+PHUqFEj3bbExESqVauWXXWJiIiI5FmZDo35+/sTGRmZ4TY7Ozt8fX2zpSgRERGRvCzTgPbpp5/SoUOHDLc5OTmxdevWLJ/o5MmTBAUF0apVK4KCgjh16tRt2544cYKqVasyZsyYLB9fREREJLfINKA5Ojri6Oj4QE40ZMgQgoOD2bx5M8HBwQwePDjDdhaLhSFDhtCiRYsHcl4RERGRh02Wrv43m81MnjyZli1bUq1aNVq2bMmkSZNITEzM0kni4uKIiooiICAAgICAAKKiorh06VK6trNnz6Zp06aULFky659CREREJBe540oCAEOHDuXkyZMMGDCAokWLcvbsWWbPnk1sbCyjR4++4/4xMTF4e3tjMpkAMJlMeHl5ERMTg7u7e2q7I0eO8NNPP7Fo0SKmT59+Tx/odtfMZaZmzZr3dC4xjr1799rsXOovDzf1Fbkbtuwvkvdk9jsiSwFty5YtfPPNNxQsWBCAsmXLUrVqVVq2bPlgKgSSkpIYNGgQo0ePTg1y98Lf3x9nZ+cHVpc8HPSHULJKfUXuhvqL5JQsBbRHH32UmzdvpgY0uPWYDU9PzyydxNfXl9jYWCwWCyaTCYvFwvnz59PcBXrhwgVOnz7Nm2++CcC1a9ewWq1cv36dESNG3M1nEhEReeiYkyw4Od77AIWRpSSZsXd0yukysk1Kshl7hwf7+bIU0AIDA3njjTfo2LEj3t7enDt3ji+++ILAwEC2b9+e2q5+/foZ7u/h4YGfnx/h4eEEBgYSHh6On59fmunNIkWKsHPnztTXU6dOJT4+nj59+tzrZxMREXloODmaCB68LafLyBZLhjfl2PhOOV1Gtnm814IHfswsBbRly5YBMHPmzHTv/73Nzs6OLVu23PYYQ4cOpW/fvkyfPp2CBQumPkIjJCSEHj16ULly5Xv6ACIiIiK5TZYC2t087+x2ypQpw8qVK9O9P2fOnAzbv/vuu/d9ThEREZGHUZYCGkBycjIRERHExsbi4+NDtWrVcHDI8u4iIiIikkVZSljHjx+ne/fuJCQk4OvrS0xMDM7OzsycOZMyZcpkd40iIiIieUqWAtqwYcNo164dXbt2xc7ODoC5c+cydOhQFi9enK0FioiIiOQ1WVpJ4MiRI3Tu3Dk1nAG8/vrrHDlyJNsKExEREcmrshTQvLy82LVrV5r39uzZg5eXV7YUJSIiIpKXZWmKs2fPnoSGhtK0aVOKFClCdHQ027ZtY9y4cdldn4iIiEiek6URtObNm7N69WrKlSvHjRs3KFeuHKtXr6ZFixbZXZ+IiIhInpOlEbS5c+fStWtXQkND07w/f/58OnfunC2FiYiIiORVWRpBmzZtWobvz5gx44EWIyIiIiJ3GEH7e53NlJQUduzYgdVqTd125swZ3Nzcsrc6ERERkTwo04A2YMAAABITE+nfv3/q+3Z2dnh6ejJw4MDsrU5EREQkD8o0oP29Bmfv3r0ZO3asTQoSERERyeuydA2awpmIiIiI7WQpoImIiIiI7SigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwSigiYiIiBiMApqIiIiIwTjY6kQnT56kb9++XLlyhUKFCjFmzBhKliyZps20adPYuHEjJpMJBwcHevbsSaNGjWxVooiIiIgh2CygDRkyhODgYAIDA1m7di2DBw9m0aJFadpUqVKFLl264OLiwpEjR3j11Vf56aefyJcvn63KFBEREclxNpnijIuLIyoqioCAAAACAgKIiori0qVLado1atQIFxcXAMqXL4/VauXKlSu2KFFERETEMGwyghYTE4O3tzcmkwkAk8mEl5cXMTExuLu7Z7jPl19+SYkSJfDx8bmrc0VGRt51fTVr1rzrfcRY9u7da7Nzqb883NRX5G6ov0hW3UtfyexnbrMpzruxa9cuJk+ezLx58+56X39/f5ydnbOhKjEy/WKTrFJfkbuh/iJZ9aD7ik2mOH19fYmNjcVisQBgsVg4f/48vr6+6dpGRETw0UcfMW3aNEqXLm2L8kREREQMxSYBzcPDAz8/P8LDwwEIDw/Hz88v3fTmwYMH6dmzJ1OmTKFSpUq2KE1ERETEcGz2HLShQ4cSFhZGq1atCAsLY9iwYQCEhIRw6NAhAIYNG0ZCQgKDBw8mMDCQwMBAjh49aqsSRURERAzBZteglSlThpUrV6Z7f86cOalfr1q1ylbliIiIiBiWVhIQERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDUUATERERMRgFNBERERGDsdli6TkpKSmJM2fOkJCQcNs23Vs/YsOKbOvw4cMkNeqa02Vkm8OHD9v8nLm1v6ivPHh321esVrhwzcLGPTe4abZmU1UiYnR5IqCdOXOGAgUKULJkSezs7DJsc+LsnzauynZKFy1AwrmTOV1GtsnnU8rm58yt/UV95cG7275itVrxuHEViGXVz9ezpygRMbw8McWZkJCAh4fHbcOZiIhR2NnZkc/tETwLmnK6FBHJQXkioAEKZyLy0LCzs0O/skTytjwxxflv5iQLTo5p/3VaumiB+z7uzcRkYi7evO/jiIiISN6WJwOak6OJ4MHbHvhxlwxv+sCPKSIiInlPnpnifFi0blaLmzfjc7qM+3I25hxNnmv3wI43Y/5iPp0+J8NtK9ZuYMGCBfd87GbNmnHs2DEABgwYwJ49ewC4fPky7du3JzAwkM8///yej/9vr3dow6mTvz2w44mISO6UJ0fQ8rLkZAsODsa6+Ph+amoX+OwDuzNv1KhRqV9v376dggULsmzZsgdy7NzAYrFgMhmr74iI5FYKaDmgdbNavPJaCPv27OTatSt0euNtGjZunq7dnBmTOHRwH8lJSRR8pBA9PxqMt48vseei6dGtI63bvMDunf8lMSGB9z8ajH/laumOEXsumg4vvEb7ts+yY28Ezz7VnHo1qzPi0ylcvnIFk8lEj5DOPFG3FmdjzhH8Vg++X7cCIM3rv79+qc0z/LhjNwmJCQzt3ZMaVfwBWLZmHWEr1/Couzu1qle54/dg0OjxuLq4cvrsWS5fucqyOZ8xb8kKwr/eAkCl8o/T771QXF1dADgXe563ew8iOjaWUiWKM6zPBxTI78aM+Ysx2zvTp08fVq9eTXh4OAULFuTXX3+lQIECTJ06FU9Pzyz9XDp27EiXLl1wcXFh7NixXL9+ncDAQAYNGkTJkiUZMmQIp0+fBqDNC8G0aBmQ4XG+27KJtauWkpScBMAb3d6neo06adqcOX2KEUN6M2v+CiyWZNq1bU6HV7ryUvvX+GHbN2z/aRt9Bo5i1Yowvv/uayyWZJycnHnn/b6UKVuelcsWciH2HKHv9QHg8qU4QkM6MP+LdUTs3cGieTOwtzdhsSQT2qM3VarVyrDWg/v3MPOzTylbrgInjh/D1cWJYb3eoUzJx9gdcYBxn82iemV/fjl6jJCOHfAoXIgxU2ZwMyEBl3z56NOjO/5+5QH4/uedzFwQRnJyMnZ29ozs/yGPlynNwagjTJ41jxvxt0aGQ7t0pHH9usRdvkK/EZ8Qd+kKAPVqVeejd95if2QUoydNw2q1kpSczJsdO9C6xZNZ+hmKiOQWCmg5xM7engmfzePM6VN80KMr/pWrU6iwe5o27YI7EdL9fQA2bfiSeXOm0G/QaACuXbuKX8UqdOr6Nlu//Yr5s6fw6dR5GZ7rypUrlHqsBN07dwTglW7v8WKb1rzw7NMcP/U7XXp8xJpFs+9Y85Wr16hSyY93Qzqx4ZutTJ41j4XTJnDs+Ak+X7yM5Z9Pw8O9MKMmfJal78HBqMPMnTwOV5d8/LRjN+Ffb2HRtAm4uboy8OPxzFq0hJ7dbj00dd/BSFbMnY6He2EGfzKB2YuW8GFoSLpjHjp0iHXr1uHr68vAgQMJCwujZ8+eWarnb/Xq1aNHjx5s27aNKVOmAPD+++9Trlw5pk2bxvnz53kusC1ly1WgZKmy6favWaseTZu1ws7OjjOnT9G3VyhhKzamaVOsREni429wKe4iseeieeyxMuyP2M1L7V9j/75dVKtRG4DmLZ/lxXavAhCxdydTJ45m0rQFPP3s87zV6WU6v/kuLi6ufLVhDU2btSJfvnwsnj+Lt9/ri3+V6lgsFhISMr9x5eSJX+n2bi+qVK3JgV3fMvDj8SydPRWAX0+cYkDPd+j3fihJSUkEBHdhWJ+e1KtVg517I/hw8EjCl8zj7LlYho2bxPyp43msWFHMZjNJyclc+/M6Iz+dyrSxw/H08OBCXBzBb73HqvmV2PjNVny9vZg94RMArv1563lh85es4NWXn6dNqxZYrVb+vH7jrn5+IiK5ga5ByyGtWgcCt/5Qly1XniNRh9K12bPzv7z/die6dWnHqhWLOfHbsdRtLi6u1K3fCIAKfpWJiT5723M5OzvT6snGANyIj+fobydo27olAGVKPkb5sqU5+MuRO9bs6uJCkwZ1AahSsQJ/RMcAsDviII3q1cHDvTAAL7ZpfcdjAbRo0hBXl3wA7NgbwdPNmpDfzQ07OztebNOanXsjUts2rl839fjPP9uKXfv2Z3jMGjVq4OvrC0DVqlVTR7zu1/bt22nfvj0AXl5e1KnXkAMRezJsGxN9hgG93+Gtzu0YPaI/ly/FcenSxXTtqlaryf59u4jYt4vWbV7gwvlYkpKSiNi7i6rVbwW0344d5qP3QujWpR2zZ0xM7QMFChSkboPGbPl6IxZLMps2rOHZ5166ddzqtZgzYyL/WbaIP06fxM0tf6afrUjR4lSpWhOAwMBAfj1xius3boWiEsWKUNW/IgCnTp/B0dGBerVqAFC3ZnUcHR04dfoMO/bso2Hd2jxWrCgATk5OuLm6cuCXKKLPnePt3oNo1zWUt3sPwg7442w0VSpWYPvufUyYMYfvf96Ji8ut0dLa1asy74sVzF60hEOHj1KwQOb1i4jkRhpBMwCrFf790KPYczHMnj6ByTMW4eNblKjIA4wZNTB1u6OjY+rXJpM9FksyAEvD5vLT97emCd8M/QAf3yK4uLikPgfOas146Rg7OzscTCZSUlJS3zObk9K0cXL655z29vZYLJZ7+LT/cP3rD/Lfdf37WXW3e3ZdRm3/5uzsnPq1yWS67xozq8fOzo5rV6/Qr1coAMWKP0a/waP5ZOQAQrr3pEHDpqSkpNC2dUOSzOZ0x6taow779+3m3LmzfNR/BJEH97Ft62YAfHyLkpSUxKihfRg3aQ5lH69A3MULvNrun/Ab+EIQY0YNpFDhwhQvUYpixR8D4K23P+Tkid84ELGbj4f15fmXXqF1wPP39JnT/IywYkcG33c7O27TrbBarZQrXYr5U8dnuH3F3Ols37OP8K+3MG/JchZ+NoFXX36eJg3qsmNvBJ9Mnk6D2jV4541O91S/iMjDKk8GNHOSJVseiXEzMTnLbb/ZtI4OHd/g7JnTnPjtKBX8/NNsj4+/gYOjI4XdPUhJSWHj+lVZOm6HV7vS4dV/1lKMPRedZnt+NzfKly3Nuk3f0vaZlpz8/Q+O/XaSyhXLUzB/AZKTLZw+E02JYkXY+O13WTpn7epVmL90JXGXr+BRuBBrNmzO0n7/r36tGkycOZfgFwNxdXFhzYZN1K1ZPXX7jzt2cenKFdwLFWLdpm+oXb3qXZ/jftSvX5/ly5fTo0cPLly4wO6d/+X5F4Mp+Eghps1ZkqbtjRvX8fEtAsDmjWtJSkofzgCq1ajNgs8/45FHCuPp6U31GnWY//k0atS8NUppNidisVh41MsbgPC1K9PsX7JUWQoWfIRZ0ybwdo/eqe+fOX2KUqXLUqp0WW7ejOfY0ahMA1r02T+IPBiBf5XqrF+/nnKlS5LfzS1du1IlimNOSmLXvgPUqVGVXfsOkJxsoWTxojg5OTJn8VJ+P3M2zRRnNf+KnD5zNnUfgMjDR6lU4XHOnovF2/NRWjdvSo0q/rQJ7kJKSgqnz0ZTsngxihctgquLC+s2fXunH4+ISK6TJwPavx9SC7ZfW9HR0YkP3+3C1atXePeD/umuPytVuiyNmrSgW5cgPL28qVy1JocORtzmaHdn9MA+jPh0CmErV2MymRg14CPcCxUCoPe73Xjrw34U8fHKcgh6vExp3ng1iE5vf4CHe2Ea1a9z553+pWG92hw7cZKOobeuF6tUvhxvduyQur1OjWoM+WQiZ2JiKFm8GB+GvnnX57gfAwcOZPDgwbRp0waAziHv8FipMhm2fSv0A4YP6oXHo55UrlqDggUzXizb09MbFxdXKv11c0fV6rW5cP4cVavfuqDfzS0/HTu/xXvdX8PLy4dadRukO0arZ9qy8PNp1K7XMPW9eXM+I/rsaUwmB9zy5+f9XoMz/Wylyz7Otq2bmTXtU1zyOTKyf68M2zk6OvLp8IFpbhIYP2wAjo6OPFasKIN7vUfvoR+TkpKCvb09I/v1olyZUkz+eCgTZ37OuM9mkpSUTLEiPkwZPYw9EQdZtGJV6sjtwA/fxd7eniWr1rI74gCODo44OTnSt0f3TOsXEcmN7Ky3m/N6yCQmJhIZGYm/v3+aaS6Aw4cP4+fnl+n+tgxorZvVYvWGH3BxcbXJ+bQA9oNnlMXSJ40fQbFij/FS+9fuaf+D+/fw+czJTJm5GFBfyQ732lfOnv6NGV9dfcDVPFhLhjfl2PhOOV1Gtnm81wKbnzM7HqJuBOord083CYg8hOIuXuCN117g7Jk/CGj74B4KLCIixpAnpzhz2ldbM777L7c58utxBn/yabr32z/fhhcCsnan54OwcuVKwsLC0r3/ySef3HFk1ag8HvXk80Wrs9x+6ICeXDgfm+Y9Ty9vho6amDp6JiIixqGAJtmmQrkyrJg7PafL4OWXX+bll1/O6TJy1NBRE3O6BBERuQua4hQRERExGAU0EREREYNRQBMRERExmDx5DVpKshl7B6c075UuWuC+j5uUmMgfFzN+KKmIiIhIVuXJgGbv4JQtz2O59RyU+wtotn5G2p38cuQYYSvXMHpQn0zbnY05R/BbPfh+3Yp02679eZ1V67+ic/C9XajfOug1poweTrnSJe9p/+zwzab17NzxIwOHjn0gx+vd801ebNcxdX3V/zdp/AhatAzAv0r1DPbMXOy5aHp068jyL28t//V2SDATPpuHs3M+oiIPMGXCKEwODrzZ/YPUB+Tej8z6gYiIZF2eDGiSdZUqPH7HcHYnf16/zoJlK+85oN0ti8WCyZR+tYicZLEkYzLd2/9u7/ca9MDq+P9lqbZ8s5EWLQPu+SG3uU1KSgp2dna3XedVRMSWFNByQOtmtXjltRD27dnJtWtX6PTG2zRs3DxduzkzJnHo4D6Sk5Io+Eghen40GG8f39RRkdZtXmD3zv+SmJDA+x8Nxv+vJYP+3/w5n/FYMU9eDWjO5q0/0Gf4aLasWYpH4UK83XsQr7zclga1a/Ljjl18vngZiWYzjg4OfPTOW1Sp5MfuiANMmPE5S2dPBWDp6nUs+c+XFMjvRsN6dVj+5fo0oyVT5yzgxx27SUhMYGjvntSo4s/oSdP48/p12nUNJZ+zM4umT+RCXByfTJ5BTOx5EhPNtG7elDc6tgdg34FIRk38DGdnJ6pUrHDbhbj/tnr1ajZs2IC7uzvHjx9n1KhRXLhwgQkTJmCxWHB3d2f48OE89thjrF69mm3btjFlypTUff9+vXr1asLDwylYsCC//vorBQoUYOrUqXh6emI2mxk5ciQ7d+7E29ubR72L3/Hn/HqHNrR6JpADEbvx8S1Kt3d6MWPKOI4d/QWAZk89Q7sOnVLbR+zdxaoVi7l44TyNmz5FpzfeBtKOrn06ZihOTk6c/eM0Fy7E4lexMh/2HZblUPH3CO2Gtf/hh23f4Oycj++2bGLCZ/M4dfI4/T6YyI1rV3DJl48+Pbrj71c+w+N8On0Oe/cfJCk5mUKPPMKwPj0p4uOdps3KdRv49fhJ+vd8h0OHj/Jqt/f4YuZk/P3KM2rCZ5QvW5qXnnuGfiPGcOqPM5jNSRQvWoThfXtSsEAB3u49iLbPtOSpprdGFb/94SdWrt3IrE8/ZuaCML7asg1nJyfAjs8njaFggfwZ1jpj/mJOnDpN/M0EzsVdpnTp0nz88cepP9/ff/+d+Ph4/vjjD8LCwvjuu++YO3cuACVKlGD48OF4eHgAMGvWLMLDw7Gzs8PV1ZUlS5Zgb2/PmjVrWLJkCRaLhfz58zN06FBKly7Nvn37GDhoKFarleTkZDq82oWmzZ9mY/hqvvzPEhwdnUixptB/8CcUL1EySz9DEckbFNByiJ29PRM+m8eZ06f4oEdX/CtXT7ceZ7vgToR0fx+ATRu+ZN6cKfQbNBqAa9eu4lexCp26vs3Wb79i/uwpfDp1XrrzVKtRm41rl/JqQHN27ougSsUK7Nq3nxZNGnLo8BGqV67EH2ejmb1wCTPGjyK/mxu/nTzF270HsXll2geYHjt+grlfLGfF3Gm4FyrE2Kkz02y/cvUaVSr58W5IJzZ8s5XJs+axcNoE+r3/NsFv9UjzTLSBo8bz5uvB1KxamaSkJEJ69qVShcepWdWf3sNHM3pgb2pXr8rmrT+wdPW6O34/9+3bx9q1aylRogRxcXF07tyZsLAwypYty8qVK+nVqxcrV66843EOHTrEunXr8PX1ZeDAgYSFhdGzZ0+WL1/OmTNnCA8PJzk5mZfbdcDLx/eOx7sUd5ExE2YBMHf2FFKsKcyYu5z4+Bt88E4XSpUuR+26TwBw+vcTjB4/HbPZzAfvdMavUpUMpzxPnTzO6PHTsbOz5503XyFi705q1Kp3x1r+30vtX+P3309Q7nE/nns+iKSkJEYN7c3YMZ9Qo7QvO/dG8OHgkYQvmYejo2O6/bsEt+PD0BAAVod/xaRZ8xg7pF+aNnVrVCds5RoAdu2NoGolP3bu24+/X3l27ovgtaAXgFvrvxYudGu90s8+X8C8JSt5/60uBL8YyLwlK1ID2vI14QS/GMi1P/9k4bJVfLd2GfmcnbkRH4+zU9rl3f5t38FIVsydTtGKNejXrx/Tp0+nT59bI8N79uxh9erVuLu7c+zYMcaPH8/q1avx8vJi0qRJjBgxgkmTJrFmzRq2bt3K0qVLyZ8/P5cvX8be3p49e/bw1Vdf8cUXX+Dk5MT3339P//79WbZsGXPmzOH5l4Jp3vJZrFYrN25cB2DurMnMnLscTy8fzGYzKSkpd/XzE5HcTwEth7RqHQhAsRIlKVuuPEeiDlHviSZp2uzZ+V/Wr11Jws14LBZLmm0uLq6pf7wr+FXm8xmTMjxPRf+qjBnZn6SkJPZHRvFh9xC++f5HvB99lHKlS+KSLx8/79rLH9ExdOnxUep+FouFuEuX0xxrd8RBGtWtnbqwemDrlmz4ZmvqdlcXF5o0qAtAlYoV+HT6nAxrir+ZwJ79B7l89Z91Bm/E3+TE76fxcC+Ei7Nz6kLtrZo1ZsSnkzM8zv+rUaMGJUqUAODAgQNUqFCBsmXLAvDiiy8ybNgwrl+/nqXj+PreCl5Vq1bl559/BmDnzp20bdsWR0dHHB0debJFa36J3H/H4zVv+Wzq1/v37uKtd3phZ2eHm1t+mjZryf59u1IDWotWAZhMDri4OND4yZYciNidYUCr/0RTnP4KJGXLlScm+swd67iTM3/8joODIw0aNCDh3Enq1qyOo6MDp06foVyZ9OtX/rRzN8u/DCf+5s10ffNvJYoVITHRTOz5C+zct58eb3ZmzqKlPPtUM5KSbo2WAazf/C0bv/2OpKRkbiYk8FjxogA0qFOTcZ/N4sSp09jZ2XEmOprG9esAULJEMfqPHMsTdWrRuEEd3Fwzv2azcf26eLgXBuCll15i5MiR/2xr3Bh391v/ONq5cydNmjTBy8sLgPbt2xMYeOv/1e+++44OHTqQP/+tkbrChW8db+vWrRw5ciT1YchWq5Vr164BULduXcKWLOB8bAzVa9Wjgp8/AFWr1WbC2OHUa9CYOvUa4lukWKb1i0jeo4BmAFYr8K8pqthzMcyePoHJMxbh41uUqMgDjBk1MHX7/49qmEz2WCzJACwNm8tP39+6IPzN0FsXfpcvX56vtmzD092d2tWrMH76bLw9H6V29Wq3zo+VJ+rUYtSAfwLa3078fjrN68ym0pyc/qnJ3t7+tn+4rdYUsLPji1lTcHRI2wWP/nbitsfPjJub2/8d33rbOk0mU5rRisTExDTbnZ2d07T9+zNY7zTPehsuLi7/quvfLW7z/czkMzg5/XMHsv3/1Xhfbnc+Ozv+u2sPk2fdGp19psWTtHyyMeOnzeaLWVMo5uvD/sgo+o34JMPD1q5elR927CLu0hVqVavCx5Om8cP2Xal9b9+BSFau3cDC6RNwL1SIjd98x6rwjX+d2o6g59uw/MtwAF5q80zqtYWLp09if+Qv7Np3gA4h7zJ93EgeL1M6ix817WfNat/J7Hgvvvgi7733XrptnTp1omzFuuzft5MZU8ZSo1Y9Xu8ayqDh4zh25BcOROyh7wfdeKdnv9SgLiICeTSgpSSbs2Xl+aR//bHPzDeb1tGh4xucPXOaE78dTf2X9d/i42/g4OhIYXcPUlJS2Lh+VZaO2+HVrnR4tWua9+rXr8+M+Yt56blncXJywtvzUdZt+oaPB96a4qlfuyYzF3zBbydPUbZUSQAiDx9Nd/1RrWpVWLDsP1y+cpXChR5h3aZvslRTfjc3EhISSU624OBgws3VlRpV/Jn3xXLeev0VAM6dv4CDyUSpEsVIMJvZe+AQNatW5pttP/Ln9RtZOs/fqlevzoABAzh+/DhlypRhzZo1VKxYkfz581OiRAmOHj2K2XzrbtvNmzdTsGDBOx6zfv36rF27lmeeeYbk5GS2bdmEp7fP3dVVqy6bN6ylYqWq3LwZz/fffc0b3d5P3b71m400efIpksxJ/Pj9Fl7vGnpXx78fxUqUJCnJzI4dO6hW0ptd+w6QnGyhZPGilCtdkifq/HOH56/HT+Lo4MCj7oVJSUlh5doNtz1u3ZrVmDZ3IQ3+2r+af0XmL1nOO290AuDa9evkz+9GoYIFMZvNfPnV5jT7P/d0C55/7U3MSUmsXnhrqvhGfDzxN29Sq1oValWrwoFfDvPbid8zDWg/7tjFpStXKOIDa9asoW7duhm2q1+/PnPmzOHChQt4enqyYsUKGjRoAMCTTz7J0qVLadGiReoUZ+HChWnWrBl9+vQhKCgIHx8fLBYLhw8fxt/fn5MnT1KkaDGKFC1GPhdXvt0cjsWSTOy5c5T386e8nz8x0Wc4/ttRBTQRSSNPBrR/PwMN4MTZP21ag6OjEx++24WrV6/w7gf9011/Vqp0WRo1aUG3LkF4enlTuWpNDh2MuKdz1a9fn8mTJ1O3RjXg1rVB+yOjUgPYY8WK8vHA3gwdO4nExESSkpKpVrliuoBWvmxpOrV/iddCe+LhXph6taqT//9GH27nkYIFeKbFk7zUuRsFC+Rn0fSJjB7Ym3GfzeLFTt0AcHN1YVifD3jUw50xg/qm3iRQp0Y1fL297urzuru7M3bsWHr16kVycjLu7u6MGzcOuBXe6tevT0BAAMWKFaNMmTJcuHDhjsds164dR48e5dlnn8XHx4fKVWty7tzZu6oruOMbTJ8ylu5dg4BbNwnUqtMgdXvZchXo1yuUuIsXaNSkRYbTm9nF0dGRAUPHMnHiPzcJjB82IMPrz8qVKcVTTRvxwutv4evtSc2qVdh38FCGx61ToxoDRo1L0/dWrf+KOjVuTWE3rFubDd9sJbBjCN6ej1KpfDkijxxN3d/N1ZUn6tQiwZyYOrV+/foNPhg8ksTERFKsVvzKlaV548zDTZ0a1RjyyUSiL8RRqlQp+vbtm2G7cuXK8eGHH9KlSxcAihcvzvDhwwFo27YtsbGxBAUFYTKZcHNz44svvqB27dq8//77dO/eHYvFQlJSEk8//TT+/v4sXryYH3/ajoOjA46OTnR/9yMslhQmjB3Kjet/Ymdnj6eXN51D3sm0fhHJe+ys9zp3YzCJiYlERkbi7++fZpoK4PDhw/j5+WW6vy0Dmq2fdVa6aAESzp18IMe6ER+fer3PjPmLOX02mtED7+8xHPcrn0/6a6Sym60Dva08yL7yICQnW3i5S3dG9PvwtneU3smM+YuJv5nAh6EhD1VfOXv6N2Z8dfXODXPQkuFNs+WZkkaRHTMtdxI8eJvNz2kL6it3L0+OoMm9mzxrHvsjo0hKSqZoER8G90p/3Y3Ig7Dtv9v5ZPIMmjVqcM/hTETkYaWAlgO+2ronp0u4Z/175txUTIc33013QXzlihUYNW5iDlV06/En679M/9T8D/oMoUxZ24WKqRM/5khUZJr3TCYTU2Yuvs0extf0ifo0faJ+ltrGXb5C9179073frNETdO/c8UGXJiKS7RTQ5KHx98NyjeTpZ9vy9LNtc7oM3u2ZPpzkJR6FC6V5zp6IyMPOPqcLsJVccqmdiOQBVqv1jitoiEjulicCWr58+YiLi1NIExHDs1qtJNy4yoVrD+D5diLy0MoTU5zFihXjzJkzmT5O4eKVBBtWZFuJ1/KRdO1iTpeRbRwv2/5nl1v7i/rKg3e3fcVqhQvXLGzcc3fP/xOR3CVPBDRHR0dKlcr89vrcemszwJLh1XV78wOWW/uL+sqDl1v7iohkL5tNcZ48eZKgoCBatWpFUFAQp06dStfGYrEwbNgwWrRowVNPPZWlxa1FREREchubBbQhQ4YQHBzM5s2bCQ4OZvDgwenarF+/ntOnT/P111+zfPlypk6dypkz978QtIiIiMjDxCZTnHFxcURFRTF//nwAAgICGDFiBJcuXcLd/Z8ljjZu3MjLL7+Mvb097u7utGjRgk2bNvHGG2/c8Rx/3wDw9xqLd6ugy90tkPwwSUxMJCVfgZwuI9v8e8FzW8it/UV95cHLrX0F1F+yQ27tL+ort+fk5ISdXfqfu02WeoqMjKRPnz5s2PDPosrPPPMM48aNo1KlSqnvtWnThlGjRlGlShUA5syZQ2xsLAMHDrzjOf7880+OHTv24IsXERERySYZLVEJuegmATc3Nx5//HEcHR0zTKIiIiIiRuPk5JTh+zYJaL6+vsTGxmKxWDCZTFgsFs6fP4+vr2+6dtHR0akjaDExMRQpUiRL57C3t6dAgdw7fCoiIiJ5h01uEvDw8MDPz4/w8HAAwsPD8fPzS3P9GcDTTz/NypUrSUlJ4dKlS3z77be0atXKFiWKiIiIGIZNrkEDOH78OH379uXatWsULFiQMWPGULp0aUJCQujRoweVK1fGYrEwfPhw/vvf/wIQEhJCUFCQLcoTERERMQybBTQRERERyZo8sRaniIiIyMNEAU1ERETEYBTQRERERAxGAU1ERETEYBTQcrExY8bQrFkzypcvr1UWJJ3b9Y+TJ08SFBREq1atCAoK4tSpUzlXpOSIy5cvExISQqtWrWjTpg3vvPMOly5dAjLvH+o7ece9/P5Q37lLVsm1du/ebY2OjrY++eST1qNHj+Z0OWIwt+sfHTt2tH755ZdWq9Vq/fLLL60dO3bMqRIlh1y+fNm6Y8eO1NeffPKJtV+/flarNfP+ob6Td9zL7w/1nbujEbRcrFatWulWaxD5W0b9Iy4ujqioKAICAgAICAggKioqdfRE8oZChQpRt27d1NfVqlUjOjo60/6hvpO33O3vD/Wdu5dr1uIUkfsXExODt7c3JpMJAJPJhJeXFzExMelW/pC8ISUlhaVLl9KsWbNM+4fValXfyePutX+o72RMI2giInJbI0aMwNXVlVdffTWnSxHJUzSCJiKpfH19iY2NxWKxYDKZsFgsnD9/XlPledSYMWP4/fffmTlzJvb29pn2D6vVqr6Tx91r/1DfyZhG0EQklYeHB35+foSHhwMQHh6On59fnp5myKsmTpxIZGQk06ZNw8nJCci8f6jvyL32D/WdjGktzlxs5MiRfP3111y8eJHChQtTqFAhNmzYkNNliUHcrn8cP36cvn37cu3aNQoWLMiYMWMoXbp0TpcrNvTrr78SEBBAyZIlyZcvHwDFihVj2rRpmfYP9Z28415+f6jv3B0FNBERERGD0RSniIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjAKaiIiIiMEooImIiIgYjFYSEJGHSrNmzbh48SImkwlXV1caNWrEoEGDcHNzy+nSREQeGI2gichDZ+bMmURERPDll18SFRXF7Nmzc7okAJKTk3O6BBHJJRTQROSh5enpScOGDTl8+DAA+/fvp3379tSqVYvnnnuOnTt3prZdvXo1zZs3p3r16jRr1ox169YBkJKSwvTp03nyySepX78+vXv35s8//wRg586dNG7cOM05mzVrxs8//wzA1KlT6dGjB7169aJGjRqsWbOGK1eu0K9fPxo2bEjt2rUJDQ1N3fe7774jMDCQWrVq0b59e44cOZK6bfbs2TRq1Ijq1avTqlUrtm/fnj3fNBF5KGiKU0QeWufOnePHH3+kbt26xMbG8tZbbzF27FgaNWrE9u3b6dGjB1999RX58uVj5MiR/Oc//6F06dKcP3+eq1evAreC25o1a1i0aBHu7u706dOH4cOHM27cuCzVsGXLFiZPnszYsWMxm8306NEDV1dXNmzYgKurKxEREQD88ssv9O/fn5kzZ+Lv78+6desIDQ1l06ZNnDlzhi+++IL//Oc/eHt7c+bMGVJSUrLt+yYixqcRNBF56Lz99ttUr16dJk2a4O7uTo8ePVi7di2NGzemSZMm2Nvb88QTT+Dv78/3338PgL29Pb/++isJCQl4eXlRrlw5ANavX0+nTp0oXrw4bm5ufPDBB2zcuDHL05XVqlWjRYsW2Nvbc+3aNX744QeGDRvGI488gqOjI3Xq1AFgxYoVBAUFUbVqVUwmE88//zyOjo7s378fk8mE2Wzm+PHjJCUlUaxYMUqUKJE93zwReSgooInIQ2fatGlERESwePFiTpw4weXLl4mOjmbTpk3UqlUr9b+9e/dy4cIFXF1dmThxIsuWLaNhw4a8+eabHD9+HIDz589TtGjR1GMXLVqU5ORk4uLislSLj49P6tfnzp3jkUce4ZFHHknXLjo6mvnz56ep79y5c5w/f57HHnuM/v37M3XqVBo0aEDPnj2JjY29z++SiDzMNMUpIg+tOnXq8MILLzBmzBiqVq1KYGAgI0eOzLBto0aNaNSoEQkJCUyaNIlBgwaxZMkSvLy8OHv2bGq76OhoHBwc8PDwIDY2loSEhNRtFouFS5cupTmunZ1d6tc+Pj5cvXqVa9euUbBgwTTtfH196datG927d8+wvjZt2tCmTRuuX7/O4MGDGT9+fJanWUUk99EImog81F5//XV+/vlnatasyXfffcePP/6IxWIhMTGRnTt3cu7cOS5evMiWLVuIj4/HyckJV1dXTCYTAAEBASxcuJA//viDGzduMHHiRFq3bo2DgwOlSpUiMTGRbdu2kZSUxIwZMzCbzbetxcvLi8aNGzNs2DCuXr1KUlISu3fvBuDll19m2bJlHDhwAKvVSnx8PNu2beP69eucOHGC7du3YzabcXJywtnZObU+EcmbFNBE5KHm7u5OYGAgCxcuZPr06cyaNYv69evTpEkT5s6dS0pKCikpKcyfP59GjRpRp04ddu/ezZAhQwB48cUXee6553j11Vdp3rw5Tk5ODBo0CIACBQowZMgQBg4cSOPGjXFxcUkzpZmRsWPH4uDgQOvWrWnQoAELFy4EoHLlyowYMYLhw4dTu3ZtWrZsyerVqwEwm818+umn1K1bl4YNG3Lp0iV69uyZjd81ETE6O6vVas3pIkRERETkHxpBExERETEYBTQRERERg1FAExERETEYBTQRERERg1FAExERETEYBTQRERERg1FAExERETEYBTQRERERg1FAExERETGY/wE1ySFMstKG5AAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "resources = [int(10 / c) for c in constants] \n", + "df = pd.DataFrame({\n", + " 'Model Runtime Const': resources, \n", + " **key_results\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars='Model Runtime Const').rename(columns=str.title)\n", + "seaborn.barplot(x='Model Runtime Const', y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "ax1.set(xlabel='Resources', ylabel=f'{metric} Error')\n", + "ax1.legend_.remove()\n", + "plt.legend(loc='lower left')\n", + "seaborn.despine(fig)" + ] + }, + { + "cell_type": "code", + "execution_count": 150, + "id": "1e07c3e9", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmgAAAFCCAYAAABFMCGEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABWiklEQVR4nO3dd1jV5f/H8SccQIYTE8WtqIiigBvLkVtzVY40VyaZWpgjV+5V4kTD3BP3SsXZ17Lxc+bGVblR3HvAgQO/PzCKQESF41Fej+v6XhfnfO7zud/nnPt7fHV/xm0VGxsbi4iIiIhYDOuXXYCIiIiIJKSAJiIiImJhFNBERERELIwCmoiIiIiFeW0CWmxsLJGRkeiaBxEREXnVvTYBzWg0EhoaitFofNmliIiIiLyQ1yagiYiIiLwuFNBERERELIwCmoiIiIiFUUATERERsTAKaCIiIiIWRgFNRERExMIooImIiIhYGAU0EREREQujgCYiIiJiYRTQRERERCyMApqIiIiIhVFAExEREbEwCmiS6mKiol7r/kRERNKazcsuwBIYo0zY2RrM1l9MlBFrWzuz9RdlMmJrMF9/1ra27Ore3Wz9VQoMNFtfr7soUxS2BtvXtj8RkVeFAhpgZ2ug9eDtZutv8fDq/DGug9n6K9Z7Hn229zRbfwHVJ5itr5fBnIHe3P/xYGuw1VgREbEACmgiz8icgX7x8Opm6UdERCyLzkETsWAx0caXXUKa0vmKIiJJ0wyaiAWztrEz++Fwc9L5iiIiSdMMmoiIiIiFUUATERERsTAKaCIiIiIWRgFNRERExMIooImIiIhYGAU0EREREQujgCYiIiJiYRTQRERERCyMApqIiIiIhVFAExEREbEwCmgiIiIiFkYBTURERMTCKKCJiIiIWBgFNBERERELo4AmIiIiYmFszNXRmTNn6NevH7dv3yZr1qyMGTOGggULJmhz48YN+vfvT3h4OFFRUVSqVImBAwdiY2O2MkVEREReOrPNoA0ZMoTWrVuzZcsWWrduzeDBgxO1mTZtGm5ubqxfv57169dz9OhRtm7daq4SRURSlTHK9Fr3JyJpxyxTUzdu3ODYsWPMnTsXgIYNGzJixAhu3ryJs7NzfDsrKysePHhATEwMRqORqKgocubMaY4SRURSnZ2tgdaDt5utv8XDq5utLxFJW2YJaOHh4eTMmRODwQCAwWDAxcWF8PDwBAGta9eufP7557z11ls8evSIDz/8kLJlyz5TX6Ghoc9c37P2IZZn3759ZutL4+XV9rqPFXO+PxF5Mcn9RljUyV2bN2/G3d2d+fPn8+DBA/z8/Ni8eTP16tVL8T48PT3JkCFDGlYplkihSVLqdR8rr/v7E0kvzHIOmqurK1euXMFkijs/wmQycfXqVVxdXRO0Cw4OpnHjxlhbW5MpUyZq1KjB7t27zVGiiIiIiMUwS0DLnj07Hh4ehISEABASEoKHh0eCw5sAefPm5ZdffgHAaDSyc+dOihYtao4SRURERCyG2a7iHDp0KMHBwdStW5fg4GCGDRsGgJ+fH0eOHAFgwIAB7Nu3j0aNGtG0aVMKFixIixYtzFWiiIiIiEUw2zlobm5urFixItHzM2fOjP87f/788Vd6ioiIiKRXWklARERExMIooImIyHOJMkW91v2JvEwWdZsNERF5ddgabOmzvafZ+guoPsFsfYm8bJpBExEREbEwCmgiIiIiFkYBTURERMTCKKCJiIiIWBgFNBERERELo4AmIiIiYmEU0EREREQsjAKaiIiIiIVRQBMRERGxMApoIiIiIhZGAU1ERETEwiigiYiIiFgYBTQRERERC6OAJiIiImJhFNBERERELIwCmoiIiIiFUUATERERsTAKaCIiIiIWRgFNRERExMIooImIiIhYGAU0EREREQujgCYiIiJiYRTQRERELIAxyvRa9iXPx+ZlFyAiIiJgZ2ug9eDtZulr8fDqZulHnp9m0EREREQsjAKaiIiIpKkoU9Rr3V9a0CFOERERSVO2Blv6bO9ptv4Cqk8wW19pRTNoIiIiIhZGAU1ERETEwiigiYiIiFgYBTQRERERC6OAJiIiImJhFNBERERELIwCmoiIiIiFUUATERERsTAKaCIiIiIWRgFNRERExMIooImIiIhYGAU0EREREQujgCYiIiJiYRTQRERERCyMApqIiLwSYqKiXuv+RP7N5mUXICIikhLWtrbs6t7dbP1VCgw0W18i/6UZNBERERELo4AmIiIiYmEU0EREREQsjNkC2pkzZ2jZsiV169alZcuWnD17Nsl2GzdupFGjRjRs2JBGjRpx/fp1c5UoIiIiYhHMdpHAkCFDaN26NU2aNGHt2rUMHjyYBQsWJGhz5MgRvv32W+bPn0+OHDm4d+8ednZ25ipRRERExCKYZQbtxo0bHDt2jIYNGwLQsGFDjh07xs2bNxO0mzdvHh07diRHjhwAZMqUiQwZMpijRBGRV15MtPFllyAiqcQsM2jh4eHkzJkTg8EAgMFgwMXFhfDwcJydnePbnTp1irx58/Lhhx/y8OFDateuTZcuXbCysjJHmSIirzRrGzv+GNfBbP0V6z3PbH2JpDcWdR80k8nEyZMnmTt3LkajkU6dOpE7d26aNm2a4n2EhoY+c79ly5Z95teIZdm3b5/Z+tJ4ebVprMizeJ3Hy+v83sC87+95Jfe5mCWgubq6cuXKFUwmEwaDAZPJxNWrV3F1dU3QLnfu3NSrVw87Ozvs7OyoWbMmhw8ffqaA5unpqcOi6ZD+IZSU0liRZ/E6j5fX+b3Bq//+zHIOWvbs2fHw8CAkJASAkJAQPDw8EhzehLhz03777TdiY2OJiopi165dFC9e3BwlioiIiFgMs91mY+jQoQQHB1O3bl2Cg4MZNmwYAH5+fhw5cgSAd955h+zZs9OgQQOaNm1KkSJFaNasmblKFBEREbEIZjsHzc3NjRUrViR6fubMmfF/W1tb079/f/r372+uskREREQsjlYSEBEREbEwCmgiIiIiFiZFAS0mJiat6xARERGRx54a0EwmE97e3hiNukO1iIiIiDk8NaAZDAYKFizIrVu3zFGPiIiISLqXoqs4GzVqxKeffkq7du3IlStXgm2+vr5pUpiIiIhIepWigLZkyRIApkyZkuB5Kysrtm3blvpViYiIiKRjKQpoP/74Y1rXISIiIiKPpfhGtdHR0Rw4cIArV66QK1cuvL29sbGxqLXWRURERF4LKUpYp06dokuXLkRERODq6kp4eDgZMmRg2rRpuLm5pXWNIiIiIulKigLasGHDaNGiBR9//DFWVlYAzJ49m6FDh7Jw4cI0LVBEREQkvUnRjWpPnDjBRx99FB/OANq3b8+JEyfSrDARERGR9CpFAc3FxYU9e/YkeO7333/HxcUlTYoSERGRtBMTrZvPW7oUHeLs0aMHXbt2pXr16uTOnZtLly6xfft2xo4dm9b1iYiISCqztrHjj3EdzNZfsd7zzNYXQExUFNa2tq90fykKaG+//TZr1qxh48aNXL16laJFi+Lv70+hQoVStRgRERGRF2Vta8uu7t3N1l+lwMBU3+dTA5rJZMLHx4fff/+drl27pnoBIiIiIpKQ1uIUERERsTBai1NERETEwmgtThEREREL89SAFhMTw6hRoyhbtix2dnbmqElEREQkXXvqOWjW1tZ07dpV4UxERETETFJ0o9ry5ctz8ODBNC5FRERERCCF56Dlzp0bPz8/atasSa5cuRIs+dTdjPcZEREREUkPUhTQIiMjqVWrFgBXrlxJ04JERERE0rsUBbSvv/46resQERERkceSPQdt06ZNCR6fPn06weN58+alekEiIiIi6V2yAe2rr75K8PiDDz5I8Hjy5MmpX5GIiIhIOpdsQIuNjX2mxyIiIiLy4pINaP++WjMlj0VERETkxT31IoHY2Nj4/yX1WERERERSV7IB7eHDh5QoUSL+cWxsbPzj2NhYzaCJiIiIpIFkA5oWQhcRERExv2QDWp48ecxVh4iIiIg8lqK1OEVERETEfBTQRERERCyMApqIiIiIhVFAExEREbEwT70P2rJly1izZg1//vknDx8+xNHRkaJFi/Lee+/RokULc9QoIiIikq4kG9DGjh3L9u3b+eijjyhevDiZMmXi/v37HD9+nHnz5nHhwgV69eplrlpFRERE0oVkA9qqVatYt24dLi4uCZ4vWbIkVapUoXHjxgpoIiIiIqnsmRZLFxEREZG0l+wMWrNmzWjfvj0dO3bE3d09/hDniRMnmDdvHs2bNzdXnSIiIiLpRrIB7csvvyRfvnysWrWKv/76K/4igSJFitC2bVs++OADc9UpIiIikm489SrODz74QEFMRERExIxe6D5oly5dSq06REREROSx5w5oRqORmjVrpmYtIiIiIsJTDnHu3bv3iduMRmOqFyMiIiIiTwlobdu2JUeOHFhba0UoEREREXNJNqDlzp2bcePGUaZMmUTbIiMj8fb2Tqu6RERERNKtZKfGPD09CQ0NTXKblZUVrq6uaVKUiIiISHqWbEAbP348rVq1SnKbnZ0dP/74Y4o7OnPmDC1btqRu3bq0bNmSs2fPPrHt6dOn8fLyYsyYMSnev4iIiMjrItmAZmtri62tbap0NGTIEFq3bs2WLVto3bo1gwcPTrKdyWRiyJAh1KpVK1X6FREREXnVpOjsf6PRSGBgIHXq1MHb25s6deowadIkIiMjU9TJjRs3OHbsGA0bNgSgYcOGHDt2jJs3byZqO2PGDKpXr07BggVT/i5EREREXiMpCmhDhw5l165dfPXVV6xcuZKvvvqKvXv3MnTo0BR1Eh4eTs6cOTEYDAAYDAZcXFwIDw9P0O7EiRP89ttvdOjQ4ZnehIiIiMjr5KlLPQFs27aNH374gcyZMwNQpEgRvLy8qFOnTqoVEhUVxaBBg/j666/jg9zzeNJFDckpW7bsc/cnlmHfvn1m60vj5dWmsSLPQuNFUup5xkpy33mKAtobb7zBo0eP4gMaxN1mI0eOHCkqwNXVlStXrmAymTAYDJhMJq5evZrgKtBr165x/vx5PvnkEwDu3r1LbGws9+/fZ8SIESnqB+KuPM2QIUOK28vrQT9sklIaK/IsNF4kpVJ7rKQooDVp0oROnTrRtm1bcubMyeXLl1m0aBFNmjRh586d8e18fX2TfH327Nnx8PAgJCSEJk2aEBISgoeHB87OzvFtcufOze7du+MfT5kyhYcPH9K3b9/nfW8iIiIir6QUBbSlS5cCMG3atETP/73NysqKbdu2PXEfQ4cOpV+/fkydOpXMmTPH30LDz88Pf39/SpUq9VxvQEREROR1k6KA9iz3O3sSNzc3VqxYkej5mTNnJtn+888/f+E+RURERF5FKQpoANHR0Rw4cIArV66QK1cuvL29sbFJ8ctFREREJIVSlLBOnTpFly5diIiIwNXVlfDwcDJkyMC0adNwc3NL6xpFRERE0pUUBbRhw4bRokULPv74Y6ysrACYPXs2Q4cOZeHChWlaoIiIiEh6k6Ib1Z44cYKPPvooPpwBtG/fnhMnTqRZYSIiIiLpVYoCmouLC3v27Enw3O+//46Li0uaFCUiIiKSnqXoEGePHj3o2rUr1atXJ3fu3Fy6dInt27czduzYtK5PREREJN1J0QxazZo1Wb16NUWLFuXBgwcULVqU1atXU6tWrbSuT0RERCTdSdEM2uzZs/n444/p2rVrgufnzp3LRx99lCaFiYiIiKRXKZpBCwoKSvL57777LlWLEREREZGnzKD9vc5mTEwMu3btIjY2Nn5bWFgYTk5OaVudiIiISDqUbED76quvAIiMjGTAgAHxz1tZWZEjRw4GDhyYttWJiIiIpEPJBrS/1+Ds06cPAQEBZilIREREJL1L0TloCmciIiIi5pOigCYiIiIi5qOAJiIiImJhFNBERERELIwCmoiIiIiFUUATERERsTAKaCIiIiIWRgFNRERExMIooImIiIhYGAU0EREREQujgCYiIiJiYRTQRERERCyMApqIiIiIhVFAExEREbEwCmgiIiIiFkYBTURERMTCKKCJiIiIWBgFNBERERELo4AmIiIiYmEU0EREREQsjAKaiIiIiIVRQBMRERGxMApoIiIiIhZGAU1ERETEwiigiYiIiFgYBTQRERERC2Pzsgswh6ioKMLCwoiIiHhimy71s5itnuPHjxNV5WOz9tc0y/tm7c++RQuz9mdu5hovGiup35+5Pc9YiY2Fa3dNbPz9AY+MsWlQlYhYunQR0MLCwsiUKRMFCxbEysoqyTanL94zWz2F82Qi4vIZs/Vnn6sQYfcumK2/vJnycf/8ebP1lzF/frP19TdzjReNldT1qoyV2NhYsj+4A1xh1Y77qV+UiFi8dHGIMyIiguzZsz8xnImIWBIrKyvsnbKQI7PhZZciIi9JughogMKZiLxSrKys0M+WSPqVbgKaiIiIyKsiXZyD9l/GKBN2tgkPHRTOk+mF9/soMprw649eeD8iIiKSvqXLgGZna6D14O2pvt/Fw6u/8D68qtVj56Y1ODo6vHhBaaznJz1p3rYFvlUqpWk/58PC6DdyJABtmjfn94MHaVSnDj6lSnH77l06ffABjx49olGjRnTq1ClV+qxRowbTpk2jWLFiqbI/ERGRZ5EuA1p6Zoo2YbB5tU48/vG33yhdogT9/P0BaFCzZvy2Pfv3kzlzZpYuXfqyyrM4JpMJg+HV+o5FRCQhBbSXwN3dnU87fMjOvfu5ffcu/n4fUavaW4najZ86k30HDxMVHU3WLFkY1rcHuXPl5GL4ZVp39qdZowb8umsvEZERDO3TgzKlPRPt42L4ZVo3+YDGzRuxf89+atWvRZ78eZjz3VyiIo2YTCZad/yQGnXfBuJmxdxLunPs8DFuXL9BtVrV8PvcD4Czp88xdthYoqOjKVC4AEaj8Z9+Llxk4uhJ3L51Gwc7B7q0aUPl8uUBKFu7Nl07dGD7jh3cuXuXgT16sPvAAXbu3Uu0ycSYgQMpVKBAkp/Vxm3bWLx6NTGxsRw6epSAIUMYMX48bZo3x8HenkkzZvAwIoImTZowaNAgChYsyJAhQzj/+NYNH3/8MU2bNk1y3+vXr2fBggVERUUB0LdvX3x9fRO0OX36NJ9//jkbNmwgOjqaihUr0qVLF2rUb8kv239g52/b6TtwFKuWB/PzT1sxmaKxs8vAZ1/0w62IOyuWzufalct07d4XgFs3b9DVrxVzF63jwL5dLJjzHdbWBkymaLr696G0d7kka9174BABU6ZRvGgR/jh1GoPBwIj+vXArWIC9Bw4x9tvp+JTy5OjJP/Br24rs2bIyZvJ3PIqIwMHenr7+XfD0cAfg5x27mTYvmOjoaKysrBk5oBfF3Apz+NgJpnw5mFt3bwHQ4dP2VHqrErdu3mL0wNHcuhH3fJkKZejaqytHDx1lcsAUYmNjiY6Opk3HD6lRr0aS9YuIyLNRQHtJrKysWTB1ImfPX6Bdt574lPYke7asCdp0bN2CXl3jwtHqkE1Mmj6HgCH9Abh95y6lS3rwuV8HNvzwI4HT5zA/aEKSfd2+fZv8hfLTvnN7AO7dvUfgrEkYDAZu3rhFl7ZdKO9bjkyZ487Du3r5KhNnTuThw4e0bdKO+k3qkzd/Xr4Z/A3vfvAudRvW4diRY3T/+Iv4PkYP/Jp33n2HBk3rE3ElktatWrFq9myyZY17T5kyZmRhUBA//PwzPYcM4euBA/n844+Zv2wZs5csYWS/fknW3qBmTS5cvMjDR4/o0blzgm3lvb35tH17dh05wuTJkwH44osvKFq0KEFBQVy9epX33nuPEiVKJHmo8q233qJhw4ZYWVlx+vRpOnTowC+//JKgTeHChbl//z5Xr17l4sWLFC1alJ07d1KjfksO7t+Dd5m4EFqzzju836INAAf27WbKxK+ZFDSPeu+8S+cOzfnok89xcHBk04Y1VK9RF3t7exbOnU637v3wLO2DyWQiIiL58xf/OHWGvv5dKOddmnWbf2Dg6HEsmTEFgD9Pn+WrHp/R/4uuREVF0bB1R4b17UGlcmXYve8AvQaPJGTxHC5evsKwsZOYO2UcBfLmwWg0EhUdzd179xk5fgqz5s7D6BDJjes36NquG7OXzWLbpm245MrJ2Klj48cPwNL5S2nW+n1qv1Ob2NhYHtx/kGz9IiKScgpoL8m779QFoGD+fHgULcKRY8ep/mbC2Zvfdu9l2fchPHz0CJPJlGCbo4MD1SpXBKB0ieKMnzrziX1lyJCB6rWrxz++fesOY4eP4+L5ixhsDNy7c48L5y5QolQJAKrWrIa1tTUZM2Ykf6H8XAq7RDbnbJw9dZbaDWoBUKJUCQoVKQTAwwcPOfXHKeo1jntPRYoUwd3NjSPHj1P18YxU7epx/RcvWhQrKyuqVIyr3aNoUX787bdn/vyeZOfOnfR7HPZcXFyoVq0au3fvTjKgXbhwgV69enHlyhVsbGy4fv06165dI0eOHAnaVaxYkZ07dxIWFkbLli2ZNWsWUVFRHNi3hxatOgDw1x/HWbZoLvfu3cHK2pqLF+Jm8DJlykzFylXZtnUj9Rs2ZfOGNYweOxUAL59yzPxuIlWq1aJcxcoULFQk2feWP09uynmXBqBhnZoMHzeZ+w/iQlH+vLnx8oz7/s6eD8PW1oZK5crE1V/WB1tbG86eD2Pf4SO8VbE8BfLmAcDOzg47Ozt+3bWHS5cv4+fnR1RM3IyilZUVFy9cxKNUCVYuXsX0wOmULuNFed+4WT7vct4snreEK+FXKFupLB6eHs/wTYmISHIU0CxALLFAwhseXbp8hXFBM1g0fTJ5XXNxMPQY/Ud8E7/dzs42/m9ra+v4ADdz4RJ+2P4rAL27dSaPa04cHBwS3Acu8JtAfKv6MmzsUKysrGj3XnuMkf8crrTLYBf/t8HaGpMpJu7BE+7JFBv7hKVo/tVnBju7+P3Z2v6rdoMhUfh8Uf+9552VlRW3bt2iQ4cOABQqVIhJkybRs2dP+vXrR61atYiJicHLy4vIyMhE+/P19WXXrl2EhYUxduxY9u7dy/YftwCQyzUPUVFRjBral7GTZlKkWHFuXL9Gmxb141/f5L2WjBk1kKzZspEvfyHy5os7nNu5Wy/OnP6LQwf2MnpYP95t9iH1G777XO/Z0eGfi0piicUqqS/LyoonfVWxsbEULVyIJStWJbmSwPRF09i3ez//2/gDS+ctIXB2IO+3fh/fqr7s272fKQHfUq5SWTp27fhc9YuISEJmC2hnzpyhX79+3L59m6xZszJmzBgKFiyYoE1QUBAbN27EYDBgY2NDjx49qFKlSqrXYowypcoVl//1KDI6xW3XbtrKJ+1acy7sIif/Ok2pEsUTbH/w4CG2Nja84ZyNmJgYVqzdkKL9+rVthV/bVvGPL4ZfTtTm/r375HLNiZWVFb/v2selC5eeul+njE4UcivEts0/UrtBLU6EnuDMX2fit7kVc2NryFbqNa7HqVOn+OP0aUoVL/6UvaY+X19fli1bhr+/P9euXePnn3+mQ4cOZMuWjbVr1yZoe+/ePfLmzQvAypUrE5xT9999jh8/HmdnZ3LlykXlypUZEzCOMmXjZgGNxkhMJhNvuOQEIGTtigSvL1ioCJkzZ2F60AS6+feJfz7s/FkKFS5CocJFePToIX+cPJZsQDt/8RL7D4VSxsuTjf/7iaKFC5LRySlRu0L582GMimLP/kNUKOPFnv2HiI42UTBfHuzsbJm5cAnnwi4mOMTp7VmC82EX2bVrF3lLxs2unTh6AvcS7ly+dJkcOXNQo+7blPIpRbt32xETE8PFC5fIVyAvufPmxsHRga0hW5/29YiISAqZLaANGTKE1q1b06RJE9auXcvgwYNZsGBBgjalS5emY8eOODg4cOLECdq0acNvv/2Gvb19qtby33uggXnX4oyrwZb23Xpy684dBvXyT3T+WVG3QtSuXoX32nfGNWcOynqVZv/hI6nSd6fPOjF5zGSWzF9K4SKFKVy0cIpe13dYX8YOG8vKRSsp5lE0wSGtASP7M3H0JFYuXoWDnQMj+vaNP//MnAYOHMjgwYNp1KgRAL1796Zo0aJJtu3fvz9du3YlZ86cVKhQgaxPqDdXrlw4OTlRtmxZACpVqsS1q5fx8ok71OfklJG2H3Wme5d2uLjkolzFyon2UbdBU+bPCqJ8pX8uBpkz81suXTyPwWCDU8aMfNF7cLLvzb2IG5u2bSfg22lYW1szckDvJNvZ2toyfvjABBcJjBv2Fba2thTIm4fBvbvTZ+hoYmJi4vbTvzdF3QoROHoogUFBXL91neioaFzz5GLkxJEc2neIFcErMdgYiImJ4Yv+X2Btbc2apWs4uO8gtjY22NrZ8tmXnyVbv4iIpJxV7BOPT6WeGzduULduXXbv3o3h8SGtihUrsnXrVpydnZN8TWxsLOXKlWPDhg3kypXrqX1ERkYSGhqKp6cnGTJkSLDt+PHjeHgkf36MOQNa/RrlzHqvMy2AnfqedbxMGjeCvHkL0OyDds/0ur8XS9974BATvpsVf1FAWtFYSX0v8tty8fxffLfpTorbLx5enT/GdXju/p5Vsd7z6LO9p9n6C6g+gV3du5utv0qBgWbr629pcY/OpGispK60GCtmmUELDw8nZ86c8fdmMhgMuLi4EB4e/sSA9v3335M/f/4UhbN/Cw0NTfScjY0NDx48+QozpyQOE8mrJbnvN7U9y3i5cf0afXt2JpvzG3z62ZdpWJWklKWOFbFM+/btM1tff8/Sy6vpecZKct+5RV4ksGfPHgIDA5kzZ84zv/ZJM2iW9EN58uRJIi6fedllWJTfdu8mKInvu1vHjrz1+IrP5FjS9/tv2d/IwawFq1PcfuhXPbh29Ur8Yztba3I6Z2Hy18PSfPYsvbDUsSKWSaFJUiq1x4pZApqrqytXrlyJv8O5yWTi6tWruLq6Jmp74MABvvzyS6ZOnUrhwik7N0pefW9VrJiiIPa6GzpqYoLHfx/iFBGR9MXaHJ1kz54dDw8PQkJCAAgJCcHDwyPR4c3Dhw/To0cPJk+eTMmSJc1RmoiIiIjFMUtAAxg6dCjBwcHUrVuX4OBghg0bBoCfnx9HjsRdnThs2DAiIiIYPHgwTZo0oUmTJpw8edJcJYqIiIhYBLOdg+bm5saKFSsSPT9z5j93wF+1apW5yhERERGxWBZ5kUBai4k2Ym1jl+C5wnkyvfB+oyIjuXA96ZudioiIiKRUugxo1jZ2aXL/l2K95wEvFtC8qtUz6z3SnubksZOsWryKASMHJNvu8qXLdGnblTXbEl+xeO/+fVZv2ED7li2fq4aGbdowacQIihQq9FyvfxHu7u7s378/za/82/HbdubN+hZbOzv6DxzN1yMGMOHbOUAmDoYeY/i4QGwMNvTu9gkVyni9cH8Xwy/TurM/P69b/uLFi4hIqkuXAU1Szr2E+1PD2dPcu3+f+cuXP3dAe1Z/Xy38KtkUspq2HT6lSvW4xeiDZi6O3xayZRuN69aiQ6vmL6s8ixITE4OVlVWiNVdFRF4nCmgvgbu7O592+JCde/dz++5d/P0+ola1txK1Gz91JvsOHiYqOpqsWbIwrG8PcufKGT/70axRA37dtZeIyAiG9ulBmdKeifYROGMO2XMXoF7Lumz/YTsjB4xixZblZHPORn//Abzf+j3KVSrH7t92s2jOYoxGIza2NnTt2YUSpUpw8PeDTA+cwXcLpwLw/bLvWb10DRkzOVHhzYqsXb42wazZ7KA5HNh1gIf37jGoVy98PD35ZsoU7t+/T6vOnbG3t2duYCDXbtxgbFAQl69eJSIyknpvv03H1q0BOHDkCF9Pnox9hgx4eng8eTH2x1avXs2GDRtwdnbm1KlTjBo1ip07d7JhwwZMJhMZMmRg6NCh8atJuLu706NHD3744Qdu375Nnz59qFu3LgBbt25lwoQJZM2alapVqybo55dffmHChAmYTCYcnLLg33MAufPk4/DB35n27Xjci5fkxPEjGAw2fNl/OIsWzOTs2VPkyJGTQcPGYu+Q9Kzo9KDxhB4+QNiFc4SsW8GYCdOpX6Mcqzf8wqxZy9jy08/Y29uz8X8/sWDqRP46fTbBMk59/bvg6eGe5L6fNIb+bcW6Dfx56gwDenzG4cOHad68OUHzv6V4yeIEfhOIWzE3Gr7XkNEDR3PhXBhRRiO58+Xhy8G9yZQ5E/39B1CvcT2q1Yr7vH798VfWrwohIGgMC2Ys4MctP2GXwQ4rYPz08WTMlDHpz2HBAk6fO8ejR48Iv3qVgvnyMbh3bzI5OTF9wQIuXLrEw0ePCLt0iVkTJvDLrl0sWL4cKysr8ubOzVfdu+OcLRsAc5YsYfOPP2JtbY2DvT2zJ06MW55qzRoWL16MyWQiY8aMDB06lMKFC7N//35GjBhBTEwM0dHRdOnShYYNG7Js2TLmzZuHnZ0dMTExTJo0CTc3t2THo4hIalBAe0msrKxZMHUiZ89foF23nviU9ky0HmfH1i3o1dUPgNUhm5g0fQ4BQ/oDcPvOXUqX9OBzvw5s+OFHAqfPYX7QhET9VCzjzcI1G6nXsi779xzAo5QHB/YepGrNKhw/ehxPb08uhV1i4exgxkz5BqeMTpw9dZb+/v1ZsmFJgn2d+vM0i+ctYcbi6WTNlpWg8VMTbL975y4lSpdgSL8hLJ87lykzZzInMJB+n39Om27dWDJ9enzbIQEBdPrwQ8qULk1UVBSf9ulDCXd3ypQqRf9RoxjZvz/lvLzY+vPPLPv++6d+nvv372ft2rXkf7yUT86cOenYsSMAO3bsYMiQISxf/s/hvIwZM7Jq1Sr27dvHF198Qd26dblx4waDBg1iyZIlFC5cOMEFLDdu3KBPnz4EBwdTpEgRvpu5kIBRA5k0dT4A58+dple/oXTvPZCgwDF81fdzJgbNjQtn/fzZ/uMW6r3TNMnaO3frxam/TvJ+i7ZU9K2SYFunTp04eeQAJdyL0eq9xkRFRdFr8EiG9e1BpXJl2L3vAL0GjyRk8RxsbW0T7Tu5MfS3imV8CF6xBoCdO3dSonQJDuw9QPGSxdm/5wDN28TN3HXr3Y0sWbMAMGfqHJbOX4rf5368+0FTls5fFh/Q1q5Yx7stm3Lv7j2WB69g1daVZLDPwMMHDxPdRPq/DoSGsmTaNLJny8awceOYFRxMj86d477jw4dZ9N13ZMuShb/OnGHKrFkET51KjuzZmTpvHgFBQXwzcCDrt27ll507mTNpEhmdnLh99y7W1tYcOHKETZs2sWjRIuzs7Pj5558ZMGAAS5cuZebMmbRv356mTZsSGxvLvXtxyzMFBAQQEhKCq6srRqMRk8mUbP0iIqlFAe0lefeduBmbgvnz4VG0CEeOHaf6m74J2vy2ey/Lvg/h4aNHif5hcHRwoFrluBu7li5RnPFTZ5IU71Il6TN8DFFRURw9dJTOX3zCL9t+5Q2XNyjsVgh7e3v27txLeFg4PT75Z500kymGmzduJdjXoX2HqPhmBbI+DpL1GtXlfxv/F7/dwdEB3yqVACjl4cGkfwWyf3v06BG/HzrErdu345978OgRZ86fJ3u2bNhnyEA5r7jzrOpUq8aoiROT3M+/lSlTJj6cQdySX9OnT+fOnTtYWVlx9uzZBO0bNGgQ9/l4e3P16lUiIyM5ePAgJUqUiL9BcsuWLRk3blzcez90iOLFi1OkSBEAatdrzLeBY3j4MG7ZoLz5CuBWJG4Wq0hRd65eCSdHjriZqqLFPLh0MXXWtzx7PgxbWxsqlSsDQMWyPtja2nD2fBhF3RKfo5fcGPpb/ry5iYw0cuXqNXbu3Emnbh8TPHsRNevVJCoqitx5cwOwNeQHtm3eRnRUFBEREeTNnxeA8r7l+W7CNM6dOYeVlRWXwi5R6fE4yFcgH18P+prylctTqUolHJ0ck31/VSpWJPvjWbAm9eoREBQUv+2tChXIliUuIP5+6BBvVqhAjuzZAXj/nXdo9TjI/bprF80aNSLj4/MGs2bODMAvu3Zx4sQJmjePC5yxsbHcvXs37nOsWJEZM2Zw6dIl3nzzTbwej79KlSrRv39/atasSfXq1cmXL1+y9YuIpBYFNAsQSyyQ8HyaS5evMC5oBoumTyavay4Ohh6j/4hv4rfb2f0zW2JtbR3/j+/MhUv4YfuvAPTu1pkKZbxwd3fnxy0/4fyGM97lvJk2aTo5XN7Au7xPXP+xUN63HP2G90tU2/kz5/6pMzY22fN+/j2DY7C2JvoJgSDm8X4WBAVha5NwCP5x6tQT95+cf5/EbzQa6d69O8HBwZQsWZIrV64kOlz590zO3+eqRUdHJ3so9Wnv3c7un5kha2sDtnZ2/3psnWozL7HEYkUSdVhZ8X97fidwetxyWQ1qvU2dt6smO4b+rbyPF7/s2sONGzfwKuvF5DFT2P3bbnzKeQNw+MAR1q9az+Q5gWTNlpVtm7exYfWGx11b0aR5Y9atWAdAw/feif9cv507hdBDoRz4/SBd2nTl6ylf41Y0ZSuExD7e998c/nWIOMnv4ynnpMXGxvL+++/TPYkFlDt06ECNGjXYsWMHI0aM4M0336RHjx58++23HDlyhF27dtGuXTuGDh1KtWrVUlS/iMiLMNuNaiWhtZu2AnAu7CIn/zpNqRLFE2x/8OAhtjY2vOGcjZiYGFas3ZCi/fq1bcXy2VNZPntq/NV+vr6+zJ8+nzIVfLCzsyOHyxtsCdlKmccBrVylsuzd+TtnT52N38+JoycS7du7rBe7/28Pd27fAWBLyNYU1eTk6EhEZGR8YHNydMTH05N5S5fGt7l89SrXb96kYL58RBqN7D98GID//fIL959xcWuj0Uh0dHT8UmKLFy9+yivi+Pj4cOzYsfjZtn/ft8/Hx4fjx49z6nGA/N+WENyKuOPoaN51HQvlz4cxKoo9+w8BsGf/IaKjTRTMl4c3K5SL/+47tGr+TGOoYllv5ixaho9P3Jgo6VWSJfOX4lMhbqbu/r37OGV0InOWzBiNRjav25zg9XUa1uH/ft7B9h9+pkHTuNnJhw8ecvv2bbzKetGhc3sKuhXk7Knkl636bffu+JnV9Vu2xM+k/lcFHx/+b88ert+8CcCajRupWCau1iqVKrFy/XoePHwIwO3Hs2RVK1Vi7dq1XL58GYi7mCQ0NBSAM2fOkD9/fj744APatWvHkSNHiI6O5sKFC5QuXZpPPvmEN998k+PHjydbv4hIakmXM2gx0cbHt8RIXVGRkSlua2drS/tuPbl15w6DevknOv+sqFshalevwnvtO+OaMwdlvUqz//CR56rL19eXwMBAfB4HMp/yZQg9dJTinnGhMG/+vPQf0Y9xI8YRGWkkOiqakl4lKV4yYWh0K+ZGy3Yt+fwjf7Jlz0bZCmVwyvj0gJIlc2bq16hBSz8/MmfKxNzAQEb278+E776jhV/c+VFOjo4M7tWLN5ydGT1gQPxFAuW8vcnl4vJM7zdjxoz4+/vTrFkzXF1dE82ePUn27NkZMWIEn376KVmzZqVevXrx25ydnQkICKB3795ER0fj4JSFPgNGPFNdqcHW1pbxwwcmuEhg3LCvkjz/7FnGUIUy3nw1aiy+vnGH2ctU8GHDmg34lPcGoGLlCmzb9D86NPuIHC45KOZRjJP/CvGOTo6U9y1HZKQx/hD4g/sPGNpnGJGRkcTGxFK0eBGqvF3lv10nrMPHh2Hjx3MxPJwCefPS49NPk2znVrAgn338MV379sXKyoo8rq589cUXADSsXZtr16/Twd8fg8GAo4MDsyZMoEzp0nzxxRd06dIFk8lEVFQU9erVw9PTk4ULF7J7925sbW2xs7Nj4MCBxMTE0K9fP+7du4eVlRWurq706tUr2fpFRFKLVezTLpF7RURGRhIaGoqnp2eiE5GPHz8efwXfk5y+eC8ty0ugfo1yZr3XmX2uQoTdS51zoB4+eBh/HtH86fO5GHaJASMSnnSeN1M+7p8/nyr9pUTGf517Zi7mGi/mXiz9eceKKdpEp1Z+9B3aJ1GwT86/x8r0BQt4+OhR/EUBaeFVGysXz//Fd5vupLj94uHV0+Qej09SrPc8+mzv+fSGqSSg+gR2JXGIOq1UCgw0W19/az14u1n60VhJXWkxVtLlDJo8v5nfzuLooaNERUXhmseVnl/1eNklyUu24+cdTBn7LW+9/dYzhTMREXkyBbSX4OTJk2adFUlN3fv6v7S+23Ttmuhk+1IeHoyekPj2IpZo0YKZ7Pj1p0TPjwr4lqzZnF9CRamjcrXKVK5WOUVtb928Rd/P/rkYxdbalhijkbffeovO7dqlVYkiIq8cBTR5ZQRPnfr0Rhbsw3Z+fNjO72WX8VJlc87GjMX/3H7F3IfDRUReFbqKU0RERMTCKKCJiIiIWBgFNBERERELky7PQYsyRWFrSHjfqMJ5Mr3wfiOijFy6mvJ7oYmIiIgkJV0GNFuDbZrcjyWg+gRAAU1ERERejA5xWhivavV4+PDRyy7jhYSFhVHj/fdTbX/TFyxg4hMWXl+5fj3z5s177n3v37+fhg0b0rRpU3bt2oWfnx/nH19VePbsWZo2bUrTpk1Zt27dc/fxX/VrlOPRo4eptj8REXn9pMsZtPTMFG3CYGN42WUkEG0yYWN4vpqaNWr0QneHX7t2LU2bNqVTp04AVKpUKX7b1q1b8fHxYciQIc+9/9eNyWSKXwhdRETSjgLaS+Du7s6nHT5k59793L57F3+/j6hV7a1E7cZPncm+g4eJio4ma5YsDOvbg9y5cnIx/DKtO/vTrFEDft21l4jICIb26UGZ0p6J9nEx/DKtm3xA4+aN2L9nP7Xq16JMxTJMHD2J27duYzAY+LhbRypUrsDlS5fp0rYra7atBkjw+O+/G77XkD3/t5uIiEh6D+5FKe9SAHy/fC2rFq/C+Q1nqvgmv94iwJCAAJwcHTl/8SK37txh0dSpzFu6lI3btgFQolgx+nz2GY4OccthXb56Ff8BAwi/epWC+fIxuHdvMjk5MX3BAqJtbenbty+rV68mJCSEzJkz8+eff5IpUyamTJlCjhw5kqxh1qxZbNq0CXt7e9avX8+yZcto0KAB06ZN48SJE8yfP5+YmBj279/PlClTiI2NZfDgwdy8eRNTjBUdOnWjXIWkb9C6ankwP/+0FZMpGju7DHz2RT/cirgnaLNv707Wrl7K8K8DuX3rJq3er8OAwd9QpXotViydz4P79xk+pN8Tx8GoCd+SN3cu2n/QDIDjf/xF3+Ffs3bhLFat30TwijXY2dkSExPD2KFfUahAviRrXbtpKxt/+AknJ0fCrlzHIZMD/Yb3I4fLG2xev4XtW38iS9asnDtzjt6DenHzxk1mfzsbU0wMWbNlpceAL8iTLw8Am9ZuYvXSNQDY2towcuIonLNnY/dvu1k0ZzFGoxEbWxu69uxCiVIlOH36NH169iQiMpKYmBga1qlDu+bN2b5jB1PnzsVgbY0pJoY+n332xIXTRUReRwpoL4mVlTULpk7k7PkLtOvWE5/SnokWTO/YugW9usbd2HR1yCYmTZ9DwJC4dS9v37lL6ZIefO7XgQ0//Ejg9DnMD0r6jvq3b98mf6H8tO/cHoBu7T/jnXffoUHT+pw9fY4efj2Yu3LOU2u+e+cuJUqX4ONuHfnfpm3MnDyLyXMCOfXnaRbPWcy0RdNwzp6NWeNnp+gzOHzsGDPHj8fBwYH/27OHjdu2MWfSJJwcHRkSEMCs4GD8Hy+mfiA0lCXTppE9WzaGjRvHrODgJNdsPHLkCOvWrcPV1ZWBAwcSHBxMjx5JL0fVqVMn/vrrLzw9PWnTpk2CbY0bN+bcuXM8fPiQvn37AtC8eXNatGhB8+bN+en/DtHnCz+mz1tJ1qzZEu27Zp13eL9F3D4P7NvNlIlfMyloXoI2JUv5MGbUQKKjozm4fw8eJUpz8MAeqlSvxcH9e2n2Qdyd9Z80Dlq91xj/AUNo1/J9rKysWLpmHS2bNsLKyoqJ02axat50crnkwGg0YoqJSfa7OHDkKMtnB1G8QlVGjx9F0LgghgbEzRweORjKzCUzyJ03N7du3qJPt75MmDGBgoULsPH7TYwe+DVB87/l4O8HWTx3CYGzJuH8hjOPHj7CYDBwKewSC2cHM2bKNzhldOLsqbP09+/Pkg1LWLx4MW9WqIDf48//7r24dSunzZ9Pf39/fEqVwmQy8SgiItn6RUReNwpoL8m779QFoGD+fHgULcKRY8ep/qZvgja/7d7Lsu9DePjoUaIljhwdHKhWuSIApUsUZ/zUmU/sK0OGDFSvXR2IW+z81B+nqNf4cf+FC1DE3Y1jR45TuEihZGt2cHTAt0rcIcASpTyYNmkaAIf2HaLiWxVwzh4XVFq2bMnGDRue+hnUrFoVh8czZLv376dO9epkdHIC4N133mHcv1YOqFKxItmzxe2/Sb16BAQFJbnPMmXK4OrqCoCXlxc7dux4ah0pcf/+fY4fP877j8+tK1CwMIWLuHPi2BEqVa6aqP1ffxxn2aK53Lt3Bytray5eSHy3fHt7ewoUKMyJ46Ec2L+H1u06MXt6IFFRUfz5x3FKesbNGD1pHBQumJ88rq783+7fKV2yOD//3256d4sLreV9vBn8zXiqv+lLVd8K5M3tmuz78ylVkoL542bY6jdtgN8H/6x4UMrbk9x5cwNwPPQEbsUKU7BwAQDqNa7L5DGTefjgIbv/bze136mN8xtxy1Y5OMZ9t3t37iU8LJwen/xzYY7JFMPNG7coX74834weTVR0NOW8vCjv7R1Xv7c3E6dPp1bVqlQuX54ihZIfmyIirxsFNAsQSyxgleC5S5evMC5oBoumTyavay4Ohh6j/4hv4rfb2f1zmxBra+v4f7hnLlzCD9t/BaB3t87kcc2Jg4MDVlZx+4+NjU2yBisrKwwGQ4LtRqMxQRtb2//0GW1Kdp9P42hvn7CG/9b0hNfFPq43KRkyZIj/22AwJAq2qc3Kyoozp/9i3NeDASjtXZaOn/gzamhfxk6aSZFixblx/RptWtRP8vXeZcpzaP8eThwL5bMv+pM1W3a2b9tM4cJFsbPLwMWLF5MdB63fb8Ky70M4de48NapWJlPGuIA7ceQgQk/8wZ79B+n0RV8G9vyctyqVT9mbio1N8OHbPw7R8due8M08aRjExkJ533L0G94v0bbSdUtTLEcOdu7bx7ylS1m3ZQsj+/WjV5cu/HnmDHsPHKDviBF82KwZ7zVokLL6RUReA+kyoEWZoh7fEiN1RUQZn97osbWbtvJJu9acC7vIyb9OU6pE8QTbHzx4iK2NDW84ZyMmJoYVa58+IwXg17YVfm1bxT++GH45wXanjE64FXNja8hW6jWux/mz5zn1x2k8PIuTKVMmoqOjuXjhInny5WHb5h9T1Kd3OW+WLVjGrZu3yOacjZUrV6bodf9WsUwZAmfO5IN338XRwYHvN26kQpky8dt/272bW7dvky1rVtZv2WL285EyZsyIh4cHa9as4f333+fC+bOcPvUH7h6eZM2ajaCZi+PbPnhwH5PJxBsuOQEIWbviifv1KlOBsaMHkS9/QWxtbfEuU57g+TOo905TIG7mLrlxUKVSecYFzeD4n38RNGYEANHRJsKvXKGUhzulPNwJuxjOiT9PJRvQDoYe41zYRdxzFWLL+i14l/VOsl2J0iUYN2I858+eJ3/B/GwN2UoRdzccnRzxrVKJcSPG0/C9hjhnzxZ3iNPGQLlKZVk4cyFnT52loFtBAE4cPUHxksU5d+4c2Z2daVy3Lvnz5GHYuHEAnL1wgaKFClG0UCEePXrEsZMnFdBEJF1JlwHtvzepBTh98Z5Za7CztaV9t57cunOHQb38E51/VtStELWrV+G99p1xzZmDsl6l2X/4SKr0PWBkfyaOnsTKxaswGAz0G96XrI/779arK3269SVnLhe8y3mnaH9uRQvT+qPWdP/4C7Jlz0bdmnWfuaY3K1Tgz9On+ah7dwA8ihal04cfxm+v4OPDsPHjuRgeToG8eenx6afP3MeLGjduHIMHD2bevHmYYqz4sv/wJM8/c3LKSNuPOtO9SztcXHJRrmLSFxIAFPfw5O6d23j7xIUnb5/yzJsVFP/Y3d092XFgbW1N43q1+G33XtyLFAYgJsbEoK/Hc+/+A6ytrcjpkoPunTsm+97KepXiuzkLOT08IP4igaRkzZaVfsP7Muqr0ZhMJrJmy0r/EXHnRXqV9aJVh1b06folVtbW2NraMnLiSPLmz0v/Ef0YN2IckZFGoqOiKelVkuIli7Np0ybWrl6NrY0NWFnRu2tXAKbMns2FixcxWFuTKWNGBvXqlWz9IiKvG6vY5z0+ZWEiIyMJDQ3F09MzwWEugOPHj+Ph4ZHs680Z0OrXKMfOTWtwdHR4euNUYJ+rEGH3LpilL4C8mfJx/3zic67SyovcZuN5mWu8FM6TiYjLZ5Jt07lnf95vVJ86byc+Fy4l1m7ayi879zB++ECNlTTwImPl4vm/+G7TnRS3Xzy8On+M6/Dc/T2rYr3npclNv58koPoEdj3+jzhzqBQYaLa+/tZ68Haz9KOxkrrSYqzoRrUir6ijJ/7gnVYfkTGjU5K3aRERkVdXujzE+bKdPHnyqbMir4OTf/3F0LFjEz3fokkT3jXj+UQrVqwgODg40fPffPPNU2dWLVnJ4sXYsGRuitu3+uTzRBdNlCpRnEG9/GlSv05qlyciIi9AAU3SjHuRIix5whJN5tS8eXOaN2/+sst46ZbMmPKySxARkRTSIU4RERERC6OAJiIiImJhFNBERERELEy6PActJioKa9uE90IrnCfTC+83KsLIhRuRL7wfERERSd/SZUCztrVNk/ujxN0H5cUCmle1ema9R9rTnDx2klWLVzFg5IBk212+dJkubbuyZtvqRNvu3b/P6g0baN+y5XPV0LBNGyaNGGFR6zH+sHk9u3f9ysChAamyvz49PuH9Fm2p6Fsl0bahARNpXLc2Zbw8n2vfy78PYdGq78lgl4FJowYz6OvxzA6Mq/vHX3cwecZc7OzsCBjSj+K5XvwzPvj7QaYHzuC7hVOf3lhERJKULgOapJx7CfenhrOnuXf/PvOXL3/ugPasTCYTBoPBLH2llMkUjcHwfP93G9qnxwv1vXjVWkYN+BJPD3eA+HAGsHLdRrp2bPvcN7l93URHR2Njo59FEXn59Ev0Eri7u/Nphw/ZuXc/t+/exd/voyRvNDp+6kz2HTxMVHQ0WbNkYVjfHuTOlZOL4Zdp3dmfZo0a8OuuvURERjC0Tw/KlE48wxI4Yw7ZcxegXsu6bP9hOyMHjGLFluVkc85Gf/8BvN/6PcpVKsfu33azaM5ijEYjNrY2dO3ZhRKlSiSaDfl+2fesXrqGjJmcqPBmRdYuX5tg1mx20BwO7DrAw3v3GNSrFz6ennwzZQr379+nVefO2NvbMzcwkGs3bjA2KIjLV68SERlJvbffpmPr1gAcOHKErydPxj5DBjw9PJ66GPvq1avZsGEDzs7OnDp1ilGjRnHt2jUmTJiAyWTC2dmZ4cOHU6BAAVavXs327duZPHly/Gv/frx69WpCQkLInDkzf/75J5kyZWLKlCnkyJEDo9HIyJEj2b17Nzlz5uSNnPme+j23b9WIug2acOjAXnK55uHTz3rz3eSx/HHyKAA1ajegRasO8e0P7NvDquULuX7tKlWr16ZDp24AfNz9S9q1bEa1yhUZ9PU47OzsOHfhIleuXqN0SQ9GDuj9xMXjvxw6mguXwvlq9FhKFCvKZ53a07qzPz+vW87Yb6ez/0goZy+Esez7EGYHBvDLL78wZuw3mGJiyJotKz0GfEGefHmS3PfogaO5cC6MKKOR3Pny8OXg3mTKnPBUgVnfziJT5ky0bNcyyfHX+ePOeObJQ/evvuLO3btEGo2UdHfnqy++wNbWlhZ+fgzp3ZuS7nHhMnjlSs5euMCA7t0Z8+237D14EDtbWxzt7ZmTzJ28hwQEYGNjw5XbtwkPD6d8+fIMHjwYOzs7+vXrh5OTE2fPnuXWrVusXr2aGTNmsG7dOgBKlSrFwIEDcXJywmg0MnHiRH799Vesra3Jly8fQUFBAMycOZMtW7ZgMpnImTMnI0aMIEeOHOz8v+0smPMd1tYGTKZouvr3obR3ORbNn8H2H7dgZ5cBrGDMhOlkzPjip1qIyOtBAe0lsbKyZsHUiZw9f4F23XriU9oz0XqcHVu3oFdXPwBWh2xi0vQ5BAyJW/fw9p27lC7pwed+Hdjww48ETp/D/KDEC8BXLOPNwjUbqdeyLvv3HMCjlAcH9h6kas0qHD96HE9vTy6FXWLh7GDGTPkGp4xOnD11lv7+/VmyYUmCfZ368zSL5y1hxuLpZM2WlaDxCQ9h3b1zlxKlSzCk3xCWz53LlJkzmRMYSL/PP6dNt24J7ok2JCCATh9+SJnSpYmKiuLTPn0o4e5OmVKl6D9qFCP796eclxdbf/6ZZd9//9TPc//+/axdu5b8+fNz48YNPvroI4KDgylSpAgrVqygd+/erFjx5EXL/3bkyBHWrVuHq6srAwcOJDg4mB49erBs2TLCwsIICQkhOjqa5i1a4ZLL9an7u3njOmMmxL3v2TMmExMbw3ezl/Hw4QN6ftaRQoWLUr7imwCcP3ear8dNxWg00vOzj/AoWZrCzRLf0PevM2eZMf4brK2taNGpG7t+P4Bv+TKJ2gGMHTqA+i3bMW7YQIoWLsjF8Mvx2778rDMn/vwrPvzduHWbPn36MG76OAoWLsDG7zcxeuDXBM3/Nsl9d+vdjSxZswAwZ+ocls5fit/nfgna+JT3YUXwClq2a5nk+CtbtizRV68yasAAsmbOTGxsLEMCAli7eTPNGjWiRePGrFi3jpJffklsbCwr168nYPBg/jh9mj3797Nqzhysra25e+/pyymFnjjB8lWryJAhA5988gnLly+nTZs2ABw4cIDg4GAcHR35+eefWbduHUuXLsXJyYm+ffsydepUvvzyS2bMmMGFCxdYvXo1dnZ23Lx5E4C1a9dy/vx5li9fjrW1NYsXL+abb75h/PjxLJw7nW7d++FZ2geTyURExCPu3bvLquXBLFm9lQwZ7Hn48EGiJepEJH1TQHtJ3n0nbkHxgvnz4VG0CEeOHaf6m74J2vy2ey/Lvg/h4aNHie4A7+jgQLXKFQEoXaI446fOTLIf71Il6TN8DFFRURw9dJTOX3zCL9t+5Q2XNyjsVgh7e3v27txLeFg4PT75Z500kymGmzduJdjXoX2HqPhmhfiF1es1qsv/Nv4vfruDowO+VSoBUMrDg0lPuEnto0eP+P3QIW7dvh3/3INHjzhz/jzZs2XDPkMGynl5AVCnWjVGTZyY5H7+rUyZMuR/vM7ioUOHKF68OEWKFAHg/fffZ9iwYdy/fz9F+3F1jQteXl5e7NixA4Ddu3fTtGlTbG1tsbW15e1a9TkaevCp+6tZ5534vw/u20Pnz+Jmu5ycMlK9Rh0O7t8TH9Bq1W2IwWCDg4MNVd+uw6EDe2mVRECr8VZlMmSwA8CjaBEuXArHN1GrZ3fk2AmKFy9OwcIFAKjXuC6Tx0zm4YOHODo5Jmq/NeQHtm3eRnRUFBEREeTNnzdRG08vT0b0H/nE8efg4MCdmBgWrljBjj17MMXEcO/+fewfh5V3atdmZnAwd+7e5ejJkzhny0YxNzfuPXiAKSaG4ePHU97bmyqVKj31/dWpVg0nJycAmjZtytatW+MDWr169XB0jHuPO3fupEGDBmTMmBGAFi1aMHr0aAB++ukn+vXrh51d3Ofv7OwMwI8//khoaCjvvvsuEHeY/e/Xe/mUY+Z3E6lSrRblKlamYKEimEwm8uYrwNjRgyhbvjIVfavg6OiUkq9JRNIJBTQLEEsskPAQ1aXLVxgXNINF0yeT1zUXB0OP0X/EN/Hb7ez+uQrV2to6PsDNXLiEH7b/CkDvbp2pUMYLd3d3ftzyE85vOONdzptpk6aTw+UNvMv7xPUfC+V9y9FveL9EtZ0/c+6fOmNjn3goDcD2X1fGGqytif5PqPxbzOP9LAgKwvY/5/v8cerUE/efnL//4X1anQaDgZiYmPjHkZEJL+r49yyGwWCI/1yfdpj1SRwc/rnYI66u/7Z4wueZzHv4OxzE1WidKLw/vyf3uXndZlYvXQNAi7YtcMnlwvpV65k8J5Cs2bKybfM2NqzekOh1GewzULho4WTH3+Yff+RgaCizJk7EydGROYsXcy4sDAAHe3vq1ajBui1b2Hf4MC0aNwYgk5MTK2bNYt+hQ+w5cIDJs2ax6LvveONxYHrqO/3P5/t3OEtq239f96Tnu3TpQrNmzRJt69ytF2dO/8WhA3sZPawf7zb7kPoN32Vi0FyOhh7i0IHf+fzTNoz8ZgqF3IqmqH4Ref2ly4AWExWVJivPR0UYU9x27aatfNKuNefCLnLyr9OUKlE8wfYHDx5ia2PDG87ZiImJYcXaxP/4JcWvbSv82rZK8Jyvry/zp8+n0fsNsbOzI4fLG2wJ2cqAEXGHS8tVKsvCmQs5e+osBd0KAnDi6AmKl0xYk3dZL5YvXM6d23fIkjULW0K2pqgmJ0dHIiIjiTaZsDEYcHJ0xMfTk3lLl+L3eAbj8tWr2NjYUDBfPiKNRvYfPkyZ0qX53y+/cP/BgxT18zcfHx+++uorTp06hZubG2vWrKFEiRJkzJiR/Pnzc/LkSYzGuO9qy5YtZM6c+an79PX1Ze3atTRo0IDo6Gi2b9tMjpy5nq2uchXZsmEtJUp68ejRQ37+aSudPv0ifvuPP2yk2tu1iTJG8evP22j/cddn2v+LKl3Sg6FjJ3P+7HnyF8zP1pCtFHF3w9HJkXqN61Gvcb34tjt+2YlTRicyZ8mM0Whk87rNT9xvmfI+yY6/ew8ekDVLFpwcHbn34AGbf/oJj6L/BJUWjRvj16sXJpOJsYMHA3Dr9m0MBgOVy5enYpky/LprFxfDw5MNaP/75Rf8unfHzs6OdevW8fbbbyfZrnLlyowbN462bdvi5OTEypUrqVy5MgA1atRg/vz5eHl5xR/idHZ2pkaNGixYsIDatWuTJUsWjEYjp0+fpnjx4oSdP0uhwkUoVLgIjx495I+Tx6hWow4Rjx5R2qsspb3KcvzoYc6ePaWAJiLx0mVA++890ABOX3z6OSypyc7WlvbdenLrzh0G9fJPdP5ZUbdC1K5ehffad8Y1Zw7KepVm/+Ejz9WXr68vgYGB+DyesfApX4bQQ0cp7hkXwPLmz0v/Ef0YN2IckZFGoqOiKelVMlFAcyvmRst2Lfn8I3+yZc9G2QplcMr49MMyWTJnpn6NGrT08yNzpkzMDQxkZP/+TPjuO1r4xZ2z5OToyOBevXjD2ZnRAwbEXyRQztubXC4uz/R+nZ2dCQgIoHfv3kRHR+Ps7MzYx4u2+/j44OvrS8OGDcmbNy9ubm5cu3btqfts0aIFJ0+e5J133iFXrlyU8irL5csXn6mu1m07MXVyAF0+jruatUbtBpSrUDl+e5Gixenfuys3rl+jSrVaSd5yIy05Z81KQEAAo74ajclkImu2rPR/HKL+q2LlCmzb9D86NPuIHC45KOZRjJNHTyTZ1qdCGeZOm/fE8fdO7dr8vGMHzTt1Ikf27Hh7eiaY2czj6krBfPnwLF48fpb28rVrjJw4EZPJhMlk4s0KFSj1lIXvfUqXplu3bly6dIny5cvTokWLJNtVq1aNkydP8sEHHwDg6elJly5dAPjkk08YP358/OHuAgUKMHnyZJo2bcrt27fjD5nGxsbSqlUrihcvzpyZ33Lp4nkMBhucMmbki96DefDgPqOG9CHSGElsTAxFihbnzSpJB0YRSZ+sYp/32I2FiYyMJDQ0FE9Pz0Qn2x4/fhyPp/x4mzOg1a9Rzqz3OrPPVYiwexdSZV//Ph9p/vT5XAy7FD8T8re8mfJx//z5VOkvJTI+PvfMnMw1XgrnyUTE5TNm6QtSd6ykRErGyv0HD3i/Y0cWfPstOXPkeK5+hgQEUKJYMT7293+u17+IFxkrF8//xXeb7qS4/eLh1fljXIfn7u9ZFes9jz7bez69YSoJqD4hTe5h+SRpcaTlaVoP3m6WfjRWUldajJV0OYMmz2/mt7M4eugoUVFRuOZxpedXL3aPLpHkrFy/ntmLF9OmWbPnDmciIq8iBbSX4OTJk2adFUlN3fuafwbib226dk10QnwpDw9GT0h8exFz2bzhe9Z/vzzR8z37DsGtiLvZ6hgxfjJHjiU8xGgwGFgyY4rZakgLzRo1olmjRilqe/Kvvxj6+FD2v7Vo0oRhffqkdmkiImlKAU1eGcFTLW/poHrvNKXeO01fdhkM6vXygrOlcC9SJMG99kREXmXWL7sAc3lNTrUTkXQiNjYW/WyJpF/pIqDZ29tz48YNhTQReSXExsYS8eAO1+6m1j3uRORVky4OcebNm5ewsLBkb6dw/XaE2eqJvGtP1N3rZuvP9lYEtyJumq2/e/b3ibxpvv4yPON90lKDucaLxkrqelXGSmwsXLtrYuPv5q9XRCxDughotra2FCpUKNk25rq0GWDxcJ/X+/JmH/Ne3uz9Wl8Kr7GSml7nsSIirxezHeI8c+YMLVu2pG7durRs2ZKzZ88mamMymRg2bBi1atWidu3aKVrcWkREROR1Y7aANmTIEFq3bs2WLVto3bo1gx8v2fJv69ev5/z582zdupVly5YxZcoUwh6vySciIiKSXpjlEOeNGzc4duwYc+fOBaBhw4aMGDEifh27v23cuJHmzZtjbW2Ns7MztWrVYvPmzXTq1Ompffx9AcDfayw+q8wOT14EPLVFRkYSY5/JrP05Wj19SabU7A8nM/dnZuYaLxoradCfmem3JXX703hJHRoradDfc7Kzs8PKKvH3bpalnkJDQ+nbty8bNvyz4HeDBg0YO3YsJUuWjH+uUaNGjBo1itKlSwMwc+ZMrly5wsCBA5/ax7179/jjjz9Sv3gRERGRNJLUEpXwGl0k4OTkRLFixbC1tU0yiYqIiIhYGjs7uySfN0tAc3V15cqVK5hMJgwGAyaTiatXr+Lq6pqo3aVLl+Jn0MLDw8mdO3eK+rC2tiZTJvNN14qIiIikFbNcJJA9e3Y8PDwICQkBICQkBA8PjwTnnwHUq1ePFStWEBMTw82bN/nf//5H3bp1zVGiiIiIiMUwyzloAKdOnaJfv37cvXuXzJkzM2bMGAoXLoyfnx/+/v6UKlUKk8nE8OHD+b//+z8A/Pz8aNmypTnKExEREbEYZgtoIiIiIpIy6WItThEREZFXiQKaiIiIiIVRQBMRERGxMApoIiIiIhZGAe01lpIF6kUAxowZQ40aNXB3d9eKHJLIk8aHfmPkv27duoWfnx9169alUaNGfPbZZ9y8eRPQeHlWCmivsZQsUC8CULNmTRYtWkSePHlediligZ40PvQbI/9lZWVFp06d2LJlC+vXrydfvnyMGzcO0Hh5Vgpor6m/F6hv2LAhELdA/bFjx+L/S0bk38qVK5doZQ+RvyU1PvQbI0nJmjUrFStWjH/s7e3NpUuXNF6egwLaayo8PJycOXNiMBgAMBgMuLi4EB4e/pIrE5HXgX5j5GliYmJYsmQJNWrU0Hh5DgpoIiIikupGjBiBo6Mjbdq0edmlvJIU0F5T/16gHnjiAvUiIs9DvzGSnDFjxnDu3DkmTZqEtbW1xstzUEB7TaV0gXoRkeeh3xh5kokTJxIaGkpQUBB2dnaAxsvz0Fqcr7EnLVAv8l8jR45k69atXL9+nWzZspE1a1Y2bNjwsssSC/Gk8aHfGPmvP//8k4YNG1KwYEHs7e0ByJs3L0FBQRovz0gBTURERMTC6BCniIiIiIVRQBMRERGxMApoIiIiIhZGAU1ERETEwiigiYiIiFgYBTQRERERC2PzsgsQEXkWNWrU4Pr16xgMBhwdHalSpQqDBg3CycnpZZcmIpJqNIMmIq+cadOmceDAAb7//nuOHTvGjBkzXnZJAERHR7/sEkTkNaGAJiKvrBw5cvDWW29x/PhxAA4ePMgHH3xAuXLlaNy4Mbt3745vu3r1amrWrImPjw81atRg3bp1AMTExDB16lTefvttfH196dOnD/fu3QNg9+7dVK1aNUGfNWrUYMeOHQBMmTIFf39/evfuTZkyZVizZg23b9+mf//+vPXWW5QvX56uXbvGv/ann36iSZMmlCtXjg8++IATJ07Eb5sxYwZVqlTBx8eHunXrsnPnzrT50ETklaBDnCLyyrp8+TK//vorFStW5MqVK3Tu3JmAgACqVKnCzp078ff3Z9OmTdjb2zNy5EhWrlxJ4cKFuXr1Knfu3AHigtuaNWtYsGABzs7O9O3bl+HDhzN27NgU1bBt2zYCAwMJCAjAaDTi7++Po6MjGzZswNHRkQMHDgBw9OhRBgwYwLRp0/D09GTdunV07dqVzZs3ExYWxqJFi1i5ciU5c+YkLCyMmJiYNPvcRMTyaQZNRF453bp1w8fHh2rVquHs7Iy/vz9r166latWqVKtWDWtra9588008PT35+eefAbC2tubPP/8kIiICFxcXihYtCsD69evp0KED+fLlw8nJiZ49e7Jx48YUH6709vamVq1aWFtbc/fuXX755ReGDRtGlixZsLW1pUKFCgAsX76cli1b4uXlhcFg4N1338XW1paDBw9iMBgwGo2cOnWKqKgo8ubNS/78+dPmwxORV4ICmoi8coKCgjhw4AALFy7k9OnT3Lp1i0uXLrF582bKlSsX/799+/Zx7do1HB0dmThxIkuXLuWtt97ik08+4dSpUwBcvXqVPHnyxO87T548REdHc+PGjRTVkitXrvi/L1++TJYsWciSJUuidpcuXWLu3LkJ6rt8+TJXr16lQIECDBgwgClTplC5cmV69OjBlStXXvBTEpFXmQ5xisgrq0KFCrz33nuMGTMGLy8vmjRpwsiRI5NsW6VKFapUqUJERASTJk1i0KBBLF68GBcXFy5evBjf7tKlS9jY2JA9e3auXLlCRERE/DaTycTNmzcT7NfKyir+71y5cnHnzh3u3r1L5syZE7RzdXXl008/pUuXLknW16hRIxo1asT9+/cZPHgw48aNS/FhVhF5/WgGTUReae3bt2fHjh2ULVuWn376iV9//RWTyURkZCS7d+/m8uXLXL9+nW3btvHw4UPs7OxwdHTEYDAA0LBhQ+bPn8+FCxd48OABEydOpH79+tjY2FCoUCEiIyPZvn07UVFRfPfddxiNxifW4uLiQtWqVRk2bBh37twhKiqKvXv3AtC8eXOWLl3KoUOHiI2N5eHDh2zfvp379+9z+vRpdu7cidFoxM7OjgwZMsTXJyLpkwKaiLzSnJ2dadKkCfPnz2fq1KlMnz4dX19fqlWrxuzZs4mJiSEmJoa5c+dSpUoVKlSowN69exkyZAgA77//Po0bN6ZNmzbUrFkTOzs7Bg0aBECmTJkYMmQIAwcOpGrVqjg4OCQ4pJmUgIAAbGxsqF+/PpUrV2b+/PkAlCpVihEjRjB8+HDKly9PnTp1WL16NQBGo5Hx48dTsWJF3nrrLW7evEmPHj3S8FMTEUtnFRsbG/uyixARERGRf2gGTURERMTCKKCJiIiIWBgFNBERERELo4AmIiIiYmEU0EREREQsjAKaiIiIiIVRQBMRERGxMApoIiIiIhZGAU1ERETEwvw/xJB1rQzyBVgAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "resources = [int(1 / c) for c in constants] \n", + "df = pd.DataFrame({\n", + " 'Model Runtime Const': resources, \n", + " **all_results\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars='Model Runtime Const').rename(columns=str.title)\n", + "seaborn.barplot(x='Model Runtime Const', y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "ax1.set(xlabel='Resources', ylabel=f'{metric} Error')\n", + "ax1.legend_.remove()\n", + "plt.legend(loc='lower left')\n", + "seaborn.despine(fig)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "61d517d0", + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax = plt.subplots()\n", + "ax.set_xscale('log')\n", + "#resources = [10/c for c in constants]\n", + "resources = constants \n", + "print(resources)\n", + "ax.plot(resources, plan_weighted_longest_queue_lifo, label=\"LIFO Weighted Queue\", c='coral', marker='.')\n", + "ax.plot(resources, plan_longest_queue_lifo, label=\"LIFO Queue\", c='coral', marker='.', linestyle='dashed')\n", + "\n", + "ax.plot(resources, plan_weighted_random_lifo, label=\"LIFO Weighted Random\", c='red', marker='.')\n", + "ax.plot(resources, plan_random_lifo, label=\"LIFO Random\", c='red', marker='.', linestyle='dashed')\n", + "\n", + "#ax.plot(resources, plan_lifo_sample_half, label=\"LIFO Sample Half\", c='dodgerblue', marker='.', linestyle='dashed')\n", + "#ax.plot(resources, plan_lifo_always_process, label=\"LIFO Always\", c='dodgerblue', marker='.')\n", + "\n", + "#ax.plot(resources, plan_round_robin_lifo, label=\"LIFO Round Robin\", c='blue', marker='.', linestyle='dashed')\n", + "\n", + "ax.grid()\n", + "ax.set(xlabel='resource constraint', ylabel=f'{metric} accuracy', title='Passage Retriever')\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": 280, + "id": "aece6567", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_1.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.06871169495648626, 'top5': 0.12280371338214406, 'top10': 0.13392345661573715, 'top100': 0.13392345661573715}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_1.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.0640895857365947, 'top5': 0.11222543964969277, 'top10': 0.12234071772174746, 'top100': 0.12234071772174746}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_2.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.07980004865378126, 'top5': 0.1408997810579843, 'top10': 0.15672010735221414, 'top100': 0.15672010735221414}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_2.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.06708728645306088, 'top5': 0.11490139761910367, 'top10': 0.1252442498293194, 'top100': 0.1252442498293194}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_4.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.08357464039362479, 'top5': 0.16605849440089146, 'top10': 0.1989547284412741, 'top100': 0.1989547284412741}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_4.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.0692296223054045, 'top5': 0.1186367524385746, 'top10': 0.1285087616043192, 'top100': 0.1285087616043192}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_8.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.12422408989963196, 'top5': 0.25539311470521303, 'top10': 0.2912321177735402, 'top100': 0.2912321177735402}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_8.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.06077798965714779, 'top5': 0.12347074102847816, 'top10': 0.1354536965102683, 'top100': 0.1354536965102683}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_16.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.19127213943232024, 'top5': 0.38433348243363075, 'top10': 0.4554621716850688, 'top100': 0.4554621716850688}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_16.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.07892114163743516, 'top5': 0.16019649849722595, 'top10': 0.17815916064379939, 'top100': 0.17815916064379939}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_32.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.18883945036921942, 'top5': 0.40024797733675477, 'top10': 0.46685657336127, 'top100': 0.46685657336127}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_32.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.12397297360924736, 'top5': 0.21128296882234307, 'top10': 0.22860214547480598, 'top100': 0.22860214547480598}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_1.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.06871169495648626, 'top5': 0.12280371338214406, 'top10': 0.13392345661573715, 'top100': 0.13392345661573715}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_1.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.0640895857365947, 'top5': 0.11222543964969277, 'top10': 0.12234071772174746, 'top100': 0.12234071772174746}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_2.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.07980004865378126, 'top5': 0.1408997810579843, 'top10': 0.15672010735221414, 'top100': 0.15672010735221414}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_2.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.06708728645306088, 'top5': 0.11490139761910367, 'top10': 0.1252442498293194, 'top100': 0.1252442498293194}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_4.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.08357464039362479, 'top5': 0.16605849440089146, 'top10': 0.1989547284412741, 'top100': 0.1989547284412741}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_4.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.0692296223054045, 'top5': 0.1186367524385746, 'top10': 0.1285087616043192, 'top100': 0.1285087616043192}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_8.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.12422408989963196, 'top5': 0.25539311470521303, 'top10': 0.2912321177735402, 'top100': 0.2912321177735402}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_8.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.06077798965714779, 'top5': 0.12347074102847816, 'top10': 0.1354536965102683, 'top100': 0.1354536965102683}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_16.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.19127213943232024, 'top5': 0.38433348243363075, 'top10': 0.4554621716850688, 'top100': 0.4554621716850688}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_16.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.07892114163743516, 'top5': 0.16019649849722595, 'top10': 0.17815916064379939, 'top100': 0.17815916064379939}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_lifo-always_process-0.25-100_replicas_32.json\n", + "plan-round_robin_lifo-always_process {'top1': 0.18883945036921942, 'top5': 0.40024797733675477, 'top10': 0.46685657336127, 'top100': 0.46685657336127}\n", + "/data/wooders/wikipedia/predictions/plan-round_robin_fifo-always_process-0.25-100_replicas_32.json\n", + "plan-round_robin_fifo-always_process {'top1': 0.12397297360924736, 'top5': 0.21128296882234307, 'top10': 0.22860214547480598, 'top100': 0.22860214547480598}\n" + ] + }, + { + "data": { + "text/plain": [ + "{'lifo_round_robin': [0.877196286617856,\n", + " 0.8877745603503072,\n", + " 0.8591002189420157,\n", + " 0.8850986023808963,\n", + " 0.8339415055991085,\n", + " 0.8813632475614254,\n", + " 0.7446068852947869,\n", + " 0.8765292589715219,\n", + " 0.6156665175663693,\n", + " 0.839803501502774,\n", + " 0.5997520226632452,\n", + " 0.7887170311776569],\n", + " 'fifo_round_robin': [0.877196286617856,\n", + " 0.8877745603503072,\n", + " 0.8591002189420157,\n", + " 0.8850986023808963,\n", + " 0.8339415055991085,\n", + " 0.8813632475614254,\n", + " 0.7446068852947869,\n", + " 0.8765292589715219,\n", + " 0.6156665175663693,\n", + " 0.839803501502774,\n", + " 0.5997520226632452,\n", + " 0.7887170311776569]}" + ] + }, + "execution_count": 280, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "constants = [0.25]\n", + "policies = [\"lifo\", \"fifo\"]\n", + "#key_policies = [\"random\", \"weighted_random\", \"round_robin\", \"weighted_round_robin\"]\n", + "key_policies = [\"round_robin\"]\n", + "replicas = [1, 2, 4, 8, 16, 32]\n", + "#key_policies = [\"weighted_random\", \"weighted_round_robin\"]\n", + "d = artifact_dir\n", + "metric = 'top5'\n", + "d = \"/data/wooders/wikipedia/predictions\"\n", + "\n", + "replica_results = {}\n", + "\n", + "for pol in policies: \n", + " \n", + " for key_policy in key_policies:\n", + " scores = []\n", + " for replica in replicas:\n", + " for policy in policies: \n", + "\n", + " name = f\"plan-{key_policy}_{policy}-always_process\"\n", + " for constant in constants: \n", + " print(f'{d}/{name}-{constant}-100_replicas_{replica}.json')\n", + " with open(f'{d}/{name}-{constant}-100_replicas_{replica}.json') as results_file:\n", + " results = json.load(results_file)\n", + " print(name, results)\n", + " scores.append(1-results[metric])\n", + " replica_results[pol + \"_\" + key_policy] = scores\n", + "replica_results" + ] + }, + { + "cell_type": "code", + "execution_count": 281, + "id": "e1822437", + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "arrays must all be same length", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m df = pd.DataFrame({\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;34m'Model Runtime Const'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mreplicas\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mreplica_results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m 527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 528\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 529\u001b[0;31m \u001b[0mmgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minit_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 530\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMaskedArray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 531\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmrecords\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmrecords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36minit_dict\u001b[0;34m(data, index, columns, dtype)\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0marr\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_datetime64tz_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0marr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 286\u001b[0m ]\n\u001b[0;32m--> 287\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0marrays_to_mgr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 289\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36marrays_to_mgr\u001b[0;34m(arrays, arr_names, index, columns, dtype, verify_integrity)\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[0;31m# figure out the index, if necessary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 81\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 82\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/data/wooders/anaconda3/lib/python3.8/site-packages/pandas/core/internals/construction.py\u001b[0m in \u001b[0;36mextract_index\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m 399\u001b[0m \u001b[0mlengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_lengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 400\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlengths\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 401\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"arrays must all be same length\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 402\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 403\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhave_dicts\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: arrays must all be same length" + ] + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn\n", + "df = pd.DataFrame({\n", + " 'Model Runtime Const': replicas, \n", + " **replica_results\n", + "})\n", + "fig, ax1 = plt.subplots(figsize=(10, 5))\n", + "tidy = df.melt(id_vars='Model Runtime Const').rename(columns=str.title)\n", + "seaborn.barplot(x='Model Runtime Const', y='Value', hue='Variable', data=tidy, ax=ax1)\n", + "ax1.set(xlabel='Resources', ylabel=f'{metric} Error')\n", + "ax1.legend_.remove()\n", + "plt.legend(loc='lower left')\n", + "seaborn.despine(fig)" + ] + }, + { + "cell_type": "markdown", + "id": "cdf98fa5", + "metadata": {}, + "source": [ + "## Observe how often each key was updated " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91736ad0", + "metadata": {}, + "outputs": [], + "source": [ + "plan_dir = '/data/wooders/wiki-plans'\n", + "diff_dir = '/data/wooders/wikipedia/diffs'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cdeefee", + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict \n", + "\n", + "def evaluate_plan(plan_file, optimal_file, start_ts=0, end_ts=37000): \n", + " plan = json.load(open(plan_file))\n", + " optimal_plan = json.load(open(optimal_file))\n", + " \n", + "\n", + " title_counts = defaultdict(lambda: 0)\n", + " title_counts_opt = defaultdict(lambda: 0)\n", + "\n", + " for ts in plan.keys(): \n", + " if float(ts) < start_ts or float(ts) > end_ts: continue \n", + " for edit in plan[ts]: \n", + " edit_file = edit[0]\n", + " edit_data = json.load(open(f\"{diff_dir}/{edit_file}\"))\n", + " title = edit_data['title']\n", + " title_counts[title] += 1\n", + " \n", + " for ts in optimal_plan.keys(): \n", + " if float(ts) < start_ts or float(ts) > end_ts: continue \n", + " for edit in optimal_plan[ts]: \n", + " edit_file = edit[0]\n", + " edit_data = json.load(open(f\"{diff_dir}/{edit_file}\"))\n", + " title = edit_data['title']\n", + " title_counts_opt[title] += 1\n", + " \n", + " #assert title_counts_opt != title_counts\n", + " \n", + " title_counts_df = pd.DataFrame({\"title\": title_counts.keys(), \"updates\": title_counts.values()})\n", + " title_counts_opt_df = pd.DataFrame({\"title\": title_counts_opt.keys(), \"optimal_updates\": title_counts_opt.values()})\n", + " \n", + " plan_data_df = title_counts_df.merge(pageview_df, on=\"title\")\n", + " plan_data_df = plan_data_df.merge(title_counts_opt_df, on=\"title\")\n", + " plan_data_df[\"pageviews\"] = plan_data_df[\"2021090300\"]\n", + " return plan_data_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6440018", + "metadata": {}, + "outputs": [], + "source": [ + "plan_names = [\n", + " 'plan-weighted_random_lifo-always_process-0.01-100',\n", + " 'plan-weighted_random_lifo-always_process-0.1-100' \n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "523bf657", + "metadata": {}, + "outputs": [], + "source": [ + "results = {}\n", + "end_ts = 37000\n", + "for plan_name in plan_names:\n", + " print(plan_name)\n", + " plan_file = f'{plan_dir}/{plan_name}.json'\n", + " plan_data_df = evaluate_plan(plan_file, f'/home/eecs/wooders/experiments/wikipedia/optimal_plan.json', end_ts=end_ts)\n", + " results[plan_name] = plan_data_df\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2775834", + "metadata": {}, + "outputs": [], + "source": [ + "results[\"plan-weighted_random_lifo-always_process-0.1-100\"].sort_values(by=\"updates\", ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41a032ee", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "df1 = results[\"plan-weighted_random_lifo-always_process-0.01-100\"]\n", + "df2 = results[\"plan-weighted_random_lifo-always_process-0.1-100\"]\n", + "for title in results[\"plan-weighted_random_lifo-always_process-0.01-100\"].title.tolist(): \n", + " u1 = df1[df1[\"title\"] == title].updates.tolist()\n", + " u2 = df2[df1[\"title\"] == title].updates.tolist()\n", + " if u1 != u2:\n", + " print(title)\n", + " print(u1)\n", + " print(u2)\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ea99ccb", + "metadata": {}, + "outputs": [], + "source": [ + "results['plan-round_robin_lifo-always_process-5-100'].set_index(\"title\").sort_values(by=\"pageviews\").head(10)[[\"updates\", \"optimal_updates\", \"pageviews\"]].plot(kind=\"bar\", title=\"Updates for Least Queried Documents (Round Robin)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "999fc591", + "metadata": {}, + "outputs": [], + "source": [ + "results['plan-round_robin_lifo-always_process-5-100'].plot(x=\"pageviews\", y=\"updates\", kind=\"hist\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00d43d3f", + "metadata": {}, + "outputs": [], + "source": [ + "results['plan-round_robin_lifo-always_process-5-100'].plot(x=\"pageviews\", y=\"updates\", kind=\"hist\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7af47144", + "metadata": {}, + "outputs": [], + "source": [ + "results['plan-round_robin_lifo-always_process-5-100'].plot(x=\"pageviews\", y=\"optimal_updates\", kind=\"hist\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d55378e", + "metadata": {}, + "outputs": [], + "source": [ + "optimal_plan_df = evaluate_plan(f'/home/eecs/wooders/experiments/wikipedia/optimal_plan.json', f'/home/eecs/wooders/experiments/wikipedia/optimal_plan.json', end_ts=end_ts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "739fdc68", + "metadata": {}, + "outputs": [], + "source": [ + "n_fits = np.array(range(0, 250, 1)) #optimal_plan_df[\"updates\"].unique()\n", + "n_fits.sort()\n", + "n_fits_map = {v: i for i, v in enumerate(n_fits)}\n", + "n_fits_ticks = {i: v for i, v in enumerate(n_fits)}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c34cc7c0", + "metadata": {}, + "outputs": [], + "source": [ + "n_fits_map" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "c768b43d", + "metadata": {}, + "outputs": [], + "source": [ + "import seaborn as sns\n", + "import numpy as np\n", + "sns.set(style=\"whitegrid\", palette=\"muted\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc803fd4", + "metadata": {}, + "outputs": [], + "source": [ + "max_fits = 60 " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfe761f5", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(12, 12))\n", + "for i, plan_name in enumerate(results.keys()):\n", + " plan_file = f'{plan_dir}/{plan_name}.json'\n", + " plan_data_df = results[plan_name]\n", + " plt.subplot(4, 2, i + 1)\n", + " #plan, loss = run_lp(df, max_n_fits=max_n_fits)\n", + " #arr = np.array([(key, n_fits_map[fits]) for (key, fits) in plan.items()])\n", + " vals = plan_data_df[\"updates\"].tolist()\n", + " arr = np.array([(i, vals[i]) for i in range(len(vals))])\n", + " plt.scatter(arr[:, 0], arr[:, 1], label=max_n_fits)\n", + " plt.yticks(ticks=list(n_fits_ticks.keys()), labels=list(n_fits_ticks.values()))\n", + " plt.xlabel(\"key\")\n", + " plt.ylabel(\"n_fits\")\n", + " plt.legend()\n", + " plt.title(plan_name)\n", + "plt.suptitle(\"Sample plan selection\")\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": 185, + "id": "ac3582ce", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_csv(\"/data/wooders/wikipedia/questions.csv\")\n", + "#df.columns = [\"question\", \"answer\", \"doc_id\", \"timestamp\", \"revid\", \"oldrevid\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 186, + "id": "ce16ddda", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0questionanswerdoc_iddatetimerevidoldrevidts_min
00what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:16:27.428572103721253210372124891299.0
11what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:32:54.857144103721253210372124891315.0
22what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:49:22.285716103721253210372124891331.0
33what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 01:05:49.714288103721253210372124891348.0
44what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 01:22:17.142860103721253210372124891364.0
...........................
127727127727who is the ayo??????Hunter B-15 (portrayed by Wunmi Mosaku) is an ...623726382021-09-01 20:46:09.2307001041650936104165081839968.0
127728127728who is the ayo??????Hunter B-15 (portrayed by Wunmi Mosaku) is an ...623726382021-09-01 21:30:27.6922361041650936104165081840013.0
127729127729who is the ayo??????Hunter B-15 (portrayed by Wunmi Mosaku) is an ...623726382021-09-01 22:14:46.1537721041650936104165081840057.0
127730127730who is the ayo??????Hunter B-15 (portrayed by Wunmi Mosaku) is an ...623726382021-09-01 22:59:04.6153081041650936104165081840101.0
127731127731who is the ayo??????Hunter B-15 (portrayed by Wunmi Mosaku) is an ...623726382021-09-01 23:43:23.0768441041650936104165081840145.0
\n", + "

127732 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 question \\\n", + "0 0 what is the most common death in 2021??????? \n", + "1 1 what is the most common death in 2021??????? \n", + "2 2 what is the most common death in 2021??????? \n", + "3 3 what is the most common death in 2021??????? \n", + "4 4 what is the most common death in 2021??????? \n", + "... ... ... \n", + "127727 127727 who is the ayo?????? \n", + "127728 127728 who is the ayo?????? \n", + "127729 127729 who is the ayo?????? \n", + "127730 127730 who is the ayo?????? \n", + "127731 127731 who is the ayo?????? \n", + "\n", + " answer doc_id \\\n", + "0 A typical entry reports information in the fol... 65984422 \n", + "1 A typical entry reports information in the fol... 65984422 \n", + "2 A typical entry reports information in the fol... 65984422 \n", + "3 A typical entry reports information in the fol... 65984422 \n", + "4 A typical entry reports information in the fol... 65984422 \n", + "... ... ... \n", + "127727 Hunter B-15 (portrayed by Wunmi Mosaku) is an ... 62372638 \n", + "127728 Hunter B-15 (portrayed by Wunmi Mosaku) is an ... 62372638 \n", + "127729 Hunter B-15 (portrayed by Wunmi Mosaku) is an ... 62372638 \n", + "127730 Hunter B-15 (portrayed by Wunmi Mosaku) is an ... 62372638 \n", + "127731 Hunter B-15 (portrayed by Wunmi Mosaku) is an ... 62372638 \n", + "\n", + " datetime revid oldrevid ts_min \n", + "0 2021-08-06 00:16:27.428572 1037212532 1037212489 1299.0 \n", + "1 2021-08-06 00:32:54.857144 1037212532 1037212489 1315.0 \n", + "2 2021-08-06 00:49:22.285716 1037212532 1037212489 1331.0 \n", + "3 2021-08-06 01:05:49.714288 1037212532 1037212489 1348.0 \n", + "4 2021-08-06 01:22:17.142860 1037212532 1037212489 1364.0 \n", + "... ... ... ... ... \n", + "127727 2021-09-01 20:46:09.230700 1041650936 1041650818 39968.0 \n", + "127728 2021-09-01 21:30:27.692236 1041650936 1041650818 40013.0 \n", + "127729 2021-09-01 22:14:46.153772 1041650936 1041650818 40057.0 \n", + "127730 2021-09-01 22:59:04.615308 1041650936 1041650818 40101.0 \n", + "127731 2021-09-01 23:43:23.076844 1041650936 1041650818 40145.0 \n", + "\n", + "[127732 rows x 8 columns]" + ] + }, + "execution_count": 186, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 191, + "id": "d699a5a5", + "metadata": {}, + "outputs": [], + "source": [ + "thresh = 20000" + ] + }, + { + "cell_type": "code", + "execution_count": 192, + "id": "c5b6cbeb", + "metadata": {}, + "outputs": [], + "source": [ + "train_df = df[df.ts_min < thresh]\n", + "test_df = df[df.ts_min < thresh]" + ] + }, + { + "cell_type": "code", + "execution_count": 193, + "id": "d0c039a3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0questionanswerdoc_iddatetimerevidoldrevidts_min
00what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:16:27.428572103721253210372124891299.0
11what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:32:54.857144103721253210372124891315.0
22what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:49:22.285716103721253210372124891331.0
33what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 01:05:49.714288103721253210372124891348.0
44what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 01:22:17.142860103721253210372124891364.0
...........................
127210127210What is the story of Soren??He first appears in the film \"Iron Man 3\" late...623726382021-08-18 23:18:001039252299103925226619960.0
127211127211What is the story of Soren??He first appears in the film \"Iron Man 3\" and ...623726382021-08-18 23:24:001039252266103923801419966.0
127212127212What is the story of Soren??He first appears in the film \"Iron Man 3.\" He ...623726382021-08-18 23:30:001039264480103925805919972.0
127213127213What is the story of Soren??He first appears in the film \"Iron Man 3\" late...623726382021-08-18 23:48:001039252299103925226619990.0
127214127214What is the story of Soren??He first appears in the film \"Iron Man 3\" and ...623726382021-08-18 23:54:001039252266103923801419996.0
\n", + "

48653 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 question \\\n", + "0 0 what is the most common death in 2021??????? \n", + "1 1 what is the most common death in 2021??????? \n", + "2 2 what is the most common death in 2021??????? \n", + "3 3 what is the most common death in 2021??????? \n", + "4 4 what is the most common death in 2021??????? \n", + "... ... ... \n", + "127210 127210 What is the story of Soren?? \n", + "127211 127211 What is the story of Soren?? \n", + "127212 127212 What is the story of Soren?? \n", + "127213 127213 What is the story of Soren?? \n", + "127214 127214 What is the story of Soren?? \n", + "\n", + " answer doc_id \\\n", + "0 A typical entry reports information in the fol... 65984422 \n", + "1 A typical entry reports information in the fol... 65984422 \n", + "2 A typical entry reports information in the fol... 65984422 \n", + "3 A typical entry reports information in the fol... 65984422 \n", + "4 A typical entry reports information in the fol... 65984422 \n", + "... ... ... \n", + "127210 He first appears in the film \"Iron Man 3\" late... 62372638 \n", + "127211 He first appears in the film \"Iron Man 3\" and ... 62372638 \n", + "127212 He first appears in the film \"Iron Man 3.\" He ... 62372638 \n", + "127213 He first appears in the film \"Iron Man 3\" late... 62372638 \n", + "127214 He first appears in the film \"Iron Man 3\" and ... 62372638 \n", + "\n", + " datetime revid oldrevid ts_min \n", + "0 2021-08-06 00:16:27.428572 1037212532 1037212489 1299.0 \n", + "1 2021-08-06 00:32:54.857144 1037212532 1037212489 1315.0 \n", + "2 2021-08-06 00:49:22.285716 1037212532 1037212489 1331.0 \n", + "3 2021-08-06 01:05:49.714288 1037212532 1037212489 1348.0 \n", + "4 2021-08-06 01:22:17.142860 1037212532 1037212489 1364.0 \n", + "... ... ... ... ... \n", + "127210 2021-08-18 23:18:00 1039252299 1039252266 19960.0 \n", + "127211 2021-08-18 23:24:00 1039252266 1039238014 19966.0 \n", + "127212 2021-08-18 23:30:00 1039264480 1039258059 19972.0 \n", + "127213 2021-08-18 23:48:00 1039252299 1039252266 19990.0 \n", + "127214 2021-08-18 23:54:00 1039252266 1039238014 19996.0 \n", + "\n", + "[48653 rows x 8 columns]" + ] + }, + "execution_count": 193, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_df" + ] + }, + { + "cell_type": "code", + "execution_count": 195, + "id": "954b1ea8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0questionanswerdoc_iddatetimerevidoldrevidts_min
00what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:16:27.428572103721253210372124891299.0
11what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:32:54.857144103721253210372124891315.0
22what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 00:49:22.285716103721253210372124891331.0
33what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 01:05:49.714288103721253210372124891348.0
44what is the most common death in 2021???????A typical entry reports information in the fol...659844222021-08-06 01:22:17.142860103721253210372124891364.0
...........................
127210127210What is the story of Soren??He first appears in the film \"Iron Man 3\" late...623726382021-08-18 23:18:001039252299103925226619960.0
127211127211What is the story of Soren??He first appears in the film \"Iron Man 3\" and ...623726382021-08-18 23:24:001039252266103923801419966.0
127212127212What is the story of Soren??He first appears in the film \"Iron Man 3.\" He ...623726382021-08-18 23:30:001039264480103925805919972.0
127213127213What is the story of Soren??He first appears in the film \"Iron Man 3\" late...623726382021-08-18 23:48:001039252299103925226619990.0
127214127214What is the story of Soren??He first appears in the film \"Iron Man 3\" and ...623726382021-08-18 23:54:001039252266103923801419996.0
\n", + "

48653 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 question \\\n", + "0 0 what is the most common death in 2021??????? \n", + "1 1 what is the most common death in 2021??????? \n", + "2 2 what is the most common death in 2021??????? \n", + "3 3 what is the most common death in 2021??????? \n", + "4 4 what is the most common death in 2021??????? \n", + "... ... ... \n", + "127210 127210 What is the story of Soren?? \n", + "127211 127211 What is the story of Soren?? \n", + "127212 127212 What is the story of Soren?? \n", + "127213 127213 What is the story of Soren?? \n", + "127214 127214 What is the story of Soren?? \n", + "\n", + " answer doc_id \\\n", + "0 A typical entry reports information in the fol... 65984422 \n", + "1 A typical entry reports information in the fol... 65984422 \n", + "2 A typical entry reports information in the fol... 65984422 \n", + "3 A typical entry reports information in the fol... 65984422 \n", + "4 A typical entry reports information in the fol... 65984422 \n", + "... ... ... \n", + "127210 He first appears in the film \"Iron Man 3\" late... 62372638 \n", + "127211 He first appears in the film \"Iron Man 3\" and ... 62372638 \n", + "127212 He first appears in the film \"Iron Man 3.\" He ... 62372638 \n", + "127213 He first appears in the film \"Iron Man 3\" late... 62372638 \n", + "127214 He first appears in the film \"Iron Man 3\" and ... 62372638 \n", + "\n", + " datetime revid oldrevid ts_min \n", + "0 2021-08-06 00:16:27.428572 1037212532 1037212489 1299.0 \n", + "1 2021-08-06 00:32:54.857144 1037212532 1037212489 1315.0 \n", + "2 2021-08-06 00:49:22.285716 1037212532 1037212489 1331.0 \n", + "3 2021-08-06 01:05:49.714288 1037212532 1037212489 1348.0 \n", + "4 2021-08-06 01:22:17.142860 1037212532 1037212489 1364.0 \n", + "... ... ... ... ... \n", + "127210 2021-08-18 23:18:00 1039252299 1039252266 19960.0 \n", + "127211 2021-08-18 23:24:00 1039252266 1039238014 19966.0 \n", + "127212 2021-08-18 23:30:00 1039264480 1039258059 19972.0 \n", + "127213 2021-08-18 23:48:00 1039252299 1039252266 19990.0 \n", + "127214 2021-08-18 23:54:00 1039252266 1039238014 19996.0 \n", + "\n", + "[48653 rows x 8 columns]" + ] + }, + "execution_count": 195, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_df" + ] + }, + { + "cell_type": "code", + "execution_count": 196, + "id": "07d5672a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1305297 8677\n", + "332667 4300\n", + "66304621 3720\n", + "17888363 3569\n", + "3259011 1581\n", + " ... \n", + "67959451 15\n", + "49474213 12\n", + "66135952 12\n", + "66074428 8\n", + "40713040 2\n", + "Name: doc_id, Length: 118, dtype: int64" + ] + }, + "execution_count": 196, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_df.doc_id.value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 215, + "id": "d9dab1e5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1305297 8677\n", + "332667 4300\n", + "66304621 3720\n", + "17888363 3569\n", + "3259011 1581\n", + " ... \n", + "67959451 15\n", + "49474213 12\n", + "66135952 12\n", + "66074428 8\n", + "40713040 2\n", + "Name: doc_id, Length: 118, dtype: int64" + ] + }, + "execution_count": 215, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_df.doc_id.value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 216, + "id": "ea220f3d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1305297 8677\n", + "332667 4300\n", + "66304621 3720\n", + "17888363 3569\n", + "3259011 1581\n", + " ... \n", + "67959451 15\n", + "49474213 12\n", + "66135952 12\n", + "66074428 8\n", + "40713040 2\n", + "Name: doc_id, Length: 118, dtype: int64" + ] + }, + "execution_count": 216, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "train_df.doc_id.value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": 204, + "id": "bfdea858", + "metadata": {}, + "outputs": [], + "source": [ + "test_df.to_csv(\"/data/wooders/wikipedia/test_questions.csv\")\n", + "train_df.to_csv(\"/data/wooders/wikipedia/train_questions.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b01259cc", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 205, + "id": "86c3ebf2", + "metadata": {}, + "outputs": [], + "source": [ + "weights = train_df.doc_id.value_counts().to_dict()" + ] + }, + { + "cell_type": "code", + "execution_count": 206, + "id": "f0941e02", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{1305297: 8677,\n", + " 332667: 4300,\n", + " 66304621: 3720,\n", + " 17888363: 3569,\n", + " 3259011: 1581,\n", + " 58112801: 1253,\n", + " 68507348: 1201,\n", + " 62808792: 1099,\n", + " 66052432: 987,\n", + " 62372638: 879,\n", + " 67089631: 860,\n", + " 67569553: 822,\n", + " 60476189: 822,\n", + " 68553225: 805,\n", + " 1425939: 774,\n", + " 66187257: 687,\n", + " 65521767: 585,\n", + " 67946554: 551,\n", + " 68498551: 453,\n", + " 442785: 446,\n", + " 734845: 401,\n", + " 487602: 381,\n", + " 66883576: 379,\n", + " 68294454: 369,\n", + " 61250187: 366,\n", + " 50170924: 360,\n", + " 61236755: 347,\n", + " 12936708: 309,\n", + " 1027173: 308,\n", + " 46754025: 276,\n", + " 58542318: 258,\n", + " 61258486: 242,\n", + " 66753136: 240,\n", + " 5575754: 240,\n", + " 65871303: 236,\n", + " 12202928: 228,\n", + " 60600284: 224,\n", + " 51150040: 222,\n", + " 34075129: 220,\n", + " 58385279: 217,\n", + " 61049392: 215,\n", + " 60203476: 212,\n", + " 66341639: 212,\n", + " 63129286: 210,\n", + " 26833: 210,\n", + " 24689651: 208,\n", + " 66629866: 207,\n", + " 33385984: 204,\n", + " 58113491: 200,\n", + " 63170193: 200,\n", + " 63395714: 193,\n", + " 55055575: 191,\n", + " 67918135: 190,\n", + " 68284887: 188,\n", + " 65984422: 187,\n", + " 66040815: 181,\n", + " 57798785: 168,\n", + " 67131229: 167,\n", + " 53943680: 166,\n", + " 20304678: 164,\n", + " 64783122: 160,\n", + " 51345275: 155,\n", + " 39734558: 150,\n", + " 65666080: 148,\n", + " 31243078: 147,\n", + " 36567599: 145,\n", + " 67742925: 144,\n", + " 67674654: 142,\n", + " 6063379: 140,\n", + " 66293350: 139,\n", + " 912080: 135,\n", + " 2514174: 134,\n", + " 67843993: 130,\n", + " 26000816: 127,\n", + " 67037597: 120,\n", + " 67903070: 117,\n", + " 65760352: 114,\n", + " 67711917: 112,\n", + " 404323: 111,\n", + " 68107833: 110,\n", + " 57817558: 108,\n", + " 49632909: 105,\n", + " 65770543: 103,\n", + " 67928132: 100,\n", + " 68207325: 100,\n", + " 25743896: 100,\n", + " 68475822: 100,\n", + " 21537193: 98,\n", + " 67334964: 97,\n", + " 68187748: 96,\n", + " 66461741: 93,\n", + " 56185392: 90,\n", + " 68315181: 80,\n", + " 61293820: 75,\n", + " 2656208: 71,\n", + " 60070859: 70,\n", + " 68076456: 70,\n", + " 65708437: 64,\n", + " 68420852: 60,\n", + " 61243245: 60,\n", + " 68463873: 40,\n", + " 60043578: 40,\n", + " 49588: 39,\n", + " 68229696: 36,\n", + " 737: 36,\n", + " 18097883: 34,\n", + " 66459202: 33,\n", + " 66128424: 30,\n", + " 58542328: 30,\n", + " 66916437: 30,\n", + " 67688633: 25,\n", + " 63417935: 24,\n", + " 20306953: 20,\n", + " 67959451: 15,\n", + " 49474213: 12,\n", + " 66135952: 12,\n", + " 66074428: 8,\n", + " 40713040: 2}" + ] + }, + "execution_count": 206, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "weights" + ] + }, + { + "cell_type": "code", + "execution_count": 210, + "id": "2c255c62", + "metadata": {}, + "outputs": [], + "source": [ + "buckets = [1, 2, 4, 8, 16, 32, 64]" + ] + }, + { + "cell_type": "code", + "execution_count": 213, + "id": "af72ae1a", + "metadata": {}, + "outputs": [], + "source": [ + "for key in weights: \n", + " w = int(weights[key]/10)\n", + " if w == 0: \n", + " w = 1\n", + " for b in buckets: \n", + " if w <= b:\n", + " w = b\n", + " break\n", + " weights[key] = b\n", + " #print(weights[key], b)" + ] + }, + { + "cell_type": "code", + "execution_count": 201, + "id": "500bd6ec", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{1305297: 867,\n", + " 332667: 430,\n", + " 66304621: 372,\n", + " 17888363: 356,\n", + " 3259011: 158,\n", + " 58112801: 125,\n", + " 68507348: 120,\n", + " 62808792: 109,\n", + " 66052432: 98,\n", + " 62372638: 87,\n", + " 67089631: 86,\n", + " 67569553: 82,\n", + " 60476189: 82,\n", + " 68553225: 80,\n", + " 1425939: 77,\n", + " 66187257: 68,\n", + " 65521767: 58,\n", + " 67946554: 55,\n", + " 68498551: 45,\n", + " 442785: 44,\n", + " 734845: 40,\n", + " 487602: 38,\n", + " 66883576: 37,\n", + " 68294454: 36,\n", + " 61250187: 36,\n", + " 50170924: 36,\n", + " 61236755: 34,\n", + " 12936708: 30,\n", + " 1027173: 30,\n", + " 46754025: 27,\n", + " 58542318: 25,\n", + " 61258486: 24,\n", + " 66753136: 24,\n", + " 5575754: 24,\n", + " 65871303: 23,\n", + " 12202928: 22,\n", + " 60600284: 22,\n", + " 51150040: 22,\n", + " 34075129: 22,\n", + " 58385279: 21,\n", + " 61049392: 21,\n", + " 60203476: 21,\n", + " 66341639: 21,\n", + " 63129286: 21,\n", + " 26833: 21,\n", + " 24689651: 20,\n", + " 66629866: 20,\n", + " 33385984: 20,\n", + " 58113491: 20,\n", + " 63170193: 20,\n", + " 63395714: 19,\n", + " 55055575: 19,\n", + " 67918135: 19,\n", + " 68284887: 18,\n", + " 65984422: 18,\n", + " 66040815: 18,\n", + " 57798785: 16,\n", + " 67131229: 16,\n", + " 53943680: 16,\n", + " 20304678: 16,\n", + " 64783122: 16,\n", + " 51345275: 15,\n", + " 39734558: 15,\n", + " 65666080: 14,\n", + " 31243078: 14,\n", + " 36567599: 14,\n", + " 67742925: 14,\n", + " 67674654: 14,\n", + " 6063379: 14,\n", + " 66293350: 13,\n", + " 912080: 13,\n", + " 2514174: 13,\n", + " 67843993: 13,\n", + " 26000816: 12,\n", + " 67037597: 12,\n", + " 67903070: 11,\n", + " 65760352: 11,\n", + " 67711917: 11,\n", + " 404323: 11,\n", + " 68107833: 11,\n", + " 57817558: 10,\n", + " 49632909: 10,\n", + " 65770543: 10,\n", + " 67928132: 10,\n", + " 68207325: 10,\n", + " 25743896: 10,\n", + " 68475822: 10,\n", + " 21537193: 9,\n", + " 67334964: 9,\n", + " 68187748: 9,\n", + " 66461741: 9,\n", + " 56185392: 9,\n", + " 68315181: 8,\n", + " 61293820: 7,\n", + " 2656208: 7,\n", + " 60070859: 7,\n", + " 68076456: 7,\n", + " 65708437: 6,\n", + " 68420852: 6,\n", + " 61243245: 6,\n", + " 68463873: 4,\n", + " 60043578: 4,\n", + " 49588: 3,\n", + " 68229696: 3,\n", + " 737: 3,\n", + " 18097883: 3,\n", + " 66459202: 3,\n", + " 66128424: 3,\n", + " 58542328: 3,\n", + " 66916437: 3,\n", + " 67688633: 2,\n", + " 63417935: 2,\n", + " 20306953: 2,\n", + " 67959451: 1,\n", + " 49474213: 1,\n", + " 66135952: 1,\n", + " 66074428: 1,\n", + " 40713040: 1}" + ] + }, + "execution_count": 201, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "weights" + ] + }, + { + "cell_type": "code", + "execution_count": 202, + "id": "db8ae3c9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1834" + ] + }, + "execution_count": 202, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "open(\"/home/eecs/wooders/experiments/wikipedia/weights.json\", \"w\").write(json.dumps(weights))" + ] + }, + { + "cell_type": "code", + "execution_count": 214, + "id": "0901f729", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1831" + ] + }, + "execution_count": 214, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "open(\"/home/eecs/wooders/experiments/wikipedia/bucket_weights.json\", \"w\").write(json.dumps(weights))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a356582", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/wikipedia/offline/download_data.sh b/wikipedia/offline/download_data.sh new file mode 100644 index 0000000..208e477 --- /dev/null +++ b/wikipedia/offline/download_data.sh @@ -0,0 +1,21 @@ +data_dir=/data/wooders/wikipedia + +# download diffs +mkdir -p ${data_dir}/diffs; +aws s3 sync s3://feature-store-datasets/wikipedia/diffs diffs; + +# download model +aws s3 cp s3://feature-store-datasets/wikipedia/models/bert-base-encoder.cp ${data_dir}; + +# download questions +aws s3 cp s3://feature-store-datasets/wikipedia/10062021_filtered_questions.csv ${data_dir}; + +# download embeddings +mkdir -p ${data_dir}/embeddings; +aws s3 sync s3://feature-store-datasets/wikipedia/embeddings embeddings; + +## download raw api data +#mkdir -p ${data_dir}/recentchanges; +#mkdir -p ${data_dir}/doc_xml; +#aws s3 sync s3://feature-store-datasets/wikipedia/recentchanges recentchanges; +#aws s3 sync s3://feature-store-datasets/wikipedia/doc_xml doc_xml; diff --git a/wikipedia/wiki_eval.py b/wikipedia/offline/prepare_dpr_data.py similarity index 64% rename from wikipedia/wiki_eval.py rename to wikipedia/offline/prepare_dpr_data.py index c47d2ff..5f454a1 100644 --- a/wikipedia/wiki_eval.py +++ b/wikipedia/offline/prepare_dpr_data.py @@ -31,6 +31,7 @@ ) from dpr.utils.data_utils import Tensorizer +from preprocessing.log_data import log_plan_data """ @@ -47,14 +48,21 @@ """ # simulation data +import wandb +run = wandb.init(project='wiki-workload', job_type="dataset-creation") +simulation_dir = run.use_artifact('ucb-ralf/wiki-workload /simulation:v2', type='dataset').download() +question_dir = run.use_artifact('ucb-ralf/wiki-workload /questions:v2', type='dataset').download() + +init_data_file = f"{simulation_dir}/init_data.json" +stream_edits_file = f"{simulation_dir}/edit_stream.json" +stream_questions_file = f"{simulation_dir}/question_stream.json" + config = configparser.ConfigParser() config.read("config.yml") -plan_dir = config["simulation"]["plan_dir"] -init_data_file = config["simulation"]["init_data_file"] -stream_edits_file = config["simulation"]["stream_edits_file"] -stream_questions_file = config["simulation"]["stream_questions_file"] - -data_dir = config['files']['data_dir'] +#plan_dir = config["simulation"]["plan_dir"] +#init_data_file = config["simulation"]["init_data_file"] +#stream_edits_file = config["simulation"]["stream_edits_file"] +#stream_questions_file = config["simulation"]["stream_questions_file"] rev_dir = config['directory']['diff_dir'] embedding_dir = config['directory']['embedding_dir'] exp_dir = config['directory']['exp_dir'] @@ -64,87 +72,13 @@ parser = argparse.ArgumentParser(description="Specify experiment config") parser.add_argument("--offline-plan-path", type=str) parser.add_argument("--embed", default=False, action="store_true") +parser.add_argument("--wandb", default=False, action="store_true") +parser.add_argument("--workers", type=int) args = parser.parse_args() - -class Retriever: - def __init__(self): - - # parser = argparse.ArgumentParser(description="") - add_encoder_params(parser) - add_tokenizer_params(parser) - add_cuda_params(parser) - args = parser.parse_args() - - setup_args_gpu(args) - - saved_state = load_states_from_checkpoint(model_file) - set_encoder_params_from_state(saved_state.encoder_params, args) - - self.tensorizer, self.encoder, _ = init_biencoder_components( - args.encoder_model_type, args, inference_only=True - ) - - self.encoder = self.encoder.ctx_model - - self.encoder, _ = setup_for_distributed_mode( - self.encoder, - None, - args.device, - args.n_gpu, - args.local_rank, - args.fp16, - args.fp16_opt_level, - ) - self.encoder.eval() - - model_to_load = get_model_obj(self.encoder) - - prefix_len = len("ctx_model.") - ctx_state = { - key[prefix_len:]: value - for (key, value) in saved_state.model_dict.items() - if key.startswith("ctx_model.") - } - model_to_load.load_state_dict(ctx_state) - self.device = args.device - - def predict(self, text): - - st = time.time() - batch_token_tensors = [self.tensorizer.text_to_tensor(text)] - - ctx_ids_batch = move_to_device( - torch.stack(batch_token_tensors, dim=0), self.device - ) - ctx_seg_batch = move_to_device(torch.zeros_like(ctx_ids_batch), self.device) - ctx_attn_mask = move_to_device( - self.tensorizer.get_attn_mask(ctx_ids_batch), self.device - ) - with torch.no_grad(): - _, embedding, _ = self.encoder(ctx_ids_batch, ctx_seg_batch, ctx_attn_mask) - embedding = embedding.cpu().numpy() - return embedding - - -def assign_timestamps_min(ts): - # take in unix timestamp - covert to integer - start_ts = 1628131044000000000 # don't change - delta = ts - start_ts - if delta < 0: - return None - - return int(delta / (60 * 1000000000)) - - -def embed_passages(sents, retriever_model, num_sent_in_pass=10): - passages = [] - embeddings = [] - for i in range(0, len(sents), num_sent_in_pass): - passages.append(" ".join(sents[i : i + num_sent_in_pass])) - embeddings.append(retriever_model.predict(passages[-1])) - return passages, embeddings - +exp_id = os.path.basename(args.offline_plan_path).replace(".json", "") +run.config.update(vars(args)) +run.config.update({"plan": exp_id}) def sents_to_passages(sents, num_sent_in_pass=10): passages = [] @@ -164,8 +98,6 @@ def offline_eval(plan_json_path, exp_id, compute_embeddings=True): keys = ["51150040"] filter_keys = False - # retriever_model = Retriever() - # print("Created retriever") # compute initial passage embeddings for each document init_data = json.load(open(init_data_file)) @@ -210,7 +142,7 @@ def offline_eval(plan_json_path, exp_id, compute_embeddings=True): for version in tqdm(embed_version_keys): state = {} for task in plan[version]: - print("task", task, version) + #print("task", task, version) rev_file = task[0] doc_id = task[1] # doc_id = task[2] @@ -254,31 +186,36 @@ def offline_eval(plan_json_path, exp_id, compute_embeddings=True): print("EMBED", embed_versions.keys()) print("Num refits", count, len(missing)) - # returns latest version of document embeddings for timestep/key - def get_latest_embedding(timestep, doc_id): - - latest = 0 - for version in embed_versions.keys(): - version = float(version) - if ( - float(timestep) >= version - and version > latest - and doc_id in embed_versions[str(version)] - ): - latest = version - print(doc_id, "latest", timestep, latest, timestep - latest) - assert ( - doc_id in embed_versions[str(latest)] - ), f"Missing doc id {doc_id} {latest} {doc_id in init_data}" - doc_version = embed_versions[str(latest)][doc_id] - assert latest <= timestep - return ( - doc_version["passages"], - doc_version["embeddings"], - doc_version["rev"], - latest, - ) - + embed_filename = "embed_versions.pkl" + pickle.dump(embed_versions, open(embed_filename, "wb")) + return embed_filename + +# returns latest version of document embeddings for timestep/key +def get_latest_embedding(timestep, doc_id, embed_versions): + + latest = 0 + for version in embed_versions.keys(): + version = float(version) + if ( + float(timestep) >= version + and version > latest + and doc_id in embed_versions[str(version)] + ): + latest = version + #print(doc_id, "latest", timestep, latest, timestep - latest) + assert ( + doc_id in embed_versions[str(latest)] + ), f"Missing doc id {doc_id} {latest} {doc_id in init_data}" + doc_version = embed_versions[str(latest)][doc_id] + assert latest <= timestep + return ( + doc_version["passages"], + doc_version["embeddings"], + doc_version["rev"], + latest, + ) + +def generate_question_data_all(exp_id, embed_filename): # create experiment directory directory = os.path.join(exp_dir, exp_id) if os.path.isdir(directory): @@ -289,24 +226,48 @@ def get_latest_embedding(timestep, doc_id): # get simulation data questions questions = json.load(open(stream_questions_file)) + + for ts in range(len(questions)): + questions[ts]["ts"] = ts + print("processing questions", len(questions)) print("directory", directory) - # Get embedding version for each query, write outputs - for ts in tqdm(range(len(questions))): - timestep = ts / 100 # TODO: Watch out!! can change and mess up experiment - # print(ts, timestep) + chunk_size = 1000 + chunks = [(questions[i:i+chunk_size], embed_filename, directory) for i in range(0, len(questions), chunk_size)] + p = Pool(args.workers) + staleness_all = p.starmap(generate_question_data, chunks) + p.close() + staleness_all = [item for sublist in staleness_all for item in sublist] + staleness = np.array(staleness_all).mean() + print("all staleness", staleness) + wandb.log({"staleness": staleness}) + return directory - for doc_id in questions[ts].keys(): +def generate_question_data(questions, embed_filename, directory): + embed_versions = pickle.load(open(embed_filename, "rb")) + init_data = json.load(open(init_data_file)) + + staleness = [] + for ts_questions in questions: + ts = ts_questions["ts"] + timestep = ts / 100 # TODO: Watch out!! can change and mess up experiment + for doc_id in ts_questions.keys(): + if doc_id == "ts": continue # not considered in edits if doc_id not in init_data: print("missing", doc_id) # print(init_data.keys()) continue + # get current embedding and write + passage_texts, passage_embeddings, version, latest = get_latest_embedding( + timestep, doc_id, embed_versions + ) + # loop through questions - doc_questions = questions[ts][doc_id] + doc_questions = ts_questions[doc_id] queries = [] for q in doc_questions: question = q["question"] @@ -319,10 +280,8 @@ def get_latest_embedding(timestep, doc_id): ), f"time mismatch {q['ts_min']}, {timestep}, {ts}" queries.append([question, [answer], doc_id]) - # get current embedding and write - passage_texts, passage_embeddings, version, latest = get_latest_embedding( - timestep, doc_id - ) + # append per query + staleness.append(timestep - latest) # dump CTX/question script contex_file = f"{directory}/dpr_ctx_after_{int(ts)}_{doc_id}" @@ -347,26 +306,20 @@ def get_latest_embedding(timestep, doc_id): assert len(passage_ctx) == len(passage_texts) assert len(passage_embeddings) == len(passage_texts) - - print("done processing queries!", len(questions)) - return directory - + print("staleness", np.array(staleness).mean()) + return staleness def main(): + + + embed_filename = offline_eval(args.offline_plan_path, exp_id, compute_embeddings=args.embed) - plan_file = ( - args.offline_plan_path - ) # "wiki-plans/plan-fifo-always_process-1-0.001-60.json" - exp_id = os.path.basename(plan_file).replace(".json", "") - - output_dir = offline_eval(plan_file, exp_id, compute_embeddings=args.embed) - log_wandb = False - if log_wandb: - import wandb - - run = wandb.init(job_type="create_simulation_output") - artifact = wandb.Artifact(exp_id, type="dataset") - artifact.add_folder(output_dir) + #embed_filename = "embed_versions.pkl" + generate_question_data_all(exp_id, embed_filename) + #if args.wandb: + # import wandb + # run = wandb.init(job_type="dataset-creation", project="wiki-workload") + # log_plan_data(run, config, exp_id, output_dir) if __name__ == "__main__": diff --git a/wikipedia/preprocessing/embedding.py b/wikipedia/preprocessing/embedding.py deleted file mode 100644 index 58b66e9..0000000 --- a/wikipedia/preprocessing/embedding.py +++ /dev/null @@ -1,152 +0,0 @@ -from typing import List -import pickle -from tqdm import tqdm -import time -import numpy as np -import json -import pandas as pd -import argparse -import json -import os -from collections import defaultdict -from multiprocessing import Pool -import torch -from dpr.models import init_biencoder_components -from dpr.options import ( - add_encoder_params, - setup_args_gpu, - print_args, - set_encoder_params_from_state, - add_tokenizer_params, - add_cuda_params, -) -from dpr.utils.model_utils import ( - setup_for_distributed_mode, - load_states_from_checkpoint, - get_model_obj, - move_to_device, -) -from dpr.utils.data_utils import Tensorizer - - -class Retriever: - def __init__(self, model_file): - - parser = argparse.ArgumentParser(description="") - add_encoder_params(parser) - add_tokenizer_params(parser) - add_cuda_params(parser) - args = parser.parse_args() - - setup_args_gpu(args) - - print(args) - - saved_state = load_states_from_checkpoint(model_file) - set_encoder_params_from_state(saved_state.encoder_params, args) - - self.tensorizer, self.encoder, _ = init_biencoder_components( - args.encoder_model_type, args, inference_only=True - ) - - self.encoder = self.encoder.ctx_model - - self.encoder, _ = setup_for_distributed_mode( - self.encoder, - None, - args.device, - args.n_gpu, - args.local_rank, - args.fp16, - args.fp16_opt_level, - ) - self.encoder.eval() - - model_to_load = get_model_obj(self.encoder) - - prefix_len = len("ctx_model.") - ctx_state = { - key[prefix_len:]: value - for (key, value) in saved_state.model_dict.items() - if key.startswith("ctx_model.") - } - model_to_load.load_state_dict(ctx_state) - self.device = args.device - - def predict(self, text): - - st = time.time() - batch_token_tensors = [self.tensorizer.text_to_tensor(text)] - - ctx_ids_batch = move_to_device( - torch.stack(batch_token_tensors, dim=0), self.device - ) - ctx_seg_batch = move_to_device(torch.zeros_like(ctx_ids_batch), self.device) - ctx_attn_mask = move_to_device( - self.tensorizer.get_attn_mask(ctx_ids_batch), self.device - ) - with torch.no_grad(): - _, embedding, _ = self.encoder(ctx_ids_batch, ctx_seg_batch, ctx_attn_mask) - embedding = embedding.cpu().numpy() - return embedding - - -def embed_passages(sents, retriever_model, num_sent_in_pass=10): - passages = [] - embeddings = [] - for i in range(0, len(sents), num_sent_in_pass): - passages.append(" ".join(sents[i : i + num_sent_in_pass])) - embeddings.append(retriever_model.predict(passages[-1])) - return passages, embeddings - - -def get_passages(sents, num_sent_in_pass=10): - passages = [] - for i in range(0, len(sents), num_sent_in_pass): - passages.append(" ".join(sents[i : i + num_sent_in_pass])) - return passages - - -def generate_embeddings(model_file, diff_dir, embedding_dir): - - # create retriever - retriever_model = Retriever(model_file) - - # loop through files - index = 0 - # gpu = 2 - for filename in tqdm(os.listdir(diff_dir)): - - # index += 1 - # if index % 5 != gpu: - # continue - - new_id = filename.replace(".json", "").split("_")[0] - old_id = filename.replace(".json", "").split("_")[1] - - for revid in [new_id, old_id]: - - data = json.load(open(os.path.join(diff_dir, filename))) - - if len(data["diffs"]) == 0: - continue - - if revid == data["orig_id"]: - sents = [d["sent_a"] for d in data["diffs"][0]] - filepath = os.path.join(embedding_dir, revid + "_orig.pkl") - elif revid == data["new_id"]: - sents = [d["sent_b"] for d in data["diffs"][0]] - filepath = os.path.join(embedding_dir, revid + "_new.pkl") - - if not os.path.exists(filepath): - passages, embeddings = embed_passages(sents, retriever_model) - - pickle.dump( - { - "timestamp": data["timestamp"], - "passages": passages, - "file": filepath, - "embeddings": embeddings, - }, - open(filepath, "wb"), - ) diff --git a/wikipedia/preprocessing/generate_diffs.py b/wikipedia/preprocessing/generate_diffs.py deleted file mode 100644 index 400ed3a..0000000 --- a/wikipedia/preprocessing/generate_diffs.py +++ /dev/null @@ -1,442 +0,0 @@ -from tqdm import tqdm -import re -from collections import defaultdict -import os -from bs4 import BeautifulSoup -import pickle -import difflib -import scipy - -# import spacy -# from benepar.spacy_plugin import BeneparComponent -from nltk.translate.bleu_score import sentence_bleu - - -def read_incr_dump(d): - edit_titles = {} - for folder in tqdm(os.listdir(d)): - for file in os.listdir(os.path.join(d, folder)): - f = os.path.join(d, folder, file) - data = open(f, "r").read() - - # parse - soup = BeautifulSoup(data, "html.parser") - - for doc in soup.find_all("doc"): - id = doc.get("id") - title = doc.get("title") - url = doc.get("url") - text = doc.get_text() - assert title not in edit_titles - edit_titles[title] = { - "id": id, - "title": title, - "url": url, - "text": text, - } - return edit_titles - - -def read_dump(d, edits): - documents = [] - for file in tqdm(os.listdir(d)): - data = pickle.loads(open(os.path.join(d, file), "rb").read()) - for doc in data: - title = doc["title"] - if title in edits and edits[title]["id"] == doc["id"]: - documents.append(doc) - return documents - - -def get_spans(sent_diffs): - spans = [] - start = None - for i in range(len(sent_diffs)): - r = sent_diffs[i] - if r[:2] == "+ " or r[:2] == "- ": - if start is None: - start = i - - if start is not None: - if (r[:2] != "+ " and r[:2] != "- ") or i == len(sent_diffs) - 1: - spans.append((start, i)) - start = None - return spans - - -def get_diffs(sent_a, sent_b): - d = difflib.Differ() - result = list(d.compare(sent_a, sent_b)) - sent_a_diffs = [r for r in result if "+ " not in r] - sent_b_diffs = [r for r in result if "- " not in r] - return sent_a_diffs, sent_b_diffs - - -def split_sentences(text): - rtext = text.replace(".\n", ".\n") - rtext = rtext.replace(". ", ". ") - sentences = rtext.split("") - assert len(text) == sum( - [len(s) for s in sentences] - ), f"Invalid length {len(text)}, {sum([len(s) for s in sentences])}" - - index_to_sent = [-1] * len(text) - sent_to_index = [-1] * len(sentences) - index = 0 - for i in range(len(sentences)): - sent_to_index[i] = index - for j in range(len(sentences[i])): - # map character index to sentence index - index_to_sent[index] = i - index += 1 - - assert index == len(text), f"changed text len {index}, {len(text)}" - for i in index_to_sent: - assert i >= 0 - - return sentences, index_to_sent, sent_to_index - - -def get_diff_spans(doc, doc_diffs_raw, nlp): - - # get spans - doc_spans = get_spans(doc_diffs_raw) - - # get sentences from - sentences, index_to_sent, sent_to_index = split_sentences(doc) - - sent_diffs = [] - - for span in doc_spans: - - # print('DIFF:', doc[span[0]:span[1]]) - - # get sentence indices - start_i = index_to_sent[span[0]] - end_i = index_to_sent[span[1]] - sent_ind = range(start_i, end_i + 1, 1) if end_i > start_i else [start_i] - - # offset char indices - offset = sent_to_index[start_i] - diff_span = (span[0] - offset, span[1] - offset) - - # parse sentence - sent_comb = " ".join([sentences[i] for i in sent_ind]) - if len(sent_comb) > 10000: - print("sentence too long", len(sent_comb)) - continue - - try: - - parsed = nlp(sent_comb) - csent_all = list(parsed.sents) - - # generate word spans - word_spans = [] - words = [] - for csent in csent_all: - for constituent in csent._.constituents: - - word_spans.append((constituent.start, constituent.end)) - if DEBUG: - print("C:", const_offset, constituent) - if constituent.start + 1 == constituent.end: - words.append((constituent.start, str(constituent))) - except Exception as e: - print(e) - continue - - # map word indices to character indices - index = 0 - word_to_char_index = {len(words): len(sent_comb)} - # TODO: make sure to sort words - for word_index, word in words: - csize = len(word) - while str(sent_comb[index : index + csize]) != str(word): - index += 1 - word_to_char_index[word_index] = index - - # convert word spans to char spans - char_spans = [ - (word_to_char_index[s[0]], word_to_char_index[s[1]]) for s in word_spans - ] - - # find minimal length span - min_i = None - min_length = len(sent_comb) - for i in range(len(char_spans)): - span_len = char_spans[i][1] - char_spans[i][0] - if ( - char_spans[i][0] <= diff_span[0] - and char_spans[i][1] >= diff_span[1] - and span_len <= min_length - ): - min_i = i - min_length = span_len - - if min_i is None: - span_text = sent_comb - # print("COULD NOT DETERMINE SPAN") - # print(doc[span[0]:span[1]]) - else: - span_text = sent_comb[char_spans[min_i][0] : char_spans[min_i][1]] - - # generate span text - diff_text = doc[span[0] : span[1]] - sent_diffs.append((diff_text, span_text)) - - if DEBUG: - print("WORD SPANS", word_spans) - print("CHAR SPANS", char_spans) - print("DIFF SPAN", diff_span) - print("DIFF", diff_text) - print(char_spans[min_i], span_text) - print() - - return sent_diffs - - -def get_diffs(sent_a, sent_b): - d = difflib.Differ() - result = list(d.compare(sent_a, sent_b)) - sent_a_diffs = [r for r in result if "+ " not in r] - sent_b_diffs = [r for r in result if "- " not in r] - return sent_a_diffs, sent_b_diffs - - -def get_diffs(sent_a, sent_b): - d = difflib.Differ() - result = list(d.compare(sent_a, sent_b)) - sent_a_diffs = [r for r in result if "+ " not in r] - sent_b_diffs = [r for r in result if "- " not in r] - return sent_a_diffs, sent_b_diffs - - -def get_diffs(sent_a, sent_b): - d = difflib.Differ() - result = list(d.compare(sent_a, sent_b)) - sent_a_diffs = [r for r in result if "+ " not in r] - sent_b_diffs = [r for r in result if "- " not in r] - return sent_a_diffs, sent_b_diffs - - -def get_diffs(sent_a, sent_b): - d = difflib.Differ() - result = list(d.compare(sent_a, sent_b)) - sent_a_diffs = [r for r in result if "+ " not in r] - sent_b_diffs = [r for r in result if "- " not in r] - return sent_a_diffs, sent_b_diffs - - -def get_sentence_diff(sent_a, sent_b, nlp=None): - - # get spans from differ - sent_a_diffs_raw, sent_b_diffs_raw = get_diffs(sent_a, sent_b) - - if nlp is None: - diffs_a = sent_a_diffs_raw - diffs_b = sent_b_diffs_raw - else: - diffs_a = list(set(get_diff_spans(sent_a, sent_a_diffs_raw, nlp))) - diffs_b = list(set(get_diff_spans(sent_b, sent_b_diffs_raw, nlp))) - - span_diffs_a = [d[1] for d in diffs_a] - span_diffs_b = [d[1] for d in diffs_b] - raw_diffs_a = [d[0] for d in diffs_a] - raw_diffs_b = [d[0] for d in diffs_b] - - return { - "sent_a": sent_a, - "sent_b": sent_b, - "sent_a_diffs": span_diffs_a, - "sent_b_diffs": span_diffs_b, - "sent_a_raw_diffs": raw_diffs_a, - "sent_b_raw_diffs": raw_diffs_b, - } - - -def generate_diffs(documents, edits): - all_diffs = [] - i = 0 - for article in tqdm(documents): - title = article["title"] - edit = edits[title] - - sent_a = article["text"] - sent_b = edit["text"] - - doc_id = article["id"] - assert ( - article["id"] == edit["id"] - ), f"Mismatch article - title: {title}, {edit['title']}, id: {article['id']}, {edit['id']}" - - DEBUG = False - # run: python -m spacy download en - # nlp = spacy.load("en") - ## nlp = spacy.load("en_core_web_sm") - # nlp.add_pipe(BeneparComponent("benepar_en3")) - - diff = get_sentence_diff(sent_a, sent_b, nlp) - diff["title"] = (title,) - diff["doc_id"] = doc_id - all_diffs.append(diff) - - if len(all_diffs) > 1000: - print("Writing", i) - pickle.dump(all_diffs, open(f"output/diffs_{i}.pkl", "wb")) - all_diffs = [] - - i += 1 - - -def check_alphanumeric(s): - return re.match("(?s).*[a-zA-Z0-9]+(?s).*$", s) is not None - - -def generate_sentence_level_diffs(documents, edits): - all_diffs = [] - count = 0 - - # for article in tqdm(documents): - for article in documents: - title = article["title"] - edit = edits[title] - sent_a = article["text"] - sent_b = edit["text"] - - splits_a, index_to_sent_a, sent_to_index_a = split_sentences(sent_a) - splits_b, index_to_sent_b, sent_to_index_b = split_sentences(sent_b) - - d = difflib.Differ( - linejunk=lambda x: x in " \n", charjunk=lambda x: x in " \n \t" - ) - diff = list(d.compare(splits_a, splits_b)) - - index = 0 - last_match = 0 - options = defaultdict(list) - for i in range(len(diff)): - - code = diff[i][:2] - if code == "? ": - continue - elif code == "+ ": - options[last_match].append(diff[i]) - elif code == "- ": - options[last_match].append(diff[i]) - else: - options[index] = diff[i][2:] - last_match = index + 1 - index += 1 - - diff_data = [] - - has_diff = False - for key, value in options.items(): - # print(key, value) - if not isinstance(value, list): - diff_data.append( - { - "sent_a": value, - "sent_b": value, - "sent_a_diffs": [], - "sent_b_diffs": [], - "diff_type": None, - } - ) - continue - - diff_a = [d[2:] for d in value if "- " in d] - diff_b = [d[2:] for d in value if "+ " in d] - - has_diff = True - - # nlp = spacy.load("en") - # nlp.add_pipe(BeneparComponent("benepar_en3")) - - for da in diff_a: - match = False - for i in range(len(diff_b) - 1, -1, -1): - db = diff_b[i] - score = sentence_bleu([da.split()], db.split()) - # print(score) - if score > 0.1: - # local_a, local_b = get_diffs(da, db) - diff = get_sentence_diff(da, db, nlp=None) - diff["diff_type"] = "EDIT" - diff["score"] = score - - # filter alphanumeric - # orig_a = list( diff["sent_a_diffs"]) - # orig_b = list( diff["sent_b_diffs"]) - diff["sent_a_diffs"] = [ - d for d in diff["sent_a_diffs"] if check_alphanumeric(d) - ] - diff["sent_b_diffs"] = [ - d for d in diff["sent_b_diffs"] if check_alphanumeric(d) - ] - if ( - len(diff["sent_a_diffs"]) == 0 - and len(diff["sent_b_diffs"]) == 0 - and da == db - ): - diff["diff_type"] = None - # print("CONVERT EDIT TO NONE") - # print(diff["sent_a_raw_diffs"]) - # print(diff["sent_b_raw_diffs"]) - # print(orig_a) - # print(orig_b) - - # pprint(diff) - diff_data.append(diff) - del diff_b[i] # avoid double counting - match = True - break - - if not match: - diff_data.append( - { - "sent_a": da, - "sent_b": "", - "sent_a_diffs": [da], - "sent_b_diffs": [], - "diff_type": "DELETE", - } - ) - - for db in diff_b: - diff_data.append( - { - "sent_a": "", - "sent_b": db, - "sent_a_diffs": [], - "sent_b_diffs": [db], - "diff_type": "INSERT", - } - ) - - # pprint([d for d in diff_data if d['diff_type'] is not None]) - all_diffs.append(diff_data) - count += 1 - - # if len(all_diffs) > 1000: - # print("Writing", count) - # pickle.dump(all_diffs, open(f"output/sent_diffs_{count}.pkl", "wb")) - # all_diffs = [] - - return all_diffs, has_diff - - -def main(): - edits = read_incr_dump("/home/ubuntu/incr-enwiki-20190206/text/") - print("finished reading edits", len(edits.keys())) - documents = read_dump("/home/ubuntu/enwiki-20190201/tmp/parsed", edits) - print("finished reading docs", len(documents)) - - print("generating diffs...") - # generate_diffs(documents, edits) - generate_sentence_level_diffs(documents, edits) - - -if __name__ == "__main__": - main() diff --git a/wikipedia/preprocessing/wiki_api_data.py b/wikipedia/preprocessing/wiki_api_data.py deleted file mode 100644 index 67235c5..0000000 --- a/wikipedia/preprocessing/wiki_api_data.py +++ /dev/null @@ -1,682 +0,0 @@ -import os -import time -import pickle -import json -from tqdm import tqdm -from collections import defaultdict -import subprocess - -import configparser -import argparse - -import pandas as pd -import numpy as np - -from multiprocessing import Pool - -# from concurrent.futures import ProcessPoolExecutor -from bs4 import BeautifulSoup - -# from generate diffs file (originally from DPR repo... sorry kevin) -from generate_diffs import generate_sentence_level_diffs -from embedding import generate_embeddings - - -def query_recentchanges(start_time, end_time, revision_file): - pass - - -def query_doc_versions(titles_file, start_time, end_time, raw_doc_dir): - # TODO: query doc versions - titles_df = pd.read_csv(titles_file) - titles = list(set(top_titles.index.tolist())) - pass - - -def get_recent_changes(revisions_dir, changes_file): - changes = [] - revids = set([]) - files = os.listdir(revisions_dir) - for i in range(len(files)): - f = files[i] - f_changes = json.loads(open(os.path.join(revisions_dir, f), "r").read()) - - for change in f_changes: - if change["revid"] in revids: - continue - - changes.append(change) - revids.add(change["revid"]) - - # if i % 100 == 0: - # print(f"Read {i}/{len(files)}, changes so far: {len(changes)}") - - # create dataframe - changes_df = pd.DataFrame(changes) - - # create time index - changes_df["datetime"] = pd.to_datetime(changes_df["timestamp"]) - changes_df = changes_df.set_index("datetime").sort_index() - - # save to CSV file - changes_df.to_csv(changes_file) - - return changes_df - - -def get_titles(changes_file, titles_file, n=200): - changes_df = pd.read_csv(changes_file) - title_ids = set(changes_df[["title", "pageid"]].apply(tuple, axis=1).tolist()) - - counts = changes_df.title.value_counts().to_frame() - top_titles = counts[counts["title"] > n] - top_titles.columns = ["count"] - top_titles["title"] = top_titles.index - top_titles.to_csv(titles_file) - return top_titles - - -def get_edits(edits_file, changes_file, titles_file): - changes_df = pd.read_csv(changes_file) - titles_df = get_titles(changes_file, titles_file) - titles = list(set(titles_df.index.tolist())) - edits_df = changes_df[changes_df.title.apply(lambda x: x in titles)] - - # assign timestamps - edits_df["ts_min"] = ( - pd.to_datetime(edits_df["datetime"]) - .astype(np.int64) - .apply(assign_timestamps_min) - ) - - # write CSV - edits_df.to_csv(edits_file) - return edits_df - - -def get_questions(raw_questions_file, questions_file): - questions_df = pd.read_csv(raw_questions_file, sep="\t") - questions_df.columns = [ - "question", - "answer", - "doc_id", - "datetime", - "revid", - "oldrevid", - ] - - # assign timestamps - questions_df["ts_min"] = ( - pd.to_datetime(questions_df["datetime"]) - .astype(np.int64) - .apply(assign_timestamps_min) - ) - - # write CSV - questions_df.to_csv(questions_file) - return questions_df - - -# create diff JSON file from valid list of revision pairs, doc pkl -def create_diff_json(doc_pkl, rev_pairs, diff_dir): - - # load data for file - data = pickle.loads(open(doc_pkl, "rb").read()) - title = os.path.basename(doc_pkl).replace(".pkl", "") - - for i in range(len(data)): - orig_doc = data[i] - - for j in range(0, len(data), 1): - new_doc = data[j] - - rev_pair = orig_doc["id"] + "_" + new_doc["id"] - - if rev_pair not in rev_pairs: - continue - - diff_file = os.path.join(diff_dir, rev_pair + ".json") - if os.path.exists(diff_file): - # skip - continue - - edits = {orig_doc["title"]: new_doc} - try: - all_diffs = generate_sentence_level_diffs([orig_doc], edits) - except Exception as e: - print(e) - raise ValueError(f"Failed to parse diffs {rev_pair}") - diff = { - "title": orig_doc["title"], - "timestamp": rev_pairs[rev_pair], - "orig_id": orig_doc["id"], - "new_id": new_doc["id"], - "diffs": all_diffs, - } - open(diff_file, "w").write(json.dumps(diff, indent=2)) - - -def generate_diffs_helper(filename, diff_dir, rev_pair, timestamp): - - data = pickle.loads(open(filename, "rb").read()) - - for i in range(len(data)): - for j in range(len(data)): - orig_doc = data[i] - new_doc = data[j] - - if new_doc["id"] + "_" + orig_doc["id"] != rev_pair: - continue - - # parse diffs - diff_file = os.path.join(diff_dir, rev_pair + ".json") - - if os.path.exists(diff_file): - continue - - edits = {orig_doc["title"]: new_doc} - st = time.time() - all_diffs, has_diff = generate_sentence_level_diffs([orig_doc], edits) - # print("runtime", time.time() - st) - diff = { - "title": orig_doc["title"], - "timestamp": timestamp, - "orig_id": orig_doc["id"], - "new_id": new_doc["id"], - "diffs": all_diffs, - } - if has_diff: - diff = { - "title": orig_doc["title"], - "timestamp": timestamp, - "orig_id": orig_doc["id"], - "new_id": new_doc["id"], - "diffs": all_diffs, - } - else: - diff = { - "title": orig_doc["title"], - "timestamp": timestamp, - "orig_id": orig_doc["id"], - "new_id": new_doc["id"], - "diffs": [], - } - # TODO: write to tmp file first (make sure we dont have messed up files) - open(diff_file, "w").write(json.dumps(diff, indent=2)) - return - - -def generate_diffs( - edits_file, titles_file, parsed_doc_dir, diff_dir, revision_file, workers=32 -): - - # make sure title is in titles df - titles_df = pd.read_csv(titles_file) - titles = list(set(titles_df.title.tolist())) - - # print(titles) - - # filter out revision pairs not in edits_file - edits_df = pd.read_csv(edits_file) - title_to_rev_pairs = defaultdict(dict) - for index, row in edits_df.iterrows(): - if row["title"] not in titles: - continue # skip if not top title - - # map title -> (revid, old_revid) -> timestamp of revision - rev_pair = str(row["revid"]) + "_" + str(row["old_revid"]) - title_to_rev_pairs[row["title"]][rev_pair] = row["timestamp"] - - open(revision_file, "w").write(json.dumps(title_to_rev_pairs)) - - num_keys = len(title_to_rev_pairs.keys()) - # print( - # f"Proceessing revisions for {num_keys} titles, writing to {diff_dir}" - # ) - - inputs = [] - for title in tqdm(titles): - filename = os.path.join(parsed_doc_dir, f"{title}.pkl") - if not os.path.exists(filename): - print("missing", filename) - continue - - for rev_pair in title_to_rev_pairs[title].keys(): - if os.path.exists(os.path.join(diff_dir, rev_pair + ".json")): - continue - inputs.append( - (filename, diff_dir, rev_pair, title_to_rev_pairs[title][rev_pair]) - ) - - print("processing revids", len(inputs), diff_dir) - chunk_size = 100000 - for i in range(0, len(inputs), chunk_size): - p = Pool(128) - print("created pool", i, i + chunk_size, len(inputs)) - p.starmap(generate_diffs_helper, inputs[i : i + chunk_size]) - p.close() - - return - - # diff remaining - inputs = [ - ( - os.path.join(parsed_doc_dir, f"{title}.pkl"), - title_to_rev_pairs[title], - diff_dir, - ) - for title in titles - ] - p = Pool(workers) - p.starmap(create_diff_json, inputs) - p.close() - - -# convert wikipedia dump into single pkl file per title -def dump_to_pickle_title(top_folder, target_dir, title): - total = 0 - docs = [] - for folder in os.listdir(top_folder): - for file in os.listdir(os.path.join(top_folder, folder)): - - filename = os.path.join(top_folder, folder, file) - data = open(filename, "r").read() - soup = BeautifulSoup(data, "html.parser") - - for doc in soup.find_all("doc"): - id = doc.get("id") - title = doc.get("title") - url = doc.get("url") - text = doc.get_text() - docs.append({"id": id, "url": url, "title": title, "text": text}) - total += len(docs) - pickle.dump(docs, open(os.path.join(target_dir, title + ".pkl"), "wb")) - return os.path.join(target_dir, title + ".pkl") - - -# call wikiextractor library on XML -def extract(title, raw_doc_dir, parsed_tmp_dir, parsed_doc_dir): - f = f"{raw_doc_dir}/{title}" - bashCommand = f"wikiextractor {f} -o {parsed_tmp_dir}/tmp_parsed{title}" - - process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE) - output, error = process.communicate() - - pkl_file = dump_to_pickle_title( - f"{parsed_tmp_dir}/tmp_parsed{title}", parsed_doc_dir, title - ) - - -def parse_docs(raw_doc_dir, parsed_tmp_dir, parsed_doc_dir, workers=32): - # parse documents from raw XML - - # extract individual doc - files = os.listdir(raw_doc_dir) - # TODO: add assert to make sure titles correspond to filenames - files = [ - (f, raw_doc_dir, parsed_tmp_dir, parsed_doc_dir) - for f in files - if not os.path.isdir(f) - ] - - # create pool and run - p = Pool(workers) - p.starmap(extract, files) - p.close() - - -# assign timesteps -def assign_timestamps_min(ts): - # take in unix timestamp - covert to integer - start_ts = 1628131044000000000 # don't change - delta = ts - start_ts - if delta < 0: - return None - - return int(delta / (60 * 1000000000)) - - -def generate_simulation_data( - questions_file, - edits_file, - diff_dir, - init_data_file, - stream_edits_file, - stream_questions_file, -): - edits_df = pd.read_csv(edits_file) - questions_df = pd.read_csv(questions_file) - - # lists for questions/edits at each timestep - questions = [] - edits = [] - - # initialization data for embeddings/passages - init_data = {} - - # timestamp to stop - max_ts = int(questions_df.ts_min.max()) - - # loop through timestamps - for ts in range(max_ts + 1): - - ts_edits = defaultdict(list) - ts_queries = defaultdict(list) - for index, row in edits_df[edits_df["ts_min"] == ts].iterrows(): - filename = str(row["revid"]) + "_" + str(row["old_revid"]) + ".json" - key = row["pageid"] - - # make sure file is OK - file_path = os.path.join(diff_dir, filename) - if os.path.exists(file_path): - try: - data = json.load(open(file_path)) - if len(data["diffs"]) == 0: - continue - diffs = data["diffs"][0] - except Exception as e: - print(file_path) - print(e) - continue - diff_types = [ - d["diff_type"] for d in diffs if d["diff_type"] is not None - ] - if len(diff_types) == 0: - print(f"Invalid file {filename}") - continue - assert str(data["orig_id"]) == str( - row["old_revid"] - ), f"Invalid id {filename}, id {data['orig_id']} row {row['revid']}" - - if key not in init_data: - diffs = data["diffs"][0] - init_data[key] = { - "revid": data["orig_id"], - "sents": [d["sent_a"] for d in diffs], - "file": filename, - "ts_min": row["ts_min"], - } - ts_edits[key].append(filename) - - else: - # print("missing", file_path) - continue - - for index, row in questions_df[questions_df["ts_min"] == ts].iterrows(): - key = row["doc_id"] - ts_queries[key].append( - { - "question": row["question"], - "doc_id": key, - "answer": row["answer"], - "datetime": row["datetime"], - "ts_min": row["ts_min"], - "revid": row["revid"], - "old_revid": row["oldrevid"], - } - ) - - edits.append(ts_edits) - questions.append(ts_queries) - - if ts % 1000 == 0: - unique_files = set([]) - for e in edits: - for files in e.values(): - for f in files: - unique_files.add(f) - print(f"Num edits ts {ts}/{max_ts+1}: {len(unique_files)}") - - open(stream_edits_file, "w").write(json.dumps(edits)) - open(stream_questions_file, "w").write(json.dumps(questions)) - open(init_data_file, "w").write(json.dumps(init_data)) - - -def search_answer(rev_file, embedding_dir, question): - # read file and see if answer is contained - revid = rev_file.replace(".json", "").split("_")[0] - # assert str(revid) == str(question["revid"]), f"Invalid id {revid}, {question}" - embedding_filename = os.path.join(embedding_dir, f"{revid}_new.pkl") - try: - passages = pickle.load(open(embedding_filename, "rb"))["passages"] - except Exception as e: - print(e) - print("File error", embedding_filename) - return False - - found_answer = False - for passage in passages: - if question["answer"] in passage: - found_answer = True - return found_answer - - -def generate_key_weights(pageview_file, titles_file): - pass - - -def check_dataset( - titles_file, - edits_file, - init_data_file, - stream_edits_file, - stream_questions_file, - diff_dir, -): - # TODO: add checks (init data keys match stream keys, questions match keys, etc.) - - # load data - edits_df = pd.read_csv(edits_file) - titles_df = get_titles(changes_file, titles_file) - titles = list(set(titles_df.index.tolist())) - init_data = json.load(open(init_data_file)) - edits = json.load(open(stream_edits_file)) - questions = json.load(open(stream_questions_file)) - - # same length - assert len(questions) == len(edits) - - for ts in range(len(questions)): - for doc_id in questions[ts].keys(): - if not doc_id in init_data: - print("missing doc", doc_id) - continue - for question in questions[ts][doc_id]: - # print(question) - answer = question["answer"] - # import pdb; pdb.set_trace() - - # question = questions[ts][doc_id] - rev_file = ( - str(question["revid"]) + "_" + str(question["old_revid"]) + ".json" - ) - - if not os.path.exists(os.path.join(diff_dir, rev_file)): - print("Still missing diff", rev_file) - continue - - # question generated from document edit - assert it was created before - found = False - revision_file = None - found_index = 0 - for i in range(ts): - if doc_id in edits[ts - i]: - if rev_file in edits[ts - i][doc_id]: - found = True - revision_file = rev_file - found_index = ts - i - break - if not found: - # only option is that it was derived from original doc - assert str(init_data[doc_id]["revid"]) == str( - question["old_revid"] - ), f"Missing revision {ts}, {rev_file}, {doc_id}, init version {init_data[doc_id]['revid']}" - revision_file = init_data[doc_id]["file"] - - # search for answer in revision file - found_answer = search_answer(revision_file, embedding_dir, question) - if not found_answer: - print("NOT FOUND", found_answer, revision_file) - else: - print("FOUND", found_answer, revision_file) - - if ( - question["question"] - == "how far is hurricane ida from cuba?????????????????" - ): - print("DEBUG", question) - print(rev_file) - print("question ts", ts, "edit ts", found_index) - for i in range(found_index, ts + 1, 1): - if doc_id in edits[i]: - print( - i, - edits[i][doc_id], - search_answer( - edits[i][doc_id][-1], embedding_dir, question - ), - ) - print(found_answer) - - # docid_to_title = {} - # for index, row in edits_df.iterrows(): - # docid_to_title[row["pageid"]] = row["title"] - - # open("docid_to_title.json", "w").write(json.dumps(docid_to_title)) - - ## check matching keys - # last_doc = init_data - # for i in len(edits): - # # TODO: assert that question actually contained in this edit? - # continue - - # check each edit is contained - - # check raw edit timestamp is same as query timestamp - - -if __name__ == "__main__": - - print("starting script") - - # configuration file - config = configparser.ConfigParser() - config.read("config.yml") - - # argument flags - parser = argparse.ArgumentParser() - parser.add_argument( - "--run_query_recentchanges", action="store_true", default=False - ) # query wiki api for recentchanges - parser.add_argument( - "--run_query_doc_versions", action="store_true", default=False - ) # query wiki api for doc versions - parser.add_argument( - "--run_recent_changes", action="store_true", default=False - ) # re-processing api changes data - parser.add_argument( - "--run_parse_docs", action="store_true", default=False - ) # re-parse document versions - parser.add_argument("--run_get_questions", action="store_true", default=False) - parser.add_argument( - "--run_generate_diffs", action="store_true", default=False - ) # re-process generating diffs - parser.add_argument( - "--run_generate_simulation_data", action="store_true", default=False - ) - parser.add_argument("--run_check_dataset", action="store_true", default=False) - parser.add_argument("--run_generate_embeddings", action="store_true", default=False) - args = parser.parse_args() - - # directories - data_dir = config["directory"]["data_dir"] - revisions_dir = config["directory"]["revisions_dir"] - raw_doc_dir = config["directory"]["raw_doc_dir"] - parsed_doc_dir = config["directory"]["parsed_doc_dir"] - parsed_tmp_dir = config["directory"]["parsed_tmp_dir"] - diff_dir = config["directory"]["diff_dir"] - embedding_dir = config["directory"]["embedding_dir"] - - # intermediate files - model_file = config["files"]["model_file"] - changes_file = config["files"]["changes_file"] - titles_file = config["files"]["titles_file"] - revisions_file = config["files"]["revisions_file"] - edits_file = config["files"]["edits_file"] - raw_questions_file = config["files"]["raw_questions_file"] - questions_file = config["files"]["questions_file"] - pageview_file = config["files"]["pageview_file"] - - # simulation data - init_data_file = config["simulation"]["init_data_file"] - stream_edits_file = config["simulation"]["stream_edits_file"] - stream_questions_file = config["simulation"]["stream_questions_file"] - - if args.run_query_recentchanges: - query_edit_stream(start_time, end_time, revisions_dir) - - if args.run_query_doc_versions: - query_doc_versions(titles_file, start_time, end_time, raw_doc_dir) - - if args.run_recent_changes: - print("Generating from revisions", revisions_dir) - changes_df = get_recent_changes(revisions_dir, changes_file) - - print("Generated changes file", changes_file) - titles_df = get_titles(changes_file, titles_file) - print("Generated titles file", titles_file) - edits_df = get_edits(edits_file, changes_file, titles_file) - print("Generated edits file", edits_file) - - # query document versions for list of titles - if args.run_query_doc_versions: - if not os.path.exists(raw_doc_dir): - os.mkdir(raw_doc_dir) - query_doc_versions(titles_file, start_time, end_time, raw_doc_dir) - - # parse documents - if args.run_parse_docs: - if not os.path.exists(parsed_doc_dir): - os.mkdir(parsed_doc_dir) - if not os.path.exists(parsed_tmp_dir): - os.mkdir(parsed_tmp_dir) - parse_docs(raw_doc_dir, parsed_tmp_dir, parsed_doc_dir, workers=32) - - # get questions - if args.run_get_questions: - questions_df = get_questions(raw_questions_file, questions_file) - print("Generated questions file", questions_file) - - # generate diffs between document versions - if args.run_generate_diffs: - # if not os.path.isdir(diff_dir): - # os.mkdir(diff_dir) - generate_diffs( - edits_file, titles_file, parsed_doc_dir, diff_dir, revisions_file - ) - - # generate simulation data - if args.run_generate_simulation_data: - generate_simulation_data( - questions_file, - edits_file, - diff_dir, - init_data_file, - stream_edits_file, - stream_questions_file, - ) - - # run tests to validate simulation data - if args.run_check_dataset: - check_dataset( - titles_file, - edits_file, - init_data_file, - stream_edits_file, - stream_questions_file, - diff_dir, - ) - - # generate embeddings for revids from diffs (make passages) - if args.run_generate_embeddings: - generate_embeddings(model_file, diff_dir, embedding_dir) diff --git a/wikipedia/run_wiki.sh b/wikipedia/run_wiki.sh deleted file mode 100644 index 55a3035..0000000 --- a/wikipedia/run_wiki.sh +++ /dev/null @@ -1,10 +0,0 @@ -FILE="passages_sent_diffs_10010.pkl" -MODEL_FILE="/home/ubuntu/DPR/checkpoint/retriever/single/nq/bert-base-encoder.cp" -SEND_RATE=100 -DATA_DIR="/home/ubuntu/flink-feature-flow/RayServer/data/" -EXP_DIR="/home/ubuntu/flink-feature-flow/RayServer/experiments/" -TIMESTAMP=$(date +%s) -EXP="experiment_$TIMESTAMP" -echo $EXP; -python wiki_server.py --data-dir $DATA_DIR --send-rate $SEND_RATE --exp-dir $EXP_DIR --exp $EXP --file $FILE --model_file $MODEL_FILE -#python wiki_client.py --exp $EXP_DIR diff --git a/wikipedia/simulate.py b/wikipedia/simulate.py deleted file mode 100644 index 7b11339..0000000 --- a/wikipedia/simulate.py +++ /dev/null @@ -1,217 +0,0 @@ -import json -from typing import DefaultDict, Dict, List, Optional, Tuple -from collections import defaultdict -from dataclasses import dataclass -from functools import cmp_to_key - -import configparser - -import pandas as pd - -import simpy -from ralf.state import Record -from ralf.policies.load_shedding_policy import ( - always_process, - make_mean_policy, - make_sampling_policy, -) -from ralf.policies.processing_policy import fifo, lifo # , make_sorter_with_key_weights -from ralf.simulation.priority_queue import PerKeyPriorityQueue -from ralf.simulation.source import JSONSource -from ralf.simulation.window import WindowOperator -from ralf.simulation.mapper import ( - RalfMapper, - RoundRobinLoadBalancer, - CrossKeyLoadBalancer, -) - - -from ralf.policies.load_shedding_policy import ( - always_process, - newer_processing_time, - later_complete_time, - make_sampling_policy, - make_mean_policy, - make_cosine_policy, -) - -from typing import Dict, List, Tuple, Type - - -class WeightedLoadBalancer(CrossKeyLoadBalancer): - - # def __init__(self, keys, key_weights): - # self.keys = keys - # self.key_weights = key_weights - - def choose(self, per_key_queues: Dict[str, PerKeyPriorityQueue]) -> str: - - chosen_key = None - max_len = 0 - for key in per_key_queues.keys(): - if per_key_queues[key].size() > max_len: - chosen_key = key - max_len = per_key_queues[key].size() - # print("choose", chosen_key, max_len) - return chosen_key - - -class WikiMapper(RalfMapper): - def __init__( - self, - env: simpy.Environment, - source_queues: Dict[str, PerKeyPriorityQueue], - key_selection_policy_cls: Type[CrossKeyLoadBalancer], - model_run_time_s: float, - keys: List[str], - ) -> None: - - super().__init__(env, source_queues, key_selection_policy_cls, model_run_time_s) - self.keys = keys - - # self.env = env - # self.source_queues = source_queues - # self.key_selection_policy = key_selection_policy_cls() - # self.model_runtime_s = model_run_time_s - # self.env.process(self.run()) - - # self.ready_time_to_batch: Dict[float, List[Tuple[int, float]]] = {} - - def run(self): - while True: - if self.env.now > 387: - break - # windows = yield self.source_queue.get() - chosen_key = self.key_selection_policy.choose(self.source_queues) - - if chosen_key is not None: - - # for chosen_key in self.keys: - windows = yield self.source_queues[chosen_key].get() - print( - f"at time {self.env.now:.2f}, RalfMapper should work on {windows} (last timestamp)" - ) - edits = [(val, chosen_key) for val in windows.window[0].value] - print("edits", edits) - - if self.env.now in self.ready_time_to_batch: - self.ready_time_to_batch[self.env.now] += edits - else: - self.ready_time_to_batch[self.env.now] = edits - - yield self.env.timeout(self.model_runtime_s) - - else: # nothing to do - yield self.env.timeout(0.01) - - -policies = { - "fifo": fifo, - "lifo": lifo, - "always_process": always_process, - "sample_half": make_sampling_policy(0.5), -} - - -def run_once( - out_path: str, - prioritization_policy: str, - load_sheeding_policy: str, - keys: List[str], - per_key_records_per_second: int, - total_runtime_s: float, - model_runtime_constant: float, - data_file: str = None, -): - - env = simpy.Environment() - - source_to_window_queue = simpy.Store(env) - windows_to_mapper_queue = { - key: PerKeyPriorityQueue( - env, - processing_policy=policies[prioritization_policy], - load_shedding_policy=policies[load_sheeding_policy], - ) - for key in keys - } - - JSONSource( - env, - records_per_sec_per_key=per_key_records_per_second, - num_keys=len(keys), - next_queue=source_to_window_queue, - total_run_time=total_runtime_s, - data_file=data_file, - ) - - WindowOperator( - env, - window_size=1, - slide_size=1, - source_queue=source_to_window_queue, - next_queues=windows_to_mapper_queue, - ) - - m = WikiMapper( - env, - source_queues=windows_to_mapper_queue, - model_run_time_s=model_runtime_constant, - key_selection_policy_cls=WeightedLoadBalancer, - keys=keys, - ) - env.run(until=total_runtime_s) - - plan = m.ready_time_to_batch - with open(out_path, "w") as f: - json.dump(plan, f) - - -if __name__ == "__main__": - - # load sheding: random, drop short edits - # prioritization: prioritize most recent version - # cross-key prioritzation: historical page views, - - # configuration file - config = configparser.ConfigParser() - config.read("config.yml") - plan_dir = config["simulation"]["plan_dir"] - init_data_file = config["simulation"]["init_data_file"] - stream_edits_file = config["simulation"]["stream_edits_file"] - stream_questions_file = config["simulation"]["stream_questions_file"] - - # load simulation data - edits = json.load(open(stream_edits_file)) - init_data = json.load(open(init_data_file)) - keys = list(init_data.keys()) - - # policies - prioritization_policies = ["fifo", "lifo"] - load_shedding_policies = ["always_process"] - model_runtimes = [0.000001, 0.00001, 0.0000001, 0.000000001, 0] - records_per_second = [100] - - output_files = [] - - for prio_policy in prioritization_policies: - for load_shed_policy in load_shedding_policies: - for runtime in model_runtimes: - for rate in records_per_second: - - out_path = f"{plan_dir}/plan-{prio_policy}-{load_shed_policy}-{runtime}-{rate}.json" - print("running", out_path, runtime) - run_once( - out_path, - prio_policy, - load_shed_policy, - keys, - per_key_records_per_second=rate, - total_runtime_s=len(edits), - model_runtime_constant=runtime, - data_file=stream_edits_file, - ) - output_files.append(out_path) - print("DONE", out_path) - for f in output_files: - print(f)